From 73805113784dc836e12b6244ccc8a4387ddf50a6 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 29 Jun 2024 19:18:37 +0100 Subject: [PATCH 01/58] Everything finally compiles --- Cargo.lock | 1 + crates/red_knot_module_resolver/Cargo.toml | 1 + crates/red_knot_module_resolver/src/lib.rs | 5 + crates/red_knot_module_resolver/src/module.rs | 134 +- crates/red_knot_module_resolver/src/path.rs | 1179 +++++++++++++++++ .../red_knot_module_resolver/src/resolver.rs | 345 ++--- crates/red_knot_python_semantic/src/types.rs | 5 +- .../src/types/infer.rs | 7 +- 8 files changed, 1411 insertions(+), 266 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/path.rs diff --git a/Cargo.lock b/Cargo.lock index e7c8dcb7057a5..8a2bd6e25559f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1884,6 +1884,7 @@ name = "red_knot_module_resolver" version = "0.0.0" dependencies = [ "anyhow", + "camino", "compact_str", "insta", "path-slash", diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index ec05ec525b52f..99e69f35cc27f 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -15,6 +15,7 @@ ruff_db = { workspace = true } ruff_python_stdlib = { workspace = true } compact_str = { workspace = true } +camino = { workspace = true } rustc-hash = { workspace = true } salsa = { workspace = true } tracing = { workspace = true } diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index 72be73c55db65..ad7723fee228d 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -1,9 +1,14 @@ mod db; mod module; +pub mod path; mod resolver; mod typeshed; pub use db::{Db, Jar}; pub use module::{Module, ModuleKind, ModuleName}; +pub use path::{ + ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, SitePackagesPath, + SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, +}; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; pub use typeshed::versions::TypeshedVersions; diff --git a/crates/red_knot_module_resolver/src/module.rs b/crates/red_knot_module_resolver/src/module.rs index 8657c4a196e24..0977b6fdc0f14 100644 --- a/crates/red_knot_module_resolver/src/module.rs +++ b/crates/red_knot_module_resolver/src/module.rs @@ -3,10 +3,13 @@ use std::fmt::Formatter; use std::ops::Deref; use std::sync::Arc; -use ruff_db::file_system::FileSystemPath; -use ruff_db::vfs::{VfsFile, VfsPath}; +use ruff_db::vfs::VfsFile; use ruff_python_stdlib::identifiers::is_identifier; +use crate::path::{ + ExtraPathBuf, FirstPartyPathBuf, ModuleResolutionPathRef, SitePackagesPathBuf, + StandardLibraryPath, StandardLibraryPathBuf, +}; use crate::Db; /// A module name, e.g. `foo.bar`. @@ -62,11 +65,7 @@ impl ModuleName { } fn is_valid_name(name: &str) -> bool { - if name.is_empty() { - return false; - } - - name.split('.').all(is_identifier) + !name.is_empty() && name.split('.').all(is_identifier) } /// An iterator over the components of the module name: @@ -130,30 +129,20 @@ impl ModuleName { &self.0 } - pub(crate) fn from_relative_path(path: &FileSystemPath) -> Option { - let path = if path.ends_with("__init__.py") || path.ends_with("__init__.pyi") { - path.parent()? - } else { - path - }; - - let name = if let Some(parent) = path.parent() { - let mut name = compact_str::CompactString::with_capacity(path.as_str().len()); - - for component in parent.components() { - name.push_str(component.as_os_str().to_str()?); + pub(crate) fn from_relative_path(path: ModuleResolutionPathRef) -> Option { + let path = path.sans_dunder_init(); + let mut parts_iter = path.module_name_parts(); + let first_part = parts_iter.next()?; + if let Some(second_part) = parts_iter.next() { + let mut name = format!("{first_part}.{second_part}"); + for part in parts_iter { name.push('.'); + name.push_str(part); } - - // SAFETY: Unwrap is safe here or `parent` would have returned `None`. - name.push_str(path.file_stem().unwrap()); - - name + Self::new(&name) } else { - path.file_stem()?.to_compact_string() - }; - - Some(Self(name)) + Self::new(first_part) + } } } @@ -194,7 +183,7 @@ impl Module { pub(crate) fn new( name: ModuleName, kind: ModuleKind, - search_path: ModuleSearchPath, + search_path: Arc, file: VfsFile, ) -> Self { Self { @@ -218,7 +207,7 @@ impl Module { } /// The search path from which the module was resolved. - pub fn search_path(&self) -> &ModuleSearchPath { + pub(crate) fn search_path(&self) -> &ModuleSearchPathEntry { &self.inner.search_path } @@ -254,7 +243,7 @@ impl salsa::DebugWithDb for Module { struct ModuleInner { name: ModuleName, kind: ModuleKind, - search_path: ModuleSearchPath, + search_path: Arc, file: VfsFile, } @@ -267,77 +256,42 @@ pub enum ModuleKind { Package, } -/// A search path in which to search modules. -/// Corresponds to a path in [`sys.path`](https://docs.python.org/3/library/sys_path_init.html) at runtime. -/// -/// Cloning a search path is cheap because it's an `Arc`. -#[derive(Clone, PartialEq, Eq)] -pub struct ModuleSearchPath { - inner: Arc, -} - -impl ModuleSearchPath { - pub fn new

(path: P, kind: ModuleSearchPathKind) -> Self - where - P: Into, - { - Self { - inner: Arc::new(ModuleSearchPathInner { - path: path.into(), - kind, - }), - } - } - - /// Determine whether this is a first-party, third-party or standard-library search path - pub fn kind(&self) -> ModuleSearchPathKind { - self.inner.kind - } - - /// Return the location of the search path on the file system - pub fn path(&self) -> &VfsPath { - &self.inner.path - } -} - -impl std::fmt::Debug for ModuleSearchPath { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ModuleSearchPath") - .field("path", &self.inner.path) - .field("kind", &self.kind()) - .finish() - } -} - -#[derive(Eq, PartialEq)] -struct ModuleSearchPathInner { - path: VfsPath, - kind: ModuleSearchPathKind, -} - /// Enumeration of the different kinds of search paths type checkers are expected to support. /// /// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the -/// priority that we want to give these modules when resolving them. -/// This is roughly [the order given in the typing spec], but typeshed's stubs -/// for the standard library are moved higher up to match Python's semantics at runtime. +/// priority that we want to give these modules when resolving them, +/// as per [the order given in the typing spec] /// /// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] -pub enum ModuleSearchPathKind { +#[derive(Debug, Eq, PartialEq, Hash)] +pub(crate) enum ModuleSearchPathEntry { /// "Extra" paths provided by the user in a config file, env var or CLI flag. /// E.g. mypy's `MYPYPATH` env var, or pyright's `stubPath` configuration setting - Extra, + Extra(ExtraPathBuf), /// Files in the project we're directly being invoked on - FirstParty, + FirstParty(FirstPartyPathBuf), /// The `stdlib` directory of typeshed (either vendored or custom) - StandardLibrary, + StandardLibrary(StandardLibraryPathBuf), /// Stubs or runtime modules installed in site-packages - SitePackagesThirdParty, + SitePackagesThirdParty(SitePackagesPathBuf), + // TODO(Alex): vendor third-party stubs from typeshed as well? + // VendoredThirdParty(VendoredPathBuf), +} - /// Vendored third-party stubs from typeshed - VendoredThirdParty, +impl ModuleSearchPathEntry { + pub(crate) fn stdlib_from_typeshed_root(typeshed: &StandardLibraryPath) -> Self { + Self::StandardLibrary(StandardLibraryPath::stdlib_from_typeshed_root(typeshed)) + } + + pub(crate) fn path(&self) -> ModuleResolutionPathRef { + match self { + Self::Extra(path) => ModuleResolutionPathRef::Extra(path), + Self::FirstParty(path) => ModuleResolutionPathRef::FirstParty(path), + Self::StandardLibrary(path) => ModuleResolutionPathRef::StandardLibrary(path), + Self::SitePackagesThirdParty(path) => ModuleResolutionPathRef::SitePackages(path), + } + } } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs new file mode 100644 index 0000000000000..531cc197c4ec9 --- /dev/null +++ b/crates/red_knot_module_resolver/src/path.rs @@ -0,0 +1,1179 @@ +#![allow(unsafe_code)] +use std::iter::FusedIterator; +use std::ops::Deref; +use std::path; + +use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; +use ruff_db::vfs::VfsPath; + +use crate::Db; + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct ExtraPath(FileSystemPath); + +impl ExtraPath { + #[must_use] + pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { + let path = path.as_ref(); + if path + .extension() + .is_some_and(|extension| !matches!(extension, "pyi" | "py")) + { + return None; + } + Some(Self::new_unchecked(path)) + } + + #[must_use] + fn new_unchecked(path: &FileSystemPath) -> &Self { + // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const ExtraPath is valid. + unsafe { &*(path as *const FileSystemPath as *const ExtraPath) } + } + + #[must_use] + pub(crate) fn parent(&self) -> Option<&Self> { + Some(Self::new_unchecked(self.0.parent()?)) + } + + #[must_use] + pub(crate) fn sans_dunder_init(&self) -> &Self { + if self.0.ends_with("__init__.py") || self.0.ends_with("__init__.pyi") { + self.parent() + .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) + } else { + self + } + } + + #[must_use] + pub(crate) fn module_name_parts(&self) -> ModulePartIterator { + ModulePartIterator::from_fs_path(&self.0) + } + + #[must_use] + pub(crate) fn relative_to_search_path(&self, search_path: &ExtraPath) -> Option<&Self> { + self.0 + .strip_prefix(search_path) + .map(Self::new_unchecked) + .ok() + } + + #[must_use] + pub fn to_path_buf(&self) -> ExtraPathBuf { + ExtraPathBuf(self.0.to_path_buf()) + } + + #[must_use] + fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { + file_system.exists(&self.0.join("__init__.py")) + || file_system.exists(&self.0.join("__init__.pyi")) + } + + #[must_use] + pub(crate) fn with_pyi_extension(&self) -> ExtraPathBuf { + ExtraPathBuf(self.0.with_extension("pyi")) + } + + #[must_use] + pub(crate) fn with_py_extension(&self) -> ExtraPathBuf { + ExtraPathBuf(self.0.with_extension("py")) + } + + #[must_use] + #[inline] + pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { + &self.0 + } +} + +impl PartialEq for ExtraPath { + fn eq(&self, other: &FileSystemPath) -> bool { + self.0 == *other + } +} + +impl PartialEq for ExtraPath { + fn eq(&self, other: &VfsPath) -> bool { + match other { + VfsPath::FileSystem(path) => **path == self.0, + VfsPath::Vendored(_) => false, + } + } +} + +impl AsRef for ExtraPath { + #[inline] + fn as_ref(&self) -> &ExtraPath { + self + } +} + +impl AsRef for ExtraPath { + #[inline] + fn as_ref(&self) -> &FileSystemPath { + self.as_file_system_path() + } +} + +impl AsRef for ExtraPath { + #[inline] + fn as_ref(&self) -> &path::Path { + self.0.as_ref() + } +} + +impl AsRef for ExtraPath { + #[inline] + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct ExtraPathBuf(FileSystemPathBuf); + +impl ExtraPathBuf { + #[must_use] + #[inline] + fn as_path(&self) -> &ExtraPath { + ExtraPath::new(&self.0).unwrap() + } + + /// Push a new part to the path, + /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions + /// + /// ## Panics: + /// If a component with an invalid extension is passed + fn push(&mut self, component: &str) { + debug_assert!(matches!(component.matches('.').count(), 0 | 1)); + if cfg!(debug) { + if let Some(extension) = std::path::Path::new(component).extension() { + assert!( + matches!(extension.to_str().unwrap(), "pyi" | "py"), + "Extension must be `py` or `pyi`; got {extension:?}" + ); + } + } + self.0.push(component); + } + + #[inline] + pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { + &self.0 + } +} + +impl AsRef for ExtraPathBuf { + #[inline] + fn as_ref(&self) -> &FileSystemPathBuf { + self.as_file_system_path_buf() + } +} + +impl AsRef for ExtraPathBuf { + #[inline] + fn as_ref(&self) -> &ExtraPath { + self.as_path() + } +} + +impl Deref for ExtraPathBuf { + type Target = ExtraPath; + + #[inline] + fn deref(&self) -> &Self::Target { + self.as_path() + } +} + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct FirstPartyPath(FileSystemPath); + +impl FirstPartyPath { + #[must_use] + pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { + let path = path.as_ref(); + if path + .extension() + .is_some_and(|extension| !matches!(extension, "pyi" | "py")) + { + return None; + } + Some(Self::new_unchecked(path)) + } + + #[must_use] + fn new_unchecked(path: &FileSystemPath) -> &Self { + // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const FirstPartyPath is valid. + unsafe { &*(path as *const FileSystemPath as *const FirstPartyPath) } + } + + #[must_use] + pub(crate) fn parent(&self) -> Option<&Self> { + Some(Self::new_unchecked(self.0.parent()?)) + } + + #[must_use] + pub(crate) fn sans_dunder_init(&self) -> &Self { + if self.0.ends_with("__init__.py") || self.0.ends_with("__init__.pyi") { + self.parent() + .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) + } else { + self + } + } + + #[must_use] + pub(crate) fn module_name_parts(&self) -> ModulePartIterator { + ModulePartIterator::from_fs_path(&self.0) + } + + #[must_use] + pub(crate) fn relative_to_search_path(&self, search_path: &FirstPartyPath) -> Option<&Self> { + self.0 + .strip_prefix(search_path) + .map(Self::new_unchecked) + .ok() + } + + #[must_use] + pub fn to_path_buf(&self) -> FirstPartyPathBuf { + FirstPartyPathBuf(self.0.to_path_buf()) + } + + #[must_use] + fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { + file_system.exists(&self.0.join("__init__.py")) + || file_system.exists(&self.0.join("__init__.pyi")) + } + + #[must_use] + pub(crate) fn with_pyi_extension(&self) -> FirstPartyPathBuf { + FirstPartyPathBuf(self.0.with_extension("pyi")) + } + + #[must_use] + pub(crate) fn with_py_extension(&self) -> FirstPartyPathBuf { + FirstPartyPathBuf(self.0.with_extension("py")) + } + + #[must_use] + #[inline] + pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { + &self.0 + } + + #[cfg(test)] + #[must_use] + pub(crate) fn join(&self, path: &str) -> FirstPartyPathBuf { + let mut result = self.to_path_buf(); + result.push(path); + result + } +} + +impl PartialEq for FirstPartyPath { + fn eq(&self, other: &FileSystemPath) -> bool { + self.0 == *other + } +} + +impl PartialEq for FirstPartyPath { + fn eq(&self, other: &VfsPath) -> bool { + match other { + VfsPath::FileSystem(path) => **path == self.0, + VfsPath::Vendored(_) => false, + } + } +} + +impl AsRef for FirstPartyPath { + #[inline] + fn as_ref(&self) -> &FirstPartyPath { + self + } +} + +impl AsRef for FirstPartyPath { + #[inline] + fn as_ref(&self) -> &FileSystemPath { + self.as_file_system_path() + } +} + +impl AsRef for FirstPartyPath { + #[inline] + fn as_ref(&self) -> &path::Path { + self.0.as_ref() + } +} + +impl AsRef for FirstPartyPath { + #[inline] + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct FirstPartyPathBuf(FileSystemPathBuf); + +impl FirstPartyPathBuf { + #[must_use] + #[inline] + fn as_path(&self) -> &FirstPartyPath { + FirstPartyPath::new(&self.0).unwrap() + } + + /// Push a new part to the path, + /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions + /// + /// ## Panics: + /// If a component with an invalid extension is passed + fn push(&mut self, component: &str) { + debug_assert!(matches!(component.matches('.').count(), 0 | 1)); + if cfg!(debug) { + if let Some(extension) = std::path::Path::new(component).extension() { + assert!( + matches!(extension.to_str().unwrap(), "pyi" | "py"), + "Extension must be `py` or `pyi`; got {extension:?}" + ); + } + } + self.0.push(component); + } + + #[cfg(test)] + pub(crate) fn into_vfs_path(self) -> VfsPath { + VfsPath::FileSystem(self.0) + } + + #[inline] + pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { + &self.0 + } +} + +impl AsRef for FirstPartyPathBuf { + #[inline] + fn as_ref(&self) -> &FileSystemPathBuf { + self.as_file_system_path_buf() + } +} + +impl AsRef for FirstPartyPathBuf { + #[inline] + fn as_ref(&self) -> &FirstPartyPath { + self.as_path() + } +} + +impl Deref for FirstPartyPathBuf { + type Target = FirstPartyPath; + + #[inline] + fn deref(&self) -> &Self::Target { + self.as_path() + } +} + +// TODO(Alex): Standard-library paths could be vendored paths +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct StandardLibraryPath(FileSystemPath); + +impl StandardLibraryPath { + #[must_use] + pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { + let path = path.as_ref(); + // Only allow pyi extensions, unlike other paths + if path.extension().is_some_and(|extension| extension != "pyi") { + return None; + } + Some(Self::new_unchecked(path)) + } + + #[must_use] + fn new_unchecked(path: &(impl AsRef + ?Sized)) -> &Self { + // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const FirstPartyPath is valid. + let path = path.as_ref(); + unsafe { &*(path as *const FileSystemPath as *const StandardLibraryPath) } + } + + #[must_use] + #[inline] + pub(crate) fn stdlib_dir() -> &'static Self { + Self::new_unchecked("stdlib") + } + + pub(crate) fn stdlib_from_typeshed_root( + typeshed: &StandardLibraryPath, + ) -> StandardLibraryPathBuf { + StandardLibraryPathBuf(typeshed.0.join(Self::stdlib_dir())) + } + + #[must_use] + pub(crate) fn relative_to_search_path( + &self, + search_path: &StandardLibraryPath, + ) -> Option<&Self> { + self.0 + .strip_prefix(search_path) + .map(Self::new_unchecked) + .ok() + } + + #[must_use] + pub(crate) fn parent(&self) -> Option<&Self> { + Some(Self::new_unchecked(self.0.parent()?)) + } + + #[must_use] + pub(crate) fn sans_dunder_init(&self) -> &Self { + // Only try to strip `__init__.pyi` from the end, unlike other paths + if self.0.ends_with("__init__.pyi") { + self.parent() + .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) + } else { + self + } + } + + #[must_use] + pub(crate) fn module_name_parts(&self) -> ModulePartIterator { + ModulePartIterator::from_fs_path(&self.0) + } + + #[must_use] + pub fn to_path_buf(&self) -> StandardLibraryPathBuf { + StandardLibraryPathBuf(self.0.to_path_buf()) + } + + #[must_use] + fn is_regular_package(&self, db: &dyn Db) -> bool { + todo!() + } + + #[must_use] + pub(crate) fn with_pyi_extension(&self) -> StandardLibraryPathBuf { + StandardLibraryPathBuf(self.0.with_extension("pyi")) + } + + #[must_use] + #[inline] + pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { + &self.0 + } + + #[cfg(test)] + #[must_use] + pub(crate) fn join(&self, path: &str) -> StandardLibraryPathBuf { + let mut result = self.to_path_buf(); + result.push(path); + result + } +} + +impl PartialEq for StandardLibraryPath { + fn eq(&self, other: &FileSystemPath) -> bool { + self.0 == *other + } +} + +impl PartialEq for StandardLibraryPath { + fn eq(&self, other: &VfsPath) -> bool { + match other { + VfsPath::FileSystem(path) => **path == self.0, + VfsPath::Vendored(_) => false, + } + } +} + +impl AsRef for StandardLibraryPath { + fn as_ref(&self) -> &StandardLibraryPath { + self + } +} + +impl AsRef for StandardLibraryPath { + #[inline] + fn as_ref(&self) -> &FileSystemPath { + self.as_file_system_path() + } +} + +impl AsRef for StandardLibraryPath { + #[inline] + fn as_ref(&self) -> &path::Path { + self.0.as_ref() + } +} + +impl AsRef for StandardLibraryPath { + #[inline] + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + +// TODO(Alex): Standard-library paths could also be vendored paths +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct StandardLibraryPathBuf(FileSystemPathBuf); + +impl StandardLibraryPathBuf { + #[must_use] + #[inline] + fn as_path(&self) -> &StandardLibraryPath { + StandardLibraryPath::new(&self.0).unwrap() + } + + /// Push a new part to the path, + /// while maintaining the invariant that the path can only have `.pyi` extensions + /// + /// ## Panics: + /// If a component with an invalid extension is passed + fn push(&mut self, component: &str) { + debug_assert!(matches!(component.matches('.').count(), 0 | 1)); + if cfg!(debug) { + if let Some(extension) = std::path::Path::new(component).extension() { + assert_eq!( + extension.to_str().unwrap(), + "pyi", + "Extension must be `pyi`; got {extension:?}" + ); + } + } + self.0.push(component); + } + + #[cfg(test)] + pub(crate) fn into_vfs_path(self) -> VfsPath { + VfsPath::FileSystem(self.0) + } + + #[inline] + pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { + &self.0 + } +} + +impl AsRef for StandardLibraryPathBuf { + #[inline] + fn as_ref(&self) -> &StandardLibraryPath { + self.as_path() + } +} + +impl AsRef for StandardLibraryPathBuf { + #[inline] + fn as_ref(&self) -> &FileSystemPathBuf { + self.as_file_system_path_buf() + } +} + +impl Deref for StandardLibraryPathBuf { + type Target = StandardLibraryPath; + + #[inline] + fn deref(&self) -> &Self::Target { + self.as_path() + } +} + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SitePackagesPath(FileSystemPath); + +impl SitePackagesPath { + #[must_use] + pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { + let path = path.as_ref(); + if path + .extension() + .is_some_and(|extension| !matches!(extension, "pyi" | "py")) + { + return None; + } + Some(Self::new_unchecked(path)) + } + + #[must_use] + fn new_unchecked(path: &FileSystemPath) -> &Self { + // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const SitePackagesPath is valid. + unsafe { &*(path as *const FileSystemPath as *const SitePackagesPath) } + } + + #[must_use] + pub(crate) fn parent(&self) -> Option<&Self> { + Some(Self::new_unchecked(self.0.parent()?)) + } + + #[must_use] + pub(crate) fn sans_dunder_init(&self) -> &Self { + // Only try to strip `__init__.pyi` from the end, unlike other paths + if self.0.ends_with("__init__.pyi") || self.0.ends_with("__init__.py") { + self.parent() + .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) + } else { + self + } + } + + #[must_use] + pub(crate) fn module_name_parts(&self) -> ModulePartIterator { + ModulePartIterator::from_fs_path(&self.0) + } + + #[must_use] + pub(crate) fn relative_to_search_path(&self, search_path: &SitePackagesPath) -> Option<&Self> { + self.0 + .strip_prefix(search_path) + .map(Self::new_unchecked) + .ok() + } + + #[must_use] + pub fn to_path_buf(&self) -> SitePackagesPathBuf { + SitePackagesPathBuf(self.0.to_path_buf()) + } + + #[must_use] + fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { + file_system.exists(&self.0.join("__init__.py")) + || file_system.exists(&self.0.join("__init__.pyi")) + } + + #[must_use] + pub(crate) fn with_pyi_extension(&self) -> SitePackagesPathBuf { + SitePackagesPathBuf(self.0.with_extension("pyi")) + } + + #[must_use] + pub(crate) fn with_py_extension(&self) -> SitePackagesPathBuf { + SitePackagesPathBuf(self.0.with_extension("py")) + } + + #[must_use] + #[inline] + pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { + &self.0 + } + + #[cfg(test)] + #[must_use] + pub(crate) fn join(&self, path: &str) -> SitePackagesPathBuf { + let mut result = self.to_path_buf(); + result.push(path); + result + } +} + +impl PartialEq for SitePackagesPath { + fn eq(&self, other: &FileSystemPath) -> bool { + self.0 == *other + } +} + +impl PartialEq for SitePackagesPath { + fn eq(&self, other: &VfsPath) -> bool { + match other { + VfsPath::FileSystem(path) => **path == self.0, + VfsPath::Vendored(_) => false, + } + } +} + +impl AsRef for SitePackagesPath { + fn as_ref(&self) -> &SitePackagesPath { + self + } +} + +impl AsRef for SitePackagesPath { + #[inline] + fn as_ref(&self) -> &FileSystemPath { + self.as_file_system_path() + } +} + +impl AsRef for SitePackagesPath { + #[inline] + fn as_ref(&self) -> &path::Path { + self.0.as_ref() + } +} + +impl AsRef for SitePackagesPath { + #[inline] + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct SitePackagesPathBuf(FileSystemPathBuf); + +impl SitePackagesPathBuf { + #[must_use] + #[inline] + fn as_path(&self) -> &SitePackagesPath { + SitePackagesPath::new(&self.0).unwrap() + } + + /// Push a new part to the path, + /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions + /// + /// ## Panics: + /// If a component with an invalid extension is passed + fn push(&mut self, component: &str) { + debug_assert!(matches!(component.matches('.').count(), 0 | 1)); + if cfg!(debug) { + if let Some(extension) = std::path::Path::new(component).extension() { + assert!( + matches!(extension.to_str().unwrap(), "pyi" | "py"), + "Extension must be `py` or `pyi`; got {extension:?}" + ); + } + } + self.0.push(component); + } + + #[cfg(test)] + pub(crate) fn into_vfs_path(self) -> VfsPath { + VfsPath::FileSystem(self.0) + } + + #[inline] + pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { + &self.0 + } +} + +impl AsRef for SitePackagesPathBuf { + #[inline] + fn as_ref(&self) -> &FileSystemPathBuf { + self.as_file_system_path_buf() + } +} + +impl AsRef for SitePackagesPathBuf { + #[inline] + fn as_ref(&self) -> &SitePackagesPath { + self.as_path() + } +} + +impl Deref for SitePackagesPathBuf { + type Target = SitePackagesPath; + + #[inline] + fn deref(&self) -> &Self::Target { + self.as_path() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub(crate) enum ModuleResolutionPath { + Extra(ExtraPathBuf), + FirstParty(FirstPartyPathBuf), + StandardLibrary(StandardLibraryPathBuf), + SitePackages(SitePackagesPathBuf), +} + +impl ModuleResolutionPath { + pub(crate) fn push(&mut self, component: &str) { + match self { + Self::Extra(ref mut path) => path.push(component), + Self::FirstParty(ref mut path) => path.push(component), + Self::StandardLibrary(ref mut path) => path.push(component), + Self::SitePackages(ref mut path) => path.push(component), + } + } + + pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { + ModuleResolutionPathRef::from(self).is_regular_package(db) + } + + pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { + ModuleResolutionPathRef::from(self).is_regular_package(db) + } + + pub(crate) fn with_pyi_extension(&self) -> Self { + ModuleResolutionPathRef::from(self).with_pyi_extension() + } + + pub(crate) fn with_py_extension(&self) -> Option { + ModuleResolutionPathRef::from(self).with_py_extension() + } +} + +impl AsRef for ModuleResolutionPath { + fn as_ref(&self) -> &FileSystemPath { + match self { + Self::Extra(path) => path.as_file_system_path(), + Self::FirstParty(path) => path.as_file_system_path(), + Self::StandardLibrary(path) => path.as_file_system_path(), + Self::SitePackages(path) => path.as_file_system_path(), + } + } +} + +impl AsRef for ModuleResolutionPath { + fn as_ref(&self) -> &FileSystemPathBuf { + match self { + Self::Extra(path) => path.as_file_system_path_buf(), + Self::FirstParty(path) => path.as_file_system_path_buf(), + Self::StandardLibrary(path) => path.as_file_system_path_buf(), + Self::SitePackages(path) => path.as_file_system_path_buf(), + } + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &ExtraPath) -> bool { + if let ModuleResolutionPath::Extra(path) = self { + **path == *other + } else { + false + } + } +} + +impl PartialEq for ExtraPath { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &ExtraPathBuf) -> bool { + self.eq(&**other) + } +} + +impl PartialEq for ExtraPathBuf { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &FirstPartyPath) -> bool { + if let ModuleResolutionPath::FirstParty(path) = self { + **path == *other + } else { + false + } + } +} + +impl PartialEq for FirstPartyPath { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &FirstPartyPathBuf) -> bool { + self.eq(&**other) + } +} + +impl PartialEq for FirstPartyPathBuf { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &StandardLibraryPath) -> bool { + if let ModuleResolutionPath::StandardLibrary(path) = self { + **path == *other + } else { + false + } + } +} + +impl PartialEq for StandardLibraryPath { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &StandardLibraryPathBuf) -> bool { + self.eq(&**other) + } +} + +impl PartialEq for StandardLibraryPathBuf { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &SitePackagesPath) -> bool { + if let ModuleResolutionPath::SitePackages(path) = self { + **path == *other + } else { + false + } + } +} + +impl PartialEq for SitePackagesPath { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &SitePackagesPathBuf) -> bool { + self.eq(&**other) + } +} + +impl PartialEq for SitePackagesPathBuf { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + other.eq(self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum ModuleResolutionPathRef<'a> { + Extra(&'a ExtraPath), + FirstParty(&'a FirstPartyPath), + StandardLibrary(&'a StandardLibraryPath), + SitePackages(&'a SitePackagesPath), +} + +impl<'a> ModuleResolutionPathRef<'a> { + #[must_use] + pub(crate) fn sans_dunder_init(self) -> Self { + match self { + Self::Extra(path) => Self::Extra(path.sans_dunder_init()), + Self::FirstParty(path) => Self::FirstParty(path.sans_dunder_init()), + Self::StandardLibrary(path) => Self::StandardLibrary(path.sans_dunder_init()), + Self::SitePackages(path) => Self::SitePackages(path.sans_dunder_init()), + } + } + + #[must_use] + pub(crate) fn module_name_parts(self) -> ModulePartIterator<'a> { + match self { + Self::Extra(path) => path.module_name_parts(), + Self::FirstParty(path) => path.module_name_parts(), + Self::StandardLibrary(path) => path.module_name_parts(), + Self::SitePackages(path) => path.module_name_parts(), + } + } + + #[must_use] + pub(crate) fn to_owned(self) -> ModuleResolutionPath { + match self { + Self::Extra(path) => ModuleResolutionPath::Extra(path.to_path_buf()), + Self::FirstParty(path) => ModuleResolutionPath::FirstParty(path.to_path_buf()), + Self::StandardLibrary(path) => { + ModuleResolutionPath::StandardLibrary(path.to_path_buf()) + } + Self::SitePackages(path) => ModuleResolutionPath::SitePackages(path.to_path_buf()), + } + } + + #[must_use] + pub(crate) fn is_regular_package(self, db: &dyn Db) -> bool { + match self { + Self::Extra(path) => path.is_regular_package(db.file_system()), + Self::FirstParty(path) => path.is_regular_package(db.file_system()), + Self::StandardLibrary(path) => path.is_regular_package(db), + Self::SitePackages(path) => path.is_regular_package(db.file_system()), + } + } + + #[must_use] + pub(crate) fn with_pyi_extension(self) -> ModuleResolutionPath { + match self { + Self::Extra(path) => ModuleResolutionPath::Extra(path.with_pyi_extension()), + Self::FirstParty(path) => ModuleResolutionPath::FirstParty(path.with_pyi_extension()), + Self::StandardLibrary(path) => { + ModuleResolutionPath::StandardLibrary(path.with_pyi_extension()) + } + Self::SitePackages(path) => { + ModuleResolutionPath::SitePackages(path.with_pyi_extension()) + } + } + } + + #[must_use] + pub(crate) fn with_py_extension(self) -> Option { + match self { + Self::Extra(path) => Some(ModuleResolutionPath::Extra(path.with_py_extension())), + Self::FirstParty(path) => { + Some(ModuleResolutionPath::FirstParty(path.with_py_extension())) + } + Self::StandardLibrary(_) => None, + Self::SitePackages(path) => { + Some(ModuleResolutionPath::SitePackages(path.with_py_extension())) + } + } + } +} + +impl<'a> From<&'a ModuleResolutionPath> for ModuleResolutionPathRef<'a> { + #[inline] + fn from(value: &'a ModuleResolutionPath) -> Self { + match value { + ModuleResolutionPath::Extra(path) => ModuleResolutionPathRef::Extra(path), + ModuleResolutionPath::FirstParty(path) => ModuleResolutionPathRef::FirstParty(path), + ModuleResolutionPath::StandardLibrary(path) => { + ModuleResolutionPathRef::StandardLibrary(path) + } + ModuleResolutionPath::SitePackages(path) => ModuleResolutionPathRef::SitePackages(path), + } + } +} + +impl<'a> AsRef for ModuleResolutionPathRef<'a> { + fn as_ref(&self) -> &FileSystemPath { + match self { + Self::Extra(path) => path.as_file_system_path(), + Self::FirstParty(path) => path.as_file_system_path(), + Self::StandardLibrary(path) => path.as_file_system_path(), + Self::SitePackages(path) => path.as_file_system_path(), + } + } +} + +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &ExtraPath) -> bool { + if let ModuleResolutionPathRef::Extra(path) = self { + *path == other + } else { + false + } + } +} + +impl<'a> PartialEq> for ExtraPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { + other.eq(self) + } +} + +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &FirstPartyPath) -> bool { + if let ModuleResolutionPathRef::FirstParty(path) = self { + *path == other + } else { + false + } + } +} + +impl<'a> PartialEq> for FirstPartyPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { + other.eq(self) + } +} + +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &StandardLibraryPath) -> bool { + if let ModuleResolutionPathRef::StandardLibrary(path) = self { + *path == other + } else { + false + } + } +} + +impl<'a> PartialEq> for StandardLibraryPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { + other.eq(self) + } +} + +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &SitePackagesPath) -> bool { + if let ModuleResolutionPathRef::SitePackages(path) = self { + *path == other + } else { + false + } + } +} + +impl<'a> PartialEq> for SitePackagesPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { + other.eq(self) + } +} + +pub(crate) struct ModulePartIterator<'a> { + parent_components: Option>, + stem: Option<&'a str>, +} + +impl<'a> ModulePartIterator<'a> { + #[must_use] + fn from_fs_path(path: &'a FileSystemPath) -> Self { + Self { + parent_components: path.parent().map(|path| path.components()), + stem: path.file_stem(), + } + } +} + +impl<'a> Iterator for ModulePartIterator<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + let ModulePartIterator { + parent_components, + stem, + } = self; + + if let Some(ref mut components) = parent_components { + components + .next() + .map(|component| component.as_str()) + .or_else(|| stem.take()) + } else { + stem.take() + } + } + + fn last(mut self) -> Option { + self.next_back() + } +} + +impl<'a> DoubleEndedIterator for ModulePartIterator<'a> { + fn next_back(&mut self) -> Option { + let ModulePartIterator { + parent_components, + stem, + } = self; + + if let Some(part) = stem.take() { + Some(part) + } else if let Some(components) = parent_components { + components.next_back().map(|component| component.as_str()) + } else { + None + } + } +} + +impl<'a> FusedIterator for ModulePartIterator<'a> {} diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 33f7281cf17e2..fb359b580618e 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -1,14 +1,17 @@ use std::ops::Deref; +use std::sync::Arc; -use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; -use crate::module::{Module, ModuleKind, ModuleName, ModuleSearchPath, ModuleSearchPathKind}; +use crate::module::{Module, ModuleKind, ModuleName, ModuleSearchPathEntry}; +use crate::path::{ + ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, ModuleResolutionPath, + ModuleResolutionPathRef, SitePackagesPath, SitePackagesPathBuf, StandardLibraryPath, + StandardLibraryPathBuf, +}; use crate::resolver::internal::ModuleResolverSearchPaths; use crate::Db; -const TYPESHED_STDLIB_DIRECTORY: &str = "stdlib"; - /// Configures the module search paths for the module resolver. /// /// Must be called before calling any other module resolution functions. @@ -71,7 +74,7 @@ pub fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option { /// Resolves the module for the file with the given id. /// -/// Returns `None` if the file is not a module locatable via `sys.path`. +/// Returns `None` if the file is not a module locatable via any of the PEP-561 search paths #[salsa::tracked] #[allow(unused)] pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { @@ -81,19 +84,29 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let search_paths = module_search_paths(db); - let relative_path = search_paths - .iter() - .find_map(|root| match (root.path(), path) { - (VfsPath::FileSystem(root_path), VfsPath::FileSystem(path)) => { - let relative_path = path.strip_prefix(root_path).ok()?; - Some(relative_path) - } - (VfsPath::Vendored(_), VfsPath::Vendored(_)) => { - todo!("Add support for vendored modules") - } - (VfsPath::Vendored(_), VfsPath::FileSystem(_)) - | (VfsPath::FileSystem(_), VfsPath::Vendored(_)) => None, - })?; + let relative_path = search_paths.iter().find_map(|root| match (&**root, path) { + (_, VfsPath::Vendored(_)) => todo!("VendoredPaths are not yet supported"), + (ModuleSearchPathEntry::Extra(root_path), VfsPath::FileSystem(path)) => { + Some(ModuleResolutionPathRef::Extra( + ExtraPath::new(path)?.relative_to_search_path(root_path)?, + )) + } + (ModuleSearchPathEntry::FirstParty(root_path), VfsPath::FileSystem(path)) => { + Some(ModuleResolutionPathRef::FirstParty( + FirstPartyPath::new(path)?.relative_to_search_path(root_path)?, + )) + } + (ModuleSearchPathEntry::StandardLibrary(root_path), VfsPath::FileSystem(path)) => { + Some(ModuleResolutionPathRef::StandardLibrary( + StandardLibraryPath::new(path)?.relative_to_search_path(root_path)?, + )) + } + (ModuleSearchPathEntry::SitePackagesThirdParty(root_path), VfsPath::FileSystem(path)) => { + Some(ModuleResolutionPathRef::SitePackages( + SitePackagesPath::new(path)?.relative_to_search_path(root_path)?, + )) + } + })?; let module_name = ModuleName::from_relative_path(relative_path)?; @@ -123,19 +136,18 @@ pub struct ModuleResolutionSettings { /// List of user-provided paths that should take first priority in the module resolution. /// Examples in other type checkers are mypy's MYPYPATH environment variable, /// or pyright's stubPath configuration setting. - pub extra_paths: Vec, + pub extra_paths: Vec, /// The root of the workspace, used for finding first-party modules. - pub workspace_root: FileSystemPathBuf, + pub workspace_root: FirstPartyPathBuf, - /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed. - pub site_packages: Option, + /// Optional (already validated) path to standard-library typeshed stubs. + /// If this is not provided, we will fallback to our vendored typeshed stubs + /// bundled as a zip file in the binary + pub custom_typeshed: Option, - /// Optional path to standard-library typeshed stubs. - /// Currently this has to be a directory that exists on disk. - /// - /// (TODO: fall back to vendored stubs if no custom directory is provided.) - pub custom_typeshed: Option, + /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed. + pub site_packages: Option, } impl ModuleResolutionSettings { @@ -151,41 +163,33 @@ impl ModuleResolutionSettings { let mut paths: Vec<_> = extra_paths .into_iter() - .map(|path| ModuleSearchPath::new(path, ModuleSearchPathKind::Extra)) + .map(ModuleSearchPathEntry::Extra) .collect(); - paths.push(ModuleSearchPath::new( - workspace_root, - ModuleSearchPathKind::FirstParty, - )); + paths.push(ModuleSearchPathEntry::FirstParty(workspace_root)); - // TODO fallback to vendored typeshed stubs if no custom typeshed directory is provided by the user if let Some(custom_typeshed) = custom_typeshed { - paths.push(ModuleSearchPath::new( - custom_typeshed.join(TYPESHED_STDLIB_DIRECTORY), - ModuleSearchPathKind::StandardLibrary, + paths.push(ModuleSearchPathEntry::stdlib_from_typeshed_root( + &custom_typeshed, )); } // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step if let Some(site_packages) = site_packages { - paths.push(ModuleSearchPath::new( - site_packages, - ModuleSearchPathKind::SitePackagesThirdParty, - )); + paths.push(ModuleSearchPathEntry::SitePackagesThirdParty(site_packages)); } - OrderedSearchPaths(paths) + OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()) } } /// A resolved module resolution order, implementing PEP 561 /// (with some small, deliberate differences) #[derive(Clone, Debug, Default, Eq, PartialEq)] -pub(crate) struct OrderedSearchPaths(Vec); +pub(crate) struct OrderedSearchPaths(Vec>); impl Deref for OrderedSearchPaths { - type Target = [ModuleSearchPath]; + type Target = [Arc]; fn deref(&self) -> &Self::Target { &self.0 @@ -217,31 +221,30 @@ pub(crate) mod internal { } } -fn module_search_paths(db: &dyn Db) -> &[ModuleSearchPath] { +fn module_search_paths(db: &dyn Db) -> &[Arc] { ModuleResolverSearchPaths::get(db).search_paths(db) } /// Given a module name and a list of search paths in which to lookup modules, /// attempt to resolve the module name -fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(ModuleSearchPath, VfsFile, ModuleKind)> { +fn resolve_name( + db: &dyn Db, + name: &ModuleName, +) -> Option<(Arc, VfsFile, ModuleKind)> { let search_paths = module_search_paths(db); for search_path in search_paths { let mut components = name.components(); let module_name = components.next_back()?; - let VfsPath::FileSystem(fs_search_path) = search_path.path() else { - todo!("Vendored search paths are not yet supported"); - }; - - match resolve_package(db.file_system(), fs_search_path, components) { + match resolve_package(db, search_path.path(), components) { Ok(resolved_package) => { let mut package_path = resolved_package.path; package_path.push(module_name); // Must be a `__init__.pyi` or `__init__.py` or it isn't a package. - let kind = if db.file_system().is_directory(&package_path) { + let kind = if package_path.is_directory(db) { package_path.push("__init__"); ModuleKind::Package } else { @@ -249,17 +252,17 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(ModuleSearchPath, Vfs }; // TODO Implement full https://peps.python.org/pep-0561/#type-checker-module-resolution-order resolution - let stub = package_path.with_extension("pyi"); - - if let Some(stub) = system_path_to_file(db.upcast(), &stub) { + if let Some(stub) = + system_path_to_file(db.upcast(), package_path.with_pyi_extension()) + { return Some((search_path.clone(), stub, kind)); } - let module = package_path.with_extension("py"); - - if let Some(module) = system_path_to_file(db.upcast(), &module) { - return Some((search_path.clone(), module, kind)); - } + if let Some(path_with_extension) = package_path.with_py_extension() { + if let Some(module) = system_path_to_file(db.upcast(), &path_with_extension) { + return Some((search_path.clone(), module, kind)); + } + }; // For regular packages, don't search the next search path. All files of that // package must be in the same location @@ -280,14 +283,14 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(ModuleSearchPath, Vfs } fn resolve_package<'a, I>( - fs: &dyn FileSystem, - module_search_path: &FileSystemPath, + db: &dyn Db, + module_search_path: ModuleResolutionPathRef, components: I, ) -> Result where I: Iterator, { - let mut package_path = module_search_path.to_path_buf(); + let mut package_path = module_search_path.to_owned(); // `true` if inside a folder that is a namespace package (has no `__init__.py`). // Namespace packages are special because they can be spread across multiple search paths. @@ -301,12 +304,11 @@ where for folder in components { package_path.push(folder); - let has_init_py = fs.is_file(&package_path.join("__init__.py")) - || fs.is_file(&package_path.join("__init__.pyi")); + let is_regular_package = package_path.is_regular_package(db); - if has_init_py { + if is_regular_package { in_namespace_package = false; - } else if fs.is_directory(&package_path) { + } else if package_path.is_directory(db) { // A directory without an `__init__.py` is a namespace package, continue with the next folder. in_namespace_package = true; } else if in_namespace_package { @@ -339,7 +341,7 @@ where #[derive(Debug)] struct ResolvedPackage { - path: FileSystemPathBuf, + path: ModuleResolutionPath, kind: PackageKind, } @@ -368,37 +370,37 @@ impl PackageKind { #[cfg(test)] mod tests { - use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; + use ruff_db::file_system::FileSystemPath; use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; use crate::db::tests::TestDb; use crate::module::{ModuleKind, ModuleName}; + use crate::path::{FirstPartyPath, SitePackagesPath}; - use super::{ - path_to_module, resolve_module, set_module_resolution_settings, ModuleResolutionSettings, - TYPESHED_STDLIB_DIRECTORY, - }; + use super::*; struct TestCase { db: TestDb, - src: FileSystemPathBuf, - custom_typeshed: FileSystemPathBuf, - site_packages: FileSystemPathBuf, + src: FirstPartyPathBuf, + custom_typeshed: StandardLibraryPathBuf, + site_packages: SitePackagesPathBuf, } fn create_resolver() -> std::io::Result { let mut db = TestDb::new(); - let src = FileSystemPath::new("src").to_path_buf(); - let site_packages = FileSystemPath::new("site_packages").to_path_buf(); - let custom_typeshed = FileSystemPath::new("typeshed").to_path_buf(); + let src = FirstPartyPath::new("src").unwrap().to_path_buf(); + let site_packages = SitePackagesPath::new("site_packages") + .unwrap() + .to_path_buf(); + let custom_typeshed = StandardLibraryPath::new("typeshed").unwrap().to_path_buf(); let fs = db.memory_file_system(); - fs.create_directory_all(&src)?; - fs.create_directory_all(&site_packages)?; - fs.create_directory_all(&custom_typeshed)?; + fs.create_directory_all(&*src)?; + fs.create_directory_all(&*site_packages)?; + fs.create_directory_all(&*custom_typeshed)?; let settings = ModuleResolutionSettings { extra_paths: vec![], @@ -424,7 +426,7 @@ mod tests { let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_path = src.join("foo.py"); db.memory_file_system() - .write_file(&foo_path, "print('Hello, world!')")?; + .write_file(&*foo_path, "print('Hello, world!')")?; let foo_module = resolve_module(&db, foo_module_name.clone()).unwrap(); @@ -434,13 +436,13 @@ mod tests { ); assert_eq!("foo", foo_module.name()); - assert_eq!(&src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path().path()); assert_eq!(ModuleKind::Module, foo_module.kind()); - assert_eq!(&foo_path, foo_module.file().path(&db)); + assert_eq!(*foo_path, *foo_module.file().path(&db)); assert_eq!( Some(foo_module), - path_to_module(&db, &VfsPath::FileSystem(foo_path)) + path_to_module(&db, &foo_path.into_vfs_path()) ); Ok(()) @@ -454,10 +456,10 @@ mod tests { .. } = create_resolver()?; - let stdlib_dir = custom_typeshed.join(TYPESHED_STDLIB_DIRECTORY); - let functools_path = stdlib_dir.join("functools.py"); + let stdlib_dir = StandardLibraryPath::stdlib_from_typeshed_root(&custom_typeshed); + let functools_path = stdlib_dir.join("functools.pyi"); db.memory_file_system() - .write_file(&functools_path, "def update_wrapper(): ...")?; + .write_file(&*functools_path, "def update_wrapper(): ...")?; let functools_module_name = ModuleName::new_static("functools").unwrap(); let functools_module = resolve_module(&db, functools_module_name.clone()).unwrap(); @@ -467,13 +469,14 @@ mod tests { resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(&stdlib_dir, functools_module.search_path().path()); + assert_eq!(*stdlib_dir, functools_module.search_path().path()); assert_eq!(ModuleKind::Module, functools_module.kind()); - assert_eq!(&functools_path.clone(), functools_module.file().path(&db)); + + assert_eq!(*functools_path, *functools_module.file().path(&db)); assert_eq!( Some(functools_module), - path_to_module(&db, &VfsPath::FileSystem(functools_path)) + path_to_module(&db, &functools_path.into_vfs_path()) ); Ok(()) @@ -488,13 +491,19 @@ mod tests { .. } = create_resolver()?; - let stdlib_dir = custom_typeshed.join(TYPESHED_STDLIB_DIRECTORY); - let stdlib_functools_path = stdlib_dir.join("functools.py"); + let stdlib_dir = StandardLibraryPath::stdlib_from_typeshed_root(&custom_typeshed); + let stdlib_functools_path = stdlib_dir.join("functools.pyi"); let first_party_functools_path = src.join("functools.py"); db.memory_file_system().write_files([ - (&stdlib_functools_path, "def update_wrapper(): ..."), - (&first_party_functools_path, "def update_wrapper(): ..."), + ( + stdlib_functools_path.as_file_system_path(), + "def update_wrapper(): ...", + ), + ( + first_party_functools_path.as_file_system_path(), + "def update_wrapper(): ...", + ), ])?; let functools_module_name = ModuleName::new_static("functools").unwrap(); @@ -504,16 +513,16 @@ mod tests { Some(&functools_module), resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(&src, functools_module.search_path().path()); + assert_eq!(*src, functools_module.search_path().path()); assert_eq!(ModuleKind::Module, functools_module.kind()); assert_eq!( - &first_party_functools_path.clone(), - functools_module.file().path(&db) + *first_party_functools_path, + *functools_module.file().path(&db) ); assert_eq!( Some(functools_module), - path_to_module(&db, &VfsPath::FileSystem(first_party_functools_path)) + path_to_module(&db, &first_party_functools_path.into_vfs_path()) ); Ok(()) @@ -551,21 +560,21 @@ mod tests { let foo_path = foo_dir.join("__init__.py"); db.memory_file_system() - .write_file(&foo_path, "print('Hello, world!')")?; + .write_file(&*foo_path, "print('Hello, world!')")?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); assert_eq!("foo", foo_module.name()); - assert_eq!(&src, foo_module.search_path().path()); - assert_eq!(&foo_path, foo_module.file().path(&db)); + assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*foo_path, *foo_module.file().path(&db)); assert_eq!( Some(&foo_module), - path_to_module(&db, &VfsPath::FileSystem(foo_path)).as_ref() + path_to_module(&db, &foo_path.into_vfs_path()).as_ref() ); // Resolving by directory doesn't resolve to the init file. - assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_dir))); + assert_eq!(None, path_to_module(&db, &foo_dir.into_vfs_path())); Ok(()) } @@ -578,23 +587,23 @@ mod tests { let foo_init = foo_dir.join("__init__.py"); db.memory_file_system() - .write_file(&foo_init, "print('Hello, world!')")?; + .write_file(&*foo_init, "print('Hello, world!')")?; let foo_py = src.join("foo.py"); db.memory_file_system() - .write_file(&foo_py, "print('Hello, world!')")?; + .write_file(&*foo_py, "print('Hello, world!')")?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(&src, foo_module.search_path().path()); - assert_eq!(&foo_init, foo_module.file().path(&db)); + assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*foo_init, *foo_module.file().path(&db)); assert_eq!(ModuleKind::Package, foo_module.kind()); assert_eq!( Some(foo_module), - path_to_module(&db, &VfsPath::FileSystem(foo_init)) + path_to_module(&db, &foo_init.into_vfs_path()) ); - assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_py))); + assert_eq!(None, path_to_module(&db, &foo_py.into_vfs_path())); Ok(()) } @@ -606,18 +615,15 @@ mod tests { let foo_stub = src.join("foo.pyi"); let foo_py = src.join("foo.py"); db.memory_file_system() - .write_files([(&foo_stub, "x: int"), (&foo_py, "print('Hello, world!')")])?; + .write_files([(&*foo_stub, "x: int"), (&*foo_py, "print('Hello, world!')")])?; let foo = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(&src, foo.search_path().path()); - assert_eq!(&foo_stub, foo.file().path(&db)); + assert_eq!(*src, foo.search_path().path()); + assert_eq!(*foo_stub, *foo.file().path(&db)); - assert_eq!( - Some(foo), - path_to_module(&db, &VfsPath::FileSystem(foo_stub)) - ); - assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_py))); + assert_eq!(Some(foo), path_to_module(&db, &foo_stub.into_vfs_path())); + assert_eq!(None, path_to_module(&db, &foo_py.into_vfs_path())); Ok(()) } @@ -631,21 +637,18 @@ mod tests { let baz = bar.join("baz.py"); db.memory_file_system().write_files([ - (&foo.join("__init__.py"), ""), - (&bar.join("__init__.py"), ""), - (&baz, "print('Hello, world!')"), + (&*foo.join("__init__.py"), ""), + (&*bar.join("__init__.py"), ""), + (&*baz, "print('Hello, world!')"), ])?; let baz_module = resolve_module(&db, ModuleName::new_static("foo.bar.baz").unwrap()).unwrap(); - assert_eq!(&src, baz_module.search_path().path()); - assert_eq!(&baz, baz_module.file().path(&db)); + assert_eq!(*src, baz_module.search_path().path()); + assert_eq!(*baz, *baz_module.file().path(&db)); - assert_eq!( - Some(baz_module), - path_to_module(&db, &VfsPath::FileSystem(baz)) - ); + assert_eq!(Some(baz_module), path_to_module(&db, &baz.into_vfs_path())); Ok(()) } @@ -681,24 +684,18 @@ mod tests { let two = child2.join("two.py"); db.memory_file_system().write_files([ - (&one, "print('Hello, world!')"), - (&two, "print('Hello, world!')"), + (one.as_file_system_path(), "print('Hello, world!')"), + (two.as_file_system_path(), "print('Hello, world!')"), ])?; let one_module = resolve_module(&db, ModuleName::new_static("parent.child.one").unwrap()).unwrap(); - assert_eq!( - Some(one_module), - path_to_module(&db, &VfsPath::FileSystem(one)) - ); + assert_eq!(Some(one_module), path_to_module(&db, &one.into_vfs_path())); let two_module = resolve_module(&db, ModuleName::new_static("parent.child.two").unwrap()).unwrap(); - assert_eq!( - Some(two_module), - path_to_module(&db, &VfsPath::FileSystem(two)) - ); + assert_eq!(Some(two_module), path_to_module(&db, &two.into_vfs_path())); Ok(()) } @@ -734,18 +731,18 @@ mod tests { let two = child2.join("two.py"); db.memory_file_system().write_files([ - (&child1.join("__init__.py"), "print('Hello, world!')"), - (&one, "print('Hello, world!')"), - (&two, "print('Hello, world!')"), + ( + child1.join("__init__.py").as_file_system_path(), + "print('Hello, world!')", + ), + (one.as_file_system_path(), "print('Hello, world!')"), + (two.as_file_system_path(), "print('Hello, world!')"), ])?; let one_module = resolve_module(&db, ModuleName::new_static("parent.child.one").unwrap()).unwrap(); - assert_eq!( - Some(one_module), - path_to_module(&db, &VfsPath::FileSystem(one)) - ); + assert_eq!(Some(one_module), path_to_module(&db, &one.into_vfs_path())); assert_eq!( None, @@ -766,21 +763,23 @@ mod tests { let foo_src = src.join("foo.py"); let foo_site_packages = site_packages.join("foo.py"); - db.memory_file_system() - .write_files([(&foo_src, ""), (&foo_site_packages, "")])?; + db.memory_file_system().write_files([ + (foo_src.as_file_system_path(), ""), + (foo_site_packages.as_file_system_path(), ""), + ])?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(&src, foo_module.search_path().path()); - assert_eq!(&foo_src, foo_module.file().path(&db)); + assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*foo_src, *foo_module.file().path(&db)); assert_eq!( Some(foo_module), - path_to_module(&db, &VfsPath::FileSystem(foo_src)) + path_to_module(&db, &foo_src.into_vfs_path()) ); assert_eq!( None, - path_to_module(&db, &VfsPath::FileSystem(foo_site_packages)) + path_to_module(&db, &foo_site_packages.into_vfs_path()) ); Ok(()) @@ -801,9 +800,9 @@ mod tests { let temp_dir = tempfile::tempdir()?; let root = FileSystemPath::from_std_path(temp_dir.path()).unwrap(); - let src = root.join(src); - let site_packages = root.join(site_packages); - let custom_typeshed = root.join(custom_typeshed); + let src = root.join(&*src); + let site_packages = root.join(&*site_packages); + let custom_typeshed = root.join(&*custom_typeshed); let foo = src.join("foo.py"); let bar = src.join("bar.py"); @@ -815,11 +814,17 @@ mod tests { std::fs::write(foo.as_std_path(), "")?; std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?; + let src = FirstPartyPath::new(&src).unwrap().to_path_buf(); + let site_packages = SitePackagesPath::new(&site_packages).unwrap().to_path_buf(); + let custom_typeshed = StandardLibraryPath::new(&custom_typeshed) + .unwrap() + .to_path_buf(); + let settings = ModuleResolutionSettings { extra_paths: vec![], workspace_root: src.clone(), - site_packages: Some(site_packages), - custom_typeshed: Some(custom_typeshed), + site_packages: Some(site_packages.clone()), + custom_typeshed: Some(custom_typeshed.clone()), }; set_module_resolution_settings(&mut db, settings); @@ -829,12 +834,12 @@ mod tests { assert_ne!(foo_module, bar_module); - assert_eq!(&src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path().path()); assert_eq!(&foo, foo_module.file().path(&db)); // `foo` and `bar` shouldn't resolve to the same file - assert_eq!(&src, bar_module.search_path().path()); + assert_eq!(*src, bar_module.search_path().path()); assert_eq!(&bar, bar_module.file().path(&db)); assert_eq!(&foo, foo_module.file().path(&db)); @@ -853,24 +858,24 @@ mod tests { } #[test] - fn deleting_an_unrealted_file_doesnt_change_module_resolution() -> anyhow::Result<()> { + fn deleting_an_unrelated_file_doesnt_change_module_resolution() -> anyhow::Result<()> { let TestCase { mut db, src, .. } = create_resolver()?; let foo_path = src.join("foo.py"); - let bar_path = src.join("bar.py"); + let bar_path: FirstPartyPathBuf = src.join("bar.py"); db.memory_file_system() - .write_files([(&foo_path, "x = 1"), (&bar_path, "y = 2")])?; + .write_files([(&*foo_path, "x = 1"), (&*bar_path, "y = 2")])?; let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_module = resolve_module(&db, foo_module_name.clone()).unwrap(); - let bar = system_path_to_file(&db, &bar_path).expect("bar.py to exist"); + let bar = system_path_to_file(&db, &*bar_path).expect("bar.py to exist"); db.clear_salsa_events(); // Delete `bar.py` - db.memory_file_system().remove_file(&bar_path)?; + db.memory_file_system().remove_file(&*bar_path)?; bar.touch(&mut db); // Re-query the foo module. The foo module should still be cached because `bar.py` isn't relevant @@ -898,9 +903,9 @@ mod tests { assert_eq!(resolve_module(&db, foo_module_name.clone()), None); // Now write the foo file - db.memory_file_system().write_file(&foo_path, "x = 1")?; - VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_path.clone())); - let foo_file = system_path_to_file(&db, &foo_path).expect("foo.py to exist"); + db.memory_file_system().write_file(&*foo_path, "x = 1")?; + VfsFile::touch_path(&mut db, &foo_path.clone().into_vfs_path()); + let foo_file = system_path_to_file(&db, &*foo_path).expect("foo.py to exist"); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); assert_eq!(foo_file, foo_module.file()); @@ -916,21 +921,21 @@ mod tests { let foo_init_path = src.join("foo/__init__.py"); db.memory_file_system() - .write_files([(&foo_path, "x = 1"), (&foo_init_path, "x = 2")])?; + .write_files([(&*foo_path, "x = 1"), (&*foo_init_path, "x = 2")])?; let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_module = resolve_module(&db, foo_module_name.clone()).expect("foo module to exist"); - assert_eq!(&foo_init_path, foo_module.file().path(&db)); + assert_eq!(*foo_init_path, *foo_module.file().path(&db)); // Delete `foo/__init__.py` and the `foo` folder. `foo` should now resolve to `foo.py` - db.memory_file_system().remove_file(&foo_init_path)?; + db.memory_file_system().remove_file(&*foo_init_path)?; db.memory_file_system() .remove_directory(foo_init_path.parent().unwrap())?; - VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_init_path.clone())); + VfsFile::touch_path(&mut db, &foo_init_path.into_vfs_path()); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); - assert_eq!(&foo_path, foo_module.file().path(&db)); + assert_eq!(*foo_path, *foo_module.file().path(&db)); Ok(()) } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index a5fe056c26124..57cd8e7b5d745 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -510,8 +510,7 @@ impl<'db, 'inference> TypingContext<'db, 'inference> { #[cfg(test)] mod tests { - use red_knot_module_resolver::{set_module_resolution_settings, ModuleResolutionSettings}; - use ruff_db::file_system::FileSystemPathBuf; + use red_knot_module_resolver::{FirstPartyPath, set_module_resolution_settings, ModuleResolutionSettings}; use ruff_db::parsed::parsed_module; use ruff_db::vfs::system_path_to_file; @@ -528,7 +527,7 @@ mod tests { &mut db, ModuleResolutionSettings { extra_paths: vec![], - workspace_root: FileSystemPathBuf::from("/src"), + workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), site_packages: None, custom_typeshed: None, }, diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 6830a58d6be6f..4717eb1e90d0d 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -700,12 +700,13 @@ impl<'db> TypeInferenceBuilder<'db> { #[cfg(test)] mod tests { - use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::system_path_to_file; use crate::db::tests::TestDb; use crate::types::{public_symbol_ty_by_name, Type, TypingContext}; - use red_knot_module_resolver::{set_module_resolution_settings, ModuleResolutionSettings}; + use red_knot_module_resolver::{ + set_module_resolution_settings, FirstPartyPath, ModuleResolutionSettings, + }; use ruff_python_ast::name::Name; fn setup_db() -> TestDb { @@ -715,7 +716,7 @@ mod tests { &mut db, ModuleResolutionSettings { extra_paths: Vec::new(), - workspace_root: FileSystemPathBuf::from("/src"), + workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), site_packages: None, custom_typeshed: None, }, From 75a140a27bcbec013d431f8fcfa1b68d6615cae4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 29 Jun 2024 20:05:10 +0100 Subject: [PATCH 02/58] Move `ModuleName` into a new file --- crates/red_knot_module_resolver/src/lib.rs | 6 +- crates/red_knot_module_resolver/src/module.rs | 165 +----------------- .../src/module_name.rs | 151 ++++++++++++++++ crates/red_knot_module_resolver/src/path.rs | 17 ++ .../red_knot_module_resolver/src/resolver.rs | 10 +- .../src/typeshed/versions.rs | 2 +- 6 files changed, 180 insertions(+), 171 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/module_name.rs diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index ad7723fee228d..0a22d8e3339dc 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -1,11 +1,13 @@ mod db; mod module; -pub mod path; +mod module_name; +mod path; mod resolver; mod typeshed; pub use db::{Db, Jar}; -pub use module::{Module, ModuleKind, ModuleName}; +pub use module::{Module, ModuleKind}; +pub use module_name::ModuleName; pub use path::{ ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, SitePackagesPath, SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, diff --git a/crates/red_knot_module_resolver/src/module.rs b/crates/red_knot_module_resolver/src/module.rs index 0977b6fdc0f14..c14a328ca0f66 100644 --- a/crates/red_knot_module_resolver/src/module.rs +++ b/crates/red_knot_module_resolver/src/module.rs @@ -1,178 +1,15 @@ -use compact_str::ToCompactString; use std::fmt::Formatter; -use std::ops::Deref; use std::sync::Arc; use ruff_db::vfs::VfsFile; -use ruff_python_stdlib::identifiers::is_identifier; +use crate::module_name::ModuleName; use crate::path::{ ExtraPathBuf, FirstPartyPathBuf, ModuleResolutionPathRef, SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, }; use crate::Db; -/// A module name, e.g. `foo.bar`. -/// -/// Always normalized to the absolute form (never a relative module name, i.e., never `.foo`). -#[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)] -pub struct ModuleName(compact_str::CompactString); - -impl ModuleName { - /// Creates a new module name for `name`. Returns `Some` if `name` is a valid, absolute - /// module name and `None` otherwise. - /// - /// The module name is invalid if: - /// - /// * The name is empty - /// * The name is relative - /// * The name ends with a `.` - /// * The name contains a sequence of multiple dots - /// * A component of a name (the part between two dots) isn't a valid python identifier. - #[inline] - pub fn new(name: &str) -> Option { - Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) - } - - /// Creates a new module name for `name` where `name` is a static string. - /// Returns `Some` if `name` is a valid, absolute module name and `None` otherwise. - /// - /// The module name is invalid if: - /// - /// * The name is empty - /// * The name is relative - /// * The name ends with a `.` - /// * The name contains a sequence of multiple dots - /// * A component of a name (the part between two dots) isn't a valid python identifier. - /// - /// ## Examples - /// - /// ``` - /// use red_knot_module_resolver::ModuleName; - /// - /// assert_eq!(ModuleName::new_static("foo.bar").as_deref(), Some("foo.bar")); - /// assert_eq!(ModuleName::new_static(""), None); - /// assert_eq!(ModuleName::new_static("..foo"), None); - /// assert_eq!(ModuleName::new_static(".foo"), None); - /// assert_eq!(ModuleName::new_static("foo."), None); - /// assert_eq!(ModuleName::new_static("foo..bar"), None); - /// assert_eq!(ModuleName::new_static("2000"), None); - /// ``` - #[inline] - pub fn new_static(name: &'static str) -> Option { - // TODO(Micha): Use CompactString::const_new once we upgrade to 0.8 https://github.com/ParkMyCar/compact_str/pull/336 - Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) - } - - fn is_valid_name(name: &str) -> bool { - !name.is_empty() && name.split('.').all(is_identifier) - } - - /// An iterator over the components of the module name: - /// - /// # Examples - /// - /// ``` - /// use red_knot_module_resolver::ModuleName; - /// - /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::>(), vec!["foo", "bar", "baz"]); - /// ``` - pub fn components(&self) -> impl DoubleEndedIterator { - self.0.split('.') - } - - /// The name of this module's immediate parent, if it has a parent. - /// - /// # Examples - /// - /// ``` - /// use red_knot_module_resolver::ModuleName; - /// - /// assert_eq!(ModuleName::new_static("foo.bar").unwrap().parent(), Some(ModuleName::new_static("foo").unwrap())); - /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap())); - /// assert_eq!(ModuleName::new_static("root").unwrap().parent(), None); - /// ``` - pub fn parent(&self) -> Option { - let (parent, _) = self.0.rsplit_once('.')?; - Some(Self(parent.to_compact_string())) - } - - /// Returns `true` if the name starts with `other`. - /// - /// This is equivalent to checking if `self` is a sub-module of `other`. - /// - /// # Examples - /// - /// ``` - /// use red_knot_module_resolver::ModuleName; - /// - /// assert!(ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap())); - /// - /// assert!(!ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("bar").unwrap())); - /// assert!(!ModuleName::new_static("foo_bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap())); - /// ``` - pub fn starts_with(&self, other: &ModuleName) -> bool { - let mut self_components = self.components(); - let other_components = other.components(); - - for other_component in other_components { - if self_components.next() != Some(other_component) { - return false; - } - } - - true - } - - #[inline] - pub fn as_str(&self) -> &str { - &self.0 - } - - pub(crate) fn from_relative_path(path: ModuleResolutionPathRef) -> Option { - let path = path.sans_dunder_init(); - let mut parts_iter = path.module_name_parts(); - let first_part = parts_iter.next()?; - if let Some(second_part) = parts_iter.next() { - let mut name = format!("{first_part}.{second_part}"); - for part in parts_iter { - name.push('.'); - name.push_str(part); - } - Self::new(&name) - } else { - Self::new(first_part) - } - } -} - -impl Deref for ModuleName { - type Target = str; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_str() - } -} - -impl PartialEq for ModuleName { - fn eq(&self, other: &str) -> bool { - self.as_str() == other - } -} - -impl PartialEq for str { - fn eq(&self, other: &ModuleName) -> bool { - self == other.as_str() - } -} - -impl std::fmt::Display for ModuleName { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - /// Representation of a Python module. #[derive(Clone, PartialEq, Eq)] pub struct Module { diff --git a/crates/red_knot_module_resolver/src/module_name.rs b/crates/red_knot_module_resolver/src/module_name.rs new file mode 100644 index 0000000000000..19c52a3884b69 --- /dev/null +++ b/crates/red_knot_module_resolver/src/module_name.rs @@ -0,0 +1,151 @@ +use std::fmt; +use std::ops::Deref; + +use compact_str::ToCompactString; + +use ruff_python_stdlib::identifiers::is_identifier; + +/// A module name, e.g. `foo.bar`. +/// +/// Always normalized to the absolute form (never a relative module name, i.e., never `.foo`). +#[derive(Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)] +pub struct ModuleName(compact_str::CompactString); + +impl ModuleName { + /// Creates a new module name for `name`. Returns `Some` if `name` is a valid, absolute + /// module name and `None` otherwise. + /// + /// The module name is invalid if: + /// + /// * The name is empty + /// * The name is relative + /// * The name ends with a `.` + /// * The name contains a sequence of multiple dots + /// * A component of a name (the part between two dots) isn't a valid python identifier. + #[inline] + pub fn new(name: &str) -> Option { + Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) + } + + /// Creates a new module name for `name` where `name` is a static string. + /// Returns `Some` if `name` is a valid, absolute module name and `None` otherwise. + /// + /// The module name is invalid if: + /// + /// * The name is empty + /// * The name is relative + /// * The name ends with a `.` + /// * The name contains a sequence of multiple dots + /// * A component of a name (the part between two dots) isn't a valid python identifier. + /// + /// ## Examples + /// + /// ``` + /// use red_knot_module_resolver::ModuleName; + /// + /// assert_eq!(ModuleName::new_static("foo.bar").as_deref(), Some("foo.bar")); + /// assert_eq!(ModuleName::new_static(""), None); + /// assert_eq!(ModuleName::new_static("..foo"), None); + /// assert_eq!(ModuleName::new_static(".foo"), None); + /// assert_eq!(ModuleName::new_static("foo."), None); + /// assert_eq!(ModuleName::new_static("foo..bar"), None); + /// assert_eq!(ModuleName::new_static("2000"), None); + /// ``` + #[inline] + pub fn new_static(name: &'static str) -> Option { + // TODO(Micha): Use CompactString::const_new once we upgrade to 0.8 https://github.com/ParkMyCar/compact_str/pull/336 + Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) + } + + fn is_valid_name(name: &str) -> bool { + !name.is_empty() && name.split('.').all(is_identifier) + } + + /// An iterator over the components of the module name: + /// + /// # Examples + /// + /// ``` + /// use red_knot_module_resolver::ModuleName; + /// + /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::>(), vec!["foo", "bar", "baz"]); + /// ``` + pub fn components(&self) -> impl DoubleEndedIterator { + self.0.split('.') + } + + /// The name of this module's immediate parent, if it has a parent. + /// + /// # Examples + /// + /// ``` + /// use red_knot_module_resolver::ModuleName; + /// + /// assert_eq!(ModuleName::new_static("foo.bar").unwrap().parent(), Some(ModuleName::new_static("foo").unwrap())); + /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap())); + /// assert_eq!(ModuleName::new_static("root").unwrap().parent(), None); + /// ``` + pub fn parent(&self) -> Option { + let (parent, _) = self.0.rsplit_once('.')?; + Some(Self(parent.to_compact_string())) + } + + /// Returns `true` if the name starts with `other`. + /// + /// This is equivalent to checking if `self` is a sub-module of `other`. + /// + /// # Examples + /// + /// ``` + /// use red_knot_module_resolver::ModuleName; + /// + /// assert!(ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap())); + /// + /// assert!(!ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("bar").unwrap())); + /// assert!(!ModuleName::new_static("foo_bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap())); + /// ``` + pub fn starts_with(&self, other: &ModuleName) -> bool { + let mut self_components = self.components(); + let other_components = other.components(); + + for other_component in other_components { + if self_components.next() != Some(other_component) { + return false; + } + } + + true + } + + #[inline] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Deref for ModuleName { + type Target = str; + + #[inline] + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +impl PartialEq for ModuleName { + fn eq(&self, other: &str) -> bool { + self.as_str() == other + } +} + +impl PartialEq for str { + fn eq(&self, other: &ModuleName) -> bool { + self == other.as_str() + } +} + +impl std::fmt::Display for ModuleName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 531cc197c4ec9..9ec0a0e9b1e88 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -6,6 +6,7 @@ use std::path; use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::VfsPath; +use crate::module_name::ModuleName; use crate::Db; #[repr(transparent)] @@ -1029,6 +1030,22 @@ impl<'a> ModuleResolutionPathRef<'a> { } } } + + pub(crate) fn as_module_name(&self) -> Option { + let path = self.sans_dunder_init(); + let mut parts_iter = path.module_name_parts(); + let first_part = parts_iter.next()?; + if let Some(second_part) = parts_iter.next() { + let mut name = format!("{first_part}.{second_part}"); + for part in parts_iter { + name.push('.'); + name.push_str(part); + } + ModuleName::new(&name) + } else { + ModuleName::new(first_part) + } + } } impl<'a> From<&'a ModuleResolutionPath> for ModuleResolutionPathRef<'a> { diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index fb359b580618e..aad17361be884 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; -use crate::module::{Module, ModuleKind, ModuleName, ModuleSearchPathEntry}; +use crate::module::{Module, ModuleKind, ModuleSearchPathEntry}; +use crate::module_name::ModuleName; use crate::path::{ ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, ModuleResolutionPath, ModuleResolutionPathRef, SitePackagesPath, SitePackagesPathBuf, StandardLibraryPath, @@ -108,7 +109,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { } })?; - let module_name = ModuleName::from_relative_path(relative_path)?; + let module_name = relative_path.as_module_name()?; // Resolve the module name to see if Python would resolve the name to the same path. // If it doesn't, then that means that multiple modules have the same name in different @@ -202,7 +203,7 @@ impl Deref for OrderedSearchPaths { // TODO(micha): Contribute a fix for this upstream where the singleton methods have the same visibility as the struct. #[allow(unreachable_pub, clippy::used_underscore_binding)] pub(crate) mod internal { - use crate::module::ModuleName; + use crate::module_name::ModuleName; use crate::resolver::OrderedSearchPaths; #[salsa::input(singleton)] @@ -374,7 +375,8 @@ mod tests { use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; use crate::db::tests::TestDb; - use crate::module::{ModuleKind, ModuleName}; + use crate::module::ModuleKind; + use crate::module_name::ModuleName; use crate::path::{FirstPartyPath, SitePackagesPath}; use super::*; diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index aea7b2cab494c..66554e314fa9c 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use rustc_hash::FxHashMap; -use crate::module::ModuleName; +use crate::module_name::ModuleName; #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { From 8cd568caaa021e6f87bc55e64976cec1b3238772 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 29 Jun 2024 20:21:59 +0100 Subject: [PATCH 03/58] Specify that the `Db` must allow typeshed versions to be queried --- crates/red_knot_module_resolver/src/db.rs | 15 +++++++-- crates/red_knot_module_resolver/src/path.rs | 32 ++++++++++++++++++- .../src/typeshed/versions.rs | 2 +- crates/red_knot_python_semantic/src/db.rs | 12 +++++-- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index c1d4e274ec3c3..872230fc9f5aa 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -5,6 +5,7 @@ use crate::resolver::{ internal::{ModuleNameIngredient, ModuleResolverSearchPaths}, resolve_module_query, }; +use crate::typeshed::versions::TypeshedVersions; #[salsa::jar(db=Db)] pub struct Jar( @@ -14,9 +15,12 @@ pub struct Jar( file_to_module, ); -pub trait Db: salsa::DbWithJar + ruff_db::Db + Upcast {} +pub trait Db: salsa::DbWithJar + ruff_db::Db + Upcast { + fn typeshed_versions(&self) -> &TypeshedVersions; +} pub(crate) mod tests { + use std::str::FromStr; use std::sync; use salsa::DebugWithDb; @@ -32,6 +36,7 @@ pub(crate) mod tests { file_system: TestFileSystem, events: sync::Arc>>, vfs: Vfs, + typeshed_versions: TypeshedVersions, } impl TestDb { @@ -42,6 +47,7 @@ pub(crate) mod tests { file_system: TestFileSystem::Memory(MemoryFileSystem::default()), events: sync::Arc::default(), vfs: Vfs::with_stubbed_vendored(), + typeshed_versions: TypeshedVersions::from_str("").unwrap(), } } @@ -111,7 +117,11 @@ pub(crate) mod tests { } } - impl Db for TestDb {} + impl Db for TestDb { + fn typeshed_versions(&self) -> &TypeshedVersions { + &self.typeshed_versions + } + } impl salsa::Database for TestDb { fn salsa_event(&self, event: salsa::Event) { @@ -128,6 +138,7 @@ pub(crate) mod tests { file_system: self.file_system.snapshot(), events: self.events.clone(), vfs: self.vfs.snapshot(), + typeshed_versions: self.typeshed_versions.clone(), }) } } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 9ec0a0e9b1e88..25f4f76eef057 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -72,6 +72,11 @@ impl ExtraPath { || file_system.exists(&self.0.join("__init__.pyi")) } + #[must_use] + fn is_directory(&self, file_system: &dyn FileSystem) -> bool { + file_system.is_directory(&self.0) + } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> ExtraPathBuf { ExtraPathBuf(self.0.with_extension("pyi")) @@ -253,6 +258,11 @@ impl FirstPartyPath { || file_system.exists(&self.0.join("__init__.pyi")) } + #[must_use] + fn is_directory(&self, file_system: &dyn FileSystem) -> bool { + file_system.is_directory(&self.0) + } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> FirstPartyPathBuf { FirstPartyPathBuf(self.0.with_extension("pyi")) @@ -462,6 +472,11 @@ impl StandardLibraryPath { todo!() } + #[must_use] + fn is_directory(&self, db: &dyn Db) -> bool { + todo!() + } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> StandardLibraryPathBuf { StandardLibraryPathBuf(self.0.with_extension("pyi")) @@ -653,6 +668,11 @@ impl SitePackagesPath { || file_system.exists(&self.0.join("__init__.pyi")) } + #[must_use] + fn is_directory(&self, file_system: &dyn FileSystem) -> bool { + file_system.is_directory(&self.0) + } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> SitePackagesPathBuf { SitePackagesPathBuf(self.0.with_extension("pyi")) @@ -806,7 +826,7 @@ impl ModuleResolutionPath { } pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { - ModuleResolutionPathRef::from(self).is_regular_package(db) + ModuleResolutionPathRef::from(self).is_directory(db) } pub(crate) fn with_pyi_extension(&self) -> Self { @@ -1003,6 +1023,16 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] + pub(crate) fn is_directory(self, db: &dyn Db) -> bool { + match self { + Self::Extra(path) => path.is_directory(db.file_system()), + Self::FirstParty(path) => path.is_directory(db.file_system()), + Self::StandardLibrary(path) => path.is_directory(db), + Self::SitePackages(path) => path.is_directory(db.file_system()), + } + } + #[must_use] pub(crate) fn with_pyi_extension(self) -> ModuleResolutionPath { match self { diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 66554e314fa9c..b817088b9808b 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -80,7 +80,7 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct TypeshedVersions(FxHashMap); impl TypeshedVersions { diff --git a/crates/red_knot_python_semantic/src/db.rs b/crates/red_knot_python_semantic/src/db.rs index 11c7a88352236..73ba39f021d77 100644 --- a/crates/red_knot_python_semantic/src/db.rs +++ b/crates/red_knot_python_semantic/src/db.rs @@ -31,6 +31,7 @@ pub trait Db: pub(crate) mod tests { use std::fmt::Formatter; use std::marker::PhantomData; + use std::str::FromStr; use std::sync::Arc; use salsa::id::AsId; @@ -38,7 +39,7 @@ pub(crate) mod tests { use salsa::storage::HasIngredientsFor; use salsa::DebugWithDb; - use red_knot_module_resolver::{Db as ResolverDb, Jar as ResolverJar}; + use red_knot_module_resolver::{Db as ResolverDb, Jar as ResolverJar, TypeshedVersions}; use ruff_db::file_system::{FileSystem, MemoryFileSystem, OsFileSystem}; use ruff_db::vfs::Vfs; use ruff_db::{Db as SourceDb, Jar as SourceJar, Upcast}; @@ -51,6 +52,7 @@ pub(crate) mod tests { vfs: Vfs, file_system: TestFileSystem, events: std::sync::Arc>>, + typeshed_versions: TypeshedVersions, } impl TestDb { @@ -60,6 +62,7 @@ pub(crate) mod tests { file_system: TestFileSystem::Memory(MemoryFileSystem::default()), events: std::sync::Arc::default(), vfs: Vfs::with_stubbed_vendored(), + typeshed_versions: TypeshedVersions::from_str("").unwrap(), } } @@ -125,7 +128,11 @@ pub(crate) mod tests { } } - impl red_knot_module_resolver::Db for TestDb {} + impl red_knot_module_resolver::Db for TestDb { + fn typeshed_versions(&self) -> &TypeshedVersions { + &self.typeshed_versions + } + } impl Db for TestDb {} impl salsa::Database for TestDb { @@ -146,6 +153,7 @@ pub(crate) mod tests { TestFileSystem::Os(fs) => TestFileSystem::Os(fs.snapshot()), }, events: self.events.clone(), + typeshed_versions: self.typeshed_versions.clone(), }) } } From eb6a3770d589f68b21d20b265d5afb50cc8b89e4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 29 Jun 2024 20:41:01 +0100 Subject: [PATCH 04/58] The supported Python version is set at the same time as the search paths --- crates/red_knot_module_resolver/src/db.rs | 5 +-- crates/red_knot_module_resolver/src/lib.rs | 2 +- .../red_knot_module_resolver/src/resolver.rs | 35 +++++++++++++++---- .../red_knot_module_resolver/src/typeshed.rs | 6 +++- .../src/typeshed/supported_py_version.rs | 12 +++++++ .../src/typeshed/versions.rs | 14 +------- crates/red_knot_python_semantic/src/types.rs | 6 +++- .../src/types/infer.rs | 2 ++ 8 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 872230fc9f5aa..1db96acc42adf 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -2,15 +2,16 @@ use ruff_db::Upcast; use crate::resolver::{ file_to_module, - internal::{ModuleNameIngredient, ModuleResolverSearchPaths}, + internal::{ModuleNameIngredient, ModuleResolverSearchPaths, TargetPyVersion}, resolve_module_query, }; -use crate::typeshed::versions::TypeshedVersions; +use crate::typeshed::TypeshedVersions; #[salsa::jar(db=Db)] pub struct Jar( ModuleNameIngredient<'_>, ModuleResolverSearchPaths, + TargetPyVersion, resolve_module_query, file_to_module, ); diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index 0a22d8e3339dc..fc808a5af5630 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -13,4 +13,4 @@ pub use path::{ SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, }; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; -pub use typeshed::versions::TypeshedVersions; +pub use typeshed::{SupportedPyVersion, TypeshedVersions}; diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index aad17361be884..d7a1c5b02ddc2 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -1,3 +1,4 @@ +use internal::TargetPyVersion; use std::ops::Deref; use std::sync::Arc; @@ -11,7 +12,7 @@ use crate::path::{ StandardLibraryPathBuf, }; use crate::resolver::internal::ModuleResolverSearchPaths; -use crate::Db; +use crate::{Db, SupportedPyVersion}; /// Configures the module search paths for the module resolver. /// @@ -20,12 +21,16 @@ pub fn set_module_resolution_settings(db: &mut dyn Db, config: ModuleResolutionS // There's no concurrency issue here because we hold a `&mut dyn Db` reference. No other // thread can mutate the `Db` while we're in this call, so using `try_get` to test if // the settings have already been set is safe. + let (target_version, search_paths) = config.into_ordered_search_paths(); if let Some(existing) = ModuleResolverSearchPaths::try_get(db) { - existing - .set_search_paths(db) - .to(config.into_ordered_search_paths()); + existing.set_search_paths(db).to(search_paths); } else { - ModuleResolverSearchPaths::new(db, config.into_ordered_search_paths()); + ModuleResolverSearchPaths::new(db, search_paths); + } + if let Some(existing) = TargetPyVersion::try_get(db) { + existing.set_target_py_version(db).to(target_version); + } else { + TargetPyVersion::new(db, target_version); } } @@ -134,6 +139,9 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { /// Configures the search paths that are used to resolve modules. #[derive(Eq, PartialEq, Debug)] pub struct ModuleResolutionSettings { + /// The target Python version the user has specified + pub target_version: SupportedPyVersion, + /// List of user-provided paths that should take first priority in the module resolution. /// Examples in other type checkers are mypy's MYPYPATH environment variable, /// or pyright's stubPath configuration setting. @@ -154,8 +162,9 @@ pub struct ModuleResolutionSettings { impl ModuleResolutionSettings { /// Implementation of PEP 561's module resolution order /// (with some small, deliberate, differences) - fn into_ordered_search_paths(self) -> OrderedSearchPaths { + fn into_ordered_search_paths(self) -> (SupportedPyVersion, OrderedSearchPaths) { let ModuleResolutionSettings { + target_version, extra_paths, workspace_root, site_packages, @@ -180,7 +189,10 @@ impl ModuleResolutionSettings { paths.push(ModuleSearchPathEntry::SitePackagesThirdParty(site_packages)); } - OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()) + ( + target_version, + OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()), + ) } } @@ -205,6 +217,7 @@ impl Deref for OrderedSearchPaths { pub(crate) mod internal { use crate::module_name::ModuleName; use crate::resolver::OrderedSearchPaths; + use crate::SupportedPyVersion; #[salsa::input(singleton)] pub(crate) struct ModuleResolverSearchPaths { @@ -212,6 +225,12 @@ pub(crate) mod internal { pub(super) search_paths: OrderedSearchPaths, } + #[salsa::input(singleton)] + pub(crate) struct TargetPyVersion { + #[return_ref] + pub(super) target_py_version: SupportedPyVersion, + } + /// A thin wrapper around `ModuleName` to make it a Salsa ingredient. /// /// This is needed because Salsa requires that all query arguments are salsa ingredients. @@ -405,6 +424,7 @@ mod tests { fs.create_directory_all(&*custom_typeshed)?; let settings = ModuleResolutionSettings { + target_version: SupportedPyVersion::Py38, extra_paths: vec![], workspace_root: src.clone(), site_packages: Some(site_packages.clone()), @@ -823,6 +843,7 @@ mod tests { .to_path_buf(); let settings = ModuleResolutionSettings { + target_version: SupportedPyVersion::Py38, extra_paths: vec![], workspace_root: src.clone(), site_packages: Some(site_packages.clone()), diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index fa49261d5f814..7da30a8e09bc2 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,4 +1,8 @@ -pub(crate) mod versions; +mod supported_py_version; +mod versions; + +pub use versions::TypeshedVersions; +pub use supported_py_version::SupportedPyVersion; #[cfg(test)] mod tests { diff --git a/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs b/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs new file mode 100644 index 0000000000000..84fffa122e853 --- /dev/null +++ b/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs @@ -0,0 +1,12 @@ +// TODO: unify with the PythonVersion enum in the linter/formatter crates? +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub enum SupportedPyVersion { + Py37, + #[default] + Py38, + Py39, + Py310, + Py311, + Py312, + Py313, +} diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index b817088b9808b..70b66b89b50e8 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -7,6 +7,7 @@ use std::str::FromStr; use rustc_hash::FxHashMap; use crate::module_name::ModuleName; +use super::supported_py_version::SupportedPyVersion; #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { @@ -266,19 +267,6 @@ impl fmt::Display for PyVersion { } } -// TODO: unify with the PythonVersion enum in the linter/formatter crates? -#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub enum SupportedPyVersion { - Py37, - #[default] - Py38, - Py39, - Py310, - Py311, - Py312, - Py313, -} - impl From for PyVersion { fn from(value: SupportedPyVersion) -> Self { match value { diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 57cd8e7b5d745..6174aede48887 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -510,7 +510,10 @@ impl<'db, 'inference> TypingContext<'db, 'inference> { #[cfg(test)] mod tests { - use red_knot_module_resolver::{FirstPartyPath, set_module_resolution_settings, ModuleResolutionSettings}; + use red_knot_module_resolver::{ + set_module_resolution_settings, FirstPartyPath, ModuleResolutionSettings, + SupportedPyVersion, + }; use ruff_db::parsed::parsed_module; use ruff_db::vfs::system_path_to_file; @@ -526,6 +529,7 @@ mod tests { set_module_resolution_settings( &mut db, ModuleResolutionSettings { + target_version: SupportedPyVersion::Py38, extra_paths: vec![], workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), site_packages: None, diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 4717eb1e90d0d..e1ebca8c4acef 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -706,6 +706,7 @@ mod tests { use crate::types::{public_symbol_ty_by_name, Type, TypingContext}; use red_knot_module_resolver::{ set_module_resolution_settings, FirstPartyPath, ModuleResolutionSettings, + SupportedPyVersion, }; use ruff_python_ast::name::Name; @@ -715,6 +716,7 @@ mod tests { set_module_resolution_settings( &mut db, ModuleResolutionSettings { + target_version: SupportedPyVersion::Py38, extra_paths: Vec::new(), workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), site_packages: None, From 87419ce4f74c2c8ff86314f1c990dfeb78e48384 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 1 Jul 2024 12:01:56 +0100 Subject: [PATCH 05/58] `TypeshedVersions` returns a three-variant enum --- .../src/typeshed/versions.rs | 164 ++++++++++++++---- 1 file changed, 126 insertions(+), 38 deletions(-) diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 70b66b89b50e8..e5ee513a0957e 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -6,8 +6,8 @@ use std::str::FromStr; use rustc_hash::FxHashMap; -use crate::module_name::ModuleName; use super::supported_py_version::SupportedPyVersion; +use crate::module_name::ModuleName; #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { @@ -93,27 +93,54 @@ impl TypeshedVersions { self.0.is_empty() } - pub fn contains_module(&self, module_name: &ModuleName) -> bool { - self.0.contains_key(module_name) + fn get_exact(&self, module_name: &ModuleName) -> Option<&PyVersionRange> { + self.0.get(module_name) + } + + /// Helper function for testing purposes + #[cfg(test)] + fn contains_exact(&self, module: &ModuleName) -> bool { + self.get_exact(module).is_some() } - pub fn module_exists_on_version( + pub fn query_module( &self, - module: ModuleName, + module: &ModuleName, version: impl Into, - ) -> bool { + ) -> TypeshedVersionsQueryResult { let version = version.into(); - let mut module: Option = Some(module); - while let Some(module_to_try) = module { - if let Some(range) = self.0.get(&module_to_try) { - return range.contains(version); + if let Some(range) = self.get_exact(module) { + if range.contains(version) { + TypeshedVersionsQueryResult::Exists + } else { + TypeshedVersionsQueryResult::DoesNotExist } - module = module_to_try.parent(); + } else { + let mut module = module.parent(); + while let Some(module_to_try) = module { + if let Some(range) = self.get_exact(&module_to_try) { + return { + if range.contains(version) { + TypeshedVersionsQueryResult::MaybeExists + } else { + TypeshedVersionsQueryResult::DoesNotExist + } + }; + } + module = module_to_try.parent(); + } + TypeshedVersionsQueryResult::DoesNotExist } - false } } +#[derive(Debug, Copy, PartialEq, Eq, Clone, Hash)] +pub enum TypeshedVersionsQueryResult { + Exists, + DoesNotExist, + MaybeExists, +} + impl FromStr for TypeshedVersions { type Err = TypeshedVersionsParseError; @@ -181,7 +208,7 @@ impl fmt::Display for TypeshedVersions { } } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Hash)] enum PyVersionRange { AvailableFrom(RangeFrom), AvailableWithin(RangeInclusive), @@ -322,18 +349,31 @@ mod tests { let asyncio_staggered = ModuleName::new_static("asyncio.staggered").unwrap(); let audioop = ModuleName::new_static("audioop").unwrap(); - assert!(versions.contains_module(&asyncio)); - assert!(versions.module_exists_on_version(asyncio, SupportedPyVersion::Py310)); + assert!(versions.contains_exact(&asyncio)); + assert_eq!( + versions.query_module(&asyncio, SupportedPyVersion::Py310), + TypeshedVersionsQueryResult::Exists + ); - assert!(versions.contains_module(&asyncio_staggered)); - assert!( - versions.module_exists_on_version(asyncio_staggered.clone(), SupportedPyVersion::Py38) + assert!(versions.contains_exact(&asyncio_staggered)); + assert_eq!( + versions.query_module(&asyncio_staggered, SupportedPyVersion::Py38), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + versions.query_module(&asyncio_staggered, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::DoesNotExist ); - assert!(!versions.module_exists_on_version(asyncio_staggered, SupportedPyVersion::Py37)); - assert!(versions.contains_module(&audioop)); - assert!(versions.module_exists_on_version(audioop.clone(), SupportedPyVersion::Py312)); - assert!(!versions.module_exists_on_version(audioop, SupportedPyVersion::Py313)); + assert!(versions.contains_exact(&audioop)); + assert_eq!( + versions.query_module(&audioop, SupportedPyVersion::Py312), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + versions.query_module(&audioop, SupportedPyVersion::Py313), + TypeshedVersionsQueryResult::DoesNotExist + ); } #[test] @@ -381,7 +421,7 @@ mod tests { let top_level_module = ModuleName::new(top_level_module) .unwrap_or_else(|| panic!("{top_level_module:?} was not a valid module name!")); - assert!(vendored_typeshed_versions.contains_module(&top_level_module)); + assert!(vendored_typeshed_versions.contains_exact(&top_level_module)); } assert!( @@ -418,26 +458,74 @@ foo: 3.8- # trailing comment let foo = ModuleName::new_static("foo").unwrap(); let bar = ModuleName::new_static("bar").unwrap(); let bar_baz = ModuleName::new_static("bar.baz").unwrap(); + let bar_eggs = ModuleName::new_static("bar.eggs").unwrap(); let spam = ModuleName::new_static("spam").unwrap(); - assert!(parsed_versions.contains_module(&foo)); - assert!(!parsed_versions.module_exists_on_version(foo.clone(), SupportedPyVersion::Py37)); - assert!(parsed_versions.module_exists_on_version(foo.clone(), SupportedPyVersion::Py38)); - assert!(parsed_versions.module_exists_on_version(foo, SupportedPyVersion::Py311)); + assert!(parsed_versions.contains_exact(&foo)); + assert_eq!( + parsed_versions.query_module(&foo, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::DoesNotExist + ); + assert_eq!( + parsed_versions.query_module(&foo, SupportedPyVersion::Py38), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + parsed_versions.query_module(&foo, SupportedPyVersion::Py311), + TypeshedVersionsQueryResult::Exists + ); + + assert!(parsed_versions.contains_exact(&bar)); + assert_eq!( + parsed_versions.query_module(&bar, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + parsed_versions.query_module(&bar, SupportedPyVersion::Py310), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + parsed_versions.query_module(&bar, SupportedPyVersion::Py311), + TypeshedVersionsQueryResult::DoesNotExist + ); - assert!(parsed_versions.contains_module(&bar)); - assert!(parsed_versions.module_exists_on_version(bar.clone(), SupportedPyVersion::Py37)); - assert!(parsed_versions.module_exists_on_version(bar.clone(), SupportedPyVersion::Py310)); - assert!(!parsed_versions.module_exists_on_version(bar, SupportedPyVersion::Py311)); + assert!(parsed_versions.contains_exact(&bar_baz)); + assert_eq!( + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py39), + TypeshedVersionsQueryResult::Exists + ); + assert_eq!( + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py310), + TypeshedVersionsQueryResult::DoesNotExist + ); - assert!(parsed_versions.contains_module(&bar_baz)); - assert!(parsed_versions.module_exists_on_version(bar_baz.clone(), SupportedPyVersion::Py37)); - assert!(parsed_versions.module_exists_on_version(bar_baz.clone(), SupportedPyVersion::Py39)); - assert!(!parsed_versions.module_exists_on_version(bar_baz, SupportedPyVersion::Py310)); + assert!(!parsed_versions.contains_exact(&spam)); + assert_eq!( + parsed_versions.query_module(&spam, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::DoesNotExist + ); + assert_eq!( + parsed_versions.query_module(&spam, SupportedPyVersion::Py313), + TypeshedVersionsQueryResult::DoesNotExist + ); - assert!(!parsed_versions.contains_module(&spam)); - assert!(!parsed_versions.module_exists_on_version(spam.clone(), SupportedPyVersion::Py37)); - assert!(!parsed_versions.module_exists_on_version(spam, SupportedPyVersion::Py313)); + assert!(!parsed_versions.contains_exact(&bar_eggs)); + assert_eq!( + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py37), + TypeshedVersionsQueryResult::MaybeExists + ); + assert_eq!( + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py310), + TypeshedVersionsQueryResult::MaybeExists + ); + assert_eq!( + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py311), + TypeshedVersionsQueryResult::DoesNotExist + ); } #[test] From 5d30cf826ccb313d34b63b0a23eea74400c4a407 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 1 Jul 2024 14:08:10 +0100 Subject: [PATCH 06/58] Silly implementation for the `todo!()`s --- crates/red_knot_module_resolver/src/path.rs | 33 +++++++++++++++++-- .../red_knot_module_resolver/src/typeshed.rs | 1 + 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 25f4f76eef057..deff8d0c6fad0 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -7,7 +7,8 @@ use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::VfsPath; use crate::module_name::ModuleName; -use crate::Db; +use crate::typeshed::TypeshedVersionsQueryResult; +use crate::{Db, SupportedPyVersion}; #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] @@ -467,14 +468,40 @@ impl StandardLibraryPath { StandardLibraryPathBuf(self.0.to_path_buf()) } + pub(crate) fn as_module_name(&self) -> Option { + ModuleResolutionPathRef::StandardLibrary(self).as_module_name() + } + #[must_use] fn is_regular_package(&self, db: &dyn Db) -> bool { - todo!() + let Some(module_name) = self.as_module_name() else { + return false; + }; + match db + .typeshed_versions() + .query_module(&module_name, SupportedPyVersion::Py37) + { + TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { + db.file_system().exists(&self.0.join("__init__.pyi")) + } + TypeshedVersionsQueryResult::DoesNotExist => false, + } } #[must_use] fn is_directory(&self, db: &dyn Db) -> bool { - todo!() + let Some(module_name) = self.as_module_name() else { + return false; + }; + match db + .typeshed_versions() + .query_module(&module_name, SupportedPyVersion::Py37) + { + TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { + db.file_system().is_directory(&self.0) + } + TypeshedVersionsQueryResult::DoesNotExist => false, + } } #[must_use] diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index 7da30a8e09bc2..159eaf81c089d 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,6 +1,7 @@ mod supported_py_version; mod versions; +pub(crate) use versions::TypeshedVersionsQueryResult; pub use versions::TypeshedVersions; pub use supported_py_version::SupportedPyVersion; From 5c1f4e872deef99693cb373f6d44cd750b8bc941 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 1 Jul 2024 14:34:19 +0100 Subject: [PATCH 07/58] Implementation that actually looks at the target version we set --- crates/red_knot_module_resolver/src/db.rs | 3 +- crates/red_knot_module_resolver/src/lib.rs | 4 ++- crates/red_knot_module_resolver/src/path.rs | 7 ++-- .../red_knot_module_resolver/src/resolver.rs | 17 ++------- .../src/supported_py_version.rs | 35 +++++++++++++++++++ .../red_knot_module_resolver/src/typeshed.rs | 4 +-- .../src/typeshed/supported_py_version.rs | 12 ------- .../src/typeshed/versions.rs | 2 +- 8 files changed, 49 insertions(+), 35 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/supported_py_version.rs delete mode 100644 crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 1db96acc42adf..6ce74590a4be6 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -2,9 +2,10 @@ use ruff_db::Upcast; use crate::resolver::{ file_to_module, - internal::{ModuleNameIngredient, ModuleResolverSearchPaths, TargetPyVersion}, + internal::{ModuleNameIngredient, ModuleResolverSearchPaths}, resolve_module_query, }; +use crate::supported_py_version::TargetPyVersion; use crate::typeshed::TypeshedVersions; #[salsa::jar(db=Db)] diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index fc808a5af5630..d8f62fc03d979 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -3,6 +3,7 @@ mod module; mod module_name; mod path; mod resolver; +mod supported_py_version; mod typeshed; pub use db::{Db, Jar}; @@ -13,4 +14,5 @@ pub use path::{ SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, }; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; -pub use typeshed::{SupportedPyVersion, TypeshedVersions}; +pub use supported_py_version::SupportedPyVersion; +pub use typeshed::TypeshedVersions; diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index deff8d0c6fad0..d8e2532ed2e93 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -7,8 +7,9 @@ use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::VfsPath; use crate::module_name::ModuleName; +use crate::supported_py_version::get_target_py_version; use crate::typeshed::TypeshedVersionsQueryResult; -use crate::{Db, SupportedPyVersion}; +use crate::Db; #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] @@ -479,7 +480,7 @@ impl StandardLibraryPath { }; match db .typeshed_versions() - .query_module(&module_name, SupportedPyVersion::Py37) + .query_module(&module_name, get_target_py_version(db)) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().exists(&self.0.join("__init__.pyi")) @@ -495,7 +496,7 @@ impl StandardLibraryPath { }; match db .typeshed_versions() - .query_module(&module_name, SupportedPyVersion::Py37) + .query_module(&module_name, get_target_py_version(db)) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().is_directory(&self.0) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index d7a1c5b02ddc2..40a8543112046 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -1,4 +1,3 @@ -use internal::TargetPyVersion; use std::ops::Deref; use std::sync::Arc; @@ -12,9 +11,10 @@ use crate::path::{ StandardLibraryPathBuf, }; use crate::resolver::internal::ModuleResolverSearchPaths; +use crate::supported_py_version::set_target_py_version; use crate::{Db, SupportedPyVersion}; -/// Configures the module search paths for the module resolver. +/// Configures the module resolver settings. /// /// Must be called before calling any other module resolution functions. pub fn set_module_resolution_settings(db: &mut dyn Db, config: ModuleResolutionSettings) { @@ -27,11 +27,7 @@ pub fn set_module_resolution_settings(db: &mut dyn Db, config: ModuleResolutionS } else { ModuleResolverSearchPaths::new(db, search_paths); } - if let Some(existing) = TargetPyVersion::try_get(db) { - existing.set_target_py_version(db).to(target_version); - } else { - TargetPyVersion::new(db, target_version); - } + set_target_py_version(db, target_version); } /// Resolves a module name to a module. @@ -217,7 +213,6 @@ impl Deref for OrderedSearchPaths { pub(crate) mod internal { use crate::module_name::ModuleName; use crate::resolver::OrderedSearchPaths; - use crate::SupportedPyVersion; #[salsa::input(singleton)] pub(crate) struct ModuleResolverSearchPaths { @@ -225,12 +220,6 @@ pub(crate) mod internal { pub(super) search_paths: OrderedSearchPaths, } - #[salsa::input(singleton)] - pub(crate) struct TargetPyVersion { - #[return_ref] - pub(super) target_py_version: SupportedPyVersion, - } - /// A thin wrapper around `ModuleName` to make it a Salsa ingredient. /// /// This is needed because Salsa requires that all query arguments are salsa ingredients. diff --git a/crates/red_knot_module_resolver/src/supported_py_version.rs b/crates/red_knot_module_resolver/src/supported_py_version.rs new file mode 100644 index 0000000000000..c32af4d6486db --- /dev/null +++ b/crates/red_knot_module_resolver/src/supported_py_version.rs @@ -0,0 +1,35 @@ +#![allow(clippy::used_underscore_binding)] // necessary for Salsa inputs +#![allow(unreachable_pub)] + +use crate::Db; + +// TODO: unify with the PythonVersion enum in the linter/formatter crates? +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] +pub enum SupportedPyVersion { + Py37, + #[default] + Py38, + Py39, + Py310, + Py311, + Py312, + Py313, +} + +#[salsa::input(singleton)] +pub(crate) struct TargetPyVersion { + #[return_ref] + pub(crate) target_py_version: SupportedPyVersion, +} + +pub(crate) fn set_target_py_version(db: &mut dyn Db, target_version: SupportedPyVersion) { + if let Some(existing) = TargetPyVersion::try_get(db) { + existing.set_target_py_version(db).to(target_version); + } else { + TargetPyVersion::new(db, target_version); + } +} + +pub(crate) fn get_target_py_version(db: &dyn Db) -> SupportedPyVersion { + *TargetPyVersion::get(db).target_py_version(db) +} diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index 159eaf81c089d..a4627ff749f90 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,9 +1,7 @@ -mod supported_py_version; mod versions; -pub(crate) use versions::TypeshedVersionsQueryResult; pub use versions::TypeshedVersions; -pub use supported_py_version::SupportedPyVersion; +pub(crate) use versions::TypeshedVersionsQueryResult; #[cfg(test)] mod tests { diff --git a/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs b/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs deleted file mode 100644 index 84fffa122e853..0000000000000 --- a/crates/red_knot_module_resolver/src/typeshed/supported_py_version.rs +++ /dev/null @@ -1,12 +0,0 @@ -// TODO: unify with the PythonVersion enum in the linter/formatter crates? -#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub enum SupportedPyVersion { - Py37, - #[default] - Py38, - Py39, - Py310, - Py311, - Py312, - Py313, -} diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index e5ee513a0957e..1190d03715dec 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -6,8 +6,8 @@ use std::str::FromStr; use rustc_hash::FxHashMap; -use super::supported_py_version::SupportedPyVersion; use crate::module_name::ModuleName; +use crate::supported_py_version::SupportedPyVersion; #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { From 1a82d218980d58928a5287c74cacdd53dcb97d2e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 1 Jul 2024 16:22:33 +0100 Subject: [PATCH 08/58] Address easy review comments --- .../src/typeshed/versions.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 1190d03715dec..9f6d720608a69 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -3,6 +3,7 @@ use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; +use std::sync::Arc; use rustc_hash::FxHashMap; @@ -82,7 +83,7 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } #[derive(Debug, PartialEq, Eq, Clone)] -pub struct TypeshedVersions(FxHashMap); +pub struct TypeshedVersions(Arc>); impl TypeshedVersions { pub fn len(&self) -> usize { @@ -93,14 +94,14 @@ impl TypeshedVersions { self.0.is_empty() } - fn get_exact(&self, module_name: &ModuleName) -> Option<&PyVersionRange> { + fn exact(&self, module_name: &ModuleName) -> Option<&PyVersionRange> { self.0.get(module_name) } /// Helper function for testing purposes #[cfg(test)] fn contains_exact(&self, module: &ModuleName) -> bool { - self.get_exact(module).is_some() + self.exact(module).is_some() } pub fn query_module( @@ -109,7 +110,7 @@ impl TypeshedVersions { version: impl Into, ) -> TypeshedVersionsQueryResult { let version = version.into(); - if let Some(range) = self.get_exact(module) { + if let Some(range) = self.exact(module) { if range.contains(version) { TypeshedVersionsQueryResult::Exists } else { @@ -118,7 +119,7 @@ impl TypeshedVersions { } else { let mut module = module.parent(); while let Some(module_to_try) = module { - if let Some(range) = self.get_exact(&module_to_try) { + if let Some(range) = self.exact(&module_to_try) { return { if range.contains(version) { TypeshedVersionsQueryResult::MaybeExists @@ -194,7 +195,7 @@ impl FromStr for TypeshedVersions { }; } - Ok(Self(map)) + Ok(Self(Arc::new(map))) } } From d91626c5301ffb9e79653cf5afb6592c5f0ce8c2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 12:03:26 +0100 Subject: [PATCH 09/58] Get rid of a lot of boilerplate --- crates/red_knot_module_resolver/src/lib.rs | 4 - crates/red_knot_module_resolver/src/module.rs | 53 +- crates/red_knot_module_resolver/src/path.rs | 1354 +++++------------ .../red_knot_module_resolver/src/resolver.rs | 205 ++- crates/red_knot_python_semantic/src/types.rs | 6 +- .../src/types/infer.rs | 6 +- 6 files changed, 517 insertions(+), 1111 deletions(-) diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index d8f62fc03d979..c1f5993974370 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -9,10 +9,6 @@ mod typeshed; pub use db::{Db, Jar}; pub use module::{Module, ModuleKind}; pub use module_name::ModuleName; -pub use path::{ - ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, SitePackagesPath, - SitePackagesPathBuf, StandardLibraryPath, StandardLibraryPathBuf, -}; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; pub use supported_py_version::SupportedPyVersion; pub use typeshed::TypeshedVersions; diff --git a/crates/red_knot_module_resolver/src/module.rs b/crates/red_knot_module_resolver/src/module.rs index c14a328ca0f66..024d104a9f2b2 100644 --- a/crates/red_knot_module_resolver/src/module.rs +++ b/crates/red_knot_module_resolver/src/module.rs @@ -4,10 +4,7 @@ use std::sync::Arc; use ruff_db::vfs::VfsFile; use crate::module_name::ModuleName; -use crate::path::{ - ExtraPathBuf, FirstPartyPathBuf, ModuleResolutionPathRef, SitePackagesPathBuf, - StandardLibraryPath, StandardLibraryPathBuf, -}; +use crate::path::{ModuleResolutionPath, ModuleResolutionPathRef}; use crate::Db; /// Representation of a Python module. @@ -20,7 +17,7 @@ impl Module { pub(crate) fn new( name: ModuleName, kind: ModuleKind, - search_path: Arc, + search_path: Arc, file: VfsFile, ) -> Self { Self { @@ -44,8 +41,8 @@ impl Module { } /// The search path from which the module was resolved. - pub(crate) fn search_path(&self) -> &ModuleSearchPathEntry { - &self.inner.search_path + pub(crate) fn search_path(&self) -> ModuleResolutionPathRef { + ModuleResolutionPathRef::from(&*self.inner.search_path) } /// Determine whether this module is a single-file module or a package @@ -80,7 +77,7 @@ impl salsa::DebugWithDb for Module { struct ModuleInner { name: ModuleName, kind: ModuleKind, - search_path: Arc, + search_path: Arc, file: VfsFile, } @@ -92,43 +89,3 @@ pub enum ModuleKind { /// A python package (`foo/__init__.py` or `foo/__init__.pyi`) Package, } - -/// Enumeration of the different kinds of search paths type checkers are expected to support. -/// -/// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the -/// priority that we want to give these modules when resolving them, -/// as per [the order given in the typing spec] -/// -/// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering -#[derive(Debug, Eq, PartialEq, Hash)] -pub(crate) enum ModuleSearchPathEntry { - /// "Extra" paths provided by the user in a config file, env var or CLI flag. - /// E.g. mypy's `MYPYPATH` env var, or pyright's `stubPath` configuration setting - Extra(ExtraPathBuf), - - /// Files in the project we're directly being invoked on - FirstParty(FirstPartyPathBuf), - - /// The `stdlib` directory of typeshed (either vendored or custom) - StandardLibrary(StandardLibraryPathBuf), - - /// Stubs or runtime modules installed in site-packages - SitePackagesThirdParty(SitePackagesPathBuf), - // TODO(Alex): vendor third-party stubs from typeshed as well? - // VendoredThirdParty(VendoredPathBuf), -} - -impl ModuleSearchPathEntry { - pub(crate) fn stdlib_from_typeshed_root(typeshed: &StandardLibraryPath) -> Self { - Self::StandardLibrary(StandardLibraryPath::stdlib_from_typeshed_root(typeshed)) - } - - pub(crate) fn path(&self) -> ModuleResolutionPathRef { - match self { - Self::Extra(path) => ModuleResolutionPathRef::Extra(path), - Self::FirstParty(path) => ModuleResolutionPathRef::FirstParty(path), - Self::StandardLibrary(path) => ModuleResolutionPathRef::StandardLibrary(path), - Self::SitePackagesThirdParty(path) => ModuleResolutionPathRef::SitePackages(path), - } - } -} diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index d8e2532ed2e93..26a17c9aae1b9 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,9 +1,8 @@ #![allow(unsafe_code)] use std::iter::FusedIterator; use std::ops::Deref; -use std::path; -use ruff_db::file_system::{FileSystem, FileSystemPath, FileSystemPathBuf}; +use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::VfsPath; use crate::module_name::ModuleName; @@ -13,842 +12,213 @@ use crate::Db; #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] -pub struct ExtraPath(FileSystemPath); +pub(crate) struct ExtraPath(FileSystemPath); impl ExtraPath { - #[must_use] - pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { - let path = path.as_ref(); - if path - .extension() - .is_some_and(|extension| !matches!(extension, "pyi" | "py")) - { - return None; - } - Some(Self::new_unchecked(path)) - } - #[must_use] fn new_unchecked(path: &FileSystemPath) -> &Self { // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const ExtraPath is valid. - unsafe { &*(path as *const FileSystemPath as *const ExtraPath) } - } - - #[must_use] - pub(crate) fn parent(&self) -> Option<&Self> { - Some(Self::new_unchecked(self.0.parent()?)) - } - - #[must_use] - pub(crate) fn sans_dunder_init(&self) -> &Self { - if self.0.ends_with("__init__.py") || self.0.ends_with("__init__.pyi") { - self.parent() - .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) - } else { - self - } - } - - #[must_use] - pub(crate) fn module_name_parts(&self) -> ModulePartIterator { - ModulePartIterator::from_fs_path(&self.0) - } - - #[must_use] - pub(crate) fn relative_to_search_path(&self, search_path: &ExtraPath) -> Option<&Self> { - self.0 - .strip_prefix(search_path) - .map(Self::new_unchecked) - .ok() - } - - #[must_use] - pub fn to_path_buf(&self) -> ExtraPathBuf { - ExtraPathBuf(self.0.to_path_buf()) - } - - #[must_use] - fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { - file_system.exists(&self.0.join("__init__.py")) - || file_system.exists(&self.0.join("__init__.pyi")) - } - - #[must_use] - fn is_directory(&self, file_system: &dyn FileSystem) -> bool { - file_system.is_directory(&self.0) - } - - #[must_use] - pub(crate) fn with_pyi_extension(&self) -> ExtraPathBuf { - ExtraPathBuf(self.0.with_extension("pyi")) - } - - #[must_use] - pub(crate) fn with_py_extension(&self) -> ExtraPathBuf { - ExtraPathBuf(self.0.with_extension("py")) - } - - #[must_use] - #[inline] - pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { - &self.0 - } -} - -impl PartialEq for ExtraPath { - fn eq(&self, other: &FileSystemPath) -> bool { - self.0 == *other - } -} - -impl PartialEq for ExtraPath { - fn eq(&self, other: &VfsPath) -> bool { - match other { - VfsPath::FileSystem(path) => **path == self.0, - VfsPath::Vendored(_) => false, - } - } -} - -impl AsRef for ExtraPath { - #[inline] - fn as_ref(&self) -> &ExtraPath { - self - } -} - -impl AsRef for ExtraPath { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - self.as_file_system_path() - } -} - -impl AsRef for ExtraPath { - #[inline] - fn as_ref(&self) -> &path::Path { - self.0.as_ref() - } -} - -impl AsRef for ExtraPath { - #[inline] - fn as_ref(&self) -> &str { - self.0.as_str() + unsafe { &*(path as *const FileSystemPath as *const Self) } } } #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct ExtraPathBuf(FileSystemPathBuf); - -impl ExtraPathBuf { - #[must_use] - #[inline] - fn as_path(&self) -> &ExtraPath { - ExtraPath::new(&self.0).unwrap() - } - - /// Push a new part to the path, - /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions - /// - /// ## Panics: - /// If a component with an invalid extension is passed - fn push(&mut self, component: &str) { - debug_assert!(matches!(component.matches('.').count(), 0 | 1)); - if cfg!(debug) { - if let Some(extension) = std::path::Path::new(component).extension() { - assert!( - matches!(extension.to_str().unwrap(), "pyi" | "py"), - "Extension must be `py` or `pyi`; got {extension:?}" - ); - } - } - self.0.push(component); - } - - #[inline] - pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { - &self.0 - } -} - -impl AsRef for ExtraPathBuf { - #[inline] - fn as_ref(&self) -> &FileSystemPathBuf { - self.as_file_system_path_buf() - } -} - -impl AsRef for ExtraPathBuf { - #[inline] - fn as_ref(&self) -> &ExtraPath { - self.as_path() - } -} +pub(crate) struct ExtraPathBuf(FileSystemPathBuf); impl Deref for ExtraPathBuf { type Target = ExtraPath; - #[inline] fn deref(&self) -> &Self::Target { - self.as_path() + ExtraPath::new_unchecked(&self.0) } } #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] -pub struct FirstPartyPath(FileSystemPath); +pub(crate) struct FirstPartyPath(FileSystemPath); impl FirstPartyPath { - #[must_use] - pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { - let path = path.as_ref(); - if path - .extension() - .is_some_and(|extension| !matches!(extension, "pyi" | "py")) - { - return None; - } - Some(Self::new_unchecked(path)) - } - #[must_use] fn new_unchecked(path: &FileSystemPath) -> &Self { // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const FirstPartyPath is valid. - unsafe { &*(path as *const FileSystemPath as *const FirstPartyPath) } - } - - #[must_use] - pub(crate) fn parent(&self) -> Option<&Self> { - Some(Self::new_unchecked(self.0.parent()?)) - } - - #[must_use] - pub(crate) fn sans_dunder_init(&self) -> &Self { - if self.0.ends_with("__init__.py") || self.0.ends_with("__init__.pyi") { - self.parent() - .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) - } else { - self - } - } - - #[must_use] - pub(crate) fn module_name_parts(&self) -> ModulePartIterator { - ModulePartIterator::from_fs_path(&self.0) - } - - #[must_use] - pub(crate) fn relative_to_search_path(&self, search_path: &FirstPartyPath) -> Option<&Self> { - self.0 - .strip_prefix(search_path) - .map(Self::new_unchecked) - .ok() - } - - #[must_use] - pub fn to_path_buf(&self) -> FirstPartyPathBuf { - FirstPartyPathBuf(self.0.to_path_buf()) - } - - #[must_use] - fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { - file_system.exists(&self.0.join("__init__.py")) - || file_system.exists(&self.0.join("__init__.pyi")) - } - - #[must_use] - fn is_directory(&self, file_system: &dyn FileSystem) -> bool { - file_system.is_directory(&self.0) - } - - #[must_use] - pub(crate) fn with_pyi_extension(&self) -> FirstPartyPathBuf { - FirstPartyPathBuf(self.0.with_extension("pyi")) - } - - #[must_use] - pub(crate) fn with_py_extension(&self) -> FirstPartyPathBuf { - FirstPartyPathBuf(self.0.with_extension("py")) - } - - #[must_use] - #[inline] - pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { - &self.0 - } - - #[cfg(test)] - #[must_use] - pub(crate) fn join(&self, path: &str) -> FirstPartyPathBuf { - let mut result = self.to_path_buf(); - result.push(path); - result - } -} - -impl PartialEq for FirstPartyPath { - fn eq(&self, other: &FileSystemPath) -> bool { - self.0 == *other - } -} - -impl PartialEq for FirstPartyPath { - fn eq(&self, other: &VfsPath) -> bool { - match other { - VfsPath::FileSystem(path) => **path == self.0, - VfsPath::Vendored(_) => false, - } - } -} - -impl AsRef for FirstPartyPath { - #[inline] - fn as_ref(&self) -> &FirstPartyPath { - self - } -} - -impl AsRef for FirstPartyPath { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - self.as_file_system_path() - } -} - -impl AsRef for FirstPartyPath { - #[inline] - fn as_ref(&self) -> &path::Path { - self.0.as_ref() - } -} - -impl AsRef for FirstPartyPath { - #[inline] - fn as_ref(&self) -> &str { - self.0.as_str() + unsafe { &*(path as *const FileSystemPath as *const Self) } } } #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct FirstPartyPathBuf(FileSystemPathBuf); - -impl FirstPartyPathBuf { - #[must_use] - #[inline] - fn as_path(&self) -> &FirstPartyPath { - FirstPartyPath::new(&self.0).unwrap() - } - - /// Push a new part to the path, - /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions - /// - /// ## Panics: - /// If a component with an invalid extension is passed - fn push(&mut self, component: &str) { - debug_assert!(matches!(component.matches('.').count(), 0 | 1)); - if cfg!(debug) { - if let Some(extension) = std::path::Path::new(component).extension() { - assert!( - matches!(extension.to_str().unwrap(), "pyi" | "py"), - "Extension must be `py` or `pyi`; got {extension:?}" - ); - } - } - self.0.push(component); - } - - #[cfg(test)] - pub(crate) fn into_vfs_path(self) -> VfsPath { - VfsPath::FileSystem(self.0) - } - - #[inline] - pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { - &self.0 - } -} - -impl AsRef for FirstPartyPathBuf { - #[inline] - fn as_ref(&self) -> &FileSystemPathBuf { - self.as_file_system_path_buf() - } -} - -impl AsRef for FirstPartyPathBuf { - #[inline] - fn as_ref(&self) -> &FirstPartyPath { - self.as_path() - } -} +pub(crate) struct FirstPartyPathBuf(FileSystemPathBuf); impl Deref for FirstPartyPathBuf { type Target = FirstPartyPath; - #[inline] fn deref(&self) -> &Self::Target { - self.as_path() + FirstPartyPath::new_unchecked(&self.0) } } -// TODO(Alex): Standard-library paths could be vendored paths #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] -pub struct StandardLibraryPath(FileSystemPath); +pub(crate) struct StandardLibraryPath(FileSystemPath); impl StandardLibraryPath { #[must_use] - pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { - let path = path.as_ref(); - // Only allow pyi extensions, unlike other paths - if path.extension().is_some_and(|extension| extension != "pyi") { - return None; - } - Some(Self::new_unchecked(path)) - } - - #[must_use] - fn new_unchecked(path: &(impl AsRef + ?Sized)) -> &Self { - // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const FirstPartyPath is valid. - let path = path.as_ref(); - unsafe { &*(path as *const FileSystemPath as *const StandardLibraryPath) } - } - - #[must_use] - #[inline] - pub(crate) fn stdlib_dir() -> &'static Self { - Self::new_unchecked("stdlib") - } - - pub(crate) fn stdlib_from_typeshed_root( - typeshed: &StandardLibraryPath, - ) -> StandardLibraryPathBuf { - StandardLibraryPathBuf(typeshed.0.join(Self::stdlib_dir())) - } - - #[must_use] - pub(crate) fn relative_to_search_path( - &self, - search_path: &StandardLibraryPath, - ) -> Option<&Self> { - self.0 - .strip_prefix(search_path) - .map(Self::new_unchecked) - .ok() - } - - #[must_use] - pub(crate) fn parent(&self) -> Option<&Self> { - Some(Self::new_unchecked(self.0.parent()?)) - } - - #[must_use] - pub(crate) fn sans_dunder_init(&self) -> &Self { - // Only try to strip `__init__.pyi` from the end, unlike other paths - if self.0.ends_with("__init__.pyi") { - self.parent() - .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) - } else { - self - } - } - - #[must_use] - pub(crate) fn module_name_parts(&self) -> ModulePartIterator { - ModulePartIterator::from_fs_path(&self.0) - } - - #[must_use] - pub fn to_path_buf(&self) -> StandardLibraryPathBuf { - StandardLibraryPathBuf(self.0.to_path_buf()) - } - - pub(crate) fn as_module_name(&self) -> Option { - ModuleResolutionPathRef::StandardLibrary(self).as_module_name() - } - - #[must_use] - fn is_regular_package(&self, db: &dyn Db) -> bool { - let Some(module_name) = self.as_module_name() else { - return false; - }; - match db - .typeshed_versions() - .query_module(&module_name, get_target_py_version(db)) - { - TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { - db.file_system().exists(&self.0.join("__init__.pyi")) - } - TypeshedVersionsQueryResult::DoesNotExist => false, - } - } - - #[must_use] - fn is_directory(&self, db: &dyn Db) -> bool { - let Some(module_name) = self.as_module_name() else { - return false; - }; - match db - .typeshed_versions() - .query_module(&module_name, get_target_py_version(db)) - { - TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { - db.file_system().is_directory(&self.0) - } - TypeshedVersionsQueryResult::DoesNotExist => false, - } - } - - #[must_use] - pub(crate) fn with_pyi_extension(&self) -> StandardLibraryPathBuf { - StandardLibraryPathBuf(self.0.with_extension("pyi")) - } - - #[must_use] - #[inline] - pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { - &self.0 - } - - #[cfg(test)] - #[must_use] - pub(crate) fn join(&self, path: &str) -> StandardLibraryPathBuf { - let mut result = self.to_path_buf(); - result.push(path); - result - } -} - -impl PartialEq for StandardLibraryPath { - fn eq(&self, other: &FileSystemPath) -> bool { - self.0 == *other - } -} - -impl PartialEq for StandardLibraryPath { - fn eq(&self, other: &VfsPath) -> bool { - match other { - VfsPath::FileSystem(path) => **path == self.0, - VfsPath::Vendored(_) => false, - } - } -} - -impl AsRef for StandardLibraryPath { - fn as_ref(&self) -> &StandardLibraryPath { - self - } -} - -impl AsRef for StandardLibraryPath { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - self.as_file_system_path() - } -} - -impl AsRef for StandardLibraryPath { - #[inline] - fn as_ref(&self) -> &path::Path { - self.0.as_ref() - } -} - -impl AsRef for StandardLibraryPath { - #[inline] - fn as_ref(&self) -> &str { - self.0.as_str() + fn new_unchecked(path: &FileSystemPath) -> &Self { + // SAFETY: StandardLibraryPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const StandardLibraryPath is valid. + unsafe { &*(path as *const FileSystemPath as *const Self) } } } -// TODO(Alex): Standard-library paths could also be vendored paths #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct StandardLibraryPathBuf(FileSystemPathBuf); - -impl StandardLibraryPathBuf { - #[must_use] - #[inline] - fn as_path(&self) -> &StandardLibraryPath { - StandardLibraryPath::new(&self.0).unwrap() - } - - /// Push a new part to the path, - /// while maintaining the invariant that the path can only have `.pyi` extensions - /// - /// ## Panics: - /// If a component with an invalid extension is passed - fn push(&mut self, component: &str) { - debug_assert!(matches!(component.matches('.').count(), 0 | 1)); - if cfg!(debug) { - if let Some(extension) = std::path::Path::new(component).extension() { - assert_eq!( - extension.to_str().unwrap(), - "pyi", - "Extension must be `pyi`; got {extension:?}" - ); - } - } - self.0.push(component); - } - - #[cfg(test)] - pub(crate) fn into_vfs_path(self) -> VfsPath { - VfsPath::FileSystem(self.0) - } - - #[inline] - pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { - &self.0 - } -} - -impl AsRef for StandardLibraryPathBuf { - #[inline] - fn as_ref(&self) -> &StandardLibraryPath { - self.as_path() - } -} - -impl AsRef for StandardLibraryPathBuf { - #[inline] - fn as_ref(&self) -> &FileSystemPathBuf { - self.as_file_system_path_buf() - } -} - -impl Deref for StandardLibraryPathBuf { - type Target = StandardLibraryPath; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_path() - } -} - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct SitePackagesPath(FileSystemPath); - -impl SitePackagesPath { - #[must_use] - pub fn new(path: &(impl AsRef + ?Sized)) -> Option<&Self> { - let path = path.as_ref(); - if path - .extension() - .is_some_and(|extension| !matches!(extension, "pyi" | "py")) - { - return None; - } - Some(Self::new_unchecked(path)) - } - - #[must_use] - fn new_unchecked(path: &FileSystemPath) -> &Self { - // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const SitePackagesPath is valid. - unsafe { &*(path as *const FileSystemPath as *const SitePackagesPath) } - } - - #[must_use] - pub(crate) fn parent(&self) -> Option<&Self> { - Some(Self::new_unchecked(self.0.parent()?)) - } - - #[must_use] - pub(crate) fn sans_dunder_init(&self) -> &Self { - // Only try to strip `__init__.pyi` from the end, unlike other paths - if self.0.ends_with("__init__.pyi") || self.0.ends_with("__init__.py") { - self.parent() - .unwrap_or_else(|| Self::new_unchecked(FileSystemPath::new(""))) - } else { - self - } - } - - #[must_use] - pub(crate) fn module_name_parts(&self) -> ModulePartIterator { - ModulePartIterator::from_fs_path(&self.0) - } - - #[must_use] - pub(crate) fn relative_to_search_path(&self, search_path: &SitePackagesPath) -> Option<&Self> { - self.0 - .strip_prefix(search_path) - .map(Self::new_unchecked) - .ok() - } +pub(crate) struct StandardLibraryPathBuf(FileSystemPathBuf); - #[must_use] - pub fn to_path_buf(&self) -> SitePackagesPathBuf { - SitePackagesPathBuf(self.0.to_path_buf()) - } - - #[must_use] - fn is_regular_package(&self, file_system: &dyn FileSystem) -> bool { - file_system.exists(&self.0.join("__init__.py")) - || file_system.exists(&self.0.join("__init__.pyi")) - } - - #[must_use] - fn is_directory(&self, file_system: &dyn FileSystem) -> bool { - file_system.is_directory(&self.0) - } - - #[must_use] - pub(crate) fn with_pyi_extension(&self) -> SitePackagesPathBuf { - SitePackagesPathBuf(self.0.with_extension("pyi")) - } - - #[must_use] - pub(crate) fn with_py_extension(&self) -> SitePackagesPathBuf { - SitePackagesPathBuf(self.0.with_extension("py")) - } - - #[must_use] - #[inline] - pub(crate) fn as_file_system_path(&self) -> &FileSystemPath { - &self.0 - } - - #[cfg(test)] - #[must_use] - pub(crate) fn join(&self, path: &str) -> SitePackagesPathBuf { - let mut result = self.to_path_buf(); - result.push(path); - result - } -} - -impl PartialEq for SitePackagesPath { - fn eq(&self, other: &FileSystemPath) -> bool { - self.0 == *other - } -} - -impl PartialEq for SitePackagesPath { - fn eq(&self, other: &VfsPath) -> bool { - match other { - VfsPath::FileSystem(path) => **path == self.0, - VfsPath::Vendored(_) => false, - } - } -} - -impl AsRef for SitePackagesPath { - fn as_ref(&self) -> &SitePackagesPath { - self - } -} - -impl AsRef for SitePackagesPath { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - self.as_file_system_path() - } -} +impl Deref for StandardLibraryPathBuf { + type Target = StandardLibraryPath; -impl AsRef for SitePackagesPath { - #[inline] - fn as_ref(&self) -> &path::Path { - self.0.as_ref() + fn deref(&self) -> &Self::Target { + StandardLibraryPath::new_unchecked(&self.0) } } -impl AsRef for SitePackagesPath { - #[inline] - fn as_ref(&self) -> &str { - self.0.as_str() +#[repr(transparent)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) struct SitePackagesPath(FileSystemPath); + +impl SitePackagesPath { + #[must_use] + fn new_unchecked(path: &FileSystemPath) -> &Self { + // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const SitePackagesPath is valid. + unsafe { &*(path as *const FileSystemPath as *const Self) } } } #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct SitePackagesPathBuf(FileSystemPathBuf); +pub(crate) struct SitePackagesPathBuf(FileSystemPathBuf); -impl SitePackagesPathBuf { - #[must_use] - #[inline] - fn as_path(&self) -> &SitePackagesPath { - SitePackagesPath::new(&self.0).unwrap() +impl Deref for SitePackagesPathBuf { + type Target = SitePackagesPath; + + fn deref(&self) -> &Self::Target { + SitePackagesPath::new_unchecked(&self.0) } +} + +/// Enumeration of the different kinds of search paths type checkers are expected to support. +/// +/// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the +/// priority that we want to give these modules when resolving them, +/// as per [the order given in the typing spec] +/// +/// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum ModuleResolutionPath { + Extra(ExtraPathBuf), + FirstParty(FirstPartyPathBuf), + StandardLibrary(StandardLibraryPathBuf), + SitePackages(SitePackagesPathBuf), +} +impl ModuleResolutionPath { /// Push a new part to the path, - /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions + /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions. + /// For the stdlib variant specifically, it may only have a `.pyi` extension. /// /// ## Panics: /// If a component with an invalid extension is passed - fn push(&mut self, component: &str) { + pub(crate) fn push(&mut self, component: &str) { debug_assert!(matches!(component.matches('.').count(), 0 | 1)); if cfg!(debug) { if let Some(extension) = std::path::Path::new(component).extension() { - assert!( - matches!(extension.to_str().unwrap(), "pyi" | "py"), - "Extension must be `py` or `pyi`; got {extension:?}" - ); + match self { + Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( + matches!(extension.to_str().unwrap(), "pyi" | "py"), + "Extension must be `py` or `pyi`; got {extension:?}" + ), + Self::StandardLibrary(_) => assert_eq!( + extension.to_str().unwrap(), + "pyi", + "Extension must be `py` or `pyi`; got {extension:?}" + ), + }; } } - self.0.push(component); + let inner = match self { + Self::Extra(ExtraPathBuf(ref mut path)) => path, + Self::FirstParty(FirstPartyPathBuf(ref mut path)) => path, + Self::StandardLibrary(StandardLibraryPathBuf(ref mut path)) => path, + Self::SitePackages(SitePackagesPathBuf(ref mut path)) => path, + }; + inner.push(component); } - #[cfg(test)] - pub(crate) fn into_vfs_path(self) -> VfsPath { - VfsPath::FileSystem(self.0) + pub(crate) fn extra(path: FileSystemPathBuf) -> Option { + if path + .extension() + .map_or(true, |ext| matches!(ext, "py" | "pyi")) + { + Some(Self::extra_unchecked(path)) + } else { + None + } } - #[inline] - pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { - &self.0 + fn extra_unchecked(path: FileSystemPathBuf) -> Self { + Self::Extra(ExtraPathBuf(path)) } -} -impl AsRef for SitePackagesPathBuf { - #[inline] - fn as_ref(&self) -> &FileSystemPathBuf { - self.as_file_system_path_buf() + pub(crate) fn first_party(path: FileSystemPathBuf) -> Option { + if path + .extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + { + Some(Self::first_party_unchecked(path)) + } else { + None + } } -} -impl AsRef for SitePackagesPathBuf { - #[inline] - fn as_ref(&self) -> &SitePackagesPath { - self.as_path() + fn first_party_unchecked(path: FileSystemPathBuf) -> Self { + Self::FirstParty(FirstPartyPathBuf(path)) } -} -impl Deref for SitePackagesPathBuf { - type Target = SitePackagesPath; + pub(crate) fn standard_library(path: FileSystemPathBuf) -> Option { + if path.extension().map_or(true, |ext| ext == "pyi") { + Some(Self::standard_library_unchecked(path)) + } else { + None + } + } - #[inline] - fn deref(&self) -> &Self::Target { - self.as_path() + pub(crate) fn stdlib_from_typeshed_root(typeshed_root: &FileSystemPath) -> Option { + Self::standard_library(typeshed_root.join(FileSystemPath::new("stdlib"))) } -} -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) enum ModuleResolutionPath { - Extra(ExtraPathBuf), - FirstParty(FirstPartyPathBuf), - StandardLibrary(StandardLibraryPathBuf), - SitePackages(SitePackagesPathBuf), -} + fn standard_library_unchecked(path: FileSystemPathBuf) -> Self { + Self::StandardLibrary(StandardLibraryPathBuf(path)) + } -impl ModuleResolutionPath { - pub(crate) fn push(&mut self, component: &str) { - match self { - Self::Extra(ref mut path) => path.push(component), - Self::FirstParty(ref mut path) => path.push(component), - Self::StandardLibrary(ref mut path) => path.push(component), - Self::SitePackages(ref mut path) => path.push(component), + pub(crate) fn site_packages(path: FileSystemPathBuf) -> Option { + if path + .extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + { + Some(Self::site_packages_unchecked(path)) + } else { + None } } + fn site_packages_unchecked(path: FileSystemPathBuf) -> Self { + Self::SitePackages(SitePackagesPathBuf(path)) + } + pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { ModuleResolutionPathRef::from(self).is_regular_package(db) } @@ -864,244 +234,345 @@ impl ModuleResolutionPath { pub(crate) fn with_py_extension(&self) -> Option { ModuleResolutionPathRef::from(self).with_py_extension() } -} -impl AsRef for ModuleResolutionPath { - fn as_ref(&self) -> &FileSystemPath { + #[cfg(test)] + pub(crate) fn join(&self, component: &(impl AsRef + ?Sized)) -> Self { + ModuleResolutionPathRef::from(self).join(component) + } + + pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { match self { - Self::Extra(path) => path.as_file_system_path(), - Self::FirstParty(path) => path.as_file_system_path(), - Self::StandardLibrary(path) => path.as_file_system_path(), - Self::SitePackages(path) => path.as_file_system_path(), + Self::Extra(ExtraPathBuf(path)) => path, + Self::FirstParty(FirstPartyPathBuf(path)) => path, + Self::StandardLibrary(StandardLibraryPathBuf(path)) => path, + Self::SitePackages(SitePackagesPathBuf(path)) => path, } } -} -impl AsRef for ModuleResolutionPath { - fn as_ref(&self) -> &FileSystemPathBuf { + #[cfg(test)] + #[must_use] + #[inline] + fn into_file_system_path_buf(self) -> FileSystemPathBuf { match self { - Self::Extra(path) => path.as_file_system_path_buf(), - Self::FirstParty(path) => path.as_file_system_path_buf(), - Self::StandardLibrary(path) => path.as_file_system_path_buf(), - Self::SitePackages(path) => path.as_file_system_path_buf(), + Self::Extra(ExtraPathBuf(path)) => path, + Self::FirstParty(FirstPartyPathBuf(path)) => path, + Self::StandardLibrary(StandardLibraryPathBuf(path)) => path, + Self::SitePackages(SitePackagesPathBuf(path)) => path, } } + + #[cfg(test)] + #[must_use] + pub(crate) fn into_vfs_path(self) -> VfsPath { + VfsPath::FileSystem(self.into_file_system_path_buf()) + } } -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &ExtraPath) -> bool { - if let ModuleResolutionPath::Extra(path) = self { - **path == *other - } else { - false +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &VfsPath) -> bool { + match other { + VfsPath::FileSystem(path) => self.as_file_system_path_buf() == path, + VfsPath::Vendored(_) => false, } } } -impl PartialEq for ExtraPath { +impl PartialEq for VfsPath { fn eq(&self, other: &ModuleResolutionPath) -> bool { other.eq(self) } } -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &ExtraPathBuf) -> bool { - self.eq(&**other) +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &FileSystemPathBuf) -> bool { + self.as_file_system_path_buf() == other } } -impl PartialEq for ExtraPathBuf { +impl PartialEq for FileSystemPathBuf { fn eq(&self, other: &ModuleResolutionPath) -> bool { other.eq(self) } } -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &FirstPartyPath) -> bool { - if let ModuleResolutionPath::FirstParty(path) = self { - **path == *other - } else { - false - } +impl PartialEq for ModuleResolutionPath { + fn eq(&self, other: &FileSystemPath) -> bool { + ModuleResolutionPathRef::from(self) == *other } } -impl PartialEq for FirstPartyPath { +impl PartialEq for FileSystemPath { fn eq(&self, other: &ModuleResolutionPath) -> bool { other.eq(self) } } -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &FirstPartyPathBuf) -> bool { - self.eq(&**other) +impl AsRef for ModuleResolutionPath { + fn as_ref(&self) -> &FileSystemPathBuf { + self.as_file_system_path_buf() } } -impl PartialEq for FirstPartyPathBuf { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) +impl AsRef for ModuleResolutionPath { + fn as_ref(&self) -> &FileSystemPath { + ModuleResolutionPathRef::from(self).as_file_system_path() } } -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &StandardLibraryPath) -> bool { - if let ModuleResolutionPath::StandardLibrary(path) = self { - **path == *other +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub(crate) enum ModuleResolutionPathRef<'a> { + Extra(&'a ExtraPath), + FirstParty(&'a FirstPartyPath), + StandardLibrary(&'a StandardLibraryPath), + SitePackages(&'a SitePackagesPath), +} + +impl<'a> ModuleResolutionPathRef<'a> { + pub(crate) fn extra(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + if path + .extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + { + Some(Self::extra_unchecked(path)) } else { - false + None } } -} -impl PartialEq for StandardLibraryPath { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) + fn extra_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { + Self::Extra(ExtraPath::new_unchecked(path.as_ref())) } -} -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &StandardLibraryPathBuf) -> bool { - self.eq(&**other) + pub(crate) fn first_party(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + if path + .extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + { + Some(Self::first_party_unchecked(path)) + } else { + None + } } -} -impl PartialEq for StandardLibraryPathBuf { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) + fn first_party_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { + Self::FirstParty(FirstPartyPath::new_unchecked(path.as_ref())) } -} -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &SitePackagesPath) -> bool { - if let ModuleResolutionPath::SitePackages(path) = self { - **path == *other + pub(crate) fn standard_library( + path: &'a (impl AsRef + ?Sized), + ) -> Option { + let path = path.as_ref(); + if path.extension().map_or(true, |ext| ext == "pyi") { + Some(Self::standard_library_unchecked(path)) } else { - false + None } } -} -impl PartialEq for SitePackagesPath { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) + fn standard_library_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { + Self::StandardLibrary(StandardLibraryPath::new_unchecked(path.as_ref())) } -} -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &SitePackagesPathBuf) -> bool { - self.eq(&**other) + pub(crate) fn site_packages(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + if path + .extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + { + Some(Self::site_packages_unchecked(path)) + } else { + None + } } -} -impl PartialEq for SitePackagesPathBuf { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) + fn site_packages_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { + Self::SitePackages(SitePackagesPath::new_unchecked(path.as_ref())) } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum ModuleResolutionPathRef<'a> { - Extra(&'a ExtraPath), - FirstParty(&'a FirstPartyPath), - StandardLibrary(&'a StandardLibraryPath), - SitePackages(&'a SitePackagesPath), -} -impl<'a> ModuleResolutionPathRef<'a> { - #[must_use] - pub(crate) fn sans_dunder_init(self) -> Self { + pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { match self { - Self::Extra(path) => Self::Extra(path.sans_dunder_init()), - Self::FirstParty(path) => Self::FirstParty(path.sans_dunder_init()), - Self::StandardLibrary(path) => Self::StandardLibrary(path.sans_dunder_init()), - Self::SitePackages(path) => Self::SitePackages(path.sans_dunder_init()), + Self::Extra(ExtraPath(path)) => db.file_system().is_directory(path), + Self::FirstParty(FirstPartyPath(path)) => db.file_system().is_directory(path), + Self::SitePackages(SitePackagesPath(path)) => db.file_system().is_directory(path), + Self::StandardLibrary(StandardLibraryPath(path)) => { + let Some(module_name) = self.as_module_name() else { + return false; + }; + match db + .typeshed_versions() + .query_module(&module_name, get_target_py_version(db)) + { + TypeshedVersionsQueryResult::Exists + | TypeshedVersionsQueryResult::MaybeExists => { + db.file_system().is_directory(path) + } + TypeshedVersionsQueryResult::DoesNotExist => false, + } + } } } - #[must_use] - pub(crate) fn module_name_parts(self) -> ModulePartIterator<'a> { + pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { match self { - Self::Extra(path) => path.module_name_parts(), - Self::FirstParty(path) => path.module_name_parts(), - Self::StandardLibrary(path) => path.module_name_parts(), - Self::SitePackages(path) => path.module_name_parts(), + Self::Extra(ExtraPath(fs_path)) + | Self::FirstParty(FirstPartyPath(fs_path)) + | Self::SitePackages(SitePackagesPath(fs_path)) => { + let file_system = db.file_system(); + file_system.exists(&fs_path.join("__init__.py")) + || file_system.exists(&fs_path.join("__init__.pyi")) + } + // Unlike the other variants: + // (1) Account for VERSIONS + // (2) Only test for `__init__.pyi`, not `__init__.py` + Self::StandardLibrary(StandardLibraryPath(fs_path)) => { + let Some(module_name) = self.as_module_name() else { + return false; + }; + match db + .typeshed_versions() + .query_module(&module_name, get_target_py_version(db)) + { + TypeshedVersionsQueryResult::Exists + | TypeshedVersionsQueryResult::MaybeExists => { + db.file_system().exists(&fs_path.join("__init__.pyi")) + } + TypeshedVersionsQueryResult::DoesNotExist => false, + } + } } } - #[must_use] - pub(crate) fn to_owned(self) -> ModuleResolutionPath { + pub(crate) fn parent(&self) -> Option { + Some(match self { + Self::Extra(ExtraPath(path)) => Self::extra_unchecked(path.parent()?), + Self::FirstParty(FirstPartyPath(path)) => Self::first_party_unchecked(path.parent()?), + Self::StandardLibrary(StandardLibraryPath(path)) => { + Self::standard_library_unchecked(path.parent()?) + } + Self::SitePackages(SitePackagesPath(path)) => { + Self::site_packages_unchecked(path.parent()?) + } + }) + } + + fn ends_with_dunder_init(&self) -> bool { match self { - Self::Extra(path) => ModuleResolutionPath::Extra(path.to_path_buf()), - Self::FirstParty(path) => ModuleResolutionPath::FirstParty(path.to_path_buf()), - Self::StandardLibrary(path) => { - ModuleResolutionPath::StandardLibrary(path.to_path_buf()) + Self::Extra(ExtraPath(path)) + | Self::FirstParty(FirstPartyPath(path)) + | Self::SitePackages(SitePackagesPath(path)) => { + path.ends_with("__init__.py") || path.ends_with("__init__.pyi") } - Self::SitePackages(path) => ModuleResolutionPath::SitePackages(path.to_path_buf()), + Self::StandardLibrary(StandardLibraryPath(path)) => path.ends_with("__init__.pyi"), } } - #[must_use] - pub(crate) fn is_regular_package(self, db: &dyn Db) -> bool { - match self { - Self::Extra(path) => path.is_regular_package(db.file_system()), - Self::FirstParty(path) => path.is_regular_package(db.file_system()), - Self::StandardLibrary(path) => path.is_regular_package(db), - Self::SitePackages(path) => path.is_regular_package(db.file_system()), + fn sans_dunder_init(self) -> Self { + if self.ends_with_dunder_init() { + self.parent().unwrap_or_else(|| match self { + Self::Extra(_) => Self::extra_unchecked(""), + Self::FirstParty(_) => Self::first_party_unchecked(""), + Self::StandardLibrary(_) => Self::standard_library_unchecked(""), + Self::SitePackages(_) => Self::site_packages_unchecked(""), + }) + } else { + self } } - #[must_use] - pub(crate) fn is_directory(self, db: &dyn Db) -> bool { - match self { - Self::Extra(path) => path.is_directory(db.file_system()), - Self::FirstParty(path) => path.is_directory(db.file_system()), - Self::StandardLibrary(path) => path.is_directory(db), - Self::SitePackages(path) => path.is_directory(db.file_system()), + pub(crate) fn as_module_name(&self) -> Option { + let mut parts_iter = match self.sans_dunder_init() { + Self::Extra(ExtraPath(path)) => ModulePartIterator::from_fs_path(path), + Self::FirstParty(FirstPartyPath(path)) => ModulePartIterator::from_fs_path(path), + Self::StandardLibrary(StandardLibraryPath(path)) => { + ModulePartIterator::from_fs_path(path) + } + Self::SitePackages(SitePackagesPath(path)) => ModulePartIterator::from_fs_path(path), + }; + let first_part = parts_iter.next()?; + if let Some(second_part) = parts_iter.next() { + let mut name = format!("{first_part}.{second_part}"); + for part in parts_iter { + name.push('.'); + name.push_str(part); + } + ModuleName::new(&name) + } else { + ModuleName::new(first_part) } } - #[must_use] - pub(crate) fn with_pyi_extension(self) -> ModuleResolutionPath { + pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPath { match self { - Self::Extra(path) => ModuleResolutionPath::Extra(path.with_pyi_extension()), - Self::FirstParty(path) => ModuleResolutionPath::FirstParty(path.with_pyi_extension()), - Self::StandardLibrary(path) => { - ModuleResolutionPath::StandardLibrary(path.with_pyi_extension()) + Self::Extra(ExtraPath(path)) => { + ModuleResolutionPath::extra_unchecked(path.with_extension("pyi")) + } + Self::FirstParty(FirstPartyPath(path)) => { + ModuleResolutionPath::first_party_unchecked(path.with_extension("pyi")) } - Self::SitePackages(path) => { - ModuleResolutionPath::SitePackages(path.with_pyi_extension()) + Self::StandardLibrary(StandardLibraryPath(path)) => { + ModuleResolutionPath::standard_library_unchecked(path.with_extension("pyi")) } + Self::SitePackages(SitePackagesPath(path)) => { + ModuleResolutionPath::site_packages_unchecked(path.with_extension("pyi")) + } + } + } + + pub(crate) fn with_py_extension(&self) -> Option { + match self { + Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPath::extra_unchecked( + path.with_extension("py"), + )), + Self::FirstParty(FirstPartyPath(path)) => Some( + ModuleResolutionPath::first_party_unchecked(path.with_extension("py")), + ), + Self::StandardLibrary(_) => None, + Self::SitePackages(SitePackagesPath(path)) => Some( + ModuleResolutionPath::site_packages_unchecked(path.with_extension("py")), + ), } } + #[cfg(test)] #[must_use] - pub(crate) fn with_py_extension(self) -> Option { + pub(crate) fn to_module_resolution_path(self) -> ModuleResolutionPath { match self { - Self::Extra(path) => Some(ModuleResolutionPath::Extra(path.with_py_extension())), - Self::FirstParty(path) => { - Some(ModuleResolutionPath::FirstParty(path.with_py_extension())) + Self::Extra(ExtraPath(path)) => { + ModuleResolutionPath::extra_unchecked(path.to_path_buf()) } - Self::StandardLibrary(_) => None, - Self::SitePackages(path) => { - Some(ModuleResolutionPath::SitePackages(path.with_py_extension())) + Self::FirstParty(FirstPartyPath(path)) => { + ModuleResolutionPath::first_party_unchecked(path.to_path_buf()) + } + Self::StandardLibrary(StandardLibraryPath(path)) => { + ModuleResolutionPath::standard_library_unchecked(path.to_path_buf()) + } + Self::SitePackages(SitePackagesPath(path)) => { + ModuleResolutionPath::site_packages_unchecked(path.to_path_buf()) } } } - pub(crate) fn as_module_name(&self) -> Option { - let path = self.sans_dunder_init(); - let mut parts_iter = path.module_name_parts(); - let first_part = parts_iter.next()?; - if let Some(second_part) = parts_iter.next() { - let mut name = format!("{first_part}.{second_part}"); - for part in parts_iter { - name.push('.'); - name.push_str(part); - } - ModuleName::new(&name) - } else { - ModuleName::new(first_part) + #[cfg(test)] + #[must_use] + pub(crate) fn join( + &self, + component: &'a (impl AsRef + ?Sized), + ) -> ModuleResolutionPath { + let mut result = self.to_module_resolution_path(); + result.push(component.as_ref().as_str()); + result + } + + #[must_use] + #[inline] + fn as_file_system_path(self) -> &'a FileSystemPath { + match self { + Self::Extra(ExtraPath(path)) => path, + Self::FirstParty(FirstPartyPath(path)) => path, + Self::StandardLibrary(StandardLibraryPath(path)) => path, + Self::SitePackages(SitePackagesPath(path)) => path, } } } @@ -1121,76 +592,65 @@ impl<'a> From<&'a ModuleResolutionPath> for ModuleResolutionPathRef<'a> { } impl<'a> AsRef for ModuleResolutionPathRef<'a> { + #[inline] fn as_ref(&self) -> &FileSystemPath { - match self { - Self::Extra(path) => path.as_file_system_path(), - Self::FirstParty(path) => path.as_file_system_path(), - Self::StandardLibrary(path) => path.as_file_system_path(), - Self::SitePackages(path) => path.as_file_system_path(), - } - } -} - -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &ExtraPath) -> bool { - if let ModuleResolutionPathRef::Extra(path) = self { - *path == other - } else { - false - } - } -} - -impl<'a> PartialEq> for ExtraPath { - fn eq(&self, other: &ModuleResolutionPathRef) -> bool { - other.eq(self) - } -} - -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &FirstPartyPath) -> bool { - if let ModuleResolutionPathRef::FirstParty(path) = self { - *path == other - } else { - false - } + self.as_file_system_path() } } -impl<'a> PartialEq> for FirstPartyPath { - fn eq(&self, other: &ModuleResolutionPathRef) -> bool { +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &ModuleResolutionPath) -> bool { + match (self, other) { + ( + ModuleResolutionPathRef::Extra(ExtraPath(self_path)), + ModuleResolutionPath::Extra(ExtraPathBuf(other_path)), + ) + | ( + ModuleResolutionPathRef::FirstParty(FirstPartyPath(self_path)), + ModuleResolutionPath::FirstParty(FirstPartyPathBuf(other_path)), + ) + | ( + ModuleResolutionPathRef::StandardLibrary(StandardLibraryPath(self_path)), + ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(other_path)), + ) + | ( + ModuleResolutionPathRef::SitePackages(SitePackagesPath(self_path)), + ModuleResolutionPath::SitePackages(SitePackagesPathBuf(other_path)), + ) => *self_path == **other_path, + _ => false, + } + } +} + +impl<'a> PartialEq> for ModuleResolutionPath { + fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { other.eq(self) } } -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &StandardLibraryPath) -> bool { - if let ModuleResolutionPathRef::StandardLibrary(path) = self { - *path == other - } else { - false - } +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &FileSystemPath) -> bool { + self.as_file_system_path() == other } } -impl<'a> PartialEq> for StandardLibraryPath { - fn eq(&self, other: &ModuleResolutionPathRef) -> bool { - other.eq(self) +impl<'a> PartialEq> for FileSystemPath { + fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { + self == other.as_file_system_path() } } -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &SitePackagesPath) -> bool { - if let ModuleResolutionPathRef::SitePackages(path) = self { - *path == other - } else { - false - } +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &VfsPath) -> bool { + let VfsPath::FileSystem(other) = other else { + return false; + }; + self.as_file_system_path() == &**other } } -impl<'a> PartialEq> for SitePackagesPath { - fn eq(&self, other: &ModuleResolutionPathRef) -> bool { +impl<'a> PartialEq> for VfsPath { + fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { other.eq(self) } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 40a8543112046..ae91b60acc04a 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -1,15 +1,12 @@ use std::ops::Deref; use std::sync::Arc; +use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; -use crate::module::{Module, ModuleKind, ModuleSearchPathEntry}; +use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; -use crate::path::{ - ExtraPath, ExtraPathBuf, FirstPartyPath, FirstPartyPathBuf, ModuleResolutionPath, - ModuleResolutionPathRef, SitePackagesPath, SitePackagesPathBuf, StandardLibraryPath, - StandardLibraryPathBuf, -}; +use crate::path::{ModuleResolutionPath, ModuleResolutionPathRef}; use crate::resolver::internal::ModuleResolverSearchPaths; use crate::supported_py_version::set_target_py_version; use crate::{Db, SupportedPyVersion}; @@ -88,25 +85,23 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let relative_path = search_paths.iter().find_map(|root| match (&**root, path) { (_, VfsPath::Vendored(_)) => todo!("VendoredPaths are not yet supported"), - (ModuleSearchPathEntry::Extra(root_path), VfsPath::FileSystem(path)) => { - Some(ModuleResolutionPathRef::Extra( - ExtraPath::new(path)?.relative_to_search_path(root_path)?, - )) + (ModuleResolutionPath::Extra(_), VfsPath::FileSystem(path)) => { + ModuleResolutionPathRef::extra(path.strip_prefix(root.as_file_system_path_buf()).ok()?) } - (ModuleSearchPathEntry::FirstParty(root_path), VfsPath::FileSystem(path)) => { - Some(ModuleResolutionPathRef::FirstParty( - FirstPartyPath::new(path)?.relative_to_search_path(root_path)?, - )) + (ModuleResolutionPath::FirstParty(_), VfsPath::FileSystem(path)) => { + ModuleResolutionPathRef::first_party( + path.strip_prefix(root.as_file_system_path_buf()).ok()?, + ) } - (ModuleSearchPathEntry::StandardLibrary(root_path), VfsPath::FileSystem(path)) => { - Some(ModuleResolutionPathRef::StandardLibrary( - StandardLibraryPath::new(path)?.relative_to_search_path(root_path)?, - )) + (ModuleResolutionPath::StandardLibrary(_), VfsPath::FileSystem(path)) => { + ModuleResolutionPathRef::standard_library( + path.strip_prefix(root.as_file_system_path_buf()).ok()?, + ) } - (ModuleSearchPathEntry::SitePackagesThirdParty(root_path), VfsPath::FileSystem(path)) => { - Some(ModuleResolutionPathRef::SitePackages( - SitePackagesPath::new(path)?.relative_to_search_path(root_path)?, - )) + (ModuleResolutionPath::SitePackages(_), VfsPath::FileSystem(path)) => { + ModuleResolutionPathRef::site_packages( + path.strip_prefix(root.as_file_system_path_buf()).ok()?, + ) } })?; @@ -141,18 +136,18 @@ pub struct ModuleResolutionSettings { /// List of user-provided paths that should take first priority in the module resolution. /// Examples in other type checkers are mypy's MYPYPATH environment variable, /// or pyright's stubPath configuration setting. - pub extra_paths: Vec, + pub extra_paths: Vec, /// The root of the workspace, used for finding first-party modules. - pub workspace_root: FirstPartyPathBuf, + pub workspace_root: FileSystemPathBuf, /// Optional (already validated) path to standard-library typeshed stubs. /// If this is not provided, we will fallback to our vendored typeshed stubs /// bundled as a zip file in the binary - pub custom_typeshed: Option, + pub custom_typeshed: Option, /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed. - pub site_packages: Option, + pub site_packages: Option, } impl ModuleResolutionSettings { @@ -167,22 +162,21 @@ impl ModuleResolutionSettings { custom_typeshed, } = self; - let mut paths: Vec<_> = extra_paths + let mut paths = extra_paths .into_iter() - .map(ModuleSearchPathEntry::Extra) - .collect(); + .map(ModuleResolutionPath::extra) + .collect::>>() + .unwrap(); - paths.push(ModuleSearchPathEntry::FirstParty(workspace_root)); + paths.push(ModuleResolutionPath::first_party(workspace_root).unwrap()); if let Some(custom_typeshed) = custom_typeshed { - paths.push(ModuleSearchPathEntry::stdlib_from_typeshed_root( - &custom_typeshed, - )); + paths.push(ModuleResolutionPath::stdlib_from_typeshed_root(&custom_typeshed).unwrap()); } // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step if let Some(site_packages) = site_packages { - paths.push(ModuleSearchPathEntry::SitePackagesThirdParty(site_packages)); + paths.push(ModuleResolutionPath::site_packages(site_packages).unwrap()); } ( @@ -195,10 +189,10 @@ impl ModuleResolutionSettings { /// A resolved module resolution order, implementing PEP 561 /// (with some small, deliberate differences) #[derive(Clone, Debug, Default, Eq, PartialEq)] -pub(crate) struct OrderedSearchPaths(Vec>); +pub(crate) struct OrderedSearchPaths(Vec>); impl Deref for OrderedSearchPaths { - type Target = [Arc]; + type Target = [Arc]; fn deref(&self) -> &Self::Target { &self.0 @@ -230,7 +224,7 @@ pub(crate) mod internal { } } -fn module_search_paths(db: &dyn Db) -> &[Arc] { +fn module_search_paths(db: &dyn Db) -> &[Arc] { ModuleResolverSearchPaths::get(db).search_paths(db) } @@ -239,14 +233,14 @@ fn module_search_paths(db: &dyn Db) -> &[Arc] { fn resolve_name( db: &dyn Db, name: &ModuleName, -) -> Option<(Arc, VfsFile, ModuleKind)> { +) -> Option<(Arc, VfsFile, ModuleKind)> { let search_paths = module_search_paths(db); for search_path in search_paths { let mut components = name.components(); let module_name = components.next_back()?; - match resolve_package(db, search_path.path(), components) { + match resolve_package(db, search_path, components) { Ok(resolved_package) => { let mut package_path = resolved_package.path; @@ -293,13 +287,13 @@ fn resolve_name( fn resolve_package<'a, I>( db: &dyn Db, - module_search_path: ModuleResolutionPathRef, + module_search_path: &ModuleResolutionPath, components: I, ) -> Result where I: Iterator, { - let mut package_path = module_search_path.to_owned(); + let mut package_path = module_search_path.clone(); // `true` if inside a folder that is a namespace package (has no `__init__.py`). // Namespace packages are special because they can be spread across multiple search paths. @@ -385,26 +379,23 @@ mod tests { use crate::db::tests::TestDb; use crate::module::ModuleKind; use crate::module_name::ModuleName; - use crate::path::{FirstPartyPath, SitePackagesPath}; use super::*; struct TestCase { db: TestDb, - src: FirstPartyPathBuf, - custom_typeshed: StandardLibraryPathBuf, - site_packages: SitePackagesPathBuf, + src: FileSystemPathBuf, + custom_typeshed: FileSystemPathBuf, + site_packages: FileSystemPathBuf, } fn create_resolver() -> std::io::Result { let mut db = TestDb::new(); - let src = FirstPartyPath::new("src").unwrap().to_path_buf(); - let site_packages = SitePackagesPath::new("site_packages") - .unwrap() - .to_path_buf(); - let custom_typeshed = StandardLibraryPath::new("typeshed").unwrap().to_path_buf(); + let src = FileSystemPath::new("src").to_path_buf(); + let site_packages = FileSystemPath::new("site_packages").to_path_buf(); + let custom_typeshed = FileSystemPath::new("typeshed").to_path_buf(); let fs = db.memory_file_system(); @@ -447,13 +438,13 @@ mod tests { ); assert_eq!("foo", foo_module.name()); - assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path()); assert_eq!(ModuleKind::Module, foo_module.kind()); assert_eq!(*foo_path, *foo_module.file().path(&db)); assert_eq!( Some(foo_module), - path_to_module(&db, &foo_path.into_vfs_path()) + path_to_module(&db, &VfsPath::FileSystem(foo_path)) ); Ok(()) @@ -467,10 +458,10 @@ mod tests { .. } = create_resolver()?; - let stdlib_dir = StandardLibraryPath::stdlib_from_typeshed_root(&custom_typeshed); + let stdlib_dir = ModuleResolutionPath::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); let functools_path = stdlib_dir.join("functools.pyi"); db.memory_file_system() - .write_file(&*functools_path, "def update_wrapper(): ...")?; + .write_file(&functools_path, "def update_wrapper(): ...")?; let functools_module_name = ModuleName::new_static("functools").unwrap(); let functools_module = resolve_module(&db, functools_module_name.clone()).unwrap(); @@ -480,10 +471,10 @@ mod tests { resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(*stdlib_dir, functools_module.search_path().path()); + assert_eq!(stdlib_dir, functools_module.search_path()); assert_eq!(ModuleKind::Module, functools_module.kind()); - assert_eq!(*functools_path, *functools_module.file().path(&db)); + assert_eq!(functools_path, *functools_module.file().path(&db)); assert_eq!( Some(functools_module), @@ -502,19 +493,13 @@ mod tests { .. } = create_resolver()?; - let stdlib_dir = StandardLibraryPath::stdlib_from_typeshed_root(&custom_typeshed); + let stdlib_dir = custom_typeshed.join("stdlib"); let stdlib_functools_path = stdlib_dir.join("functools.pyi"); let first_party_functools_path = src.join("functools.py"); db.memory_file_system().write_files([ - ( - stdlib_functools_path.as_file_system_path(), - "def update_wrapper(): ...", - ), - ( - first_party_functools_path.as_file_system_path(), - "def update_wrapper(): ...", - ), + (&stdlib_functools_path, "def update_wrapper(): ..."), + (&first_party_functools_path, "def update_wrapper(): ..."), ])?; let functools_module_name = ModuleName::new_static("functools").unwrap(); @@ -524,7 +509,7 @@ mod tests { Some(&functools_module), resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(*src, functools_module.search_path().path()); + assert_eq!(*src, functools_module.search_path()); assert_eq!(ModuleKind::Module, functools_module.kind()); assert_eq!( *first_party_functools_path, @@ -533,7 +518,7 @@ mod tests { assert_eq!( Some(functools_module), - path_to_module(&db, &first_party_functools_path.into_vfs_path()) + path_to_module(&db, &VfsPath::FileSystem(first_party_functools_path)) ); Ok(()) @@ -576,16 +561,16 @@ mod tests { let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); assert_eq!("foo", foo_module.name()); - assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path()); assert_eq!(*foo_path, *foo_module.file().path(&db)); assert_eq!( Some(&foo_module), - path_to_module(&db, &foo_path.into_vfs_path()).as_ref() + path_to_module(&db, &VfsPath::FileSystem(foo_path)).as_ref() ); // Resolving by directory doesn't resolve to the init file. - assert_eq!(None, path_to_module(&db, &foo_dir.into_vfs_path())); + assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_dir))); Ok(()) } @@ -606,15 +591,15 @@ mod tests { let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path()); assert_eq!(*foo_init, *foo_module.file().path(&db)); assert_eq!(ModuleKind::Package, foo_module.kind()); assert_eq!( Some(foo_module), - path_to_module(&db, &foo_init.into_vfs_path()) + path_to_module(&db, &VfsPath::FileSystem(foo_init)) ); - assert_eq!(None, path_to_module(&db, &foo_py.into_vfs_path())); + assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_py))); Ok(()) } @@ -630,11 +615,14 @@ mod tests { let foo = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo.search_path().path()); + assert_eq!(*src, foo.search_path()); assert_eq!(*foo_stub, *foo.file().path(&db)); - assert_eq!(Some(foo), path_to_module(&db, &foo_stub.into_vfs_path())); - assert_eq!(None, path_to_module(&db, &foo_py.into_vfs_path())); + assert_eq!( + Some(foo), + path_to_module(&db, &VfsPath::FileSystem(foo_stub)) + ); + assert_eq!(None, path_to_module(&db, &VfsPath::FileSystem(foo_py))); Ok(()) } @@ -656,10 +644,13 @@ mod tests { let baz_module = resolve_module(&db, ModuleName::new_static("foo.bar.baz").unwrap()).unwrap(); - assert_eq!(*src, baz_module.search_path().path()); + assert_eq!(*src, baz_module.search_path()); assert_eq!(*baz, *baz_module.file().path(&db)); - assert_eq!(Some(baz_module), path_to_module(&db, &baz.into_vfs_path())); + assert_eq!( + Some(baz_module), + path_to_module(&db, &VfsPath::FileSystem(baz)) + ); Ok(()) } @@ -695,18 +686,24 @@ mod tests { let two = child2.join("two.py"); db.memory_file_system().write_files([ - (one.as_file_system_path(), "print('Hello, world!')"), - (two.as_file_system_path(), "print('Hello, world!')"), + (&one, "print('Hello, world!')"), + (&two, "print('Hello, world!')"), ])?; let one_module = resolve_module(&db, ModuleName::new_static("parent.child.one").unwrap()).unwrap(); - assert_eq!(Some(one_module), path_to_module(&db, &one.into_vfs_path())); + assert_eq!( + Some(one_module), + path_to_module(&db, &VfsPath::FileSystem(one)) + ); let two_module = resolve_module(&db, ModuleName::new_static("parent.child.two").unwrap()).unwrap(); - assert_eq!(Some(two_module), path_to_module(&db, &two.into_vfs_path())); + assert_eq!( + Some(two_module), + path_to_module(&db, &VfsPath::FileSystem(two)) + ); Ok(()) } @@ -742,18 +739,18 @@ mod tests { let two = child2.join("two.py"); db.memory_file_system().write_files([ - ( - child1.join("__init__.py").as_file_system_path(), - "print('Hello, world!')", - ), - (one.as_file_system_path(), "print('Hello, world!')"), - (two.as_file_system_path(), "print('Hello, world!')"), + (&child1.join("__init__.py"), "print('Hello, world!')"), + (&one, "print('Hello, world!')"), + (&two, "print('Hello, world!')"), ])?; let one_module = resolve_module(&db, ModuleName::new_static("parent.child.one").unwrap()).unwrap(); - assert_eq!(Some(one_module), path_to_module(&db, &one.into_vfs_path())); + assert_eq!( + Some(one_module), + path_to_module(&db, &VfsPath::FileSystem(one)) + ); assert_eq!( None, @@ -774,23 +771,21 @@ mod tests { let foo_src = src.join("foo.py"); let foo_site_packages = site_packages.join("foo.py"); - db.memory_file_system().write_files([ - (foo_src.as_file_system_path(), ""), - (foo_site_packages.as_file_system_path(), ""), - ])?; + db.memory_file_system() + .write_files([(&foo_src, ""), (&foo_site_packages, "")])?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path()); assert_eq!(*foo_src, *foo_module.file().path(&db)); assert_eq!( Some(foo_module), - path_to_module(&db, &foo_src.into_vfs_path()) + path_to_module(&db, &VfsPath::FileSystem(foo_src)) ); assert_eq!( None, - path_to_module(&db, &foo_site_packages.into_vfs_path()) + path_to_module(&db, &VfsPath::FileSystem(foo_site_packages)) ); Ok(()) @@ -825,11 +820,9 @@ mod tests { std::fs::write(foo.as_std_path(), "")?; std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?; - let src = FirstPartyPath::new(&src).unwrap().to_path_buf(); - let site_packages = SitePackagesPath::new(&site_packages).unwrap().to_path_buf(); - let custom_typeshed = StandardLibraryPath::new(&custom_typeshed) - .unwrap() - .to_path_buf(); + let src = src.to_path_buf(); + let site_packages = site_packages.to_path_buf(); + let custom_typeshed = custom_typeshed.to_path_buf(); let settings = ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, @@ -846,12 +839,12 @@ mod tests { assert_ne!(foo_module, bar_module); - assert_eq!(*src, foo_module.search_path().path()); + assert_eq!(*src, foo_module.search_path()); assert_eq!(&foo, foo_module.file().path(&db)); // `foo` and `bar` shouldn't resolve to the same file - assert_eq!(*src, bar_module.search_path().path()); + assert_eq!(*src, bar_module.search_path()); assert_eq!(&bar, bar_module.file().path(&db)); assert_eq!(&foo, foo_module.file().path(&db)); @@ -874,7 +867,7 @@ mod tests { let TestCase { mut db, src, .. } = create_resolver()?; let foo_path = src.join("foo.py"); - let bar_path: FirstPartyPathBuf = src.join("bar.py"); + let bar_path = src.join("bar.py"); db.memory_file_system() .write_files([(&*foo_path, "x = 1"), (&*bar_path, "y = 2")])?; @@ -916,7 +909,7 @@ mod tests { // Now write the foo file db.memory_file_system().write_file(&*foo_path, "x = 1")?; - VfsFile::touch_path(&mut db, &foo_path.clone().into_vfs_path()); + VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_path.clone())); let foo_file = system_path_to_file(&db, &*foo_path).expect("foo.py to exist"); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); @@ -944,7 +937,7 @@ mod tests { db.memory_file_system().remove_file(&*foo_init_path)?; db.memory_file_system() .remove_directory(foo_init_path.parent().unwrap())?; - VfsFile::touch_path(&mut db, &foo_init_path.into_vfs_path()); + VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_init_path)); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); assert_eq!(*foo_path, *foo_module.file().path(&db)); diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 6174aede48887..6bbbb9f59a8a8 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -511,9 +511,9 @@ impl<'db, 'inference> TypingContext<'db, 'inference> { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, FirstPartyPath, ModuleResolutionSettings, - SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, }; + use ruff_db::file_system::FileSystemPath; use ruff_db::parsed::parsed_module; use ruff_db::vfs::system_path_to_file; @@ -531,7 +531,7 @@ mod tests { ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, extra_paths: vec![], - workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), + workspace_root: FileSystemPath::new("/src").to_path_buf(), site_packages: None, custom_typeshed: None, }, diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index e1ebca8c4acef..c8449682bb6fa 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -700,13 +700,13 @@ impl<'db> TypeInferenceBuilder<'db> { #[cfg(test)] mod tests { + use ruff_db::file_system::FileSystemPath; use ruff_db::vfs::system_path_to_file; use crate::db::tests::TestDb; use crate::types::{public_symbol_ty_by_name, Type, TypingContext}; use red_knot_module_resolver::{ - set_module_resolution_settings, FirstPartyPath, ModuleResolutionSettings, - SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, }; use ruff_python_ast::name::Name; @@ -718,7 +718,7 @@ mod tests { ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, extra_paths: Vec::new(), - workspace_root: FirstPartyPath::new("/src").unwrap().to_path_buf(), + workspace_root: FileSystemPath::new("/src").to_path_buf(), site_packages: None, custom_typeshed: None, }, From 1dc038ec81ea25384e5232ef080ff6aa1449f34a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 12:17:38 +0100 Subject: [PATCH 10/58] Cleanup --- crates/red_knot_module_resolver/src/path.rs | 141 ++++++++---------- crates/red_knot_python_semantic/src/types.rs | 4 +- .../src/types/infer.rs | 4 +- 3 files changed, 67 insertions(+), 82 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 26a17c9aae1b9..c1b3c40e0943d 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,6 +1,5 @@ #![allow(unsafe_code)] use std::iter::FusedIterator; -use std::ops::Deref; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::VfsPath; @@ -14,102 +13,34 @@ use crate::Db; #[derive(Debug, PartialEq, Eq, Hash)] pub(crate) struct ExtraPath(FileSystemPath); -impl ExtraPath { - #[must_use] - fn new_unchecked(path: &FileSystemPath) -> &Self { - // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const ExtraPath is valid. - unsafe { &*(path as *const FileSystemPath as *const Self) } - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub(crate) struct ExtraPathBuf(FileSystemPathBuf); -impl Deref for ExtraPathBuf { - type Target = ExtraPath; - - fn deref(&self) -> &Self::Target { - ExtraPath::new_unchecked(&self.0) - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] pub(crate) struct FirstPartyPath(FileSystemPath); -impl FirstPartyPath { - #[must_use] - fn new_unchecked(path: &FileSystemPath) -> &Self { - // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const FirstPartyPath is valid. - unsafe { &*(path as *const FileSystemPath as *const Self) } - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub(crate) struct FirstPartyPathBuf(FileSystemPathBuf); -impl Deref for FirstPartyPathBuf { - type Target = FirstPartyPath; - - fn deref(&self) -> &Self::Target { - FirstPartyPath::new_unchecked(&self.0) - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] pub(crate) struct StandardLibraryPath(FileSystemPath); -impl StandardLibraryPath { - #[must_use] - fn new_unchecked(path: &FileSystemPath) -> &Self { - // SAFETY: StandardLibraryPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const StandardLibraryPath is valid. - unsafe { &*(path as *const FileSystemPath as *const Self) } - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub(crate) struct StandardLibraryPathBuf(FileSystemPathBuf); -impl Deref for StandardLibraryPathBuf { - type Target = StandardLibraryPath; - - fn deref(&self) -> &Self::Target { - StandardLibraryPath::new_unchecked(&self.0) - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] pub(crate) struct SitePackagesPath(FileSystemPath); -impl SitePackagesPath { - #[must_use] - fn new_unchecked(path: &FileSystemPath) -> &Self { - // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const SitePackagesPath is valid. - unsafe { &*(path as *const FileSystemPath as *const Self) } - } -} - #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub(crate) struct SitePackagesPathBuf(FileSystemPathBuf); -impl Deref for SitePackagesPathBuf { - type Target = SitePackagesPath; - - fn deref(&self) -> &Self::Target { - SitePackagesPath::new_unchecked(&self.0) - } -} - /// Enumeration of the different kinds of search paths type checkers are expected to support. /// /// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the @@ -158,6 +89,7 @@ impl ModuleResolutionPath { inner.push(component); } + #[must_use] pub(crate) fn extra(path: FileSystemPathBuf) -> Option { if path .extension() @@ -169,10 +101,12 @@ impl ModuleResolutionPath { } } + #[must_use] fn extra_unchecked(path: FileSystemPathBuf) -> Self { Self::Extra(ExtraPathBuf(path)) } + #[must_use] pub(crate) fn first_party(path: FileSystemPathBuf) -> Option { if path .extension() @@ -184,10 +118,12 @@ impl ModuleResolutionPath { } } + #[must_use] fn first_party_unchecked(path: FileSystemPathBuf) -> Self { Self::FirstParty(FirstPartyPathBuf(path)) } + #[must_use] pub(crate) fn standard_library(path: FileSystemPathBuf) -> Option { if path.extension().map_or(true, |ext| ext == "pyi") { Some(Self::standard_library_unchecked(path)) @@ -196,14 +132,17 @@ impl ModuleResolutionPath { } } + #[must_use] pub(crate) fn stdlib_from_typeshed_root(typeshed_root: &FileSystemPath) -> Option { Self::standard_library(typeshed_root.join(FileSystemPath::new("stdlib"))) } + #[must_use] fn standard_library_unchecked(path: FileSystemPathBuf) -> Self { Self::StandardLibrary(StandardLibraryPathBuf(path)) } + #[must_use] pub(crate) fn site_packages(path: FileSystemPathBuf) -> Option { if path .extension() @@ -215,31 +154,38 @@ impl ModuleResolutionPath { } } + #[must_use] fn site_packages_unchecked(path: FileSystemPathBuf) -> Self { Self::SitePackages(SitePackagesPathBuf(path)) } + #[must_use] pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { ModuleResolutionPathRef::from(self).is_regular_package(db) } + #[must_use] pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { ModuleResolutionPathRef::from(self).is_directory(db) } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> Self { ModuleResolutionPathRef::from(self).with_pyi_extension() } + #[must_use] pub(crate) fn with_py_extension(&self) -> Option { ModuleResolutionPathRef::from(self).with_py_extension() } #[cfg(test)] + #[must_use] pub(crate) fn join(&self, component: &(impl AsRef + ?Sized)) -> Self { ModuleResolutionPathRef::from(self).join(component) } + #[must_use] pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { match self { Self::Extra(ExtraPathBuf(path)) => path, @@ -308,12 +254,14 @@ impl PartialEq for FileSystemPath { } impl AsRef for ModuleResolutionPath { + #[inline] fn as_ref(&self) -> &FileSystemPathBuf { self.as_file_system_path_buf() } } impl AsRef for ModuleResolutionPath { + #[inline] fn as_ref(&self) -> &FileSystemPath { ModuleResolutionPathRef::from(self).as_file_system_path() } @@ -328,6 +276,7 @@ pub(crate) enum ModuleResolutionPathRef<'a> { } impl<'a> ModuleResolutionPathRef<'a> { + #[must_use] pub(crate) fn extra(path: &'a (impl AsRef + ?Sized)) -> Option { let path = path.as_ref(); if path @@ -340,10 +289,14 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] fn extra_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - Self::Extra(ExtraPath::new_unchecked(path.as_ref())) + // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const ExtraPath is valid. + Self::Extra(unsafe { &*(path.as_ref() as *const FileSystemPath as *const ExtraPath) }) } + #[must_use] pub(crate) fn first_party(path: &'a (impl AsRef + ?Sized)) -> Option { let path = path.as_ref(); if path @@ -356,14 +309,21 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] fn first_party_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - Self::FirstParty(FirstPartyPath::new_unchecked(path.as_ref())) + // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const FirstPartyPath is valid. + Self::FirstParty(unsafe { + &*(path.as_ref() as *const FileSystemPath as *const FirstPartyPath) + }) } + #[must_use] pub(crate) fn standard_library( path: &'a (impl AsRef + ?Sized), ) -> Option { let path = path.as_ref(); + // Unlike other variants, only `.pyi` extensions are permitted if path.extension().map_or(true, |ext| ext == "pyi") { Some(Self::standard_library_unchecked(path)) } else { @@ -371,10 +331,16 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] fn standard_library_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - Self::StandardLibrary(StandardLibraryPath::new_unchecked(path.as_ref())) + // SAFETY: StandardLibraryPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const StandardLibraryPath is valid. + Self::StandardLibrary(unsafe { + &*(path.as_ref() as *const FileSystemPath as *const StandardLibraryPath) + }) } + #[must_use] pub(crate) fn site_packages(path: &'a (impl AsRef + ?Sized)) -> Option { let path = path.as_ref(); if path @@ -387,10 +353,16 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] fn site_packages_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - Self::SitePackages(SitePackagesPath::new_unchecked(path.as_ref())) + // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a + // *const FileSystemPath to a *const SitePackagesPath is valid. + Self::SitePackages(unsafe { + &*(path.as_ref() as *const FileSystemPath as *const SitePackagesPath) + }) } + #[must_use] pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { match self { Self::Extra(ExtraPath(path)) => db.file_system().is_directory(path), @@ -414,6 +386,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { match self { Self::Extra(ExtraPath(fs_path)) @@ -444,6 +417,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] pub(crate) fn parent(&self) -> Option { Some(match self { Self::Extra(ExtraPath(path)) => Self::extra_unchecked(path.parent()?), @@ -457,6 +431,7 @@ impl<'a> ModuleResolutionPathRef<'a> { }) } + #[must_use] fn ends_with_dunder_init(&self) -> bool { match self { Self::Extra(ExtraPath(path)) @@ -468,6 +443,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] fn sans_dunder_init(self) -> Self { if self.ends_with_dunder_init() { self.parent().unwrap_or_else(|| match self { @@ -481,6 +457,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] pub(crate) fn as_module_name(&self) -> Option { let mut parts_iter = match self.sans_dunder_init() { Self::Extra(ExtraPath(path)) => ModulePartIterator::from_fs_path(path), @@ -503,6 +480,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPath { match self { Self::Extra(ExtraPath(path)) => { @@ -520,6 +498,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] pub(crate) fn with_py_extension(&self) -> Option { match self { Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPath::extra_unchecked( @@ -581,12 +560,18 @@ impl<'a> From<&'a ModuleResolutionPath> for ModuleResolutionPathRef<'a> { #[inline] fn from(value: &'a ModuleResolutionPath) -> Self { match value { - ModuleResolutionPath::Extra(path) => ModuleResolutionPathRef::Extra(path), - ModuleResolutionPath::FirstParty(path) => ModuleResolutionPathRef::FirstParty(path), - ModuleResolutionPath::StandardLibrary(path) => { - ModuleResolutionPathRef::StandardLibrary(path) + ModuleResolutionPath::Extra(ExtraPathBuf(path)) => { + ModuleResolutionPathRef::extra_unchecked(path) + } + ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path)) => { + ModuleResolutionPathRef::first_party_unchecked(path) + } + ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(path)) => { + ModuleResolutionPathRef::standard_library_unchecked(path) + } + ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path)) => { + ModuleResolutionPathRef::site_packages_unchecked(path) } - ModuleResolutionPath::SitePackages(path) => ModuleResolutionPathRef::SitePackages(path), } } } diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 6bbbb9f59a8a8..e0847b4f23595 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -513,7 +513,7 @@ mod tests { use red_knot_module_resolver::{ set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, }; - use ruff_db::file_system::FileSystemPath; + use ruff_db::file_system::FileSystemPathBuf; use ruff_db::parsed::parsed_module; use ruff_db::vfs::system_path_to_file; @@ -531,7 +531,7 @@ mod tests { ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, extra_paths: vec![], - workspace_root: FileSystemPath::new("/src").to_path_buf(), + workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, custom_typeshed: None, }, diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index c8449682bb6fa..1cb1ca6892e88 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -700,7 +700,7 @@ impl<'db> TypeInferenceBuilder<'db> { #[cfg(test)] mod tests { - use ruff_db::file_system::FileSystemPath; + use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::system_path_to_file; use crate::db::tests::TestDb; @@ -718,7 +718,7 @@ mod tests { ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, extra_paths: Vec::new(), - workspace_root: FileSystemPath::new("/src").to_path_buf(), + workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, custom_typeshed: None, }, From 583e503679a043a6bf7b3e97f2d7de7d51f7a38a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 12:55:15 +0100 Subject: [PATCH 11/58] Make some public methods private --- crates/red_knot_module_resolver/src/path.rs | 16 +++++++++------- crates/red_knot_module_resolver/src/resolver.rs | 16 +++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index c1b3c40e0943d..c7b9e82fe6d22 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,4 +1,3 @@ -#![allow(unsafe_code)] use std::iter::FusedIterator; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; @@ -186,7 +185,7 @@ impl ModuleResolutionPath { } #[must_use] - pub(crate) fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { + fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { match self { Self::Extra(ExtraPathBuf(path)) => path, Self::FirstParty(FirstPartyPathBuf(path)) => path, @@ -195,7 +194,6 @@ impl ModuleResolutionPath { } } - #[cfg(test)] #[must_use] #[inline] fn into_file_system_path_buf(self) -> FileSystemPathBuf { @@ -206,11 +204,11 @@ impl ModuleResolutionPath { Self::SitePackages(SitePackagesPathBuf(path)) => path, } } +} - #[cfg(test)] - #[must_use] - pub(crate) fn into_vfs_path(self) -> VfsPath { - VfsPath::FileSystem(self.into_file_system_path_buf()) +impl From for VfsPath { + fn from(value: ModuleResolutionPath) -> Self { + VfsPath::FileSystem(value.into_file_system_path_buf()) } } @@ -290,6 +288,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] + #[allow(unsafe_code)] fn extra_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const ExtraPath is valid. @@ -310,6 +309,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] + #[allow(unsafe_code)] fn first_party_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const FirstPartyPath is valid. @@ -332,6 +332,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] + #[allow(unsafe_code)] fn standard_library_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { // SAFETY: StandardLibraryPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const StandardLibraryPath is valid. @@ -354,6 +355,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] + #[allow(unsafe_code)] fn site_packages_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a // *const FileSystemPath to a *const SitePackagesPath is valid. diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index ae91b60acc04a..e22ccb3bbe335 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -86,22 +86,16 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let relative_path = search_paths.iter().find_map(|root| match (&**root, path) { (_, VfsPath::Vendored(_)) => todo!("VendoredPaths are not yet supported"), (ModuleResolutionPath::Extra(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::extra(path.strip_prefix(root.as_file_system_path_buf()).ok()?) + ModuleResolutionPathRef::extra(path.strip_prefix(&**root).ok()?) } (ModuleResolutionPath::FirstParty(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::first_party( - path.strip_prefix(root.as_file_system_path_buf()).ok()?, - ) + ModuleResolutionPathRef::first_party(path.strip_prefix(&**root).ok()?) } (ModuleResolutionPath::StandardLibrary(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::standard_library( - path.strip_prefix(root.as_file_system_path_buf()).ok()?, - ) + ModuleResolutionPathRef::standard_library(path.strip_prefix(&**root).ok()?) } (ModuleResolutionPath::SitePackages(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::site_packages( - path.strip_prefix(root.as_file_system_path_buf()).ok()?, - ) + ModuleResolutionPathRef::site_packages(path.strip_prefix(&**root).ok()?) } })?; @@ -478,7 +472,7 @@ mod tests { assert_eq!( Some(functools_module), - path_to_module(&db, &functools_path.into_vfs_path()) + path_to_module(&db, &VfsPath::from(functools_path)) ); Ok(()) From 906d801e4455ab05d81227f0377bf623a5748fc3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 14:38:25 +0100 Subject: [PATCH 12/58] Inline some simple helper methods --- crates/red_knot_module_resolver/src/path.rs | 60 ++++++++------------- 1 file changed, 21 insertions(+), 39 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index c7b9e82fe6d22..48e3c04ad7463 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -94,38 +94,28 @@ impl ModuleResolutionPath { .extension() .map_or(true, |ext| matches!(ext, "py" | "pyi")) { - Some(Self::extra_unchecked(path)) + Some(Self::Extra(ExtraPathBuf(path))) } else { None } } - #[must_use] - fn extra_unchecked(path: FileSystemPathBuf) -> Self { - Self::Extra(ExtraPathBuf(path)) - } - #[must_use] pub(crate) fn first_party(path: FileSystemPathBuf) -> Option { if path .extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) { - Some(Self::first_party_unchecked(path)) + Some(Self::FirstParty(FirstPartyPathBuf(path))) } else { None } } - #[must_use] - fn first_party_unchecked(path: FileSystemPathBuf) -> Self { - Self::FirstParty(FirstPartyPathBuf(path)) - } - #[must_use] pub(crate) fn standard_library(path: FileSystemPathBuf) -> Option { if path.extension().map_or(true, |ext| ext == "pyi") { - Some(Self::standard_library_unchecked(path)) + Some(Self::StandardLibrary(StandardLibraryPathBuf(path))) } else { None } @@ -136,28 +126,18 @@ impl ModuleResolutionPath { Self::standard_library(typeshed_root.join(FileSystemPath::new("stdlib"))) } - #[must_use] - fn standard_library_unchecked(path: FileSystemPathBuf) -> Self { - Self::StandardLibrary(StandardLibraryPathBuf(path)) - } - #[must_use] pub(crate) fn site_packages(path: FileSystemPathBuf) -> Option { if path .extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) { - Some(Self::site_packages_unchecked(path)) + Some(Self::SitePackages(SitePackagesPathBuf(path))) } else { None } } - #[must_use] - fn site_packages_unchecked(path: FileSystemPathBuf) -> Self { - Self::SitePackages(SitePackagesPathBuf(path)) - } - #[must_use] pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { ModuleResolutionPathRef::from(self).is_regular_package(db) @@ -486,16 +466,18 @@ impl<'a> ModuleResolutionPathRef<'a> { pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPath { match self { Self::Extra(ExtraPath(path)) => { - ModuleResolutionPath::extra_unchecked(path.with_extension("pyi")) + ModuleResolutionPath::Extra(ExtraPathBuf(path.with_extension("pyi"))) } Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPath::first_party_unchecked(path.with_extension("pyi")) + ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path.with_extension("pyi"))) } Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPath::standard_library_unchecked(path.with_extension("pyi")) + ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf( + path.with_extension("pyi"), + )) } Self::SitePackages(SitePackagesPath(path)) => { - ModuleResolutionPath::site_packages_unchecked(path.with_extension("pyi")) + ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path.with_extension("pyi"))) } } } @@ -503,16 +485,16 @@ impl<'a> ModuleResolutionPathRef<'a> { #[must_use] pub(crate) fn with_py_extension(&self) -> Option { match self { - Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPath::extra_unchecked( + Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPath::Extra(ExtraPathBuf( path.with_extension("py"), + ))), + Self::FirstParty(FirstPartyPath(path)) => Some(ModuleResolutionPath::FirstParty( + FirstPartyPathBuf(path.with_extension("py")), )), - Self::FirstParty(FirstPartyPath(path)) => Some( - ModuleResolutionPath::first_party_unchecked(path.with_extension("py")), - ), Self::StandardLibrary(_) => None, - Self::SitePackages(SitePackagesPath(path)) => Some( - ModuleResolutionPath::site_packages_unchecked(path.with_extension("py")), - ), + Self::SitePackages(SitePackagesPath(path)) => Some(ModuleResolutionPath::SitePackages( + SitePackagesPathBuf(path.with_extension("py")), + )), } } @@ -521,16 +503,16 @@ impl<'a> ModuleResolutionPathRef<'a> { pub(crate) fn to_module_resolution_path(self) -> ModuleResolutionPath { match self { Self::Extra(ExtraPath(path)) => { - ModuleResolutionPath::extra_unchecked(path.to_path_buf()) + ModuleResolutionPath::Extra(ExtraPathBuf(path.to_path_buf())) } Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPath::first_party_unchecked(path.to_path_buf()) + ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path.to_path_buf())) } Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPath::standard_library_unchecked(path.to_path_buf()) + ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(path.to_path_buf())) } Self::SitePackages(SitePackagesPath(path)) => { - ModuleResolutionPath::site_packages_unchecked(path.to_path_buf()) + ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path.to_path_buf())) } } } From ed088e3be562680ac5f166538b9938e2bf2d0fb1 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 19:36:54 +0100 Subject: [PATCH 13/58] Use Salsa queries to load `TypeshedVersions` --- crates/red_knot_module_resolver/src/db.rs | 17 ++--- crates/red_knot_module_resolver/src/path.rs | 64 +++++++++++-------- .../red_knot_module_resolver/src/resolver.rs | 6 +- .../src/supported_py_version.rs | 5 +- .../red_knot_module_resolver/src/typeshed.rs | 2 +- .../src/typeshed/versions.rs | 8 +++ crates/red_knot_python_semantic/src/db.rs | 12 +--- 7 files changed, 58 insertions(+), 56 deletions(-) diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 6ce74590a4be6..498eba690de3d 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -6,7 +6,7 @@ use crate::resolver::{ resolve_module_query, }; use crate::supported_py_version::TargetPyVersion; -use crate::typeshed::TypeshedVersions; +use crate::typeshed::parse_typeshed_versions; #[salsa::jar(db=Db)] pub struct Jar( @@ -15,14 +15,12 @@ pub struct Jar( TargetPyVersion, resolve_module_query, file_to_module, + parse_typeshed_versions, ); -pub trait Db: salsa::DbWithJar + ruff_db::Db + Upcast { - fn typeshed_versions(&self) -> &TypeshedVersions; -} +pub trait Db: salsa::DbWithJar + ruff_db::Db + Upcast {} pub(crate) mod tests { - use std::str::FromStr; use std::sync; use salsa::DebugWithDb; @@ -38,7 +36,6 @@ pub(crate) mod tests { file_system: TestFileSystem, events: sync::Arc>>, vfs: Vfs, - typeshed_versions: TypeshedVersions, } impl TestDb { @@ -49,7 +46,6 @@ pub(crate) mod tests { file_system: TestFileSystem::Memory(MemoryFileSystem::default()), events: sync::Arc::default(), vfs: Vfs::with_stubbed_vendored(), - typeshed_versions: TypeshedVersions::from_str("").unwrap(), } } @@ -119,11 +115,7 @@ pub(crate) mod tests { } } - impl Db for TestDb { - fn typeshed_versions(&self) -> &TypeshedVersions { - &self.typeshed_versions - } - } + impl Db for TestDb {} impl salsa::Database for TestDb { fn salsa_event(&self, event: salsa::Event) { @@ -140,7 +132,6 @@ pub(crate) mod tests { file_system: self.file_system.snapshot(), events: self.events.clone(), vfs: self.vfs.snapshot(), - typeshed_versions: self.typeshed_versions.clone(), }) } } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 48e3c04ad7463..81fcacf210b75 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,12 +1,12 @@ use std::iter::FusedIterator; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; -use ruff_db::vfs::VfsPath; +use ruff_db::vfs::{system_path_to_file, VfsPath}; use crate::module_name::ModuleName; use crate::supported_py_version::get_target_py_version; -use crate::typeshed::TypeshedVersionsQueryResult; -use crate::Db; +use crate::typeshed::{parse_typeshed_versions, TypeshedVersionsQueryResult}; +use crate::{Db, TypeshedVersions}; #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] @@ -139,13 +139,13 @@ impl ModuleResolutionPath { } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { - ModuleResolutionPathRef::from(self).is_regular_package(db) + pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: &Self) -> bool { + ModuleResolutionPathRef::from(self).is_regular_package(db, search_path) } #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { - ModuleResolutionPathRef::from(self).is_directory(db) + pub(crate) fn is_directory(&self, db: &dyn Db, search_path: &Self) -> bool { + ModuleResolutionPathRef::from(self).is_directory(db, search_path) } #[must_use] @@ -345,19 +345,25 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db) -> bool { - match self { - Self::Extra(ExtraPath(path)) => db.file_system().is_directory(path), - Self::FirstParty(FirstPartyPath(path)) => db.file_system().is_directory(path), - Self::SitePackages(SitePackagesPath(path)) => db.file_system().is_directory(path), - Self::StandardLibrary(StandardLibraryPath(path)) => { + fn load_typeshed_versions(db: &dyn Db, stdlib_root: &StandardLibraryPath) -> TypeshedVersions { + let StandardLibraryPath(stdlib_fs_path) = stdlib_root; + let versions_path = stdlib_fs_path.join("VERSIONS"); + let versions_file = system_path_to_file(db.upcast(), versions_path).unwrap(); + parse_typeshed_versions(db, versions_file) + } + + #[must_use] + pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { + match (self, search_path.into()) { + (Self::Extra(ExtraPath(path)), Self::Extra(_)) => db.file_system().is_directory(path), + (Self::FirstParty(FirstPartyPath(path)), Self::FirstParty(_)) => db.file_system().is_directory(path), + (Self::SitePackages(SitePackagesPath(path)), Self::SitePackages(_)) => db.file_system().is_directory(path), + (Self::StandardLibrary(StandardLibraryPath(path)), Self::StandardLibrary(stdlib_root)) => { let Some(module_name) = self.as_module_name() else { return false; }; - match db - .typeshed_versions() - .query_module(&module_name, get_target_py_version(db)) - { + let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); + match typeshed_versions.query_module(&module_name, get_target_py_version(db)) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().is_directory(path) @@ -365,15 +371,18 @@ impl<'a> ModuleResolutionPathRef<'a> { TypeshedVersionsQueryResult::DoesNotExist => false, } } + (path, root) => unreachable!( + "The search path should always be the same variant as `self` (got: {path:?}, {root:?})" + ) } } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db) -> bool { - match self { - Self::Extra(ExtraPath(fs_path)) - | Self::FirstParty(FirstPartyPath(fs_path)) - | Self::SitePackages(SitePackagesPath(fs_path)) => { + pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { + match (self, search_path.into()) { + (Self::Extra(ExtraPath(fs_path)), Self::Extra(_)) + | (Self::FirstParty(FirstPartyPath(fs_path)), Self::FirstParty(_)) + | (Self::SitePackages(SitePackagesPath(fs_path)), Self::SitePackages(_)) => { let file_system = db.file_system(); file_system.exists(&fs_path.join("__init__.py")) || file_system.exists(&fs_path.join("__init__.pyi")) @@ -381,14 +390,12 @@ impl<'a> ModuleResolutionPathRef<'a> { // Unlike the other variants: // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` - Self::StandardLibrary(StandardLibraryPath(fs_path)) => { + (Self::StandardLibrary(StandardLibraryPath(fs_path)), Self::StandardLibrary(stdlib_root)) => { let Some(module_name) = self.as_module_name() else { return false; }; - match db - .typeshed_versions() - .query_module(&module_name, get_target_py_version(db)) - { + let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); + match typeshed_versions.query_module(&module_name, get_target_py_version(db)) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().exists(&fs_path.join("__init__.pyi")) @@ -396,6 +403,9 @@ impl<'a> ModuleResolutionPathRef<'a> { TypeshedVersionsQueryResult::DoesNotExist => false, } } + (path, root) => unreachable!( + "The search path should always be the same variant as `self` (got: {path:?}, {root:?})" + ) } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index e22ccb3bbe335..debcdcf1d447e 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -241,7 +241,7 @@ fn resolve_name( package_path.push(module_name); // Must be a `__init__.pyi` or `__init__.py` or it isn't a package. - let kind = if package_path.is_directory(db) { + let kind = if package_path.is_directory(db, search_path) { package_path.push("__init__"); ModuleKind::Package } else { @@ -301,11 +301,11 @@ where for folder in components { package_path.push(folder); - let is_regular_package = package_path.is_regular_package(db); + let is_regular_package = package_path.is_regular_package(db, module_search_path); if is_regular_package { in_namespace_package = false; - } else if package_path.is_directory(db) { + } else if package_path.is_directory(db, module_search_path) { // A directory without an `__init__.py` is a namespace package, continue with the next folder. in_namespace_package = true; } else if in_namespace_package { diff --git a/crates/red_knot_module_resolver/src/supported_py_version.rs b/crates/red_knot_module_resolver/src/supported_py_version.rs index c32af4d6486db..5a2c8f527afbe 100644 --- a/crates/red_knot_module_resolver/src/supported_py_version.rs +++ b/crates/red_knot_module_resolver/src/supported_py_version.rs @@ -1,5 +1,7 @@ #![allow(clippy::used_underscore_binding)] // necessary for Salsa inputs #![allow(unreachable_pub)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::clone_on_copy)] use crate::Db; @@ -18,7 +20,6 @@ pub enum SupportedPyVersion { #[salsa::input(singleton)] pub(crate) struct TargetPyVersion { - #[return_ref] pub(crate) target_py_version: SupportedPyVersion, } @@ -31,5 +32,5 @@ pub(crate) fn set_target_py_version(db: &mut dyn Db, target_version: SupportedPy } pub(crate) fn get_target_py_version(db: &dyn Db) -> SupportedPyVersion { - *TargetPyVersion::get(db).target_py_version(db) + TargetPyVersion::get(db).target_py_version(db) } diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index a4627ff749f90..a34a2d3a9e06c 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,7 +1,7 @@ mod versions; pub use versions::TypeshedVersions; -pub(crate) use versions::TypeshedVersionsQueryResult; +pub(crate) use versions::{parse_typeshed_versions, TypeshedVersionsQueryResult}; #[cfg(test)] mod tests { diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 9f6d720608a69..1a0ee1b6e2bc7 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -5,11 +5,19 @@ use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; use std::sync::Arc; +use ruff_db::{source::source_text, vfs::VfsFile}; use rustc_hash::FxHashMap; +use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::SupportedPyVersion; +#[salsa::tracked] +pub(crate) fn parse_typeshed_versions(db: &dyn Db, versions_file: VfsFile) -> TypeshedVersions { + let file_content = source_text(db.upcast(), versions_file); + file_content.parse().unwrap() +} + #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { line_number: NonZeroU16, diff --git a/crates/red_knot_python_semantic/src/db.rs b/crates/red_knot_python_semantic/src/db.rs index 73ba39f021d77..11c7a88352236 100644 --- a/crates/red_knot_python_semantic/src/db.rs +++ b/crates/red_knot_python_semantic/src/db.rs @@ -31,7 +31,6 @@ pub trait Db: pub(crate) mod tests { use std::fmt::Formatter; use std::marker::PhantomData; - use std::str::FromStr; use std::sync::Arc; use salsa::id::AsId; @@ -39,7 +38,7 @@ pub(crate) mod tests { use salsa::storage::HasIngredientsFor; use salsa::DebugWithDb; - use red_knot_module_resolver::{Db as ResolverDb, Jar as ResolverJar, TypeshedVersions}; + use red_knot_module_resolver::{Db as ResolverDb, Jar as ResolverJar}; use ruff_db::file_system::{FileSystem, MemoryFileSystem, OsFileSystem}; use ruff_db::vfs::Vfs; use ruff_db::{Db as SourceDb, Jar as SourceJar, Upcast}; @@ -52,7 +51,6 @@ pub(crate) mod tests { vfs: Vfs, file_system: TestFileSystem, events: std::sync::Arc>>, - typeshed_versions: TypeshedVersions, } impl TestDb { @@ -62,7 +60,6 @@ pub(crate) mod tests { file_system: TestFileSystem::Memory(MemoryFileSystem::default()), events: std::sync::Arc::default(), vfs: Vfs::with_stubbed_vendored(), - typeshed_versions: TypeshedVersions::from_str("").unwrap(), } } @@ -128,11 +125,7 @@ pub(crate) mod tests { } } - impl red_knot_module_resolver::Db for TestDb { - fn typeshed_versions(&self) -> &TypeshedVersions { - &self.typeshed_versions - } - } + impl red_knot_module_resolver::Db for TestDb {} impl Db for TestDb {} impl salsa::Database for TestDb { @@ -153,7 +146,6 @@ pub(crate) mod tests { TestFileSystem::Os(fs) => TestFileSystem::Os(fs.snapshot()), }, events: self.events.clone(), - typeshed_versions: self.typeshed_versions.clone(), }) } } From dbe882669d3b2a42c9fa699055c040bd5dad6606 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 2 Jul 2024 19:56:14 +0100 Subject: [PATCH 14/58] fix test --- crates/red_knot_module_resolver/src/resolver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index debcdcf1d447e..978a8c9cc06a5 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -396,6 +396,8 @@ mod tests { fs.create_directory_all(&*src)?; fs.create_directory_all(&*site_packages)?; fs.create_directory_all(&*custom_typeshed)?; + fs.create_directory_all(custom_typeshed.join("stdlib"))?; + fs.touch(custom_typeshed.join("stdlib/VERSIONS"))?; let settings = ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, From 0a0a334fbbb342e913f61e404e8b1285b8a402ca Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 10:38:46 +0100 Subject: [PATCH 15/58] Address easy review comments --- crates/red_knot_module_resolver/src/lib.rs | 2 +- crates/red_knot_module_resolver/src/module.rs | 8 +- .../src/module_name.rs | 22 ++ crates/red_knot_module_resolver/src/path.rs | 233 ++++++------------ .../red_knot_module_resolver/src/resolver.rs | 45 ++-- .../src/supported_py_version.rs | 2 +- .../red_knot_module_resolver/src/typeshed.rs | 4 +- .../src/typeshed/versions.rs | 36 +-- 8 files changed, 149 insertions(+), 203 deletions(-) diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index c1f5993974370..11f2a6a4b5b07 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -11,4 +11,4 @@ pub use module::{Module, ModuleKind}; pub use module_name::ModuleName; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; pub use supported_py_version::SupportedPyVersion; -pub use typeshed::TypeshedVersions; +pub use typeshed::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; diff --git a/crates/red_knot_module_resolver/src/module.rs b/crates/red_knot_module_resolver/src/module.rs index 024d104a9f2b2..bc2eb4358f0ab 100644 --- a/crates/red_knot_module_resolver/src/module.rs +++ b/crates/red_knot_module_resolver/src/module.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use ruff_db::vfs::VfsFile; +use crate::db::Db; use crate::module_name::ModuleName; -use crate::path::{ModuleResolutionPath, ModuleResolutionPathRef}; -use crate::Db; +use crate::path::{ModuleResolutionPathBuf, ModuleResolutionPathRef}; /// Representation of a Python module. #[derive(Clone, PartialEq, Eq)] @@ -17,7 +17,7 @@ impl Module { pub(crate) fn new( name: ModuleName, kind: ModuleKind, - search_path: Arc, + search_path: Arc, file: VfsFile, ) -> Self { Self { @@ -77,7 +77,7 @@ impl salsa::DebugWithDb for Module { struct ModuleInner { name: ModuleName, kind: ModuleKind, - search_path: Arc, + search_path: Arc, file: VfsFile, } diff --git a/crates/red_knot_module_resolver/src/module_name.rs b/crates/red_knot_module_resolver/src/module_name.rs index 19c52a3884b69..9a9ab79b57903 100644 --- a/crates/red_knot_module_resolver/src/module_name.rs +++ b/crates/red_knot_module_resolver/src/module_name.rs @@ -23,6 +23,7 @@ impl ModuleName { /// * The name contains a sequence of multiple dots /// * A component of a name (the part between two dots) isn't a valid python identifier. #[inline] + #[must_use] pub fn new(name: &str) -> Option { Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) } @@ -52,11 +53,13 @@ impl ModuleName { /// assert_eq!(ModuleName::new_static("2000"), None); /// ``` #[inline] + #[must_use] pub fn new_static(name: &'static str) -> Option { // TODO(Micha): Use CompactString::const_new once we upgrade to 0.8 https://github.com/ParkMyCar/compact_str/pull/336 Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) } + #[must_use] fn is_valid_name(name: &str) -> bool { !name.is_empty() && name.split('.').all(is_identifier) } @@ -70,6 +73,7 @@ impl ModuleName { /// /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::>(), vec!["foo", "bar", "baz"]); /// ``` + #[must_use] pub fn components(&self) -> impl DoubleEndedIterator { self.0.split('.') } @@ -85,6 +89,7 @@ impl ModuleName { /// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap())); /// assert_eq!(ModuleName::new_static("root").unwrap().parent(), None); /// ``` + #[must_use] pub fn parent(&self) -> Option { let (parent, _) = self.0.rsplit_once('.')?; Some(Self(parent.to_compact_string())) @@ -104,6 +109,7 @@ impl ModuleName { /// assert!(!ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("bar").unwrap())); /// assert!(!ModuleName::new_static("foo_bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap())); /// ``` + #[must_use] pub fn starts_with(&self, other: &ModuleName) -> bool { let mut self_components = self.components(); let other_components = other.components(); @@ -117,10 +123,26 @@ impl ModuleName { true } + #[must_use] #[inline] pub fn as_str(&self) -> &str { &self.0 } + + #[must_use] + pub fn from_components<'a>(mut components: impl Iterator) -> Option { + let first_part = components.next()?; + if let Some(second_part) = components.next() { + let mut name = format!("{first_part}.{second_part}"); + for part in components { + name.push('.'); + name.push_str(part); + } + ModuleName::new(&name) + } else { + ModuleName::new(first_part) + } + } } impl Deref for ModuleName { diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 81fcacf210b75..94fc5a5bdf21e 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -3,10 +3,10 @@ use std::iter::FusedIterator; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::{system_path_to_file, VfsPath}; +use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::get_target_py_version; -use crate::typeshed::{parse_typeshed_versions, TypeshedVersionsQueryResult}; -use crate::{Db, TypeshedVersions}; +use crate::typeshed::{parse_typeshed_versions, TypeshedVersions, TypeshedVersionsQueryResult}; #[repr(transparent)] #[derive(Debug, PartialEq, Eq, Hash)] @@ -48,14 +48,14 @@ pub(crate) struct SitePackagesPathBuf(FileSystemPathBuf); /// /// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) enum ModuleResolutionPath { +pub(crate) enum ModuleResolutionPathBuf { Extra(ExtraPathBuf), FirstParty(FirstPartyPathBuf), StandardLibrary(StandardLibraryPathBuf), SitePackages(SitePackagesPathBuf), } -impl ModuleResolutionPath { +impl ModuleResolutionPathBuf { /// Push a new part to the path, /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions. /// For the stdlib variant specifically, it may only have a `.pyi` extension. @@ -65,15 +65,14 @@ impl ModuleResolutionPath { pub(crate) fn push(&mut self, component: &str) { debug_assert!(matches!(component.matches('.').count(), 0 | 1)); if cfg!(debug) { - if let Some(extension) = std::path::Path::new(component).extension() { + if let Some(extension) = camino::Utf8Path::new(component).extension() { match self { Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( - matches!(extension.to_str().unwrap(), "pyi" | "py"), + matches!(extension, "pyi" | "py"), "Extension must be `py` or `pyi`; got {extension:?}" ), Self::StandardLibrary(_) => assert_eq!( - extension.to_str().unwrap(), - "pyi", + extension, "pyi", "Extension must be `py` or `pyi`; got {extension:?}" ), }; @@ -163,82 +162,20 @@ impl ModuleResolutionPath { pub(crate) fn join(&self, component: &(impl AsRef + ?Sized)) -> Self { ModuleResolutionPathRef::from(self).join(component) } - - #[must_use] - fn as_file_system_path_buf(&self) -> &FileSystemPathBuf { - match self { - Self::Extra(ExtraPathBuf(path)) => path, - Self::FirstParty(FirstPartyPathBuf(path)) => path, - Self::StandardLibrary(StandardLibraryPathBuf(path)) => path, - Self::SitePackages(SitePackagesPathBuf(path)) => path, - } - } - - #[must_use] - #[inline] - fn into_file_system_path_buf(self) -> FileSystemPathBuf { - match self { - Self::Extra(ExtraPathBuf(path)) => path, - Self::FirstParty(FirstPartyPathBuf(path)) => path, - Self::StandardLibrary(StandardLibraryPathBuf(path)) => path, - Self::SitePackages(SitePackagesPathBuf(path)) => path, - } - } } -impl From for VfsPath { - fn from(value: ModuleResolutionPath) -> Self { - VfsPath::FileSystem(value.into_file_system_path_buf()) - } -} - -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &VfsPath) -> bool { - match other { - VfsPath::FileSystem(path) => self.as_file_system_path_buf() == path, - VfsPath::Vendored(_) => false, - } - } -} - -impl PartialEq for VfsPath { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) - } -} - -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &FileSystemPathBuf) -> bool { - self.as_file_system_path_buf() == other - } -} - -impl PartialEq for FileSystemPathBuf { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) - } -} - -impl PartialEq for ModuleResolutionPath { - fn eq(&self, other: &FileSystemPath) -> bool { - ModuleResolutionPathRef::from(self) == *other - } -} - -impl PartialEq for FileSystemPath { - fn eq(&self, other: &ModuleResolutionPath) -> bool { - other.eq(self) - } -} - -impl AsRef for ModuleResolutionPath { - #[inline] - fn as_ref(&self) -> &FileSystemPathBuf { - self.as_file_system_path_buf() +impl From for VfsPath { + fn from(value: ModuleResolutionPathBuf) -> Self { + VfsPath::FileSystem(match value { + ModuleResolutionPathBuf::Extra(ExtraPathBuf(path)) => path, + ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path)) => path, + ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path)) => path, + ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path)) => path, + }) } } -impl AsRef for ModuleResolutionPath { +impl AsRef for ModuleResolutionPathBuf { #[inline] fn as_ref(&self) -> &FileSystemPath { ModuleResolutionPathRef::from(self).as_file_system_path() @@ -345,16 +282,21 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - fn load_typeshed_versions(db: &dyn Db, stdlib_root: &StandardLibraryPath) -> TypeshedVersions { + fn load_typeshed_versions<'db>( + db: &'db dyn Db, + stdlib_root: &StandardLibraryPath, + ) -> &'db TypeshedVersions { let StandardLibraryPath(stdlib_fs_path) = stdlib_root; let versions_path = stdlib_fs_path.join("VERSIONS"); let versions_file = system_path_to_file(db.upcast(), versions_path).unwrap(); parse_typeshed_versions(db, versions_file) } + // Private helper function with concrete inputs, + // to avoid monomorphization #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { - match (self, search_path.into()) { + fn is_directory_impl(&self, db: &dyn Db, search_path: Self) -> bool { + match (self, search_path) { (Self::Extra(ExtraPath(path)), Self::Extra(_)) => db.file_system().is_directory(path), (Self::FirstParty(FirstPartyPath(path)), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(SitePackagesPath(path)), Self::SitePackages(_)) => db.file_system().is_directory(path), @@ -378,8 +320,15 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { - match (self, search_path.into()) { + pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { + self.is_directory_impl(db, search_path.into()) + } + + // Private helper function with concrete inputs, + // to avoid monomorphization + #[must_use] + fn is_regular_package_impl(&self, db: &dyn Db, search_path: Self) -> bool { + match (self, search_path) { (Self::Extra(ExtraPath(fs_path)), Self::Extra(_)) | (Self::FirstParty(FirstPartyPath(fs_path)), Self::FirstParty(_)) | (Self::SitePackages(SitePackagesPath(fs_path)), Self::SitePackages(_)) => { @@ -409,6 +358,11 @@ impl<'a> ModuleResolutionPathRef<'a> { } } + #[must_use] + pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { + self.is_regular_package_impl(db, search_path.into()) + } + #[must_use] pub(crate) fn parent(&self) -> Option { Some(match self { @@ -436,7 +390,7 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - fn sans_dunder_init(self) -> Self { + fn with_dunder_init_stripped(self) -> Self { if self.ends_with_dunder_init() { self.parent().unwrap_or_else(|| match self { Self::Extra(_) => Self::extra_unchecked(""), @@ -451,78 +405,69 @@ impl<'a> ModuleResolutionPathRef<'a> { #[must_use] pub(crate) fn as_module_name(&self) -> Option { - let mut parts_iter = match self.sans_dunder_init() { + ModuleName::from_components(match self.with_dunder_init_stripped() { Self::Extra(ExtraPath(path)) => ModulePartIterator::from_fs_path(path), Self::FirstParty(FirstPartyPath(path)) => ModulePartIterator::from_fs_path(path), Self::StandardLibrary(StandardLibraryPath(path)) => { ModulePartIterator::from_fs_path(path) } Self::SitePackages(SitePackagesPath(path)) => ModulePartIterator::from_fs_path(path), - }; - let first_part = parts_iter.next()?; - if let Some(second_part) = parts_iter.next() { - let mut name = format!("{first_part}.{second_part}"); - for part in parts_iter { - name.push('.'); - name.push_str(part); - } - ModuleName::new(&name) - } else { - ModuleName::new(first_part) - } + }) } #[must_use] - pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPath { + pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPathBuf { match self { Self::Extra(ExtraPath(path)) => { - ModuleResolutionPath::Extra(ExtraPathBuf(path.with_extension("pyi"))) + ModuleResolutionPathBuf::Extra(ExtraPathBuf(path.with_extension("pyi"))) } Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path.with_extension("pyi"))) + ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path.with_extension("pyi"))) } Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf( + ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf( path.with_extension("pyi"), )) } - Self::SitePackages(SitePackagesPath(path)) => { - ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path.with_extension("pyi"))) - } + Self::SitePackages(SitePackagesPath(path)) => ModuleResolutionPathBuf::SitePackages( + SitePackagesPathBuf(path.with_extension("pyi")), + ), } } #[must_use] - pub(crate) fn with_py_extension(&self) -> Option { + pub(crate) fn with_py_extension(&self) -> Option { match self { - Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPath::Extra(ExtraPathBuf( + Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPathBuf::Extra(ExtraPathBuf( path.with_extension("py"), ))), - Self::FirstParty(FirstPartyPath(path)) => Some(ModuleResolutionPath::FirstParty( + Self::FirstParty(FirstPartyPath(path)) => Some(ModuleResolutionPathBuf::FirstParty( FirstPartyPathBuf(path.with_extension("py")), )), Self::StandardLibrary(_) => None, - Self::SitePackages(SitePackagesPath(path)) => Some(ModuleResolutionPath::SitePackages( - SitePackagesPathBuf(path.with_extension("py")), - )), + Self::SitePackages(SitePackagesPath(path)) => { + Some(ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf( + path.with_extension("py"), + ))) + } } } #[cfg(test)] #[must_use] - pub(crate) fn to_module_resolution_path(self) -> ModuleResolutionPath { + pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { match self { Self::Extra(ExtraPath(path)) => { - ModuleResolutionPath::Extra(ExtraPathBuf(path.to_path_buf())) + ModuleResolutionPathBuf::Extra(ExtraPathBuf(path.to_path_buf())) } Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path.to_path_buf())) + ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path.to_path_buf())) } Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(path.to_path_buf())) + ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path.to_path_buf())) } Self::SitePackages(SitePackagesPath(path)) => { - ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path.to_path_buf())) + ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path.to_path_buf())) } } } @@ -532,8 +477,8 @@ impl<'a> ModuleResolutionPathRef<'a> { pub(crate) fn join( &self, component: &'a (impl AsRef + ?Sized), - ) -> ModuleResolutionPath { - let mut result = self.to_module_resolution_path(); + ) -> ModuleResolutionPathBuf { + let mut result = self.to_path_buf(); result.push(component.as_ref().as_str()); result } @@ -550,63 +495,50 @@ impl<'a> ModuleResolutionPathRef<'a> { } } -impl<'a> From<&'a ModuleResolutionPath> for ModuleResolutionPathRef<'a> { +impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { #[inline] - fn from(value: &'a ModuleResolutionPath) -> Self { + fn from(value: &'a ModuleResolutionPathBuf) -> Self { match value { - ModuleResolutionPath::Extra(ExtraPathBuf(path)) => { + ModuleResolutionPathBuf::Extra(ExtraPathBuf(path)) => { ModuleResolutionPathRef::extra_unchecked(path) } - ModuleResolutionPath::FirstParty(FirstPartyPathBuf(path)) => { + ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path)) => { ModuleResolutionPathRef::first_party_unchecked(path) } - ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(path)) => { + ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path)) => { ModuleResolutionPathRef::standard_library_unchecked(path) } - ModuleResolutionPath::SitePackages(SitePackagesPathBuf(path)) => { + ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path)) => { ModuleResolutionPathRef::site_packages_unchecked(path) } } } } -impl<'a> AsRef for ModuleResolutionPathRef<'a> { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - self.as_file_system_path() - } -} - -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &ModuleResolutionPath) -> bool { +impl<'a> PartialEq for ModuleResolutionPathRef<'a> { + fn eq(&self, other: &ModuleResolutionPathBuf) -> bool { match (self, other) { ( ModuleResolutionPathRef::Extra(ExtraPath(self_path)), - ModuleResolutionPath::Extra(ExtraPathBuf(other_path)), + ModuleResolutionPathBuf::Extra(ExtraPathBuf(other_path)), ) | ( ModuleResolutionPathRef::FirstParty(FirstPartyPath(self_path)), - ModuleResolutionPath::FirstParty(FirstPartyPathBuf(other_path)), + ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(other_path)), ) | ( ModuleResolutionPathRef::StandardLibrary(StandardLibraryPath(self_path)), - ModuleResolutionPath::StandardLibrary(StandardLibraryPathBuf(other_path)), + ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(other_path)), ) | ( ModuleResolutionPathRef::SitePackages(SitePackagesPath(self_path)), - ModuleResolutionPath::SitePackages(SitePackagesPathBuf(other_path)), + ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(other_path)), ) => *self_path == **other_path, _ => false, } } } -impl<'a> PartialEq> for ModuleResolutionPath { - fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { - other.eq(self) - } -} - impl<'a> PartialEq for ModuleResolutionPathRef<'a> { fn eq(&self, other: &FileSystemPath) -> bool { self.as_file_system_path() == other @@ -619,21 +551,6 @@ impl<'a> PartialEq> for FileSystemPath { } } -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { - fn eq(&self, other: &VfsPath) -> bool { - let VfsPath::FileSystem(other) = other else { - return false; - }; - self.as_file_system_path() == &**other - } -} - -impl<'a> PartialEq> for VfsPath { - fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { - other.eq(self) - } -} - pub(crate) struct ModulePartIterator<'a> { parent_components: Option>, stem: Option<&'a str>, diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 978a8c9cc06a5..46de032ef8c85 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -6,7 +6,7 @@ use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; -use crate::path::{ModuleResolutionPath, ModuleResolutionPathRef}; +use crate::path::{ModuleResolutionPathBuf, ModuleResolutionPathRef}; use crate::resolver::internal::ModuleResolverSearchPaths; use crate::supported_py_version::set_target_py_version; use crate::{Db, SupportedPyVersion}; @@ -85,16 +85,16 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let relative_path = search_paths.iter().find_map(|root| match (&**root, path) { (_, VfsPath::Vendored(_)) => todo!("VendoredPaths are not yet supported"), - (ModuleResolutionPath::Extra(_), VfsPath::FileSystem(path)) => { + (ModuleResolutionPathBuf::Extra(_), VfsPath::FileSystem(path)) => { ModuleResolutionPathRef::extra(path.strip_prefix(&**root).ok()?) } - (ModuleResolutionPath::FirstParty(_), VfsPath::FileSystem(path)) => { + (ModuleResolutionPathBuf::FirstParty(_), VfsPath::FileSystem(path)) => { ModuleResolutionPathRef::first_party(path.strip_prefix(&**root).ok()?) } - (ModuleResolutionPath::StandardLibrary(_), VfsPath::FileSystem(path)) => { + (ModuleResolutionPathBuf::StandardLibrary(_), VfsPath::FileSystem(path)) => { ModuleResolutionPathRef::standard_library(path.strip_prefix(&**root).ok()?) } - (ModuleResolutionPath::SitePackages(_), VfsPath::FileSystem(path)) => { + (ModuleResolutionPathBuf::SitePackages(_), VfsPath::FileSystem(path)) => { ModuleResolutionPathRef::site_packages(path.strip_prefix(&**root).ok()?) } })?; @@ -158,19 +158,21 @@ impl ModuleResolutionSettings { let mut paths = extra_paths .into_iter() - .map(ModuleResolutionPath::extra) - .collect::>>() + .map(ModuleResolutionPathBuf::extra) + .collect::>>() .unwrap(); - paths.push(ModuleResolutionPath::first_party(workspace_root).unwrap()); + paths.push(ModuleResolutionPathBuf::first_party(workspace_root).unwrap()); if let Some(custom_typeshed) = custom_typeshed { - paths.push(ModuleResolutionPath::stdlib_from_typeshed_root(&custom_typeshed).unwrap()); + paths.push( + ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(), + ); } // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step if let Some(site_packages) = site_packages { - paths.push(ModuleResolutionPath::site_packages(site_packages).unwrap()); + paths.push(ModuleResolutionPathBuf::site_packages(site_packages).unwrap()); } ( @@ -183,10 +185,10 @@ impl ModuleResolutionSettings { /// A resolved module resolution order, implementing PEP 561 /// (with some small, deliberate differences) #[derive(Clone, Debug, Default, Eq, PartialEq)] -pub(crate) struct OrderedSearchPaths(Vec>); +pub(crate) struct OrderedSearchPaths(Vec>); impl Deref for OrderedSearchPaths { - type Target = [Arc]; + type Target = [Arc]; fn deref(&self) -> &Self::Target { &self.0 @@ -218,7 +220,7 @@ pub(crate) mod internal { } } -fn module_search_paths(db: &dyn Db) -> &[Arc] { +fn module_search_paths(db: &dyn Db) -> &[Arc] { ModuleResolverSearchPaths::get(db).search_paths(db) } @@ -227,7 +229,7 @@ fn module_search_paths(db: &dyn Db) -> &[Arc] { fn resolve_name( db: &dyn Db, name: &ModuleName, -) -> Option<(Arc, VfsFile, ModuleKind)> { +) -> Option<(Arc, VfsFile, ModuleKind)> { let search_paths = module_search_paths(db); for search_path in search_paths { @@ -281,7 +283,7 @@ fn resolve_name( fn resolve_package<'a, I>( db: &dyn Db, - module_search_path: &ModuleResolutionPath, + module_search_path: &ModuleResolutionPathBuf, components: I, ) -> Result where @@ -338,7 +340,7 @@ where #[derive(Debug)] struct ResolvedPackage { - path: ModuleResolutionPath, + path: ModuleResolutionPathBuf, kind: PackageKind, } @@ -454,7 +456,8 @@ mod tests { .. } = create_resolver()?; - let stdlib_dir = ModuleResolutionPath::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + let stdlib_dir = + ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); let functools_path = stdlib_dir.join("functools.pyi"); db.memory_file_system() .write_file(&functools_path, "def update_wrapper(): ...")?; @@ -467,14 +470,16 @@ mod tests { resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(stdlib_dir, functools_module.search_path()); + assert_eq!(stdlib_dir, functools_module.search_path().to_path_buf()); assert_eq!(ModuleKind::Module, functools_module.kind()); - assert_eq!(functools_path, *functools_module.file().path(&db)); + let functools_path_vfs = VfsPath::from(functools_path); + + assert_eq!(&functools_path_vfs, functools_module.file().path(&db)); assert_eq!( Some(functools_module), - path_to_module(&db, &VfsPath::from(functools_path)) + path_to_module(&db, &functools_path_vfs) ); Ok(()) diff --git a/crates/red_knot_module_resolver/src/supported_py_version.rs b/crates/red_knot_module_resolver/src/supported_py_version.rs index 5a2c8f527afbe..96d3946694b61 100644 --- a/crates/red_knot_module_resolver/src/supported_py_version.rs +++ b/crates/red_knot_module_resolver/src/supported_py_version.rs @@ -3,7 +3,7 @@ #![allow(clippy::needless_lifetimes)] #![allow(clippy::clone_on_copy)] -use crate::Db; +use crate::db::Db; // TODO: unify with the PythonVersion enum in the linter/formatter crates? #[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index a34a2d3a9e06c..5fde8297f00a3 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,7 +1,7 @@ mod versions; -pub use versions::TypeshedVersions; -pub(crate) use versions::{parse_typeshed_versions, TypeshedVersionsQueryResult}; +pub(crate) use versions::{parse_typeshed_versions, TypeshedVersions, TypeshedVersionsQueryResult}; +pub use versions::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; #[cfg(test)] mod tests { diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 1a0ee1b6e2bc7..86cc5c1628fe7 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -3,7 +3,6 @@ use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; -use std::sync::Arc; use ruff_db::{source::source_text, vfs::VfsFile}; use rustc_hash::FxHashMap; @@ -12,7 +11,7 @@ use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::SupportedPyVersion; -#[salsa::tracked] +#[salsa::tracked(return_ref)] pub(crate) fn parse_typeshed_versions(db: &dyn Db, versions_file: VfsFile) -> TypeshedVersions { let file_content = source_text(db.upcast(), versions_file); file_content.parse().unwrap() @@ -90,29 +89,31 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } } -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct TypeshedVersions(Arc>); +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct TypeshedVersions(FxHashMap); impl TypeshedVersions { - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - + #[must_use] fn exact(&self, module_name: &ModuleName) -> Option<&PyVersionRange> { self.0.get(module_name) } - /// Helper function for testing purposes + /// Helper functions for testing purposes #[cfg(test)] + #[must_use] fn contains_exact(&self, module: &ModuleName) -> bool { self.exact(module).is_some() } - pub fn query_module( + #[cfg(test)] + #[must_use] + fn len(&self) -> usize { + self.0.len() + } + + // The only public API for this struct: + #[must_use] + pub(crate) fn query_module( &self, module: &ModuleName, version: impl Into, @@ -144,7 +145,7 @@ impl TypeshedVersions { } #[derive(Debug, Copy, PartialEq, Eq, Clone, Hash)] -pub enum TypeshedVersionsQueryResult { +pub(crate) enum TypeshedVersionsQueryResult { Exists, DoesNotExist, MaybeExists, @@ -203,7 +204,7 @@ impl FromStr for TypeshedVersions { }; } - Ok(Self(Arc::new(map))) + Ok(Self(map)) } } @@ -224,6 +225,7 @@ enum PyVersionRange { } impl PyVersionRange { + #[must_use] fn contains(&self, version: PyVersion) -> bool { match self { Self::AvailableFrom(inner) => inner.contains(&version), @@ -259,7 +261,7 @@ impl fmt::Display for PyVersionRange { } #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct PyVersion { +pub(crate) struct PyVersion { major: u8, minor: u8, } From ad2e987e27cc000f089d1ab602c5256887d6c8f2 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 12:12:14 +0100 Subject: [PATCH 16/58] Cleanup VERSIONS-parsing and add a todo for better error handling --- crates/red_knot_module_resolver/src/path.rs | 13 +++++- .../red_knot_module_resolver/src/resolver.rs | 2 +- .../src/typeshed/versions.rs | 44 +++++++++++++------ 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 94fc5a5bdf21e..1dcfcb3cb1bbf 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -288,8 +288,17 @@ impl<'a> ModuleResolutionPathRef<'a> { ) -> &'db TypeshedVersions { let StandardLibraryPath(stdlib_fs_path) = stdlib_root; let versions_path = stdlib_fs_path.join("VERSIONS"); - let versions_file = system_path_to_file(db.upcast(), versions_path).unwrap(); - parse_typeshed_versions(db, versions_file) + let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { + todo!( + "Still need to figure out how to handle VERSIONS files being deleted \ + from custom typeshed directories! Expected a file to exist at {versions_path}" + ) + }; + // TODO(Alex/Micha): If VERSIONS is invalid, + // this should invalidate not just the specific module resolution we're currently attempting, + // but all type inference that depends on any standard-library types. + // Unwrapping here is not correct... + parse_typeshed_versions(db, versions_file).as_ref().unwrap() } // Private helper function with concrete inputs, diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 46de032ef8c85..1e1d01e4d8be6 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -399,7 +399,7 @@ mod tests { fs.create_directory_all(&*site_packages)?; fs.create_directory_all(&*custom_typeshed)?; fs.create_directory_all(custom_typeshed.join("stdlib"))?; - fs.touch(custom_typeshed.join("stdlib/VERSIONS"))?; + fs.write_file(custom_typeshed.join("stdlib/VERSIONS"), "functools: 3.8-")?; let settings = ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 86cc5c1628fe7..aa5927f68a8ac 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -12,14 +12,17 @@ use crate::module_name::ModuleName; use crate::supported_py_version::SupportedPyVersion; #[salsa::tracked(return_ref)] -pub(crate) fn parse_typeshed_versions(db: &dyn Db, versions_file: VfsFile) -> TypeshedVersions { +pub(crate) fn parse_typeshed_versions( + db: &dyn Db, + versions_file: VfsFile, +) -> Result { let file_content = source_text(db.upcast(), versions_file); - file_content.parse().unwrap() + file_content.parse() } #[derive(Debug, PartialEq, Eq)] pub struct TypeshedVersionsParseError { - line_number: NonZeroU16, + line_number: Option, reason: TypeshedVersionsParseErrorKind, } @@ -29,10 +32,14 @@ impl fmt::Display for TypeshedVersionsParseError { line_number, reason, } = self; - write!( - f, - "Error while parsing line {line_number} of typeshed's VERSIONS file: {reason}" - ) + if let Some(line_number) = line_number { + write!( + f, + "Error while parsing line {line_number} of typeshed's VERSIONS file: {reason}" + ) + } else { + write!(f, "Error while parsing typeshed's VERSIONS file: {reason}") + } } } @@ -57,6 +64,7 @@ pub enum TypeshedVersionsParseErrorKind { version: String, err: std::num::ParseIntError, }, + EmptyVersionsFile, } impl fmt::Display for TypeshedVersionsParseErrorKind { @@ -85,6 +93,7 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { f, "Failed to convert '{version}' to a pair of integers due to {err}", ), + Self::EmptyVersionsFile => f.write_str("Versions file was empty!"), } } } @@ -163,7 +172,7 @@ impl FromStr for TypeshedVersions { let Ok(line_number) = NonZeroU16::try_from(line_number) else { return Err(TypeshedVersionsParseError { - line_number: NonZeroU16::MAX, + line_number: None, reason: TypeshedVersionsParseErrorKind::TooManyLines(line_number), }); }; @@ -179,14 +188,14 @@ impl FromStr for TypeshedVersions { let (Some(module_name), Some(rest), None) = (parts.next(), parts.next(), parts.next()) else { return Err(TypeshedVersionsParseError { - line_number, + line_number: Some(line_number), reason: TypeshedVersionsParseErrorKind::UnexpectedNumberOfColons, }); }; let Some(module_name) = ModuleName::new(module_name) else { return Err(TypeshedVersionsParseError { - line_number, + line_number: Some(line_number), reason: TypeshedVersionsParseErrorKind::InvalidModuleName( module_name.to_string(), ), @@ -197,14 +206,21 @@ impl FromStr for TypeshedVersions { Ok(version) => map.insert(module_name, version), Err(reason) => { return Err(TypeshedVersionsParseError { - line_number, + line_number: Some(line_number), reason, }) } }; } - Ok(Self(map)) + if map.is_empty() { + Err(TypeshedVersionsParseError { + line_number: None, + reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile, + }) + } else { + Ok(Self(map)) + } } } @@ -343,7 +359,7 @@ mod tests { const TYPESHED_STDLIB_DIR: &str = "stdlib"; #[allow(unsafe_code)] - const ONE: NonZeroU16 = unsafe { NonZeroU16::new_unchecked(1) }; + const ONE: Option = Some(unsafe { NonZeroU16::new_unchecked(1) }); #[test] fn can_parse_vendored_versions_file() { @@ -552,7 +568,7 @@ foo: 3.8- # trailing comment assert_eq!( TypeshedVersions::from_str(&massive_versions_file), Err(TypeshedVersionsParseError { - line_number: NonZeroU16::MAX, + line_number: None, reason: TypeshedVersionsParseErrorKind::TooManyLines( NonZeroUsize::new(too_many + 1 - offset).unwrap() ) From 78c56f742fc82da90fc70060fc644598279cc0e1 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 12:51:17 +0100 Subject: [PATCH 17/58] Move more logic internal to `path.rs` --- crates/red_knot_module_resolver/src/path.rs | 28 +++++++++++++++++++ .../red_knot_module_resolver/src/resolver.rs | 24 +++++----------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 1dcfcb3cb1bbf..b732d79b371a7 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -162,6 +162,13 @@ impl ModuleResolutionPathBuf { pub(crate) fn join(&self, component: &(impl AsRef + ?Sized)) -> Self { ModuleResolutionPathRef::from(self).join(component) } + + pub(crate) fn relativize_path<'a>( + &'a self, + absolute_path: &'a FileSystemPath, + ) -> Option> { + ModuleResolutionPathRef::from(self).relativize_path(absolute_path) + } } impl From for VfsPath { @@ -502,6 +509,27 @@ impl<'a> ModuleResolutionPathRef<'a> { Self::SitePackages(SitePackagesPath(path)) => path, } } + + #[must_use] + pub(crate) fn relativize_path(&self, absolute_path: &'a FileSystemPath) -> Option { + match self { + Self::Extra(ExtraPath(root)) => { + absolute_path.strip_prefix(root).ok().and_then(Self::extra) + } + Self::FirstParty(FirstPartyPath(root)) => absolute_path + .strip_prefix(root) + .ok() + .and_then(Self::first_party), + Self::StandardLibrary(StandardLibraryPath(root)) => absolute_path + .strip_prefix(root) + .ok() + .and_then(Self::standard_library), + Self::SitePackages(SitePackagesPath(root)) => absolute_path + .strip_prefix(root) + .ok() + .and_then(Self::site_packages), + } + } } impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 1e1d01e4d8be6..4c5ffd17ca042 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -6,7 +6,7 @@ use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; -use crate::path::{ModuleResolutionPathBuf, ModuleResolutionPathRef}; +use crate::path::ModuleResolutionPathBuf; use crate::resolver::internal::ModuleResolverSearchPaths; use crate::supported_py_version::set_target_py_version; use crate::{Db, SupportedPyVersion}; @@ -79,25 +79,15 @@ pub fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option { pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let _span = tracing::trace_span!("file_to_module", ?file).entered(); - let path = file.path(db.upcast()); + let VfsPath::FileSystem(path) = file.path(db.upcast()) else { + todo!("VendoredPaths are not yet supported") + }; let search_paths = module_search_paths(db); - let relative_path = search_paths.iter().find_map(|root| match (&**root, path) { - (_, VfsPath::Vendored(_)) => todo!("VendoredPaths are not yet supported"), - (ModuleResolutionPathBuf::Extra(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::extra(path.strip_prefix(&**root).ok()?) - } - (ModuleResolutionPathBuf::FirstParty(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::first_party(path.strip_prefix(&**root).ok()?) - } - (ModuleResolutionPathBuf::StandardLibrary(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::standard_library(path.strip_prefix(&**root).ok()?) - } - (ModuleResolutionPathBuf::SitePackages(_), VfsPath::FileSystem(path)) => { - ModuleResolutionPathRef::site_packages(path.strip_prefix(&**root).ok()?) - } - })?; + let relative_path = search_paths + .iter() + .find_map(|root| root.relativize_path(path))?; let module_name = relative_path.as_module_name()?; From 6723c7f758273969fad024f68722dd480dd48f43 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 15:09:19 +0100 Subject: [PATCH 18/58] Get rid of the inner structs --- crates/red_knot_module_resolver/src/path.rs | 522 +++++++----------- .../red_knot_module_resolver/src/resolver.rs | 2 +- 2 files changed, 214 insertions(+), 310 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index b732d79b371a7..c5e84b0964797 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -8,38 +8,6 @@ use crate::module_name::ModuleName; use crate::supported_py_version::get_target_py_version; use crate::typeshed::{parse_typeshed_versions, TypeshedVersions, TypeshedVersionsQueryResult}; -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash)] -pub(crate) struct ExtraPath(FileSystemPath); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) struct ExtraPathBuf(FileSystemPathBuf); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash)] -pub(crate) struct FirstPartyPath(FileSystemPath); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) struct FirstPartyPathBuf(FileSystemPathBuf); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash)] -pub(crate) struct StandardLibraryPath(FileSystemPath); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) struct StandardLibraryPathBuf(FileSystemPathBuf); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash)] -pub(crate) struct SitePackagesPath(FileSystemPath); - -#[repr(transparent)] -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) struct SitePackagesPathBuf(FileSystemPathBuf); - /// Enumeration of the different kinds of search paths type checkers are expected to support. /// /// N.B. Although we don't implement `Ord` for this enum, they are ordered in terms of the @@ -48,21 +16,15 @@ pub(crate) struct SitePackagesPathBuf(FileSystemPathBuf); /// /// [the order given in the typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) enum ModuleResolutionPathBuf { - Extra(ExtraPathBuf), - FirstParty(FirstPartyPathBuf), - StandardLibrary(StandardLibraryPathBuf), - SitePackages(SitePackagesPathBuf), +enum ModuleResolutionPathBufInner { + Extra(FileSystemPathBuf), + FirstParty(FileSystemPathBuf), + StandardLibrary(FileSystemPathBuf), + SitePackages(FileSystemPathBuf), } -impl ModuleResolutionPathBuf { - /// Push a new part to the path, - /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions. - /// For the stdlib variant specifically, it may only have a `.pyi` extension. - /// - /// ## Panics: - /// If a component with an invalid extension is passed - pub(crate) fn push(&mut self, component: &str) { +impl ModuleResolutionPathBufInner { + fn push(&mut self, component: &str) { debug_assert!(matches!(component.matches('.').count(), 0 | 1)); if cfg!(debug) { if let Some(extension) = camino::Utf8Path::new(component).extension() { @@ -79,45 +41,48 @@ impl ModuleResolutionPathBuf { } } let inner = match self { - Self::Extra(ExtraPathBuf(ref mut path)) => path, - Self::FirstParty(FirstPartyPathBuf(ref mut path)) => path, - Self::StandardLibrary(StandardLibraryPathBuf(ref mut path)) => path, - Self::SitePackages(SitePackagesPathBuf(ref mut path)) => path, + Self::Extra(ref mut path) => path, + Self::FirstParty(ref mut path) => path, + Self::StandardLibrary(ref mut path) => path, + Self::SitePackages(ref mut path) => path, }; inner.push(component); } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) struct ModuleResolutionPathBuf(ModuleResolutionPathBufInner); + +impl ModuleResolutionPathBuf { + /// Push a new part to the path, + /// while maintaining the invariant that the path can only have `.py` or `.pyi` extensions. + /// For the stdlib variant specifically, it may only have a `.pyi` extension. + /// + /// ## Panics: + /// If a component with an invalid extension is passed + pub(crate) fn push(&mut self, component: &str) { + self.0.push(component); + } #[must_use] pub(crate) fn extra(path: FileSystemPathBuf) -> Option { - if path - .extension() + path.extension() .map_or(true, |ext| matches!(ext, "py" | "pyi")) - { - Some(Self::Extra(ExtraPathBuf(path))) - } else { - None - } + .then_some(Self(ModuleResolutionPathBufInner::Extra(path))) } #[must_use] pub(crate) fn first_party(path: FileSystemPathBuf) -> Option { - if path - .extension() + path.extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) - { - Some(Self::FirstParty(FirstPartyPathBuf(path))) - } else { - None - } + .then_some(Self(ModuleResolutionPathBufInner::FirstParty(path))) } #[must_use] pub(crate) fn standard_library(path: FileSystemPathBuf) -> Option { - if path.extension().map_or(true, |ext| ext == "pyi") { - Some(Self::StandardLibrary(StandardLibraryPathBuf(path))) - } else { - None - } + path.extension() + .map_or(true, |ext| ext == "pyi") + .then_some(Self(ModuleResolutionPathBufInner::StandardLibrary(path))) } #[must_use] @@ -127,14 +92,9 @@ impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn site_packages(path: FileSystemPathBuf) -> Option { - if path - .extension() + path.extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) - { - Some(Self::SitePackages(SitePackagesPathBuf(path))) - } else { - None - } + .then_some(Self(ModuleResolutionPathBufInner::SitePackages(path))) } #[must_use] @@ -159,10 +119,11 @@ impl ModuleResolutionPathBuf { #[cfg(test)] #[must_use] - pub(crate) fn join(&self, component: &(impl AsRef + ?Sized)) -> Self { - ModuleResolutionPathRef::from(self).join(component) + pub(crate) fn join(&self, component: &str) -> Self { + Self(ModuleResolutionPathRefInner::from(&self.0).join(component)) } + #[must_use] pub(crate) fn relativize_path<'a>( &'a self, absolute_path: &'a FileSystemPath, @@ -173,11 +134,11 @@ impl ModuleResolutionPathBuf { impl From for VfsPath { fn from(value: ModuleResolutionPathBuf) -> Self { - VfsPath::FileSystem(match value { - ModuleResolutionPathBuf::Extra(ExtraPathBuf(path)) => path, - ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path)) => path, - ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path)) => path, - ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path)) => path, + VfsPath::FileSystem(match value.0 { + ModuleResolutionPathBufInner::Extra(path) => path, + ModuleResolutionPathBufInner::FirstParty(path) => path, + ModuleResolutionPathBufInner::StandardLibrary(path) => path, + ModuleResolutionPathBufInner::SitePackages(path) => path, }) } } @@ -185,116 +146,25 @@ impl From for VfsPath { impl AsRef for ModuleResolutionPathBuf { #[inline] fn as_ref(&self) -> &FileSystemPath { - ModuleResolutionPathRef::from(self).as_file_system_path() + ModuleResolutionPathRefInner::from(&self.0).as_file_system_path() } } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub(crate) enum ModuleResolutionPathRef<'a> { - Extra(&'a ExtraPath), - FirstParty(&'a FirstPartyPath), - StandardLibrary(&'a StandardLibraryPath), - SitePackages(&'a SitePackagesPath), +enum ModuleResolutionPathRefInner<'a> { + Extra(&'a FileSystemPath), + FirstParty(&'a FileSystemPath), + StandardLibrary(&'a FileSystemPath), + SitePackages(&'a FileSystemPath), } -impl<'a> ModuleResolutionPathRef<'a> { - #[must_use] - pub(crate) fn extra(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - if path - .extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - { - Some(Self::extra_unchecked(path)) - } else { - None - } - } - - #[must_use] - #[allow(unsafe_code)] - fn extra_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - // SAFETY: ExtraPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const ExtraPath is valid. - Self::Extra(unsafe { &*(path.as_ref() as *const FileSystemPath as *const ExtraPath) }) - } - - #[must_use] - pub(crate) fn first_party(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - if path - .extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - { - Some(Self::first_party_unchecked(path)) - } else { - None - } - } - - #[must_use] - #[allow(unsafe_code)] - fn first_party_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - // SAFETY: FirstPartyPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const FirstPartyPath is valid. - Self::FirstParty(unsafe { - &*(path.as_ref() as *const FileSystemPath as *const FirstPartyPath) - }) - } - - #[must_use] - pub(crate) fn standard_library( - path: &'a (impl AsRef + ?Sized), - ) -> Option { - let path = path.as_ref(); - // Unlike other variants, only `.pyi` extensions are permitted - if path.extension().map_or(true, |ext| ext == "pyi") { - Some(Self::standard_library_unchecked(path)) - } else { - None - } - } - - #[must_use] - #[allow(unsafe_code)] - fn standard_library_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - // SAFETY: StandardLibraryPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const StandardLibraryPath is valid. - Self::StandardLibrary(unsafe { - &*(path.as_ref() as *const FileSystemPath as *const StandardLibraryPath) - }) - } - - #[must_use] - pub(crate) fn site_packages(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - if path - .extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - { - Some(Self::site_packages_unchecked(path)) - } else { - None - } - } - - #[must_use] - #[allow(unsafe_code)] - fn site_packages_unchecked(path: &'a (impl AsRef + ?Sized)) -> Self { - // SAFETY: SitePackagesPath is marked as #[repr(transparent)] so the conversion from a - // *const FileSystemPath to a *const SitePackagesPath is valid. - Self::SitePackages(unsafe { - &*(path.as_ref() as *const FileSystemPath as *const SitePackagesPath) - }) - } - +impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn load_typeshed_versions<'db>( db: &'db dyn Db, - stdlib_root: &StandardLibraryPath, + stdlib_root: &FileSystemPath, ) -> &'db TypeshedVersions { - let StandardLibraryPath(stdlib_fs_path) = stdlib_root; - let versions_path = stdlib_fs_path.join("VERSIONS"); + let versions_path = stdlib_root.join("VERSIONS"); let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { todo!( "Still need to figure out how to handle VERSIONS files being deleted \ @@ -308,16 +178,14 @@ impl<'a> ModuleResolutionPathRef<'a> { parse_typeshed_versions(db, versions_file).as_ref().unwrap() } - // Private helper function with concrete inputs, - // to avoid monomorphization #[must_use] - fn is_directory_impl(&self, db: &dyn Db, search_path: Self) -> bool { + fn is_directory(&self, db: &dyn Db, search_path: Self) -> bool { match (self, search_path) { - (Self::Extra(ExtraPath(path)), Self::Extra(_)) => db.file_system().is_directory(path), - (Self::FirstParty(FirstPartyPath(path)), Self::FirstParty(_)) => db.file_system().is_directory(path), - (Self::SitePackages(SitePackagesPath(path)), Self::SitePackages(_)) => db.file_system().is_directory(path), - (Self::StandardLibrary(StandardLibraryPath(path)), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = self.as_module_name() else { + (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), + (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), + (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), + (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { + let Some(module_name) = ModuleResolutionPathRef(*self).to_module_name() else { return false; }; let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); @@ -336,34 +204,27 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.is_directory_impl(db, search_path.into()) - } - - // Private helper function with concrete inputs, - // to avoid monomorphization - #[must_use] - fn is_regular_package_impl(&self, db: &dyn Db, search_path: Self) -> bool { + fn is_regular_package(&self, db: &dyn Db, search_path: Self) -> bool { match (self, search_path) { - (Self::Extra(ExtraPath(fs_path)), Self::Extra(_)) - | (Self::FirstParty(FirstPartyPath(fs_path)), Self::FirstParty(_)) - | (Self::SitePackages(SitePackagesPath(fs_path)), Self::SitePackages(_)) => { + (Self::Extra(path), Self::Extra(_)) + | (Self::FirstParty(path), Self::FirstParty(_)) + | (Self::SitePackages(path), Self::SitePackages(_)) => { let file_system = db.file_system(); - file_system.exists(&fs_path.join("__init__.py")) - || file_system.exists(&fs_path.join("__init__.pyi")) + file_system.exists(&path.join("__init__.py")) + || file_system.exists(&path.join("__init__.pyi")) } // Unlike the other variants: // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` - (Self::StandardLibrary(StandardLibraryPath(fs_path)), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = self.as_module_name() else { + (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { + let Some(module_name) = ModuleResolutionPathRef(*self).to_module_name() else { return false; }; let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); match typeshed_versions.query_module(&module_name, get_target_py_version(db)) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { - db.file_system().exists(&fs_path.join("__init__.pyi")) + db.file_system().exists(&path.join("__init__.pyi")) } TypeshedVersionsQueryResult::DoesNotExist => false, } @@ -375,33 +236,22 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.is_regular_package_impl(db, search_path.into()) - } - - #[must_use] - pub(crate) fn parent(&self) -> Option { - Some(match self { - Self::Extra(ExtraPath(path)) => Self::extra_unchecked(path.parent()?), - Self::FirstParty(FirstPartyPath(path)) => Self::first_party_unchecked(path.parent()?), - Self::StandardLibrary(StandardLibraryPath(path)) => { - Self::standard_library_unchecked(path.parent()?) - } - Self::SitePackages(SitePackagesPath(path)) => { - Self::site_packages_unchecked(path.parent()?) - } - }) + fn parent(&self) -> Option { + match self { + Self::Extra(path) => path.parent().map(Self::Extra), + Self::FirstParty(path) => path.parent().map(Self::FirstParty), + Self::StandardLibrary(path) => path.parent().map(Self::StandardLibrary), + Self::SitePackages(path) => path.parent().map(Self::SitePackages), + } } #[must_use] fn ends_with_dunder_init(&self) -> bool { match self { - Self::Extra(ExtraPath(path)) - | Self::FirstParty(FirstPartyPath(path)) - | Self::SitePackages(SitePackagesPath(path)) => { + Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => { path.ends_with("__init__.py") || path.ends_with("__init__.pyi") } - Self::StandardLibrary(StandardLibraryPath(path)) => path.ends_with("__init__.pyi"), + Self::StandardLibrary(path) => path.ends_with("__init__.pyi"), } } @@ -409,10 +259,10 @@ impl<'a> ModuleResolutionPathRef<'a> { fn with_dunder_init_stripped(self) -> Self { if self.ends_with_dunder_init() { self.parent().unwrap_or_else(|| match self { - Self::Extra(_) => Self::extra_unchecked(""), - Self::FirstParty(_) => Self::first_party_unchecked(""), - Self::StandardLibrary(_) => Self::standard_library_unchecked(""), - Self::SitePackages(_) => Self::site_packages_unchecked(""), + Self::Extra(_) => Self::Extra(FileSystemPath::new("")), + Self::FirstParty(_) => Self::FirstParty(FileSystemPath::new("")), + Self::StandardLibrary(_) => Self::StandardLibrary(FileSystemPath::new("")), + Self::SitePackages(_) => Self::SitePackages(FileSystemPath::new("")), }) } else { self @@ -420,156 +270,210 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn as_module_name(&self) -> Option { - ModuleName::from_components(match self.with_dunder_init_stripped() { - Self::Extra(ExtraPath(path)) => ModulePartIterator::from_fs_path(path), - Self::FirstParty(FirstPartyPath(path)) => ModulePartIterator::from_fs_path(path), - Self::StandardLibrary(StandardLibraryPath(path)) => { - ModulePartIterator::from_fs_path(path) - } - Self::SitePackages(SitePackagesPath(path)) => ModulePartIterator::from_fs_path(path), - }) + #[inline] + fn as_file_system_path(self) -> &'a FileSystemPath { + match self { + Self::Extra(path) => path, + Self::FirstParty(path) => path, + Self::StandardLibrary(path) => path, + Self::SitePackages(path) => path, + } } #[must_use] - pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPathBuf { + fn with_pyi_extension(&self) -> ModuleResolutionPathBufInner { match self { - Self::Extra(ExtraPath(path)) => { - ModuleResolutionPathBuf::Extra(ExtraPathBuf(path.with_extension("pyi"))) + Self::Extra(path) => ModuleResolutionPathBufInner::Extra(path.with_extension("pyi")), + Self::FirstParty(path) => { + ModuleResolutionPathBufInner::FirstParty(path.with_extension("pyi")) } - Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path.with_extension("pyi"))) + Self::StandardLibrary(path) => { + ModuleResolutionPathBufInner::StandardLibrary(path.with_extension("pyi")) } - Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf( - path.with_extension("pyi"), - )) + Self::SitePackages(path) => { + ModuleResolutionPathBufInner::SitePackages(path.with_extension("pyi")) } - Self::SitePackages(SitePackagesPath(path)) => ModuleResolutionPathBuf::SitePackages( - SitePackagesPathBuf(path.with_extension("pyi")), - ), } } #[must_use] - pub(crate) fn with_py_extension(&self) -> Option { + fn with_py_extension(&self) -> Option { match self { - Self::Extra(ExtraPath(path)) => Some(ModuleResolutionPathBuf::Extra(ExtraPathBuf( + Self::Extra(path) => Some(ModuleResolutionPathBufInner::Extra( + path.with_extension("py"), + )), + Self::FirstParty(path) => Some(ModuleResolutionPathBufInner::FirstParty( path.with_extension("py"), - ))), - Self::FirstParty(FirstPartyPath(path)) => Some(ModuleResolutionPathBuf::FirstParty( - FirstPartyPathBuf(path.with_extension("py")), )), Self::StandardLibrary(_) => None, - Self::SitePackages(SitePackagesPath(path)) => { - Some(ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf( - path.with_extension("py"), - ))) - } + Self::SitePackages(path) => Some(ModuleResolutionPathBufInner::SitePackages( + path.with_extension("py"), + )), } } #[cfg(test)] #[must_use] - pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { + fn to_path_buf(self) -> ModuleResolutionPathBufInner { match self { - Self::Extra(ExtraPath(path)) => { - ModuleResolutionPathBuf::Extra(ExtraPathBuf(path.to_path_buf())) + Self::Extra(path) => ModuleResolutionPathBufInner::Extra(path.to_path_buf()), + Self::FirstParty(path) => ModuleResolutionPathBufInner::FirstParty(path.to_path_buf()), + Self::StandardLibrary(path) => { + ModuleResolutionPathBufInner::StandardLibrary(path.to_path_buf()) } - Self::FirstParty(FirstPartyPath(path)) => { - ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path.to_path_buf())) - } - Self::StandardLibrary(StandardLibraryPath(path)) => { - ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path.to_path_buf())) - } - Self::SitePackages(SitePackagesPath(path)) => { - ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path.to_path_buf())) + Self::SitePackages(path) => { + ModuleResolutionPathBufInner::SitePackages(path.to_path_buf()) } } } #[cfg(test)] #[must_use] - pub(crate) fn join( + fn join( &self, component: &'a (impl AsRef + ?Sized), - ) -> ModuleResolutionPathBuf { + ) -> ModuleResolutionPathBufInner { let mut result = self.to_path_buf(); result.push(component.as_ref().as_str()); result } +} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ModuleResolutionPathRef<'a>(ModuleResolutionPathRefInner<'a>); + +impl<'a> ModuleResolutionPathRef<'a> { #[must_use] - #[inline] - fn as_file_system_path(self) -> &'a FileSystemPath { - match self { - Self::Extra(ExtraPath(path)) => path, - Self::FirstParty(FirstPartyPath(path)) => path, - Self::StandardLibrary(StandardLibraryPath(path)) => path, - Self::SitePackages(SitePackagesPath(path)) => path, - } + pub(crate) fn extra(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self(ModuleResolutionPathRefInner::Extra(path))) + } + + #[must_use] + pub(crate) fn first_party(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self(ModuleResolutionPathRefInner::FirstParty(path))) + } + + #[must_use] + pub(crate) fn standard_library( + path: &'a (impl AsRef + ?Sized), + ) -> Option { + let path = path.as_ref(); + // Unlike other variants, only `.pyi` extensions are permitted + path.extension() + .map_or(true, |ext| ext == "pyi") + .then_some(Self(ModuleResolutionPathRefInner::StandardLibrary(path))) + } + + #[must_use] + pub(crate) fn site_packages(path: &'a (impl AsRef + ?Sized)) -> Option { + let path = path.as_ref(); + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self(ModuleResolutionPathRefInner::SitePackages(path))) + } + + #[must_use] + pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { + self.0.is_directory(db, search_path.into().0) + } + + #[must_use] + pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { + self.0.is_regular_package(db, search_path.into().0) + } + + #[must_use] + pub(crate) fn to_module_name(self) -> Option { + ModuleName::from_components(ModulePartIterator::from_fs_path( + self.0.with_dunder_init_stripped().as_file_system_path(), + )) + } + + #[must_use] + pub(crate) fn with_pyi_extension(&self) -> ModuleResolutionPathBuf { + ModuleResolutionPathBuf(self.0.with_pyi_extension()) + } + + #[must_use] + pub(crate) fn with_py_extension(self) -> Option { + self.0.with_py_extension().map(ModuleResolutionPathBuf) } #[must_use] pub(crate) fn relativize_path(&self, absolute_path: &'a FileSystemPath) -> Option { - match self { - Self::Extra(ExtraPath(root)) => { + match self.0 { + ModuleResolutionPathRefInner::Extra(root) => { absolute_path.strip_prefix(root).ok().and_then(Self::extra) } - Self::FirstParty(FirstPartyPath(root)) => absolute_path + ModuleResolutionPathRefInner::FirstParty(root) => absolute_path .strip_prefix(root) .ok() .and_then(Self::first_party), - Self::StandardLibrary(StandardLibraryPath(root)) => absolute_path + ModuleResolutionPathRefInner::StandardLibrary(root) => absolute_path .strip_prefix(root) .ok() .and_then(Self::standard_library), - Self::SitePackages(SitePackagesPath(root)) => absolute_path + ModuleResolutionPathRefInner::SitePackages(root) => absolute_path .strip_prefix(root) .ok() .and_then(Self::site_packages), } } + + #[cfg(test)] + pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { + ModuleResolutionPathBuf(self.0.to_path_buf()) + } } -impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { +impl<'a> From<&'a ModuleResolutionPathBufInner> for ModuleResolutionPathRefInner<'a> { #[inline] - fn from(value: &'a ModuleResolutionPathBuf) -> Self { + fn from(value: &'a ModuleResolutionPathBufInner) -> Self { match value { - ModuleResolutionPathBuf::Extra(ExtraPathBuf(path)) => { - ModuleResolutionPathRef::extra_unchecked(path) - } - ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(path)) => { - ModuleResolutionPathRef::first_party_unchecked(path) + ModuleResolutionPathBufInner::Extra(path) => ModuleResolutionPathRefInner::Extra(path), + ModuleResolutionPathBufInner::FirstParty(path) => { + ModuleResolutionPathRefInner::FirstParty(path) } - ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(path)) => { - ModuleResolutionPathRef::standard_library_unchecked(path) + ModuleResolutionPathBufInner::StandardLibrary(path) => { + ModuleResolutionPathRefInner::StandardLibrary(path) } - ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(path)) => { - ModuleResolutionPathRef::site_packages_unchecked(path) + ModuleResolutionPathBufInner::SitePackages(path) => { + ModuleResolutionPathRefInner::SitePackages(path) } } } } +impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { + fn from(value: &'a ModuleResolutionPathBuf) -> Self { + ModuleResolutionPathRef(ModuleResolutionPathRefInner::from(&value.0)) + } +} + impl<'a> PartialEq for ModuleResolutionPathRef<'a> { fn eq(&self, other: &ModuleResolutionPathBuf) -> bool { - match (self, other) { + match (self.0, &other.0) { ( - ModuleResolutionPathRef::Extra(ExtraPath(self_path)), - ModuleResolutionPathBuf::Extra(ExtraPathBuf(other_path)), + ModuleResolutionPathRefInner::Extra(self_path), + ModuleResolutionPathBufInner::Extra(other_path), ) | ( - ModuleResolutionPathRef::FirstParty(FirstPartyPath(self_path)), - ModuleResolutionPathBuf::FirstParty(FirstPartyPathBuf(other_path)), + ModuleResolutionPathRefInner::FirstParty(self_path), + ModuleResolutionPathBufInner::FirstParty(other_path), ) | ( - ModuleResolutionPathRef::StandardLibrary(StandardLibraryPath(self_path)), - ModuleResolutionPathBuf::StandardLibrary(StandardLibraryPathBuf(other_path)), + ModuleResolutionPathRefInner::StandardLibrary(self_path), + ModuleResolutionPathBufInner::StandardLibrary(other_path), ) | ( - ModuleResolutionPathRef::SitePackages(SitePackagesPath(self_path)), - ModuleResolutionPathBuf::SitePackages(SitePackagesPathBuf(other_path)), + ModuleResolutionPathRefInner::SitePackages(self_path), + ModuleResolutionPathBufInner::SitePackages(other_path), ) => *self_path == **other_path, _ => false, } @@ -578,13 +482,13 @@ impl<'a> PartialEq for ModuleResolutionPathRef<'a> { impl<'a> PartialEq for ModuleResolutionPathRef<'a> { fn eq(&self, other: &FileSystemPath) -> bool { - self.as_file_system_path() == other + self.0.as_file_system_path() == other } } impl<'a> PartialEq> for FileSystemPath { fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { - self == other.as_file_system_path() + self == other.0.as_file_system_path() } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 4c5ffd17ca042..d103860d68e33 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -89,7 +89,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { .iter() .find_map(|root| root.relativize_path(path))?; - let module_name = relative_path.as_module_name()?; + let module_name = relative_path.to_module_name()?; // Resolve the module name to see if Python would resolve the name to the same path. // If it doesn't, then that means that multiple modules have the same name in different From 9ba9ac1d46dfde0a3ce974f6683c891159e9dc2c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 15:12:11 +0100 Subject: [PATCH 19/58] Bye bye `DoubleEndedIterator` --- crates/red_knot_module_resolver/src/path.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index c5e84b0964797..a8a82780972d5 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -525,27 +525,6 @@ impl<'a> Iterator for ModulePartIterator<'a> { stem.take() } } - - fn last(mut self) -> Option { - self.next_back() - } -} - -impl<'a> DoubleEndedIterator for ModulePartIterator<'a> { - fn next_back(&mut self) -> Option { - let ModulePartIterator { - parent_components, - stem, - } = self; - - if let Some(part) = stem.take() { - Some(part) - } else if let Some(components) = parent_components { - components.next_back().map(|component| component.as_str()) - } else { - None - } - } } impl<'a> FusedIterator for ModulePartIterator<'a> {} From 9c556bad58e6fc11aa0e824be2ab1c2794d2c770 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 15:53:53 +0100 Subject: [PATCH 20/58] Add tests for some helpers --- .../src/module_name.rs | 53 ++++++++++++++----- crates/red_knot_module_resolver/src/path.rs | 21 ++++++++ .../red_knot_module_resolver/src/resolver.rs | 4 +- .../src/typeshed/versions.rs | 25 +++++++++ 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/crates/red_knot_module_resolver/src/module_name.rs b/crates/red_knot_module_resolver/src/module_name.rs index 9a9ab79b57903..5b901b6d6f29f 100644 --- a/crates/red_knot_module_resolver/src/module_name.rs +++ b/crates/red_knot_module_resolver/src/module_name.rs @@ -1,7 +1,7 @@ use std::fmt; use std::ops::Deref; -use compact_str::ToCompactString; +use compact_str::{CompactString, ToCompactString}; use ruff_python_stdlib::identifiers::is_identifier; @@ -25,7 +25,7 @@ impl ModuleName { #[inline] #[must_use] pub fn new(name: &str) -> Option { - Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) + Self::is_valid_name(name).then(|| Self(CompactString::from(name))) } /// Creates a new module name for `name` where `name` is a static string. @@ -56,7 +56,7 @@ impl ModuleName { #[must_use] pub fn new_static(name: &'static str) -> Option { // TODO(Micha): Use CompactString::const_new once we upgrade to 0.8 https://github.com/ParkMyCar/compact_str/pull/336 - Self::is_valid_name(name).then(|| Self(compact_str::CompactString::from(name))) + Self::is_valid_name(name).then(|| Self(CompactString::from(name))) } #[must_use] @@ -129,19 +129,46 @@ impl ModuleName { &self.0 } + /// Construct a [`ModuleName`] from a sequence of parts. + /// + /// # Examples + /// + /// ``` + /// use red_knot_module_resolver::ModuleName; + /// + /// assert_eq!(&*ModuleName::from_components(["a"]).unwrap(), "a"); + /// assert_eq!(&*ModuleName::from_components(["a", "b"]).unwrap(), "a.b"); + /// assert_eq!(&*ModuleName::from_components(["a", "b", "c"]).unwrap(), "a.b.c"); + /// + /// assert_eq!(ModuleName::from_components(["a-b"]), None); + /// assert_eq!(ModuleName::from_components(["a", "a-b"]), None); + /// assert_eq!(ModuleName::from_components(["a", "b", "a-b-c"]), None); + /// ``` #[must_use] - pub fn from_components<'a>(mut components: impl Iterator) -> Option { + pub fn from_components<'a>(components: impl IntoIterator) -> Option { + let mut components = components.into_iter(); let first_part = components.next()?; - if let Some(second_part) = components.next() { - let mut name = format!("{first_part}.{second_part}"); - for part in components { - name.push('.'); - name.push_str(part); - } - ModuleName::new(&name) - } else { - ModuleName::new(first_part) + if !is_identifier(first_part) { + return None; } + Some(Self({ + if let Some(second_part) = components.next() { + if !is_identifier(second_part) { + return None; + } + let mut name = format!("{first_part}.{second_part}"); + for part in components { + if !is_identifier(part) { + return None; + } + name.push('.'); + name.push_str(part); + } + CompactString::from(&name) + } else { + CompactString::from(first_part) + } + })) } } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index a8a82780972d5..91e0bd4c8709c 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -492,6 +492,8 @@ impl<'a> PartialEq> for FileSystemPath { } } +/// Iterate over the "module components" of a path +/// (stripping the extension, if there is one.) pub(crate) struct ModulePartIterator<'a> { parent_components: Option>, stem: Option<&'a str>, @@ -528,3 +530,22 @@ impl<'a> Iterator for ModulePartIterator<'a> { } impl<'a> FusedIterator for ModulePartIterator<'a> {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn module_part_iterator() { + fn create_module_parts(path: &str) -> Vec<&str> { + ModulePartIterator::from_fs_path(FileSystemPath::new(path)).collect() + } + + assert_eq!(&create_module_parts("foo.pyi"), &["foo"]); + assert_eq!(&create_module_parts("foo/bar.pyi"), &["foo", "bar"]); + assert_eq!(&create_module_parts("foo/bar/baz.py"), &["foo", "bar", "baz"]); + assert_eq!(&create_module_parts("foo"), &["foo"]); + assert_eq!(&create_module_parts("foo/bar"), &["foo", "bar"]); + assert_eq!(&create_module_parts("foo/bar/baz"), &["foo", "bar", "baz"]); + } +} diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index d103860d68e33..11fc637c52dcf 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -8,8 +8,8 @@ use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; use crate::path::ModuleResolutionPathBuf; use crate::resolver::internal::ModuleResolverSearchPaths; -use crate::supported_py_version::set_target_py_version; -use crate::{Db, SupportedPyVersion}; +use crate::supported_py_version::{SupportedPyVersion, set_target_py_version}; +use crate::db::Db; /// Configures the module resolver settings. /// diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index aa5927f68a8ac..b89ce6e62bed1 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -555,6 +555,31 @@ foo: 3.8- # trailing comment ); } + #[test] + fn invalid_empty_versions_file() { + assert_eq!( + TypeshedVersions::from_str(""), + Err(TypeshedVersionsParseError { + line_number: None, + reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile + }) + ); + assert_eq!( + TypeshedVersions::from_str(" "), + Err(TypeshedVersionsParseError { + line_number: None, + reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile + }) + ); + assert_eq!( + TypeshedVersions::from_str(" \n \n \n "), + Err(TypeshedVersionsParseError { + line_number: None, + reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile + }) + ); + } + #[test] fn invalid_huge_versions_file() { let offset = 100; From 9c672f89dd9fef14bc58c3c7e166e5d5623e8653 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 16:06:47 +0100 Subject: [PATCH 21/58] Simplify `ModulePartIterator` --- crates/red_knot_module_resolver/src/path.rs | 23 +++++++++---------- .../red_knot_module_resolver/src/resolver.rs | 4 ++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 91e0bd4c8709c..b9292fb99b533 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -495,15 +495,17 @@ impl<'a> PartialEq> for FileSystemPath { /// Iterate over the "module components" of a path /// (stripping the extension, if there is one.) pub(crate) struct ModulePartIterator<'a> { - parent_components: Option>, + parent_components: camino::Utf8Components<'a>, stem: Option<&'a str>, } impl<'a> ModulePartIterator<'a> { #[must_use] fn from_fs_path(path: &'a FileSystemPath) -> Self { + let mut parent_components = path.components(); + parent_components.next_back(); Self { - parent_components: path.parent().map(|path| path.components()), + parent_components, stem: path.file_stem(), } } @@ -517,15 +519,9 @@ impl<'a> Iterator for ModulePartIterator<'a> { parent_components, stem, } = self; - - if let Some(ref mut components) = parent_components { - components - .next() - .map(|component| component.as_str()) - .or_else(|| stem.take()) - } else { - stem.take() - } + parent_components + .next() + .map_or_else(|| stem.take(), |component| Some(component.as_str())) } } @@ -543,7 +539,10 @@ mod tests { assert_eq!(&create_module_parts("foo.pyi"), &["foo"]); assert_eq!(&create_module_parts("foo/bar.pyi"), &["foo", "bar"]); - assert_eq!(&create_module_parts("foo/bar/baz.py"), &["foo", "bar", "baz"]); + assert_eq!( + &create_module_parts("foo/bar/baz.py"), + &["foo", "bar", "baz"] + ); assert_eq!(&create_module_parts("foo"), &["foo"]); assert_eq!(&create_module_parts("foo/bar"), &["foo", "bar"]); assert_eq!(&create_module_parts("foo/bar/baz"), &["foo", "bar", "baz"]); diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 11fc637c52dcf..4b696d589ac96 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -4,12 +4,12 @@ use std::sync::Arc; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; +use crate::db::Db; use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; use crate::path::ModuleResolutionPathBuf; use crate::resolver::internal::ModuleResolverSearchPaths; -use crate::supported_py_version::{SupportedPyVersion, set_target_py_version}; -use crate::db::Db; +use crate::supported_py_version::{set_target_py_version, SupportedPyVersion}; /// Configures the module resolver settings. /// From ad3bd7a7ddbad9325d491ee1588383c79bd89c9d Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 19:53:50 +0100 Subject: [PATCH 22/58] more tests --- crates/red_knot_module_resolver/src/path.rs | 434 ++++++++++++++++---- 1 file changed, 348 insertions(+), 86 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index b9292fb99b533..a37c9ff07d998 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,4 +1,4 @@ -use std::iter::FusedIterator; +use std::fmt; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::{system_path_to_file, VfsPath}; @@ -25,18 +25,23 @@ enum ModuleResolutionPathBufInner { impl ModuleResolutionPathBufInner { fn push(&mut self, component: &str) { - debug_assert!(matches!(component.matches('.').count(), 0 | 1)); - if cfg!(debug) { + if cfg!(debug_assertions) { if let Some(extension) = camino::Utf8Path::new(component).extension() { match self { Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( matches!(extension, "pyi" | "py"), - "Extension must be `py` or `pyi`; got {extension:?}" - ), - Self::StandardLibrary(_) => assert_eq!( - extension, "pyi", - "Extension must be `py` or `pyi`; got {extension:?}" + "Extension must be `py` or `pyi`; got `{extension}`" ), + Self::StandardLibrary(_) => { + assert!( + matches!(component.matches('.').count(), 0 | 1), + "Component can have at most one '.'; got {component}" + ); + assert_eq!( + extension, "pyi", + "Extension must be `pyi`; got `{extension}`" + ); + } }; } } @@ -50,7 +55,7 @@ impl ModuleResolutionPathBufInner { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash)] pub(crate) struct ModuleResolutionPathBuf(ModuleResolutionPathBufInner); impl ModuleResolutionPathBuf { @@ -65,21 +70,27 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn extra(path: FileSystemPathBuf) -> Option { + pub(crate) fn extra(path: impl Into) -> Option { + let path = path.into(); path.extension() .map_or(true, |ext| matches!(ext, "py" | "pyi")) .then_some(Self(ModuleResolutionPathBufInner::Extra(path))) } #[must_use] - pub(crate) fn first_party(path: FileSystemPathBuf) -> Option { + pub(crate) fn first_party(path: impl Into) -> Option { + let path = path.into(); path.extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) .then_some(Self(ModuleResolutionPathBufInner::FirstParty(path))) } #[must_use] - pub(crate) fn standard_library(path: FileSystemPathBuf) -> Option { + pub(crate) fn standard_library(path: impl Into) -> Option { + let path = path.into(); + if path.file_stem().is_some_and(|stem| stem.contains('.')) { + return None; + } path.extension() .map_or(true, |ext| ext == "pyi") .then_some(Self(ModuleResolutionPathBufInner::StandardLibrary(path))) @@ -91,7 +102,8 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn site_packages(path: FileSystemPathBuf) -> Option { + pub(crate) fn site_packages(path: impl Into) -> Option { + let path = path.into(); path.extension() .map_or(true, |ext| matches!(ext, "pyi" | "py")) .then_some(Self(ModuleResolutionPathBufInner::SitePackages(path))) @@ -150,6 +162,20 @@ impl AsRef for ModuleResolutionPathBuf { } } +impl fmt::Debug for ModuleResolutionPathBuf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (name, path) = match &self.0 { + ModuleResolutionPathBufInner::Extra(path) => ("Extra", path), + ModuleResolutionPathBufInner::FirstParty(path) => ("FirstParty", path), + ModuleResolutionPathBufInner::SitePackages(path) => ("SitePackages", path), + ModuleResolutionPathBufInner::StandardLibrary(path) => ("StandardLibary", path), + }; + f.debug_tuple(&format!("ModuleResolutionPath::{name}")) + .field(path) + .finish() + } +} + #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] enum ModuleResolutionPathRefInner<'a> { Extra(&'a FileSystemPath), @@ -236,36 +262,24 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn parent(&self) -> Option { - match self { - Self::Extra(path) => path.parent().map(Self::Extra), - Self::FirstParty(path) => path.parent().map(Self::FirstParty), - Self::StandardLibrary(path) => path.parent().map(Self::StandardLibrary), - Self::SitePackages(path) => path.parent().map(Self::SitePackages), - } - } + pub(crate) fn to_module_name(self) -> Option { + let (fs_path, skip_final_part) = match self { + Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => ( + path, + path.ends_with("__init__.py") || path.ends_with("__init__.pyi"), + ), + Self::StandardLibrary(path) => (path, path.ends_with("__init__.pyi")), + }; - #[must_use] - fn ends_with_dunder_init(&self) -> bool { - match self { - Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => { - path.ends_with("__init__.py") || path.ends_with("__init__.pyi") - } - Self::StandardLibrary(path) => path.ends_with("__init__.pyi"), - } - } + let parent_components = fs_path + .parent()? + .components() + .map(|component| component.as_str()); - #[must_use] - fn with_dunder_init_stripped(self) -> Self { - if self.ends_with_dunder_init() { - self.parent().unwrap_or_else(|| match self { - Self::Extra(_) => Self::Extra(FileSystemPath::new("")), - Self::FirstParty(_) => Self::FirstParty(FileSystemPath::new("")), - Self::StandardLibrary(_) => Self::StandardLibrary(FileSystemPath::new("")), - Self::SitePackages(_) => Self::SitePackages(FileSystemPath::new("")), - }) + if skip_final_part { + ModuleName::from_components(parent_components) } else { - self + ModuleName::from_components(parent_components.chain(fs_path.file_stem())) } } @@ -390,9 +404,7 @@ impl<'a> ModuleResolutionPathRef<'a> { #[must_use] pub(crate) fn to_module_name(self) -> Option { - ModuleName::from_components(ModulePartIterator::from_fs_path( - self.0.with_dunder_init_stripped().as_file_system_path(), - )) + self.0.to_module_name() } #[must_use] @@ -492,59 +504,309 @@ impl<'a> PartialEq> for FileSystemPath { } } -/// Iterate over the "module components" of a path -/// (stripping the extension, if there is one.) -pub(crate) struct ModulePartIterator<'a> { - parent_components: camino::Utf8Components<'a>, - stem: Option<&'a str>, -} +#[cfg(test)] +mod tests { + use super::*; -impl<'a> ModulePartIterator<'a> { - #[must_use] - fn from_fs_path(path: &'a FileSystemPath) -> Self { - let mut parent_components = path.components(); - parent_components.next_back(); - Self { - parent_components, - stem: path.file_stem(), - } + use insta::assert_debug_snapshot; + + #[test] + fn constructor_rejects_non_pyi_stdlib_paths() { + assert!(ModuleResolutionPathBuf::standard_library("foo.py").is_none()); + assert!(ModuleResolutionPathBuf::standard_library("foo/__init__.py").is_none()); + assert!(ModuleResolutionPathBuf::standard_library("foo.py.pyi").is_none()); } -} -impl<'a> Iterator for ModulePartIterator<'a> { - type Item = &'a str; + fn stdlib_path_test_case(path: &str) -> ModuleResolutionPathBuf { + ModuleResolutionPathBuf::standard_library(path).unwrap() + } - fn next(&mut self) -> Option { - let ModulePartIterator { - parent_components, - stem, - } = self; - parent_components - .next() - .map_or_else(|| stem.take(), |component| Some(component.as_str())) + #[test] + fn stdlib_path_no_extension() { + assert_debug_snapshot!(stdlib_path_test_case("foo"), @r###" + ModuleResolutionPath::StandardLibary( + "foo", + ) + "###); } -} -impl<'a> FusedIterator for ModulePartIterator<'a> {} + #[test] + fn stdlib_path_pyi_extension() { + assert_debug_snapshot!(stdlib_path_test_case("foo.pyi"), @r###" + ModuleResolutionPath::StandardLibary( + "foo.pyi", + ) + "###); + } -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn stdlib_path_dunder_init() { + assert_debug_snapshot!(stdlib_path_test_case("foo/__init__.pyi"), @r###" + ModuleResolutionPath::StandardLibary( + "foo/__init__.pyi", + ) + "###); + } #[test] - fn module_part_iterator() { - fn create_module_parts(path: &str) -> Vec<&str> { - ModulePartIterator::from_fs_path(FileSystemPath::new(path)).collect() - } + fn stdlib_paths_can_only_be_pyi() { + assert!(stdlib_path_test_case("foo").with_py_extension().is_none()); + } + + #[test] + fn stdlib_path_with_pyi_extension() { + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo").unwrap().with_pyi_extension(), + @r###" + ModuleResolutionPath::StandardLibary( + "foo.pyi", + ) + "### + ); + } + + #[test] + fn non_stdlib_path_with_py_extension() { + assert_debug_snapshot!( + ModuleResolutionPathBuf::first_party("foo").unwrap().with_py_extension().unwrap(), + @r###" + ModuleResolutionPath::FirstParty( + "foo.py", + ) + "### + ); + } - assert_eq!(&create_module_parts("foo.pyi"), &["foo"]); - assert_eq!(&create_module_parts("foo/bar.pyi"), &["foo", "bar"]); - assert_eq!( - &create_module_parts("foo/bar/baz.py"), - &["foo", "bar", "baz"] + #[test] + fn non_stdlib_path_with_pyi_extension() { + assert_debug_snapshot!( + ModuleResolutionPathBuf::first_party("foo").unwrap().with_pyi_extension(), + @r###" + ModuleResolutionPath::FirstParty( + "foo.pyi", + ) + "### ); - assert_eq!(&create_module_parts("foo"), &["foo"]); - assert_eq!(&create_module_parts("foo/bar"), &["foo", "bar"]); - assert_eq!(&create_module_parts("foo/bar/baz"), &["foo", "bar", "baz"]); + } + + fn non_stdlib_module_name_test_case(path: &str) -> ModuleName { + let variants = [ + ModuleResolutionPathRef::extra, + ModuleResolutionPathRef::first_party, + ModuleResolutionPathRef::site_packages, + ]; + let results: Vec> = variants + .into_iter() + .map(|variant| variant(path).unwrap().to_module_name()) + .collect(); + assert!(results + .iter() + .zip(results.iter().take(1)) + .all(|(this, next)| this == next)); + results.into_iter().next().unwrap().unwrap() + } + + #[test] + fn module_name_1_part_no_extension() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo"), @r###" + ModuleName( + "foo", + ) + "###); + } + + #[test] + fn module_name_one_part_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo.pyi"), @r###" + ModuleName( + "foo", + ) + "###); + } + + #[test] + fn module_name_one_part_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo.py"), @r###" + ModuleName( + "foo", + ) + "###); + } + + #[test] + fn module_name_2_parts_dunder_init_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/__init__.py"), @r###" + ModuleName( + "foo", + ) + "###); + } + + #[test] + fn module_name_2_parts_dunder_init_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/__init__.pyi"), @r###" + ModuleName( + "foo", + ) + "###); + } + + #[test] + fn module_name_2_parts_no_extension() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar"), @r###" + ModuleName( + "foo.bar", + ) + "###); + } + + #[test] + fn module_name_2_parts_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar.pyi"), @r###" + ModuleName( + "foo.bar", + ) + "###); + } + + #[test] + fn module_name_2_parts_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar.py"), @r###" + ModuleName( + "foo.bar", + ) + "###); + } + + #[test] + fn module_name_3_parts_dunder_init_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/__init__.pyi"), @r###" + ModuleName( + "foo.bar", + ) + "###); + } + + #[test] + fn module_name_3_parts_dunder_init_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/__init__.py"), @r###" + ModuleName( + "foo.bar", + ) + "###); + } + + #[test] + fn module_name_3_parts_no_extension() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz"), @r###" + ModuleName( + "foo.bar.baz", + ) + "###); + } + + #[test] + fn module_name_3_parts_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz.pyi"), @r###" + ModuleName( + "foo.bar.baz", + ) + "###); + } + + #[test] + fn module_name_3_parts_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz.py"), @r###" + ModuleName( + "foo.bar.baz", + ) + "###); + } + + #[test] + fn module_name_4_parts_dunder_init_pyi() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz/__init__.pyi"), @r###" + ModuleName( + "foo.bar.baz", + ) + "###); + } + + #[test] + fn module_name_4_parts_dunder_init_py() { + assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz/__init__.py"), @r###" + ModuleName( + "foo.bar.baz", + ) + "###); + } + + #[test] + fn join_1() { + assert_debug_snapshot!(ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), @r###" + ModuleResolutionPath::StandardLibary( + "foo/bar", + ) + "###); + } + + #[test] + fn join_2() { + assert_debug_snapshot!(ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), @r###" + ModuleResolutionPath::SitePackages( + "foo/bar.pyi", + ) + "###); + } + + #[test] + fn join_3() { + assert_debug_snapshot!(ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), @r###" + ModuleResolutionPath::Extra( + "foo/bar.py", + ) + "###); + } + + #[test] + fn join_4() { + assert_debug_snapshot!( + ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), + @r###" + ModuleResolutionPath::FirstParty( + "foo/bar/baz/eggs/__init__.py", + ) + "### + ); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "Extension must be `pyi`; got `py`")] + fn stdlib_path_invalid_join_py() { + stdlib_path_test_case("foo").push("bar.py"); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "Extension must be `pyi`; got `rs`")] + fn stdlib_path_invalid_join_rs() { + stdlib_path_test_case("foo").push("bar.rs"); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "Extension must be `py` or `pyi`; got `rs`")] + fn non_stdlib_path_invalid_join_rs() { + ModuleResolutionPathBuf::site_packages("foo") + .unwrap() + .push("bar.rs"); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic(expected = "Component can have at most one '.'")] + fn invalid_stdlib_join_too_many_extensions() { + stdlib_path_test_case("foo").push("bar.py.pyi"); } } From 21e60fedcd5cd4d5c73d0353c19b55dbead899dc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 22:45:36 +0100 Subject: [PATCH 23/58] fix Windows? --- crates/red_knot_module_resolver/Cargo.toml | 2 +- crates/red_knot_module_resolver/src/path.rs | 85 ++++++++++++++------- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index 99e69f35cc27f..18efb5d7c5682 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -28,7 +28,7 @@ zip = { workspace = true } [dev-dependencies] anyhow = { workspace = true } -insta = { workspace = true } +insta = { workspace = true, features = ["filters"] } tempfile = { workspace = true } walkdir = { workspace = true } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index a37c9ff07d998..9211e25d1877f 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -168,7 +168,7 @@ impl fmt::Debug for ModuleResolutionPathBuf { ModuleResolutionPathBufInner::Extra(path) => ("Extra", path), ModuleResolutionPathBufInner::FirstParty(path) => ("FirstParty", path), ModuleResolutionPathBufInner::SitePackages(path) => ("SitePackages", path), - ModuleResolutionPathBufInner::StandardLibrary(path) => ("StandardLibary", path), + ModuleResolutionPathBufInner::StandardLibrary(path) => ("StandardLibrary", path), }; f.debug_tuple(&format!("ModuleResolutionPath::{name}")) .field(path) @@ -524,7 +524,7 @@ mod tests { #[test] fn stdlib_path_no_extension() { assert_debug_snapshot!(stdlib_path_test_case("foo"), @r###" - ModuleResolutionPath::StandardLibary( + ModuleResolutionPath::StandardLibrary( "foo", ) "###); @@ -533,7 +533,7 @@ mod tests { #[test] fn stdlib_path_pyi_extension() { assert_debug_snapshot!(stdlib_path_test_case("foo.pyi"), @r###" - ModuleResolutionPath::StandardLibary( + ModuleResolutionPath::StandardLibrary( "foo.pyi", ) "###); @@ -542,7 +542,7 @@ mod tests { #[test] fn stdlib_path_dunder_init() { assert_debug_snapshot!(stdlib_path_test_case("foo/__init__.pyi"), @r###" - ModuleResolutionPath::StandardLibary( + ModuleResolutionPath::StandardLibrary( "foo/__init__.pyi", ) "###); @@ -558,7 +558,7 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::standard_library("foo").unwrap().with_pyi_extension(), @r###" - ModuleResolutionPath::StandardLibary( + ModuleResolutionPath::StandardLibrary( "foo.pyi", ) "### @@ -743,41 +743,70 @@ mod tests { #[test] fn join_1() { - assert_debug_snapshot!(ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), @r###" - ModuleResolutionPath::StandardLibary( - "foo/bar", - ) - "###); + insta::with_settings!({filters => vec![ + // Replace windows paths + (r"\\", "/"), + ]}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), + @r###" + ModuleResolutionPath::StandardLibrary( + "foo/bar", + ) + "### + ); + }); } #[test] fn join_2() { - assert_debug_snapshot!(ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), @r###" - ModuleResolutionPath::SitePackages( - "foo/bar.pyi", - ) - "###); + insta::with_settings!({filters => vec![ + // Replace windows paths + (r"\\", "/"), + ]}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), + @r###" + ModuleResolutionPath::SitePackages( + "foo/bar.pyi", + ) + "### + ); + }); } #[test] fn join_3() { - assert_debug_snapshot!(ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), @r###" - ModuleResolutionPath::Extra( - "foo/bar.py", - ) - "###); + insta::with_settings!({filters => vec![ + // Replace windows paths + (r"\\", "/"), + ]}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), + @r###" + ModuleResolutionPath::Extra( + "foo/bar.py", + ) + "### + ); + }); } #[test] fn join_4() { - assert_debug_snapshot!( - ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), - @r###" - ModuleResolutionPath::FirstParty( - "foo/bar/baz/eggs/__init__.py", - ) - "### - ); + insta::with_settings!({filters => vec![ + // Replace windows paths + (r"\\", "/"), + ]}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), + @r###" + ModuleResolutionPath::FirstParty( + "foo/bar/baz/eggs/__init__.py", + ) + "### + ); + }); } #[test] From 76f5b3f752dfe95ff5022f18b43681019eed276a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 3 Jul 2024 23:14:01 +0100 Subject: [PATCH 24/58] fix Windows??? --- crates/red_knot_module_resolver/src/path.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 9211e25d1877f..8061d9a7d16e5 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -745,7 +745,7 @@ mod tests { fn join_1() { insta::with_settings!({filters => vec![ // Replace windows paths - (r"\\", "/"), + (r"\\\\", "/"), ]}, { assert_debug_snapshot!( ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), @@ -762,7 +762,7 @@ mod tests { fn join_2() { insta::with_settings!({filters => vec![ // Replace windows paths - (r"\\", "/"), + (r"\\\\", "/"), ]}, { assert_debug_snapshot!( ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), @@ -779,7 +779,7 @@ mod tests { fn join_3() { insta::with_settings!({filters => vec![ // Replace windows paths - (r"\\", "/"), + (r"\\\\", "/"), ]}, { assert_debug_snapshot!( ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), @@ -796,7 +796,7 @@ mod tests { fn join_4() { insta::with_settings!({filters => vec![ // Replace windows paths - (r"\\", "/"), + (r"\\\\", "/"), ]}, { assert_debug_snapshot!( ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), From 116f8f901c44d4d8a57ecb45608bad71d480570f Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Jul 2024 11:09:26 +0100 Subject: [PATCH 25/58] Add tests for `relativize_path` --- crates/red_knot_module_resolver/src/path.rs | 145 +++++++++++++++++--- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 8061d9a7d16e5..b23a2c67d7696 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -138,9 +138,9 @@ impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn relativize_path<'a>( &'a self, - absolute_path: &'a FileSystemPath, + absolute_path: &'a (impl AsRef + ?Sized), ) -> Option> { - ModuleResolutionPathRef::from(self).relativize_path(absolute_path) + ModuleResolutionPathRef::from(self).relativize_path(absolute_path.as_ref()) } } @@ -510,6 +510,9 @@ mod tests { use insta::assert_debug_snapshot; + // Replace windows paths + static WINDOWS_PATH_FILTER: [(&str, &str); 1] = [(r"\\\\", "/")]; + #[test] fn constructor_rejects_non_pyi_stdlib_paths() { assert!(ModuleResolutionPathBuf::standard_library("foo.py").is_none()); @@ -743,10 +746,7 @@ mod tests { #[test] fn join_1() { - insta::with_settings!({filters => vec![ - // Replace windows paths - (r"\\\\", "/"), - ]}, { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { assert_debug_snapshot!( ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), @r###" @@ -760,10 +760,7 @@ mod tests { #[test] fn join_2() { - insta::with_settings!({filters => vec![ - // Replace windows paths - (r"\\\\", "/"), - ]}, { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { assert_debug_snapshot!( ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), @r###" @@ -777,10 +774,7 @@ mod tests { #[test] fn join_3() { - insta::with_settings!({filters => vec![ - // Replace windows paths - (r"\\\\", "/"), - ]}, { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { assert_debug_snapshot!( ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), @r###" @@ -794,10 +788,7 @@ mod tests { #[test] fn join_4() { - insta::with_settings!({filters => vec![ - // Replace windows paths - (r"\\\\", "/"), - ]}, { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { assert_debug_snapshot!( ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), @r###" @@ -838,4 +829,122 @@ mod tests { fn invalid_stdlib_join_too_many_extensions() { stdlib_path_test_case("foo").push("bar.py.pyi"); } + + #[test] + fn relativize_stdlib_path_errors() { + let root = ModuleResolutionPathBuf::standard_library("foo/stdlib").unwrap(); + + // Must have a `.pyi` extension or no extension: + let bad_absolute_path = FileSystemPath::new("foo/stdlib/x.py"); + assert!(root.relativize_path(bad_absolute_path).is_none()); + let second_bad_absolute_path = FileSystemPath::new("foo/stdlib/x.rs"); + assert!(root.relativize_path(second_bad_absolute_path).is_none()); + + // Must be a path that is a child of `root`: + let third_bad_absolute_path = FileSystemPath::new("bar/stdlib/x.pyi"); + assert!(root.relativize_path(third_bad_absolute_path).is_none()); + } + + fn non_stdlib_relativize_tester( + variant: impl FnOnce(&'static str) -> Option, + ) { + let root = variant("foo").unwrap(); + // Must have a `.py` extension, a `.pyi` extension, or no extension: + let bad_absolute_path = FileSystemPath::new("foo/stdlib/x.rs"); + assert!(root.relativize_path(bad_absolute_path).is_none()); + // Must be a path that is a child of `root`: + let second_bad_absolute_path = FileSystemPath::new("bar/stdlib/x.pyi"); + assert!(root.relativize_path(second_bad_absolute_path).is_none()); + } + + #[test] + fn relativize_site_packages_errors() { + non_stdlib_relativize_tester(ModuleResolutionPathBuf::site_packages); + } + + #[test] + fn relativize_extra_errors() { + non_stdlib_relativize_tester(ModuleResolutionPathBuf::extra); + } + + #[test] + fn relativize_first_party_errors() { + non_stdlib_relativize_tester(ModuleResolutionPathBuf::first_party); + } + + #[test] + fn relativize_stdlib_path() { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .relativize_path("foo/baz/eggs/__init__.pyi") + .unwrap(), + @r###" + ModuleResolutionPathRef( + StandardLibrary( + "baz/eggs/__init__.pyi", + ), + ) + "### + ); + }); + } + + #[test] + fn relativize_site_packages_path() { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::site_packages("foo") + .unwrap() + .relativize_path("foo/baz") + .unwrap(), + @r###" + ModuleResolutionPathRef( + SitePackages( + "baz", + ), + ) + "### + ); + }); + } + + #[test] + fn relativize_extra_path() { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::extra("foo/baz") + .unwrap() + .relativize_path("foo/baz/functools.py") + .unwrap(), + @r###" + ModuleResolutionPathRef( + Extra( + "functools.py", + ), + ) + "### + ); + }); + } + + #[test] + fn relativize_first_party_path() { + insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { + assert_debug_snapshot!( + ModuleResolutionPathBuf::site_packages("dev/src") + .unwrap() + .relativize_path("dev/src/package/bar/baz.pyi") + .unwrap(), + @r###" + ModuleResolutionPathRef( + SitePackages( + "package/bar/baz.pyi", + ), + ) + "### + ); + }); + } } From 5ecc0f34d1b1affc0dd65c666f2e05c0abefaa45 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Jul 2024 13:29:10 +0100 Subject: [PATCH 26/58] Add tests for `is_directory()` and `is_regular_package()` --- crates/red_knot_module_resolver/src/db.rs | 118 +++++++++++++++++- crates/red_knot_module_resolver/src/path.rs | 105 ++++++++++++++-- .../red_knot_module_resolver/src/resolver.rs | 71 +++-------- 3 files changed, 222 insertions(+), 72 deletions(-) diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 498eba690de3d..ead8e2b0b4ee4 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -20,14 +20,20 @@ pub struct Jar( pub trait Db: salsa::DbWithJar + ruff_db::Db + Upcast {} +#[cfg(test)] pub(crate) mod tests { use std::sync; use salsa::DebugWithDb; - use ruff_db::file_system::{FileSystem, MemoryFileSystem, OsFileSystem}; + use ruff_db::file_system::{ + FileSystem, FileSystemPath, FileSystemPathBuf, MemoryFileSystem, OsFileSystem, + }; use ruff_db::vfs::Vfs; + use crate::resolver::{set_module_resolution_settings, ModuleResolutionSettings}; + use crate::supported_py_version::SupportedPyVersion; + use super::*; #[salsa::db(Jar, ruff_db::Jar)] @@ -39,7 +45,6 @@ pub(crate) mod tests { } impl TestDb { - #[allow(unused)] pub(crate) fn new() -> Self { Self { storage: salsa::Storage::default(), @@ -53,7 +58,6 @@ pub(crate) mod tests { /// /// ## Panics /// If this test db isn't using a memory file system. - #[allow(unused)] pub(crate) fn memory_file_system(&self) -> &MemoryFileSystem { if let TestFileSystem::Memory(fs) = &self.file_system { fs @@ -67,7 +71,6 @@ pub(crate) mod tests { /// This useful for testing advanced file system features like permissions, symlinks, etc. /// /// Note that any files written to the memory file system won't be copied over. - #[allow(unused)] pub(crate) fn with_os_file_system(&mut self) { self.file_system = TestFileSystem::Os(OsFileSystem); } @@ -81,7 +84,6 @@ pub(crate) mod tests { /// /// ## Panics /// If there are any pending salsa snapshots. - #[allow(unused)] pub(crate) fn take_salsa_events(&mut self) -> Vec { let inner = sync::Arc::get_mut(&mut self.events).expect("no pending salsa snapshots"); @@ -93,7 +95,6 @@ pub(crate) mod tests { /// /// ## Panics /// If there are any pending salsa snapshots. - #[allow(unused)] pub(crate) fn clear_salsa_events(&mut self) { self.take_salsa_events(); } @@ -157,4 +158,109 @@ pub(crate) mod tests { } } } + + pub(crate) struct TestCaseBuilder { + db: TestDb, + src: FileSystemPathBuf, + custom_typeshed: FileSystemPathBuf, + site_packages: FileSystemPathBuf, + target_version: Option, + } + + impl TestCaseBuilder { + #[must_use] + pub(crate) fn with_target_version(mut self, target_version: SupportedPyVersion) -> Self { + self.target_version = Some(target_version); + self + } + + pub(crate) fn build(self) -> TestCase { + let TestCaseBuilder { + mut db, + src, + custom_typeshed, + site_packages, + target_version, + } = self; + + let settings = ModuleResolutionSettings { + target_version: target_version.unwrap_or_default(), + extra_paths: vec![], + workspace_root: src.clone(), + custom_typeshed: Some(custom_typeshed.clone()), + site_packages: Some(site_packages.clone()), + }; + + set_module_resolution_settings(&mut db, settings); + + TestCase { + db, + src, + custom_typeshed, + site_packages, + } + } + } + + pub(crate) struct TestCase { + pub(crate) db: TestDb, + pub(crate) src: FileSystemPathBuf, + pub(crate) custom_typeshed: FileSystemPathBuf, + pub(crate) site_packages: FileSystemPathBuf, + } + + pub(crate) fn create_resolver_builder() -> std::io::Result { + static VERSIONS_DATA: &str = "\ + asyncio: 3.8- # 'Regular' package on py38+ + asyncio.tasks: 3.9-3.11 + collections: 3.9- # 'Regular' package on py39+ + functools: 3.8- + importlib: 3.9- # Namespace package on py39+ + xml: 3.8-3.8 # Namespace package on py38 only + "; + + let db = TestDb::new(); + + let src = FileSystemPath::new("src").to_path_buf(); + let site_packages = FileSystemPath::new("site_packages").to_path_buf(); + let custom_typeshed = FileSystemPath::new("typeshed").to_path_buf(); + + let fs = db.memory_file_system(); + + fs.create_directory_all(&*src)?; + fs.create_directory_all(&*site_packages)?; + fs.create_directory_all(&*custom_typeshed)?; + fs.write_file(custom_typeshed.join("stdlib/VERSIONS"), VERSIONS_DATA)?; + + // Regular package on py38+ + fs.create_directory_all(custom_typeshed.join("stdlib/asyncio"))?; + fs.touch(custom_typeshed.join("stdlib/asyncio/__init__.pyi"))?; + fs.write_file( + custom_typeshed.join("stdlib/asyncio/tasks.pyi"), + "class Task: ...", + )?; + + // Regular package on py39+ + fs.create_directory_all(custom_typeshed.join("stdlib/collections"))?; + fs.touch(custom_typeshed.join("stdlib/collections/__init__.pyi"))?; + + // Namespace package on py38 only + fs.create_directory_all(custom_typeshed.join("stdlib/xml"))?; + fs.touch(custom_typeshed.join("stdlib/xml/etree.pyi"))?; + + // Namespace package on py39+ + fs.create_directory_all(custom_typeshed.join("stdlib/importlib"))?; + fs.write_file( + custom_typeshed.join("stdlib/functools.pyi"), + "def update_wrapper(): ...", + )?; + + Ok(TestCaseBuilder { + db, + src, + custom_typeshed, + site_packages, + target_version: None, + }) + } } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index b23a2c67d7696..36173dd69f9c4 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -205,13 +205,24 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_directory(&self, db: &dyn Db, search_path: Self) -> bool { - match (self, search_path) { + #[inline] + fn absolute_path_to_module_name( + absolute_path: &FileSystemPath, + stdlib_root: ModuleResolutionPathRef, + ) -> Option { + stdlib_root + .relativize_path(absolute_path) + .and_then(ModuleResolutionPathRef::to_module_name) + } + + #[must_use] + fn is_directory(&self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> bool { + match (self, search_path.0) { (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = ModuleResolutionPathRef(*self).to_module_name() else { + let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { return false; }; let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); @@ -230,8 +241,8 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_regular_package(&self, db: &dyn Db, search_path: Self) -> bool { - match (self, search_path) { + fn is_regular_package(&self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> bool { + match (self, search_path.0) { (Self::Extra(path), Self::Extra(_)) | (Self::FirstParty(path), Self::FirstParty(_)) | (Self::SitePackages(path), Self::SitePackages(_)) => { @@ -243,7 +254,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = ModuleResolutionPathRef(*self).to_module_name() else { + let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { return false; }; let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); @@ -394,12 +405,12 @@ impl<'a> ModuleResolutionPathRef<'a> { #[must_use] pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.0.is_directory(db, search_path.into().0) + self.0.is_directory(db, search_path.into()) } #[must_use] pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.0.is_regular_package(db, search_path.into().0) + self.0.is_regular_package(db, search_path.into()) } #[must_use] @@ -506,10 +517,13 @@ impl<'a> PartialEq> for FileSystemPath { #[cfg(test)] mod tests { - use super::*; - use insta::assert_debug_snapshot; + use crate::db::tests::{create_resolver_builder, TestCase}; + use crate::supported_py_version::SupportedPyVersion; + + use super::*; + // Replace windows paths static WINDOWS_PATH_FILTER: [(&str, &str); 1] = [(r"\\\\", "/")]; @@ -947,4 +961,75 @@ mod tests { ); }); } + + #[test] + fn is_directory_and_package_stdlib_py38() { + let TestCase { + db, + custom_typeshed, + .. + } = create_resolver_builder().unwrap().build(); + let stdlib_module_path = + ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + + let asyncio_regular_package = stdlib_module_path.join("asyncio"); + assert!(asyncio_regular_package.is_directory(&db, &stdlib_module_path)); + assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_module_path)); + + let xml_namespace_package = stdlib_module_path.join("xml"); + assert!(xml_namespace_package.is_directory(&db, &stdlib_module_path)); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_module_path)); + + let functools_module = stdlib_module_path.join("functools.pyi"); + assert!(!functools_module.is_directory(&db, &stdlib_module_path)); + assert!(!functools_module.is_regular_package(&db, &stdlib_module_path)); + + let asyncio_tasks_module = stdlib_module_path.join("asyncio/tasks.pyi"); + assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_module_path)); + assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_module_path)); + + let non_existent = stdlib_module_path.join("doesnt_even_exist"); + assert!(!non_existent.is_directory(&db, &stdlib_module_path)); + assert!(!non_existent.is_regular_package(&db, &stdlib_module_path)); + + // `importlib` and `collections` exist as directories in the custom stdlib, + // but don't exist yet on the default target version (3.8) according to VERSIONS + let importlib_namespace_package = stdlib_module_path.join("importlib"); + assert!(!importlib_namespace_package.is_directory(&db, &stdlib_module_path)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_module_path)); + + let collections_regular_package = stdlib_module_path.join("collections"); + assert!(!collections_regular_package.is_directory(&db, &stdlib_module_path)); + assert!(!collections_regular_package.is_regular_package(&db, &stdlib_module_path)); + } + + #[test] + fn is_directory_stdlib_py39() { + let TestCase { + db, + custom_typeshed, + .. + } = create_resolver_builder() + .unwrap() + .with_target_version(SupportedPyVersion::Py39) + .build(); + + let stdlib_module_path = + ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + + // Since we've set the target version to Py39, + // `importlib` and `collections` should now exist as directories, according to VERSIONS, + // but `xml` no longer exists + let importlib_namespace_package = stdlib_module_path.join("importlib"); + assert!(importlib_namespace_package.is_directory(&db, &stdlib_module_path)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_module_path)); + + let collections_regular_package = stdlib_module_path.join("collections"); + assert!(collections_regular_package.is_directory(&db, &stdlib_module_path)); + assert!(collections_regular_package.is_regular_package(&db, &stdlib_module_path)); + + let xml_namespace_package = stdlib_module_path.join("xml"); + assert!(!xml_namespace_package.is_directory(&db, &stdlib_module_path)); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_module_path)); + } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 4b696d589ac96..a3d102c75ec61 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -362,56 +362,15 @@ mod tests { use ruff_db::file_system::FileSystemPath; use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; - use crate::db::tests::TestDb; + use crate::db::tests::{create_resolver_builder, TestCase}; use crate::module::ModuleKind; use crate::module_name::ModuleName; use super::*; - struct TestCase { - db: TestDb, - - src: FileSystemPathBuf, - custom_typeshed: FileSystemPathBuf, - site_packages: FileSystemPathBuf, - } - - fn create_resolver() -> std::io::Result { - let mut db = TestDb::new(); - - let src = FileSystemPath::new("src").to_path_buf(); - let site_packages = FileSystemPath::new("site_packages").to_path_buf(); - let custom_typeshed = FileSystemPath::new("typeshed").to_path_buf(); - - let fs = db.memory_file_system(); - - fs.create_directory_all(&*src)?; - fs.create_directory_all(&*site_packages)?; - fs.create_directory_all(&*custom_typeshed)?; - fs.create_directory_all(custom_typeshed.join("stdlib"))?; - fs.write_file(custom_typeshed.join("stdlib/VERSIONS"), "functools: 3.8-")?; - - let settings = ModuleResolutionSettings { - target_version: SupportedPyVersion::Py38, - extra_paths: vec![], - workspace_root: src.clone(), - site_packages: Some(site_packages.clone()), - custom_typeshed: Some(custom_typeshed.clone()), - }; - - set_module_resolution_settings(&mut db, settings); - - Ok(TestCase { - db, - src, - custom_typeshed, - site_packages, - }) - } - #[test] fn first_party_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver()?; + let TestCase { db, src, .. } = create_resolver_builder()?.build(); let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_path = src.join("foo.py"); @@ -444,7 +403,7 @@ mod tests { db, custom_typeshed, .. - } = create_resolver()?; + } = create_resolver_builder()?.build(); let stdlib_dir = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); @@ -482,7 +441,7 @@ mod tests { src, custom_typeshed, .. - } = create_resolver()?; + } = create_resolver_builder()?.build(); let stdlib_dir = custom_typeshed.join("stdlib"); let stdlib_functools_path = stdlib_dir.join("functools.pyi"); @@ -541,7 +500,7 @@ mod tests { #[test] fn resolve_package() -> anyhow::Result<()> { - let TestCase { src, db, .. } = create_resolver()?; + let TestCase { src, db, .. } = create_resolver_builder()?.build(); let foo_dir = src.join("foo"); let foo_path = foo_dir.join("__init__.py"); @@ -568,7 +527,7 @@ mod tests { #[test] fn package_priority_over_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver()?; + let TestCase { db, src, .. } = create_resolver_builder()?.build(); let foo_dir = src.join("foo"); let foo_init = foo_dir.join("__init__.py"); @@ -597,7 +556,7 @@ mod tests { #[test] fn typing_stub_over_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver()?; + let TestCase { db, src, .. } = create_resolver_builder()?.build(); let foo_stub = src.join("foo.pyi"); let foo_py = src.join("foo.py"); @@ -620,7 +579,7 @@ mod tests { #[test] fn sub_packages() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver()?; + let TestCase { db, src, .. } = create_resolver_builder()?.build(); let foo = src.join("foo"); let bar = foo.join("bar"); @@ -653,7 +612,7 @@ mod tests { src, site_packages, .. - } = create_resolver()?; + } = create_resolver_builder()?.build(); // From [PEP420](https://peps.python.org/pep-0420/#nested-namespace-packages). // But uses `src` for `project1` and `site_packages2` for `project2`. @@ -706,7 +665,7 @@ mod tests { src, site_packages, .. - } = create_resolver()?; + } = create_resolver_builder()?.build(); // Adopted test case from the [PEP420 examples](https://peps.python.org/pep-0420/#nested-namespace-packages). // The `src/parent/child` package is a regular package. Therefore, `site_packages/parent/child/two.py` should not be resolved. @@ -757,7 +716,7 @@ mod tests { src, site_packages, .. - } = create_resolver()?; + } = create_resolver_builder()?.build(); let foo_src = src.join("foo.py"); let foo_site_packages = site_packages.join("foo.py"); @@ -790,7 +749,7 @@ mod tests { src, site_packages, custom_typeshed, - } = create_resolver()?; + } = create_resolver_builder()?.build(); db.with_os_file_system(); @@ -855,7 +814,7 @@ mod tests { #[test] fn deleting_an_unrelated_file_doesnt_change_module_resolution() -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver()?; + let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); let foo_path = src.join("foo.py"); let bar_path = src.join("bar.py"); @@ -892,7 +851,7 @@ mod tests { #[test] fn adding_a_file_on_which_the_module_resolution_depends_on_invalidates_the_query( ) -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver()?; + let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); let foo_path = src.join("foo.py"); let foo_module_name = ModuleName::new_static("foo").unwrap(); @@ -912,7 +871,7 @@ mod tests { #[test] fn removing_a_file_that_the_module_resolution_depends_on_invalidates_the_query( ) -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver()?; + let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); let foo_path = src.join("foo.py"); let foo_init_path = src.join("foo/__init__.py"); From c541dd04f840eb1768bc5189e9e6c399bf5ca01c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Jul 2024 15:07:21 +0100 Subject: [PATCH 27/58] Delete cruft from older tests --- crates/red_knot_module_resolver/src/path.rs | 4 +- .../red_knot_module_resolver/src/resolver.rs | 51 +++---------------- 2 files changed, 9 insertions(+), 46 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 36173dd69f9c4..77199a13975a3 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1018,8 +1018,7 @@ mod tests { ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); // Since we've set the target version to Py39, - // `importlib` and `collections` should now exist as directories, according to VERSIONS, - // but `xml` no longer exists + // `importlib` and `collections` should now exist as directories, according to VERSIONS... let importlib_namespace_package = stdlib_module_path.join("importlib"); assert!(importlib_namespace_package.is_directory(&db, &stdlib_module_path)); assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_module_path)); @@ -1028,6 +1027,7 @@ mod tests { assert!(collections_regular_package.is_directory(&db, &stdlib_module_path)); assert!(collections_regular_package.is_regular_package(&db, &stdlib_module_path)); + // ...but `xml` no longer exists let xml_namespace_package = stdlib_module_path.join("xml"); assert!(!xml_namespace_package.is_directory(&db, &stdlib_module_path)); assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_module_path)); diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 45aa3e6ea7b64..c8cc8c4316e3f 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -405,10 +405,6 @@ mod tests { let stdlib_dir = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); - let functools_path = stdlib_dir.join("functools.pyi"); - db.memory_file_system() - .write_file(&functools_path, "def update_wrapper(): ...")?; - let functools_module_name = ModuleName::new_static("functools").unwrap(); let functools_module = resolve_module(&db, functools_module_name.clone()).unwrap(); @@ -420,13 +416,14 @@ mod tests { assert_eq!(stdlib_dir, functools_module.search_path().to_path_buf()); assert_eq!(ModuleKind::Module, functools_module.kind()); - let functools_path_vfs = VfsPath::from(functools_path); + let expected_functools_path = + VfsPath::FileSystem(custom_typeshed.join("stdlib/functools.pyi")); - assert_eq!(&functools_path_vfs, functools_module.file().path(&db)); + assert_eq!(&expected_functools_path, functools_module.file().path(&db)); assert_eq!( Some(functools_module), - path_to_module(&db, &functools_path_vfs) + path_to_module(&db, &expected_functools_path) ); Ok(()) @@ -434,21 +431,11 @@ mod tests { #[test] fn first_party_precedence_over_stdlib() -> anyhow::Result<()> { - let TestCase { - db, - src, - custom_typeshed, - .. - } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = create_resolver_builder()?.build(); - let stdlib_dir = custom_typeshed.join("stdlib"); - let stdlib_functools_path = stdlib_dir.join("functools.pyi"); let first_party_functools_path = src.join("functools.py"); - - db.memory_file_system().write_files([ - (&stdlib_functools_path, "def update_wrapper(): ..."), - (&first_party_functools_path, "def update_wrapper(): ..."), - ])?; + db.memory_file_system() + .write_file(&first_party_functools_path, "def update_wrapper(): ...")?; let functools_module_name = ModuleName::new_static("functools").unwrap(); let functools_module = resolve_module(&db, functools_module_name.clone()).unwrap(); @@ -472,30 +459,6 @@ mod tests { Ok(()) } - // TODO: Port typeshed test case. Porting isn't possible at the moment because the vendored zip - // is part of the red knot crate - // #[test] - // fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { - // // The file path here is hardcoded in this crate's `build.rs` script. - // // Luckily this crate will fail to build if this file isn't available at build time. - // const TYPESHED_ZIP_BYTES: &[u8] = - // include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); - // assert!(!TYPESHED_ZIP_BYTES.is_empty()); - // let mut typeshed_zip_archive = ZipArchive::new(Cursor::new(TYPESHED_ZIP_BYTES))?; - // - // let path_to_functools = Path::new("stdlib").join("functools.pyi"); - // let mut functools_module_stub = typeshed_zip_archive - // .by_name(path_to_functools.to_str().unwrap()) - // .unwrap(); - // assert!(functools_module_stub.is_file()); - // - // let mut functools_module_stub_source = String::new(); - // functools_module_stub.read_to_string(&mut functools_module_stub_source)?; - // - // assert!(functools_module_stub_source.contains("def update_wrapper(")); - // Ok(()) - // } - #[test] fn resolve_package() -> anyhow::Result<()> { let TestCase { src, db, .. } = create_resolver_builder()?.build(); From 2f0d349dc8f414165c9980f80d8043ffc9290468 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Jul 2024 17:02:05 +0100 Subject: [PATCH 28/58] Add tests for the resolver as a whole with `VERSIONS` --- crates/red_knot_module_resolver/src/db.rs | 2 + crates/red_knot_module_resolver/src/path.rs | 51 +++++++-- .../red_knot_module_resolver/src/resolver.rs | 108 ++++++++++++++++-- 3 files changed, 143 insertions(+), 18 deletions(-) diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index ead8e2b0b4ee4..0e86b975deeb5 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -250,6 +250,8 @@ pub(crate) mod tests { // Namespace package on py39+ fs.create_directory_all(custom_typeshed.join("stdlib/importlib"))?; + fs.touch(custom_typeshed.join("stdlib/importlib/abc.pyi"))?; + fs.write_file( custom_typeshed.join("stdlib/functools.pyi"), "def update_wrapper(): ...", diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 77199a13975a3..bd6b1628b33cb 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,7 +1,7 @@ use std::fmt; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; -use ruff_db::vfs::{system_path_to_file, VfsPath}; +use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; use crate::db::Db; use crate::module_name::ModuleName; @@ -142,6 +142,10 @@ impl ModuleResolutionPathBuf { ) -> Option> { ModuleResolutionPathRef::from(self).relativize_path(absolute_path.as_ref()) } + + pub(crate) fn to_vfs_file(&self, db: &dyn Db, search_path: &Self) -> Option { + ModuleResolutionPathRef::from(self).to_vfs_file(db, search_path) + } } impl From for VfsPath { @@ -155,13 +159,6 @@ impl From for VfsPath { } } -impl AsRef for ModuleResolutionPathBuf { - #[inline] - fn as_ref(&self) -> &FileSystemPath { - ModuleResolutionPathRefInner::from(&self.0).as_file_system_path() - } -} - impl fmt::Debug for ModuleResolutionPathBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let (name, path) = match &self.0 { @@ -186,7 +183,7 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] - fn load_typeshed_versions<'db>( + pub(crate) fn load_typeshed_versions<'db>( db: &'db dyn Db, stdlib_root: &FileSystemPath, ) -> &'db TypeshedVersions { @@ -194,7 +191,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { todo!( "Still need to figure out how to handle VERSIONS files being deleted \ - from custom typeshed directories! Expected a file to exist at {versions_path}" + from custom typeshed directories! Expected a file to exist at {versions_path}" ) }; // TODO(Alex/Micha): If VERSIONS is invalid, @@ -272,6 +269,30 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } + fn to_vfs_file(self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> Option { + match (self, search_path.0) { + (Self::Extra(path), Self::Extra(_)) => system_path_to_file(db.upcast(), path), + (Self::FirstParty(path), Self::FirstParty(_)) => system_path_to_file(db.upcast(), path), + (Self::SitePackages(path), Self::SitePackages(_)) => { + system_path_to_file(db.upcast(), path) + } + (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { + let module_name = Self::absolute_path_to_module_name(path, search_path)?; + let versions = Self::load_typeshed_versions(db, stdlib_root); + match versions.query_module(&module_name, get_target_py_version(db)) { + TypeshedVersionsQueryResult::Exists + | TypeshedVersionsQueryResult::MaybeExists => { + system_path_to_file(db.upcast(), path) + } + TypeshedVersionsQueryResult::DoesNotExist => None, + } + } + (path, root) => unreachable!( + "The search path should always be the same variant as `self` (got: {path:?}, {root:?})" + ) + } + } + #[must_use] pub(crate) fn to_module_name(self) -> Option { let (fs_path, skip_final_part) = match self { @@ -413,6 +434,11 @@ impl<'a> ModuleResolutionPathRef<'a> { self.0.is_regular_package(db, search_path.into()) } + #[must_use] + pub(crate) fn to_vfs_file(self, db: &dyn Db, search_path: impl Into) -> Option { + self.0.to_vfs_file(db, search_path.into()) + } + #[must_use] pub(crate) fn to_module_name(self) -> Option { self.0.to_module_name() @@ -453,6 +479,11 @@ impl<'a> ModuleResolutionPathRef<'a> { pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { ModuleResolutionPathBuf(self.0.to_path_buf()) } + + #[cfg(test)] + pub(crate) const fn is_stdlib_search_path(&self) -> bool { + matches!(&self.0, ModuleResolutionPathRefInner::StandardLibrary(_)) + } } impl<'a> From<&'a ModuleResolutionPathBufInner> for ModuleResolutionPathRefInner<'a> { diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index c8cc8c4316e3f..b9d0c5a02c97d 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -2,7 +2,7 @@ use std::ops::Deref; use std::sync::Arc; use ruff_db::file_system::FileSystemPathBuf; -use ruff_db::vfs::{system_path_to_file, vfs_path_to_file, VfsFile, VfsPath}; +use ruff_db::vfs::{vfs_path_to_file, VfsFile, VfsPath}; use crate::db::Db; use crate::module::{Module, ModuleKind}; @@ -240,17 +240,19 @@ fn resolve_name( }; // TODO Implement full https://peps.python.org/pep-0561/#type-checker-module-resolution-order resolution - if let Some(stub) = - system_path_to_file(db.upcast(), package_path.with_pyi_extension()) + if let Some(stub) = package_path + .with_pyi_extension() + .to_vfs_file(db, search_path) { return Some((search_path.clone(), stub, kind)); } - if let Some(path_with_extension) = package_path.with_py_extension() { - if let Some(module) = system_path_to_file(db.upcast(), &path_with_extension) { - return Some((search_path.clone(), module, kind)); - } - }; + if let Some(module) = package_path + .with_py_extension() + .and_then(|path| path.to_vfs_file(db, search_path)) + { + return Some((search_path.clone(), module, kind)); + } // For regular packages, don't search the next search path. All files of that // package must be in the same location @@ -429,6 +431,96 @@ mod tests { Ok(()) } + fn create_module_names(raw_names: &[&str]) -> Vec { + raw_names + .iter() + .map(|raw| ModuleName::new(raw).unwrap()) + .collect() + } + + #[test] + fn stdlib_resolution_respects_versions_file_py38() { + let TestCase { + db, + custom_typeshed, + .. + } = create_resolver_builder().unwrap().build(); + + let existing_modules = create_module_names(&["asyncio", "functools", "xml.etree"]); + for module_name in existing_modules { + let resolved_module = resolve_module(&db, module_name.clone()).unwrap_or_else(|| { + panic!("Expected module {module_name} to exist in the mock stdlib") + }); + let search_path = resolved_module.search_path(); + assert_eq!( + *custom_typeshed.join("stdlib"), + search_path, + "Search path for {module_name} was unexpectedly {search_path:?}" + ); + assert!( + search_path.is_stdlib_search_path(), + "Expected a stdlib search path, but got {search_path:?}" + ); + } + + let nonexisting_modules = create_module_names(&[ + "collections", + "importlib", + "importlib.abc", + "xml", + "asyncio.tasks", + ]); + for module_name in nonexisting_modules { + assert!( + resolve_module(&db, module_name.clone()).is_none(), + "Unexpectedly resolved a module for {module_name}" + ); + } + } + + #[test] + fn stdlib_resolution_respects_versions_file_py39() { + let TestCase { + db, + custom_typeshed, + .. + } = create_resolver_builder() + .unwrap() + .with_target_version(SupportedPyVersion::Py39) + .build(); + + let existing_modules = create_module_names(&[ + "asyncio", + "functools", + "importlib.abc", + "collections", + "asyncio.tasks", + ]); + for module_name in existing_modules { + let resolved_module = resolve_module(&db, module_name.clone()).unwrap_or_else(|| { + panic!("Expected module {module_name} to exist in the mock stdlib") + }); + let search_path = resolved_module.search_path(); + assert_eq!( + *custom_typeshed.join("stdlib"), + search_path, + "Search path for {module_name} was unexpectedly {search_path:?}" + ); + assert!( + search_path.is_stdlib_search_path(), + "Expected a stdlib search path, but got {search_path:?}" + ); + } + + let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]); + for module_name in nonexisting_modules { + assert!( + resolve_module(&db, module_name.clone()).is_none(), + "Unexpectedly resolved a module for {module_name}" + ); + } + } + #[test] fn first_party_precedence_over_stdlib() -> anyhow::Result<()> { let TestCase { db, src, .. } = create_resolver_builder()?.build(); From 9bbe5a8cc3b3abf07d1467f6150f68d5b1447e06 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Jul 2024 17:40:43 +0100 Subject: [PATCH 29/58] More unit tests in `path.rs` --- crates/red_knot_module_resolver/src/path.rs | 195 ++++++++++++++++---- 1 file changed, 154 insertions(+), 41 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index bd6b1628b33cb..74baf44df56ff 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -143,6 +143,7 @@ impl ModuleResolutionPathBuf { ModuleResolutionPathRef::from(self).relativize_path(absolute_path.as_ref()) } + /// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory. pub(crate) fn to_vfs_file(&self, db: &dyn Db, search_path: &Self) -> Option { ModuleResolutionPathRef::from(self).to_vfs_file(db, search_path) } @@ -550,7 +551,7 @@ impl<'a> PartialEq> for FileSystemPath { mod tests { use insta::assert_debug_snapshot; - use crate::db::tests::{create_resolver_builder, TestCase}; + use crate::db::tests::{create_resolver_builder, TestCase, TestDb}; use crate::supported_py_version::SupportedPyVersion; use super::*; @@ -993,8 +994,7 @@ mod tests { }); } - #[test] - fn is_directory_and_package_stdlib_py38() { + fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { let TestCase { db, custom_typeshed, @@ -1002,40 +1002,103 @@ mod tests { } = create_resolver_builder().unwrap().build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + (db, stdlib_module_path) + } + + #[test] + fn mocked_typeshed_existing_regular_stdlib_pkg_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); + + let asyncio_regular_package = stdlib_path.join("asyncio"); + assert!(asyncio_regular_package.is_directory(&db, &stdlib_path)); + assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_path)); + // Paths to directories don't resolve to VfsFiles + assert!(asyncio_regular_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(asyncio_regular_package + .join("__init__.pyi") + .to_vfs_file(&db, &stdlib_path) + .is_some()); + + // The `asyncio` package exists on Python 3.8, but the `asyncio.tasks` submodule does not, + // according to the `VERSIONS` file in our typeshed mock: + let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); + assert!(asyncio_tasks_module + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path)); + assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path)); + } - let asyncio_regular_package = stdlib_module_path.join("asyncio"); - assert!(asyncio_regular_package.is_directory(&db, &stdlib_module_path)); - assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_module_path)); + #[test] + fn mocked_typeshed_existing_namespace_stdlib_pkg_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); - let xml_namespace_package = stdlib_module_path.join("xml"); - assert!(xml_namespace_package.is_directory(&db, &stdlib_module_path)); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_module_path)); + let xml_namespace_package = stdlib_path.join("xml"); + assert!(xml_namespace_package.is_directory(&db, &stdlib_path)); + // Paths to directories don't resolve to VfsFiles + assert!(xml_namespace_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); - let functools_module = stdlib_module_path.join("functools.pyi"); - assert!(!functools_module.is_directory(&db, &stdlib_module_path)); - assert!(!functools_module.is_regular_package(&db, &stdlib_module_path)); + let xml_etree = stdlib_path.join("xml/etree.pyi"); + assert!(!xml_etree.is_directory(&db, &stdlib_path)); + assert!(xml_etree.to_vfs_file(&db, &stdlib_path).is_some()); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path)); + } - let asyncio_tasks_module = stdlib_module_path.join("asyncio/tasks.pyi"); - assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_module_path)); - assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_module_path)); + #[test] + fn mocked_typeshed_single_file_stdlib_module_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); - let non_existent = stdlib_module_path.join("doesnt_even_exist"); - assert!(!non_existent.is_directory(&db, &stdlib_module_path)); - assert!(!non_existent.is_regular_package(&db, &stdlib_module_path)); + let functools_module = stdlib_path.join("functools.pyi"); + assert!(functools_module.to_vfs_file(&db, &stdlib_path).is_some()); + assert!(!functools_module.is_directory(&db, &stdlib_path)); + assert!(!functools_module.is_regular_package(&db, &stdlib_path)); + } + + #[test] + fn mocked_typeshed_nonexistent_regular_stdlib_pkg_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); + + let collections_regular_package = stdlib_path.join("collections"); + assert!(collections_regular_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(!collections_regular_package.is_directory(&db, &stdlib_path)); + assert!(!collections_regular_package.is_regular_package(&db, &stdlib_path)); + } + + #[test] + fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); - // `importlib` and `collections` exist as directories in the custom stdlib, - // but don't exist yet on the default target version (3.8) according to VERSIONS - let importlib_namespace_package = stdlib_module_path.join("importlib"); - assert!(!importlib_namespace_package.is_directory(&db, &stdlib_module_path)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_module_path)); + let importlib_namespace_package = stdlib_path.join("importlib"); + assert!(importlib_namespace_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(!importlib_namespace_package.is_directory(&db, &stdlib_path)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); - let collections_regular_package = stdlib_module_path.join("collections"); - assert!(!collections_regular_package.is_directory(&db, &stdlib_module_path)); - assert!(!collections_regular_package.is_regular_package(&db, &stdlib_module_path)); + let importlib_abc = stdlib_path.join("importlib/abc.pyi"); + assert!(importlib_abc.to_vfs_file(&db, &stdlib_path).is_none()); + assert!(!importlib_abc.is_directory(&db, &stdlib_path)); + assert!(!importlib_abc.is_regular_package(&db, &stdlib_path)); } #[test] - fn is_directory_stdlib_py39() { + fn mocked_typeshed_nonexistent_single_file_module_py38() { + let (db, stdlib_path) = py38_stdlib_test_case(); + + let non_existent = stdlib_path.join("doesnt_even_exist"); + assert!(non_existent.to_vfs_file(&db, &stdlib_path).is_none()); + assert!(!non_existent.is_directory(&db, &stdlib_path)); + assert!(!non_existent.is_regular_package(&db, &stdlib_path)); + } + + fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { let TestCase { db, custom_typeshed, @@ -1044,23 +1107,73 @@ mod tests { .unwrap() .with_target_version(SupportedPyVersion::Py39) .build(); - let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); + (db, stdlib_module_path) + } + + #[test] + fn mocked_typeshed_existing_regular_stdlib_pkgs_py39() { + let (db, stdlib_path) = py39_stdlib_test_case(); // Since we've set the target version to Py39, - // `importlib` and `collections` should now exist as directories, according to VERSIONS... - let importlib_namespace_package = stdlib_module_path.join("importlib"); - assert!(importlib_namespace_package.is_directory(&db, &stdlib_module_path)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_module_path)); - - let collections_regular_package = stdlib_module_path.join("collections"); - assert!(collections_regular_package.is_directory(&db, &stdlib_module_path)); - assert!(collections_regular_package.is_regular_package(&db, &stdlib_module_path)); - - // ...but `xml` no longer exists - let xml_namespace_package = stdlib_module_path.join("xml"); - assert!(!xml_namespace_package.is_directory(&db, &stdlib_module_path)); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_module_path)); + // `collections` should now exist as a directory, according to VERSIONS... + let collections_regular_package = stdlib_path.join("collections"); + assert!(collections_regular_package.is_directory(&db, &stdlib_path)); + assert!(collections_regular_package.is_regular_package(&db, &stdlib_path)); + // (This is still `None`, as directories don't resolve to `Vfs` files) + assert!(collections_regular_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(collections_regular_package + .join("__init__.pyi") + .to_vfs_file(&db, &stdlib_path) + .is_some()); + + // ...and so should the `asyncio.tasks` submodule (though it's still not a directory): + let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); + assert!(asyncio_tasks_module + .to_vfs_file(&db, &stdlib_path) + .is_some()); + assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path)); + assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path)); + } + + #[test] + fn mocked_typeshed_existing_namespace_stdlib_pkg_py39() { + let (db, stdlib_path) = py39_stdlib_test_case(); + + // The `importlib` directory now also exists... + let importlib_namespace_package = stdlib_path.join("importlib"); + assert!(importlib_namespace_package.is_directory(&db, &stdlib_path)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); + // (This is still `None`, as directories don't resolve to `Vfs` files) + assert!(importlib_namespace_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + + // ...As do submodules in the `importlib` namespace package: + let importlib_abc = importlib_namespace_package.join("abc.pyi"); + assert!(!importlib_abc.is_directory(&db, &stdlib_path)); + assert!(!importlib_abc.is_regular_package(&db, &stdlib_path)); + assert!(importlib_abc.to_vfs_file(&db, &stdlib_path).is_some()); + } + + #[test] + fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py39() { + let (db, stdlib_path) = py39_stdlib_test_case(); + + // The `xml` package no longer exists on py39: + let xml_namespace_package = stdlib_path.join("xml"); + assert!(xml_namespace_package + .to_vfs_file(&db, &stdlib_path) + .is_none()); + assert!(!xml_namespace_package.is_directory(&db, &stdlib_path)); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); + + let xml_etree = xml_namespace_package.join("etree.pyi"); + assert!(xml_etree.to_vfs_file(&db, &stdlib_path).is_none()); + assert!(!xml_etree.is_directory(&db, &stdlib_path)); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path)); } } From 36b928842f722ca4ebf07eb12e655c74874ade42 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 10:47:19 +0100 Subject: [PATCH 30/58] Reduce use of `is_none()` in tests --- crates/red_knot_module_resolver/src/path.rs | 74 +++++++++++---------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 74baf44df56ff..65e2a98140a44 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -561,9 +561,15 @@ mod tests { #[test] fn constructor_rejects_non_pyi_stdlib_paths() { - assert!(ModuleResolutionPathBuf::standard_library("foo.py").is_none()); - assert!(ModuleResolutionPathBuf::standard_library("foo/__init__.py").is_none()); - assert!(ModuleResolutionPathBuf::standard_library("foo.py.pyi").is_none()); + assert_eq!(ModuleResolutionPathBuf::standard_library("foo.py"), None); + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo/__init__.py"), + None + ); + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo.py.pyi"), + None + ); } fn stdlib_path_test_case(path: &str) -> ModuleResolutionPathBuf { @@ -599,7 +605,7 @@ mod tests { #[test] fn stdlib_paths_can_only_be_pyi() { - assert!(stdlib_path_test_case("foo").with_py_extension().is_none()); + assert_eq!(stdlib_path_test_case("foo").with_py_extension(), None); } #[test] @@ -882,13 +888,13 @@ mod tests { // Must have a `.pyi` extension or no extension: let bad_absolute_path = FileSystemPath::new("foo/stdlib/x.py"); - assert!(root.relativize_path(bad_absolute_path).is_none()); + assert_eq!(root.relativize_path(bad_absolute_path), None); let second_bad_absolute_path = FileSystemPath::new("foo/stdlib/x.rs"); - assert!(root.relativize_path(second_bad_absolute_path).is_none()); + assert_eq!(root.relativize_path(second_bad_absolute_path), None); // Must be a path that is a child of `root`: let third_bad_absolute_path = FileSystemPath::new("bar/stdlib/x.pyi"); - assert!(root.relativize_path(third_bad_absolute_path).is_none()); + assert_eq!(root.relativize_path(third_bad_absolute_path), None); } fn non_stdlib_relativize_tester( @@ -897,10 +903,10 @@ mod tests { let root = variant("foo").unwrap(); // Must have a `.py` extension, a `.pyi` extension, or no extension: let bad_absolute_path = FileSystemPath::new("foo/stdlib/x.rs"); - assert!(root.relativize_path(bad_absolute_path).is_none()); + assert_eq!(root.relativize_path(bad_absolute_path), None); // Must be a path that is a child of `root`: let second_bad_absolute_path = FileSystemPath::new("bar/stdlib/x.pyi"); - assert!(root.relativize_path(second_bad_absolute_path).is_none()); + assert_eq!(root.relativize_path(second_bad_absolute_path), None); } #[test] @@ -1013,9 +1019,7 @@ mod tests { assert!(asyncio_regular_package.is_directory(&db, &stdlib_path)); assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_path)); // Paths to directories don't resolve to VfsFiles - assert!(asyncio_regular_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!(asyncio_regular_package.to_vfs_file(&db, &stdlib_path), None); assert!(asyncio_regular_package .join("__init__.pyi") .to_vfs_file(&db, &stdlib_path) @@ -1024,9 +1028,7 @@ mod tests { // The `asyncio` package exists on Python 3.8, but the `asyncio.tasks` submodule does not, // according to the `VERSIONS` file in our typeshed mock: let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); - assert!(asyncio_tasks_module - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!(asyncio_tasks_module.to_vfs_file(&db, &stdlib_path), None); assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path)); assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path)); } @@ -1038,9 +1040,7 @@ mod tests { let xml_namespace_package = stdlib_path.join("xml"); assert!(xml_namespace_package.is_directory(&db, &stdlib_path)); // Paths to directories don't resolve to VfsFiles - assert!(xml_namespace_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!(xml_namespace_package.to_vfs_file(&db, &stdlib_path), None); assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); let xml_etree = stdlib_path.join("xml/etree.pyi"); @@ -1064,9 +1064,10 @@ mod tests { let (db, stdlib_path) = py38_stdlib_test_case(); let collections_regular_package = stdlib_path.join("collections"); - assert!(collections_regular_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!( + collections_regular_package.to_vfs_file(&db, &stdlib_path), + None + ); assert!(!collections_regular_package.is_directory(&db, &stdlib_path)); assert!(!collections_regular_package.is_regular_package(&db, &stdlib_path)); } @@ -1076,14 +1077,15 @@ mod tests { let (db, stdlib_path) = py38_stdlib_test_case(); let importlib_namespace_package = stdlib_path.join("importlib"); - assert!(importlib_namespace_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!( + importlib_namespace_package.to_vfs_file(&db, &stdlib_path), + None + ); assert!(!importlib_namespace_package.is_directory(&db, &stdlib_path)); assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); let importlib_abc = stdlib_path.join("importlib/abc.pyi"); - assert!(importlib_abc.to_vfs_file(&db, &stdlib_path).is_none()); + assert_eq!(importlib_abc.to_vfs_file(&db, &stdlib_path), None); assert!(!importlib_abc.is_directory(&db, &stdlib_path)); assert!(!importlib_abc.is_regular_package(&db, &stdlib_path)); } @@ -1093,7 +1095,7 @@ mod tests { let (db, stdlib_path) = py38_stdlib_test_case(); let non_existent = stdlib_path.join("doesnt_even_exist"); - assert!(non_existent.to_vfs_file(&db, &stdlib_path).is_none()); + assert_eq!(non_existent.to_vfs_file(&db, &stdlib_path), None); assert!(!non_existent.is_directory(&db, &stdlib_path)); assert!(!non_existent.is_regular_package(&db, &stdlib_path)); } @@ -1122,9 +1124,10 @@ mod tests { assert!(collections_regular_package.is_directory(&db, &stdlib_path)); assert!(collections_regular_package.is_regular_package(&db, &stdlib_path)); // (This is still `None`, as directories don't resolve to `Vfs` files) - assert!(collections_regular_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!( + collections_regular_package.to_vfs_file(&db, &stdlib_path), + None + ); assert!(collections_regular_package .join("__init__.pyi") .to_vfs_file(&db, &stdlib_path) @@ -1148,9 +1151,10 @@ mod tests { assert!(importlib_namespace_package.is_directory(&db, &stdlib_path)); assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); // (This is still `None`, as directories don't resolve to `Vfs` files) - assert!(importlib_namespace_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!( + importlib_namespace_package.to_vfs_file(&db, &stdlib_path), + None + ); // ...As do submodules in the `importlib` namespace package: let importlib_abc = importlib_namespace_package.join("abc.pyi"); @@ -1165,14 +1169,12 @@ mod tests { // The `xml` package no longer exists on py39: let xml_namespace_package = stdlib_path.join("xml"); - assert!(xml_namespace_package - .to_vfs_file(&db, &stdlib_path) - .is_none()); + assert_eq!(xml_namespace_package.to_vfs_file(&db, &stdlib_path), None); assert!(!xml_namespace_package.is_directory(&db, &stdlib_path)); assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); let xml_etree = xml_namespace_package.join("etree.pyi"); - assert!(xml_etree.to_vfs_file(&db, &stdlib_path).is_none()); + assert_eq!(xml_etree.to_vfs_file(&db, &stdlib_path), None); assert!(!xml_etree.is_directory(&db, &stdlib_path)); assert!(!xml_etree.is_regular_package(&db, &stdlib_path)); } From 2f0d5625a17472ec498012e7e48f4cc2b838883f Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 10:53:56 +0100 Subject: [PATCH 31/58] Add more todos for `into_ordered_search_paths()` --- crates/red_knot_module_resolver/src/resolver.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index b9d0c5a02c97d..77e86b12b85cd 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -136,6 +136,13 @@ pub struct ModuleResolutionSettings { impl ModuleResolutionSettings { /// Implementation of PEP 561's module resolution order /// (with some small, deliberate, differences) + /// + /// TODO(Alex): this method does multiple `.unwrap()` calls when it should really return an error. + /// Each `.unwrap()` call is a point where we're validating a setting that the user would pass + /// and transforming it into an internal representation for a validated path. + /// Rather than panicking if a path fails to validate, we should display an error message to the user + /// and exit the process with a nonzero exit code. + /// This validation should probably be done outside of Salsa? fn into_ordered_search_paths(self) -> (SupportedPyVersion, OrderedSearchPaths) { let ModuleResolutionSettings { target_version, @@ -145,11 +152,10 @@ impl ModuleResolutionSettings { custom_typeshed, } = self; - let mut paths = extra_paths + let mut paths: Vec = extra_paths .into_iter() - .map(ModuleResolutionPathBuf::extra) - .collect::>>() - .unwrap(); + .map(|fs_path| ModuleResolutionPathBuf::extra(fs_path).unwrap()) + .collect(); paths.push(ModuleResolutionPathBuf::first_party(workspace_root).unwrap()); From ea58e3a19053e59f3c53b14a5257dba7587e8756 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:00:49 +0100 Subject: [PATCH 32/58] Typeshed versions are looked up from the cache once per module resolution --- Cargo.lock | 1 + crates/red_knot_module_resolver/Cargo.toml | 1 + crates/red_knot_module_resolver/src/path.rs | 251 +++++++++++------- .../red_knot_module_resolver/src/resolver.rs | 23 +- .../red_knot_module_resolver/src/typeshed.rs | 4 +- .../src/typeshed/versions.rs | 105 +++++--- 6 files changed, 240 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 063413da01138..e7f401dd23e31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1885,6 +1885,7 @@ dependencies = [ "camino", "compact_str", "insta", + "once_cell", "path-slash", "ruff_db", "ruff_python_stdlib", diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index 18efb5d7c5682..121b75d15dee8 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -16,6 +16,7 @@ ruff_python_stdlib = { workspace = true } compact_str = { workspace = true } camino = { workspace = true } +once_cell = { workspace = true } rustc-hash = { workspace = true } salsa = { workspace = true } tracing = { workspace = true } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 65e2a98140a44..32a9d80c727e9 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -5,8 +5,7 @@ use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; use crate::db::Db; use crate::module_name::ModuleName; -use crate::supported_py_version::get_target_py_version; -use crate::typeshed::{parse_typeshed_versions, TypeshedVersions, TypeshedVersionsQueryResult}; +use crate::typeshed::{LazyTypeshedVersions, TypeshedVersionsQueryResult}; /// Enumeration of the different kinds of search paths type checkers are expected to support. /// @@ -110,13 +109,24 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: &Self) -> bool { - ModuleResolutionPathRef::from(self).is_regular_package(db, search_path) + pub(crate) fn is_regular_package( + &self, + db: &dyn Db, + search_path: &Self, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { + ModuleResolutionPathRef::from(self).is_regular_package(db, search_path, typeshed_versions) } #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db, search_path: &Self) -> bool { - ModuleResolutionPathRef::from(self).is_directory(db, search_path) + pub(crate) fn is_directory( + &self, + db: &dyn Db, + search_path: &Self, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { + let as_ref = ModuleResolutionPathRef::from(self); + as_ref.is_directory(db, search_path, typeshed_versions) } #[must_use] @@ -144,8 +154,13 @@ impl ModuleResolutionPathBuf { } /// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory. - pub(crate) fn to_vfs_file(&self, db: &dyn Db, search_path: &Self) -> Option { - ModuleResolutionPathRef::from(self).to_vfs_file(db, search_path) + pub(crate) fn to_vfs_file( + &self, + db: &dyn Db, + search_path: &Self, + typeshed_versions: &LazyTypeshedVersions, + ) -> Option { + ModuleResolutionPathRef::from(self).to_vfs_file(db, search_path, typeshed_versions) } } @@ -183,25 +198,6 @@ enum ModuleResolutionPathRefInner<'a> { } impl<'a> ModuleResolutionPathRefInner<'a> { - #[must_use] - pub(crate) fn load_typeshed_versions<'db>( - db: &'db dyn Db, - stdlib_root: &FileSystemPath, - ) -> &'db TypeshedVersions { - let versions_path = stdlib_root.join("VERSIONS"); - let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { - todo!( - "Still need to figure out how to handle VERSIONS files being deleted \ - from custom typeshed directories! Expected a file to exist at {versions_path}" - ) - }; - // TODO(Alex/Micha): If VERSIONS is invalid, - // this should invalidate not just the specific module resolution we're currently attempting, - // but all type inference that depends on any standard-library types. - // Unwrapping here is not correct... - parse_typeshed_versions(db, versions_file).as_ref().unwrap() - } - #[must_use] #[inline] fn absolute_path_to_module_name( @@ -214,7 +210,12 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_directory(&self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> bool { + fn is_directory( + &self, + db: &dyn Db, + search_path: ModuleResolutionPathRef<'a>, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { match (self, search_path.0) { (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), @@ -223,8 +224,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { return false; }; - let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); - match typeshed_versions.query_module(&module_name, get_target_py_version(db)) { + match typeshed_versions.query_module(&module_name, db, stdlib_root) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().is_directory(path) @@ -239,7 +239,12 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_regular_package(&self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> bool { + fn is_regular_package( + &self, + db: &dyn Db, + search_path: ModuleResolutionPathRef<'a>, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { match (self, search_path.0) { (Self::Extra(path), Self::Extra(_)) | (Self::FirstParty(path), Self::FirstParty(_)) @@ -255,8 +260,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { return false; }; - let typeshed_versions = Self::load_typeshed_versions(db, stdlib_root); - match typeshed_versions.query_module(&module_name, get_target_py_version(db)) { + match typeshed_versions.query_module(&module_name, db, stdlib_root) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { db.file_system().exists(&path.join("__init__.pyi")) @@ -270,7 +274,12 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } - fn to_vfs_file(self, db: &dyn Db, search_path: ModuleResolutionPathRef<'a>) -> Option { + fn to_vfs_file( + self, + db: &dyn Db, + search_path: ModuleResolutionPathRef<'a>, + typeshed_versions: &LazyTypeshedVersions, + ) -> Option { match (self, search_path.0) { (Self::Extra(path), Self::Extra(_)) => system_path_to_file(db.upcast(), path), (Self::FirstParty(path), Self::FirstParty(_)) => system_path_to_file(db.upcast(), path), @@ -279,8 +288,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { let module_name = Self::absolute_path_to_module_name(path, search_path)?; - let versions = Self::load_typeshed_versions(db, stdlib_root); - match versions.query_module(&module_name, get_target_py_version(db)) { + match typeshed_versions.query_module(&module_name, db, stdlib_root) { TypeshedVersionsQueryResult::Exists | TypeshedVersionsQueryResult::MaybeExists => { system_path_to_file(db.upcast(), path) @@ -426,18 +434,36 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_directory(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.0.is_directory(db, search_path.into()) + pub(crate) fn is_directory( + &self, + db: &dyn Db, + search_path: impl Into, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { + self.0 + .is_directory(db, search_path.into(), typeshed_versions) } #[must_use] - pub(crate) fn is_regular_package(&self, db: &dyn Db, search_path: impl Into) -> bool { - self.0.is_regular_package(db, search_path.into()) + pub(crate) fn is_regular_package( + &self, + db: &dyn Db, + search_path: impl Into, + typeshed_versions: &LazyTypeshedVersions, + ) -> bool { + self.0 + .is_regular_package(db, search_path.into(), typeshed_versions) } #[must_use] - pub(crate) fn to_vfs_file(self, db: &dyn Db, search_path: impl Into) -> Option { - self.0.to_vfs_file(db, search_path.into()) + pub(crate) fn to_vfs_file( + self, + db: &dyn Db, + search_path: impl Into, + typeshed_versions: &LazyTypeshedVersions, + ) -> Option { + self.0 + .to_vfs_file(db, search_path.into(), typeshed_versions) } #[must_use] @@ -1000,7 +1026,7 @@ mod tests { }); } - fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { + fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { let TestCase { db, custom_typeshed, @@ -1008,99 +1034,115 @@ mod tests { } = create_resolver_builder().unwrap().build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); - (db, stdlib_module_path) + (db, stdlib_module_path, LazyTypeshedVersions::new()) } #[test] fn mocked_typeshed_existing_regular_stdlib_pkg_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let asyncio_regular_package = stdlib_path.join("asyncio"); - assert!(asyncio_regular_package.is_directory(&db, &stdlib_path)); - assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_path)); + assert!(asyncio_regular_package.is_directory(&db, &stdlib_path, &versions)); + assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_path, &versions)); // Paths to directories don't resolve to VfsFiles - assert_eq!(asyncio_regular_package.to_vfs_file(&db, &stdlib_path), None); + assert_eq!( + asyncio_regular_package.to_vfs_file(&db, &stdlib_path, &versions), + None + ); assert!(asyncio_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path) + .to_vfs_file(&db, &stdlib_path, &versions) .is_some()); // The `asyncio` package exists on Python 3.8, but the `asyncio.tasks` submodule does not, // according to the `VERSIONS` file in our typeshed mock: let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); - assert_eq!(asyncio_tasks_module.to_vfs_file(&db, &stdlib_path), None); - assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path)); - assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path)); + assert_eq!( + asyncio_tasks_module.to_vfs_file(&db, &stdlib_path, &versions), + None + ); + assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path, &versions)); + assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_existing_namespace_stdlib_pkg_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let xml_namespace_package = stdlib_path.join("xml"); - assert!(xml_namespace_package.is_directory(&db, &stdlib_path)); + assert!(xml_namespace_package.is_directory(&db, &stdlib_path, &versions)); // Paths to directories don't resolve to VfsFiles - assert_eq!(xml_namespace_package.to_vfs_file(&db, &stdlib_path), None); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); + assert_eq!( + xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + None + ); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); let xml_etree = stdlib_path.join("xml/etree.pyi"); - assert!(!xml_etree.is_directory(&db, &stdlib_path)); - assert!(xml_etree.to_vfs_file(&db, &stdlib_path).is_some()); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path)); + assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions)); + assert!(xml_etree + .to_vfs_file(&db, &stdlib_path, &versions) + .is_some()); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_single_file_stdlib_module_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let functools_module = stdlib_path.join("functools.pyi"); - assert!(functools_module.to_vfs_file(&db, &stdlib_path).is_some()); - assert!(!functools_module.is_directory(&db, &stdlib_path)); - assert!(!functools_module.is_regular_package(&db, &stdlib_path)); + assert!(functools_module + .to_vfs_file(&db, &stdlib_path, &versions) + .is_some()); + assert!(!functools_module.is_directory(&db, &stdlib_path, &versions)); + assert!(!functools_module.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_nonexistent_regular_stdlib_pkg_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let collections_regular_package = stdlib_path.join("collections"); assert_eq!( - collections_regular_package.to_vfs_file(&db, &stdlib_path), + collections_regular_package.to_vfs_file(&db, &stdlib_path, &versions), None ); - assert!(!collections_regular_package.is_directory(&db, &stdlib_path)); - assert!(!collections_regular_package.is_regular_package(&db, &stdlib_path)); + assert!(!collections_regular_package.is_directory(&db, &stdlib_path, &versions)); + assert!(!collections_regular_package.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let importlib_namespace_package = stdlib_path.join("importlib"); assert_eq!( - importlib_namespace_package.to_vfs_file(&db, &stdlib_path), + importlib_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), None ); - assert!(!importlib_namespace_package.is_directory(&db, &stdlib_path)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); + assert!(!importlib_namespace_package.is_directory(&db, &stdlib_path, &versions)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); let importlib_abc = stdlib_path.join("importlib/abc.pyi"); - assert_eq!(importlib_abc.to_vfs_file(&db, &stdlib_path), None); - assert!(!importlib_abc.is_directory(&db, &stdlib_path)); - assert!(!importlib_abc.is_regular_package(&db, &stdlib_path)); + assert_eq!( + importlib_abc.to_vfs_file(&db, &stdlib_path, &versions), + None + ); + assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions)); + assert!(!importlib_abc.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_nonexistent_single_file_module_py38() { - let (db, stdlib_path) = py38_stdlib_test_case(); + let (db, stdlib_path, versions) = py38_stdlib_test_case(); let non_existent = stdlib_path.join("doesnt_even_exist"); - assert_eq!(non_existent.to_vfs_file(&db, &stdlib_path), None); - assert!(!non_existent.is_directory(&db, &stdlib_path)); - assert!(!non_existent.is_regular_package(&db, &stdlib_path)); + assert_eq!(non_existent.to_vfs_file(&db, &stdlib_path, &versions), None); + assert!(!non_existent.is_directory(&db, &stdlib_path, &versions)); + assert!(!non_existent.is_regular_package(&db, &stdlib_path, &versions)); } - fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { + fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { let TestCase { db, custom_typeshed, @@ -1111,71 +1153,76 @@ mod tests { .build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); - (db, stdlib_module_path) + (db, stdlib_module_path, LazyTypeshedVersions::new()) } #[test] fn mocked_typeshed_existing_regular_stdlib_pkgs_py39() { - let (db, stdlib_path) = py39_stdlib_test_case(); + let (db, stdlib_path, versions) = py39_stdlib_test_case(); // Since we've set the target version to Py39, // `collections` should now exist as a directory, according to VERSIONS... let collections_regular_package = stdlib_path.join("collections"); - assert!(collections_regular_package.is_directory(&db, &stdlib_path)); - assert!(collections_regular_package.is_regular_package(&db, &stdlib_path)); + assert!(collections_regular_package.is_directory(&db, &stdlib_path, &versions)); + assert!(collections_regular_package.is_regular_package(&db, &stdlib_path, &versions)); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - collections_regular_package.to_vfs_file(&db, &stdlib_path), + collections_regular_package.to_vfs_file(&db, &stdlib_path, &versions), None ); assert!(collections_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path) + .to_vfs_file(&db, &stdlib_path, &versions) .is_some()); // ...and so should the `asyncio.tasks` submodule (though it's still not a directory): let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); assert!(asyncio_tasks_module - .to_vfs_file(&db, &stdlib_path) + .to_vfs_file(&db, &stdlib_path, &versions) .is_some()); - assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path)); - assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path)); + assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path, &versions)); + assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path, &versions)); } #[test] fn mocked_typeshed_existing_namespace_stdlib_pkg_py39() { - let (db, stdlib_path) = py39_stdlib_test_case(); + let (db, stdlib_path, versions) = py39_stdlib_test_case(); // The `importlib` directory now also exists... let importlib_namespace_package = stdlib_path.join("importlib"); - assert!(importlib_namespace_package.is_directory(&db, &stdlib_path)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path)); + assert!(importlib_namespace_package.is_directory(&db, &stdlib_path, &versions)); + assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - importlib_namespace_package.to_vfs_file(&db, &stdlib_path), + importlib_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), None ); // ...As do submodules in the `importlib` namespace package: let importlib_abc = importlib_namespace_package.join("abc.pyi"); - assert!(!importlib_abc.is_directory(&db, &stdlib_path)); - assert!(!importlib_abc.is_regular_package(&db, &stdlib_path)); - assert!(importlib_abc.to_vfs_file(&db, &stdlib_path).is_some()); + assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions)); + assert!(!importlib_abc.is_regular_package(&db, &stdlib_path, &versions)); + assert!(importlib_abc + .to_vfs_file(&db, &stdlib_path, &versions) + .is_some()); } #[test] fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py39() { - let (db, stdlib_path) = py39_stdlib_test_case(); + let (db, stdlib_path, versions) = py39_stdlib_test_case(); // The `xml` package no longer exists on py39: let xml_namespace_package = stdlib_path.join("xml"); - assert_eq!(xml_namespace_package.to_vfs_file(&db, &stdlib_path), None); - assert!(!xml_namespace_package.is_directory(&db, &stdlib_path)); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path)); + assert_eq!( + xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + None + ); + assert!(!xml_namespace_package.is_directory(&db, &stdlib_path, &versions)); + assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); let xml_etree = xml_namespace_package.join("etree.pyi"); - assert_eq!(xml_etree.to_vfs_file(&db, &stdlib_path), None); - assert!(!xml_etree.is_directory(&db, &stdlib_path)); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path)); + assert_eq!(xml_etree.to_vfs_file(&db, &stdlib_path, &versions), None); + assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions)); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions)); } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 77e86b12b85cd..a585d8c2a1bd9 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -10,6 +10,7 @@ use crate::module_name::ModuleName; use crate::path::ModuleResolutionPathBuf; use crate::resolver::internal::ModuleResolverSearchPaths; use crate::supported_py_version::{set_target_py_version, SupportedPyVersion}; +use crate::typeshed::LazyTypeshedVersions; /// Configures the module resolver settings. /// @@ -226,19 +227,20 @@ fn resolve_name( name: &ModuleName, ) -> Option<(Arc, VfsFile, ModuleKind)> { let search_paths = module_search_paths(db); + let typeshed_versions = LazyTypeshedVersions::new(); for search_path in search_paths { let mut components = name.components(); let module_name = components.next_back()?; - match resolve_package(db, search_path, components) { + match resolve_package(db, search_path, components, &typeshed_versions) { Ok(resolved_package) => { let mut package_path = resolved_package.path; package_path.push(module_name); // Must be a `__init__.pyi` or `__init__.py` or it isn't a package. - let kind = if package_path.is_directory(db, search_path) { + let kind = if package_path.is_directory(db, search_path, &typeshed_versions) { package_path.push("__init__"); ModuleKind::Package } else { @@ -246,16 +248,17 @@ fn resolve_name( }; // TODO Implement full https://peps.python.org/pep-0561/#type-checker-module-resolution-order resolution - if let Some(stub) = package_path - .with_pyi_extension() - .to_vfs_file(db, search_path) - { + if let Some(stub) = package_path.with_pyi_extension().to_vfs_file( + db, + search_path, + &typeshed_versions, + ) { return Some((search_path.clone(), stub, kind)); } if let Some(module) = package_path .with_py_extension() - .and_then(|path| path.to_vfs_file(db, search_path)) + .and_then(|path| path.to_vfs_file(db, search_path, &typeshed_versions)) { return Some((search_path.clone(), module, kind)); } @@ -282,6 +285,7 @@ fn resolve_package<'a, I>( db: &dyn Db, module_search_path: &ModuleResolutionPathBuf, components: I, + typeshed_versions: &LazyTypeshedVersions, ) -> Result where I: Iterator, @@ -300,11 +304,12 @@ where for folder in components { package_path.push(folder); - let is_regular_package = package_path.is_regular_package(db, module_search_path); + let is_regular_package = + package_path.is_regular_package(db, module_search_path, typeshed_versions); if is_regular_package { in_namespace_package = false; - } else if package_path.is_directory(db, module_search_path) { + } else if package_path.is_directory(db, module_search_path, typeshed_versions) { // A directory without an `__init__.py` is a namespace package, continue with the next folder. in_namespace_package = true; } else if in_namespace_package { diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index 5fde8297f00a3..c8a36b46260c8 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -1,6 +1,8 @@ mod versions; -pub(crate) use versions::{parse_typeshed_versions, TypeshedVersions, TypeshedVersionsQueryResult}; +pub(crate) use versions::{ + parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsQueryResult, +}; pub use versions::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; #[cfg(test)] diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index b89ce6e62bed1..63613d4a39171 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -3,15 +3,56 @@ use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; +use std::sync::Arc; -use ruff_db::{source::source_text, vfs::VfsFile}; +use once_cell::sync::OnceCell; + +use ruff_db::file_system::FileSystemPath; +use ruff_db::source::source_text; +use ruff_db::vfs::{system_path_to_file, VfsFile}; use rustc_hash::FxHashMap; use crate::db::Db; use crate::module_name::ModuleName; -use crate::supported_py_version::SupportedPyVersion; +use crate::supported_py_version::{get_target_py_version, SupportedPyVersion}; + +pub(crate) struct LazyTypeshedVersions(OnceCell); + +impl LazyTypeshedVersions { + pub(crate) fn new() -> Self { + Self(OnceCell::new()) + } -#[salsa::tracked(return_ref)] + #[must_use] + pub(crate) fn query_module( + &self, + module: &ModuleName, + db: &dyn Db, + stdlib_root: &FileSystemPath, + ) -> TypeshedVersionsQueryResult { + let versions = self.0.get_or_init(|| { + let versions_path = stdlib_root.join("VERSIONS"); + let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { + todo!( + "Still need to figure out how to handle VERSIONS files being deleted \ + from custom typeshed directories! Expected a file to exist at {versions_path:?}" + ) + }; + // TODO(Alex/Micha): If VERSIONS is invalid, + // this should invalidate not just the specific module resolution we're currently attempting, + // but all type inference that depends on any standard-library types. + // Unwrapping here is not correct... + parse_typeshed_versions(db, versions_file) + .as_ref() + .unwrap() + .clone() + }); + let target_version = PyVersion::from(get_target_py_version(db)); + versions.query_module(module, target_version) + } +} + +#[salsa::tracked] pub(crate) fn parse_typeshed_versions( db: &dyn Db, versions_file: VfsFile, @@ -20,7 +61,7 @@ pub(crate) fn parse_typeshed_versions( file_content.parse() } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct TypeshedVersionsParseError { line_number: Option, reason: TypeshedVersionsParseErrorKind, @@ -53,7 +94,7 @@ impl std::error::Error for TypeshedVersionsParseError { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum TypeshedVersionsParseErrorKind { TooManyLines(NonZeroUsize), UnexpectedNumberOfColons, @@ -98,8 +139,8 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } } -#[derive(Debug, PartialEq, Eq)] -pub(crate) struct TypeshedVersions(FxHashMap); +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TypeshedVersions(Arc>); impl TypeshedVersions { #[must_use] @@ -120,16 +161,14 @@ impl TypeshedVersions { self.0.len() } - // The only public API for this struct: #[must_use] - pub(crate) fn query_module( + fn query_module( &self, module: &ModuleName, - version: impl Into, + target_version: PyVersion, ) -> TypeshedVersionsQueryResult { - let version = version.into(); if let Some(range) = self.exact(module) { - if range.contains(version) { + if range.contains(target_version) { TypeshedVersionsQueryResult::Exists } else { TypeshedVersionsQueryResult::DoesNotExist @@ -139,7 +178,7 @@ impl TypeshedVersions { while let Some(module_to_try) = module { if let Some(range) = self.exact(&module_to_try) { return { - if range.contains(version) { + if range.contains(target_version) { TypeshedVersionsQueryResult::MaybeExists } else { TypeshedVersionsQueryResult::DoesNotExist @@ -219,7 +258,7 @@ impl FromStr for TypeshedVersions { reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile, }) } else { - Ok(Self(map)) + Ok(Self(Arc::new(map))) } } } @@ -378,27 +417,27 @@ mod tests { assert!(versions.contains_exact(&asyncio)); assert_eq!( - versions.query_module(&asyncio, SupportedPyVersion::Py310), + versions.query_module(&asyncio, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::Exists ); assert!(versions.contains_exact(&asyncio_staggered)); assert_eq!( - versions.query_module(&asyncio_staggered, SupportedPyVersion::Py38), + versions.query_module(&asyncio_staggered, SupportedPyVersion::Py38.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - versions.query_module(&asyncio_staggered, SupportedPyVersion::Py37), + versions.query_module(&asyncio_staggered, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert!(versions.contains_exact(&audioop)); assert_eq!( - versions.query_module(&audioop, SupportedPyVersion::Py312), + versions.query_module(&audioop, SupportedPyVersion::Py312.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - versions.query_module(&audioop, SupportedPyVersion::Py313), + versions.query_module(&audioop, SupportedPyVersion::Py313.into()), TypeshedVersionsQueryResult::DoesNotExist ); } @@ -490,67 +529,67 @@ foo: 3.8- # trailing comment assert!(parsed_versions.contains_exact(&foo)); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py37), + parsed_versions.query_module(&foo, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py38), + parsed_versions.query_module(&foo, SupportedPyVersion::Py38.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py311), + parsed_versions.query_module(&foo, SupportedPyVersion::Py311.into()), TypeshedVersionsQueryResult::Exists ); assert!(parsed_versions.contains_exact(&bar)); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py37), + parsed_versions.query_module(&bar, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py310), + parsed_versions.query_module(&bar, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py311), + parsed_versions.query_module(&bar, SupportedPyVersion::Py311.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert!(parsed_versions.contains_exact(&bar_baz)); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py37), + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py39), + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py39.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py310), + parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert!(!parsed_versions.contains_exact(&spam)); assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py37), + parsed_versions.query_module(&spam, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py313), + parsed_versions.query_module(&spam, SupportedPyVersion::Py313.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert!(!parsed_versions.contains_exact(&bar_eggs)); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py37), + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py37.into()), TypeshedVersionsQueryResult::MaybeExists ); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py310), + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::MaybeExists ); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py311), + parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py311.into()), TypeshedVersionsQueryResult::DoesNotExist ); } From 8dda2e36c4d20e071cf0e56a86d49c55e831e5a1 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:10:31 +0100 Subject: [PATCH 33/58] Move test-only methods into `test` submodules --- crates/red_knot_module_resolver/src/path.rs | 84 +++++++++---------- .../src/typeshed/versions.rs | 25 +++--- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 32a9d80c727e9..ab8c309dcd5db 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -139,12 +139,6 @@ impl ModuleResolutionPathBuf { ModuleResolutionPathRef::from(self).with_py_extension() } - #[cfg(test)] - #[must_use] - pub(crate) fn join(&self, component: &str) -> Self { - Self(ModuleResolutionPathRefInner::from(&self.0).join(component)) - } - #[must_use] pub(crate) fn relativize_path<'a>( &'a self, @@ -366,32 +360,6 @@ impl<'a> ModuleResolutionPathRefInner<'a> { )), } } - - #[cfg(test)] - #[must_use] - fn to_path_buf(self) -> ModuleResolutionPathBufInner { - match self { - Self::Extra(path) => ModuleResolutionPathBufInner::Extra(path.to_path_buf()), - Self::FirstParty(path) => ModuleResolutionPathBufInner::FirstParty(path.to_path_buf()), - Self::StandardLibrary(path) => { - ModuleResolutionPathBufInner::StandardLibrary(path.to_path_buf()) - } - Self::SitePackages(path) => { - ModuleResolutionPathBufInner::SitePackages(path.to_path_buf()) - } - } - } - - #[cfg(test)] - #[must_use] - fn join( - &self, - component: &'a (impl AsRef + ?Sized), - ) -> ModuleResolutionPathBufInner { - let mut result = self.to_path_buf(); - result.push(component.as_ref().as_str()); - result - } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -501,16 +469,6 @@ impl<'a> ModuleResolutionPathRef<'a> { .and_then(Self::site_packages), } } - - #[cfg(test)] - pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { - ModuleResolutionPathBuf(self.0.to_path_buf()) - } - - #[cfg(test)] - pub(crate) const fn is_stdlib_search_path(&self) -> bool { - matches!(&self.0, ModuleResolutionPathRefInner::StandardLibrary(_)) - } } impl<'a> From<&'a ModuleResolutionPathBufInner> for ModuleResolutionPathRefInner<'a> { @@ -585,6 +543,48 @@ mod tests { // Replace windows paths static WINDOWS_PATH_FILTER: [(&str, &str); 1] = [(r"\\\\", "/")]; + impl ModuleResolutionPathBuf { + #[must_use] + pub(crate) fn join(&self, component: &str) -> Self { + ModuleResolutionPathRef::from(self).join(component) + } + } + + impl<'a> ModuleResolutionPathRef<'a> { + #[must_use] + fn join( + &self, + component: &'a (impl AsRef + ?Sized), + ) -> ModuleResolutionPathBuf { + let mut result = self.to_path_buf(); + result.push(component.as_ref().as_str()); + result + } + + #[must_use] + pub(crate) fn to_path_buf(self) -> ModuleResolutionPathBuf { + let inner = match self.0 { + ModuleResolutionPathRefInner::Extra(path) => { + ModuleResolutionPathBufInner::Extra(path.to_path_buf()) + } + ModuleResolutionPathRefInner::FirstParty(path) => { + ModuleResolutionPathBufInner::FirstParty(path.to_path_buf()) + } + ModuleResolutionPathRefInner::StandardLibrary(path) => { + ModuleResolutionPathBufInner::StandardLibrary(path.to_path_buf()) + } + ModuleResolutionPathRefInner::SitePackages(path) => { + ModuleResolutionPathBufInner::SitePackages(path.to_path_buf()) + } + }; + ModuleResolutionPathBuf(inner) + } + + pub(crate) const fn is_stdlib_search_path(&self) -> bool { + matches!(&self.0, ModuleResolutionPathRefInner::StandardLibrary(_)) + } + } + #[test] fn constructor_rejects_non_pyi_stdlib_paths() { assert_eq!(ModuleResolutionPathBuf::standard_library("foo.py"), None); diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 63613d4a39171..b8c9896f8e22e 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -148,19 +148,6 @@ impl TypeshedVersions { self.0.get(module_name) } - /// Helper functions for testing purposes - #[cfg(test)] - #[must_use] - fn contains_exact(&self, module: &ModuleName) -> bool { - self.exact(module).is_some() - } - - #[cfg(test)] - #[must_use] - fn len(&self) -> usize { - self.0.len() - } - #[must_use] fn query_module( &self, @@ -400,6 +387,18 @@ mod tests { #[allow(unsafe_code)] const ONE: Option = Some(unsafe { NonZeroU16::new_unchecked(1) }); + impl TypeshedVersions { + #[must_use] + fn contains_exact(&self, module: &ModuleName) -> bool { + self.exact(module).is_some() + } + + #[must_use] + fn len(&self) -> usize { + self.0.len() + } + } + #[test] fn can_parse_vendored_versions_file() { let versions_data = include_str!(concat!( From 7efd84c8fdc7fd1b3164e21c75a14ae929fb98d7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:12:58 +0100 Subject: [PATCH 34/58] Elide some lifetimes --- crates/red_knot_module_resolver/src/path.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index ab8c309dcd5db..7c5d9607450a4 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -495,7 +495,7 @@ impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { } } -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { +impl PartialEq for ModuleResolutionPathRef<'_> { fn eq(&self, other: &ModuleResolutionPathBuf) -> bool { match (self.0, &other.0) { ( @@ -519,14 +519,14 @@ impl<'a> PartialEq for ModuleResolutionPathRef<'a> { } } -impl<'a> PartialEq for ModuleResolutionPathRef<'a> { +impl PartialEq for ModuleResolutionPathRef<'_> { fn eq(&self, other: &FileSystemPath) -> bool { self.0.as_file_system_path() == other } } -impl<'a> PartialEq> for FileSystemPath { - fn eq(&self, other: &ModuleResolutionPathRef<'a>) -> bool { +impl PartialEq> for FileSystemPath { + fn eq(&self, other: &ModuleResolutionPathRef) -> bool { self == other.0.as_file_system_path() } } From 553ce678537e184d740180ed5bb6f8125795ba27 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:16:01 +0100 Subject: [PATCH 35/58] Delete and streamline some trait implementations --- crates/red_knot_module_resolver/src/path.rs | 53 +++------------------ 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 7c5d9607450a4..5da3c04409889 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,7 +1,7 @@ use std::fmt; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; -use ruff_db::vfs::{system_path_to_file, VfsFile, VfsPath}; +use ruff_db::vfs::{system_path_to_file, VfsFile}; use crate::db::Db; use crate::module_name::ModuleName; @@ -158,17 +158,6 @@ impl ModuleResolutionPathBuf { } } -impl From for VfsPath { - fn from(value: ModuleResolutionPathBuf) -> Self { - VfsPath::FileSystem(match value.0 { - ModuleResolutionPathBufInner::Extra(path) => path, - ModuleResolutionPathBufInner::FirstParty(path) => path, - ModuleResolutionPathBufInner::StandardLibrary(path) => path, - ModuleResolutionPathBufInner::SitePackages(path) => path, - }) - } -} - impl fmt::Debug for ModuleResolutionPathBuf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let (name, path) = match &self.0 { @@ -471,10 +460,9 @@ impl<'a> ModuleResolutionPathRef<'a> { } } -impl<'a> From<&'a ModuleResolutionPathBufInner> for ModuleResolutionPathRefInner<'a> { - #[inline] - fn from(value: &'a ModuleResolutionPathBufInner) -> Self { - match value { +impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { + fn from(value: &'a ModuleResolutionPathBuf) -> Self { + let inner = match &value.0 { ModuleResolutionPathBufInner::Extra(path) => ModuleResolutionPathRefInner::Extra(path), ModuleResolutionPathBufInner::FirstParty(path) => { ModuleResolutionPathRefInner::FirstParty(path) @@ -485,37 +473,8 @@ impl<'a> From<&'a ModuleResolutionPathBufInner> for ModuleResolutionPathRefInner ModuleResolutionPathBufInner::SitePackages(path) => { ModuleResolutionPathRefInner::SitePackages(path) } - } - } -} - -impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { - fn from(value: &'a ModuleResolutionPathBuf) -> Self { - ModuleResolutionPathRef(ModuleResolutionPathRefInner::from(&value.0)) - } -} - -impl PartialEq for ModuleResolutionPathRef<'_> { - fn eq(&self, other: &ModuleResolutionPathBuf) -> bool { - match (self.0, &other.0) { - ( - ModuleResolutionPathRefInner::Extra(self_path), - ModuleResolutionPathBufInner::Extra(other_path), - ) - | ( - ModuleResolutionPathRefInner::FirstParty(self_path), - ModuleResolutionPathBufInner::FirstParty(other_path), - ) - | ( - ModuleResolutionPathRefInner::StandardLibrary(self_path), - ModuleResolutionPathBufInner::StandardLibrary(other_path), - ) - | ( - ModuleResolutionPathRefInner::SitePackages(self_path), - ModuleResolutionPathBufInner::SitePackages(other_path), - ) => *self_path == **other_path, - _ => false, - } + }; + ModuleResolutionPathRef(inner) } } From 9fda47ad45bf0f8e9cde05fb4ee4ecad02096bb5 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:45:09 +0100 Subject: [PATCH 36/58] Remove unnecessary `*`s --- crates/red_knot_module_resolver/src/db.rs | 16 ++-- crates/red_knot_module_resolver/src/path.rs | 12 +++ .../red_knot_module_resolver/src/resolver.rs | 82 +++++++++---------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 0e86b975deeb5..5785fe8cf9e6c 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -26,9 +26,7 @@ pub(crate) mod tests { use salsa::DebugWithDb; - use ruff_db::file_system::{ - FileSystem, FileSystemPath, FileSystemPathBuf, MemoryFileSystem, OsFileSystem, - }; + use ruff_db::file_system::{FileSystem, FileSystemPathBuf, MemoryFileSystem, OsFileSystem}; use ruff_db::vfs::Vfs; use crate::resolver::{set_module_resolution_settings, ModuleResolutionSettings}; @@ -221,15 +219,15 @@ pub(crate) mod tests { let db = TestDb::new(); - let src = FileSystemPath::new("src").to_path_buf(); - let site_packages = FileSystemPath::new("site_packages").to_path_buf(); - let custom_typeshed = FileSystemPath::new("typeshed").to_path_buf(); + let src = FileSystemPathBuf::from("src"); + let site_packages = FileSystemPathBuf::from("site_packages"); + let custom_typeshed = FileSystemPathBuf::from("typeshed"); let fs = db.memory_file_system(); - fs.create_directory_all(&*src)?; - fs.create_directory_all(&*site_packages)?; - fs.create_directory_all(&*custom_typeshed)?; + fs.create_directory_all(&src)?; + fs.create_directory_all(&site_packages)?; + fs.create_directory_all(&custom_typeshed)?; fs.write_file(custom_typeshed.join("stdlib/VERSIONS"), VERSIONS_DATA)?; // Regular package on py38+ diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 5da3c04409889..70672bd7736c4 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -490,6 +490,18 @@ impl PartialEq> for FileSystemPath { } } +impl PartialEq for ModuleResolutionPathRef<'_> { + fn eq(&self, other: &FileSystemPathBuf) -> bool { + self == &**other + } +} + +impl PartialEq> for FileSystemPathBuf { + fn eq(&self, other: &ModuleResolutionPathRef<'_>) -> bool { + &**self == other + } +} + #[cfg(test)] mod tests { use insta::assert_debug_snapshot; diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index a585d8c2a1bd9..c6f32ab0966cb 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -386,7 +386,7 @@ mod tests { let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_path = src.join("foo.py"); db.memory_file_system() - .write_file(&*foo_path, "print('Hello, world!')")?; + .write_file(&foo_path, "print('Hello, world!')")?; let foo_module = resolve_module(&db, foo_module_name.clone()).unwrap(); @@ -396,10 +396,10 @@ mod tests { ); assert_eq!("foo", foo_module.name()); - assert_eq!(*src, foo_module.search_path()); + assert_eq!(&src, &foo_module.search_path()); assert_eq!(ModuleKind::Module, foo_module.kind()); - assert_eq!(*foo_path, *foo_module.file().path(&db)); + assert_eq!(&foo_path, foo_module.file().path(&db)); assert_eq!( Some(foo_module), path_to_module(&db, &VfsPath::FileSystem(foo_path)) @@ -464,8 +464,8 @@ mod tests { }); let search_path = resolved_module.search_path(); assert_eq!( - *custom_typeshed.join("stdlib"), - search_path, + &custom_typeshed.join("stdlib"), + &search_path, "Search path for {module_name} was unexpectedly {search_path:?}" ); assert!( @@ -513,8 +513,8 @@ mod tests { }); let search_path = resolved_module.search_path(); assert_eq!( - *custom_typeshed.join("stdlib"), - search_path, + &custom_typeshed.join("stdlib"), + &search_path, "Search path for {module_name} was unexpectedly {search_path:?}" ); assert!( @@ -547,11 +547,11 @@ mod tests { Some(&functools_module), resolve_module(&db, functools_module_name).as_ref() ); - assert_eq!(*src, functools_module.search_path()); + assert_eq!(&src, &functools_module.search_path()); assert_eq!(ModuleKind::Module, functools_module.kind()); assert_eq!( - *first_party_functools_path, - *functools_module.file().path(&db) + &first_party_functools_path, + functools_module.file().path(&db) ); assert_eq!( @@ -570,13 +570,13 @@ mod tests { let foo_path = foo_dir.join("__init__.py"); db.memory_file_system() - .write_file(&*foo_path, "print('Hello, world!')")?; + .write_file(&foo_path, "print('Hello, world!')")?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); assert_eq!("foo", foo_module.name()); - assert_eq!(*src, foo_module.search_path()); - assert_eq!(*foo_path, *foo_module.file().path(&db)); + assert_eq!(&src, &foo_module.search_path()); + assert_eq!(&foo_path, foo_module.file().path(&db)); assert_eq!( Some(&foo_module), @@ -597,16 +597,16 @@ mod tests { let foo_init = foo_dir.join("__init__.py"); db.memory_file_system() - .write_file(&*foo_init, "print('Hello, world!')")?; + .write_file(&foo_init, "print('Hello, world!')")?; let foo_py = src.join("foo.py"); db.memory_file_system() - .write_file(&*foo_py, "print('Hello, world!')")?; + .write_file(&foo_py, "print('Hello, world!')")?; let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo_module.search_path()); - assert_eq!(*foo_init, *foo_module.file().path(&db)); + assert_eq!(&src, &foo_module.search_path()); + assert_eq!(&foo_init, foo_module.file().path(&db)); assert_eq!(ModuleKind::Package, foo_module.kind()); assert_eq!( @@ -625,12 +625,12 @@ mod tests { let foo_stub = src.join("foo.pyi"); let foo_py = src.join("foo.py"); db.memory_file_system() - .write_files([(&*foo_stub, "x: int"), (&*foo_py, "print('Hello, world!')")])?; + .write_files([(&foo_stub, "x: int"), (&foo_py, "print('Hello, world!')")])?; let foo = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo.search_path()); - assert_eq!(*foo_stub, *foo.file().path(&db)); + assert_eq!(&src, &foo.search_path()); + assert_eq!(&foo_stub, foo.file().path(&db)); assert_eq!( Some(foo), @@ -650,16 +650,16 @@ mod tests { let baz = bar.join("baz.py"); db.memory_file_system().write_files([ - (&*foo.join("__init__.py"), ""), - (&*bar.join("__init__.py"), ""), - (&*baz, "print('Hello, world!')"), + (&foo.join("__init__.py"), ""), + (&bar.join("__init__.py"), ""), + (&baz, "print('Hello, world!')"), ])?; let baz_module = resolve_module(&db, ModuleName::new_static("foo.bar.baz").unwrap()).unwrap(); - assert_eq!(*src, baz_module.search_path()); - assert_eq!(*baz, *baz_module.file().path(&db)); + assert_eq!(&src, &baz_module.search_path()); + assert_eq!(&baz, baz_module.file().path(&db)); assert_eq!( Some(baz_module), @@ -790,8 +790,8 @@ mod tests { let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap(); - assert_eq!(*src, foo_module.search_path()); - assert_eq!(*foo_src, *foo_module.file().path(&db)); + assert_eq!(&src, &foo_module.search_path()); + assert_eq!(&foo_src, foo_module.file().path(&db)); assert_eq!( Some(foo_module), @@ -820,9 +820,9 @@ mod tests { let temp_dir = tempfile::tempdir()?; let root = FileSystemPath::from_std_path(temp_dir.path()).unwrap(); - let src = root.join(&*src); - let site_packages = root.join(&*site_packages); - let custom_typeshed = root.join(&*custom_typeshed); + let src = root.join(src); + let site_packages = root.join(site_packages); + let custom_typeshed = root.join(custom_typeshed); let foo = src.join("foo.py"); let bar = src.join("bar.py"); @@ -853,12 +853,12 @@ mod tests { assert_ne!(foo_module, bar_module); - assert_eq!(*src, foo_module.search_path()); + assert_eq!(&src, &foo_module.search_path()); assert_eq!(&foo, foo_module.file().path(&db)); // `foo` and `bar` shouldn't resolve to the same file - assert_eq!(*src, bar_module.search_path()); + assert_eq!(&src, &bar_module.search_path()); assert_eq!(&bar, bar_module.file().path(&db)); assert_eq!(&foo, foo_module.file().path(&db)); @@ -884,17 +884,17 @@ mod tests { let bar_path = src.join("bar.py"); db.memory_file_system() - .write_files([(&*foo_path, "x = 1"), (&*bar_path, "y = 2")])?; + .write_files([(&foo_path, "x = 1"), (&bar_path, "y = 2")])?; let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_module = resolve_module(&db, foo_module_name.clone()).unwrap(); - let bar = system_path_to_file(&db, &*bar_path).expect("bar.py to exist"); + let bar = system_path_to_file(&db, &bar_path).expect("bar.py to exist"); db.clear_salsa_events(); // Delete `bar.py` - db.memory_file_system().remove_file(&*bar_path)?; + db.memory_file_system().remove_file(&bar_path)?; bar.touch(&mut db); // Re-query the foo module. The foo module should still be cached because `bar.py` isn't relevant @@ -922,9 +922,9 @@ mod tests { assert_eq!(resolve_module(&db, foo_module_name.clone()), None); // Now write the foo file - db.memory_file_system().write_file(&*foo_path, "x = 1")?; + db.memory_file_system().write_file(&foo_path, "x = 1")?; VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_path.clone())); - let foo_file = system_path_to_file(&db, &*foo_path).expect("foo.py to exist"); + let foo_file = system_path_to_file(&db, &foo_path).expect("foo.py to exist"); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); assert_eq!(foo_file, foo_module.file()); @@ -940,21 +940,21 @@ mod tests { let foo_init_path = src.join("foo/__init__.py"); db.memory_file_system() - .write_files([(&*foo_path, "x = 1"), (&*foo_init_path, "x = 2")])?; + .write_files([(&foo_path, "x = 1"), (&foo_init_path, "x = 2")])?; let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_module = resolve_module(&db, foo_module_name.clone()).expect("foo module to exist"); - assert_eq!(*foo_init_path, *foo_module.file().path(&db)); + assert_eq!(&foo_init_path, foo_module.file().path(&db)); // Delete `foo/__init__.py` and the `foo` folder. `foo` should now resolve to `foo.py` - db.memory_file_system().remove_file(&*foo_init_path)?; + db.memory_file_system().remove_file(&foo_init_path)?; db.memory_file_system() .remove_directory(foo_init_path.parent().unwrap())?; VfsFile::touch_path(&mut db, &VfsPath::FileSystem(foo_init_path)); let foo_module = resolve_module(&db, foo_module_name).expect("Foo module to resolve"); - assert_eq!(*foo_path, *foo_module.file().path(&db)); + assert_eq!(&foo_path, foo_module.file().path(&db)); Ok(()) } From b819680f6c9f70f477ec1bc0ebdd40aaf0d0a7bd Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 13:53:49 +0100 Subject: [PATCH 37/58] Make `ModuleResolutionPathBuf::push()` assert invariants on release builds as well --- crates/red_knot_module_resolver/src/path.rs | 40 +++++++++------------ 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 70672bd7736c4..358abbdf693ff 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -24,25 +24,23 @@ enum ModuleResolutionPathBufInner { impl ModuleResolutionPathBufInner { fn push(&mut self, component: &str) { - if cfg!(debug_assertions) { - if let Some(extension) = camino::Utf8Path::new(component).extension() { - match self { - Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( - matches!(extension, "pyi" | "py"), - "Extension must be `py` or `pyi`; got `{extension}`" - ), - Self::StandardLibrary(_) => { - assert!( - matches!(component.matches('.').count(), 0 | 1), - "Component can have at most one '.'; got {component}" - ); - assert_eq!( - extension, "pyi", - "Extension must be `pyi`; got `{extension}`" - ); - } - }; - } + if let Some(extension) = camino::Utf8Path::new(component).extension() { + match self { + Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( + matches!(extension, "pyi" | "py"), + "Extension must be `py` or `pyi`; got `{extension}`" + ), + Self::StandardLibrary(_) => { + assert!( + matches!(component.matches('.').count(), 0 | 1), + "Component can have at most one '.'; got {component}" + ); + assert_eq!( + extension, "pyi", + "Extension must be `pyi`; got `{extension}`" + ); + } + }; } let inner = match self { Self::Extra(ref mut path) => path, @@ -850,21 +848,18 @@ mod tests { } #[test] - #[cfg(debug_assertions)] #[should_panic(expected = "Extension must be `pyi`; got `py`")] fn stdlib_path_invalid_join_py() { stdlib_path_test_case("foo").push("bar.py"); } #[test] - #[cfg(debug_assertions)] #[should_panic(expected = "Extension must be `pyi`; got `rs`")] fn stdlib_path_invalid_join_rs() { stdlib_path_test_case("foo").push("bar.rs"); } #[test] - #[cfg(debug_assertions)] #[should_panic(expected = "Extension must be `py` or `pyi`; got `rs`")] fn non_stdlib_path_invalid_join_rs() { ModuleResolutionPathBuf::site_packages("foo") @@ -873,7 +868,6 @@ mod tests { } #[test] - #[cfg(debug_assertions)] #[should_panic(expected = "Component can have at most one '.'")] fn invalid_stdlib_join_too_many_extensions() { stdlib_path_test_case("foo").push("bar.py.pyi"); From 7eebb7540c8647e93787ff41bac0e15feaa90df0 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:04:01 +0100 Subject: [PATCH 38/58] Reduce nesting in `ModuleName::from_components()` --- .../src/module_name.rs | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/red_knot_module_resolver/src/module_name.rs b/crates/red_knot_module_resolver/src/module_name.rs index 5b901b6d6f29f..8752f5577f5c4 100644 --- a/crates/red_knot_module_resolver/src/module_name.rs +++ b/crates/red_knot_module_resolver/src/module_name.rs @@ -151,24 +151,23 @@ impl ModuleName { if !is_identifier(first_part) { return None; } - Some(Self({ - if let Some(second_part) = components.next() { - if !is_identifier(second_part) { + let name = if let Some(second_part) = components.next() { + if !is_identifier(second_part) { + return None; + } + let mut name = format!("{first_part}.{second_part}"); + for part in components { + if !is_identifier(part) { return None; } - let mut name = format!("{first_part}.{second_part}"); - for part in components { - if !is_identifier(part) { - return None; - } - name.push('.'); - name.push_str(part); - } - CompactString::from(&name) - } else { - CompactString::from(first_part) + name.push('.'); + name.push_str(part); } - })) + CompactString::from(&name) + } else { + CompactString::from(first_part) + }; + Some(Self(name)) } } From 470f0c6f1595d9ee9eec65bbdbe31b6ebc1c494f Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:14:54 +0100 Subject: [PATCH 39/58] Document `TypeshedVersionsQueryResult` variants --- .../src/typeshed/versions.rs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index b8c9896f8e22e..79d51985a9871 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -179,10 +179,52 @@ impl TypeshedVersions { } } +/// Possible answers [`LazyTypeshedVersions::query_module()`] could give to the question: +/// "Does this module exist in the stdlib at runtime on a certain target version?" #[derive(Debug, Copy, PartialEq, Eq, Clone, Hash)] pub(crate) enum TypeshedVersionsQueryResult { + /// The module definitely exists in the stdlib at runtime on the user-specified target version. + /// + /// For example: + /// - The target version is Python 3.8 + /// - We're querying whether the `asyncio.tasks` module exists in the stdlib + /// - The VERSIONS file contains the line `asyncio.tasks: 3.8-` Exists, + + /// The module definitely does not exist in the stdlib on the user-specified target version. + /// + /// For example: + /// - We're querying whether the `foo` module exists in the stdlib + /// - There is no top-level `foo` module in VERSIONS + /// + /// OR: + /// - The target version is Python 3.8 + /// - We're querying whether the module `importlib.abc` exists in the stdlib + /// - The VERSIONS file contains the line `importlib.abc: 3.10-`, + /// indicating that the module was added in 3.10 + /// + /// OR: + /// - The target version is Python 3.8 + /// - We're querying whether the module `collections.abc` exists in the stdlib + /// - The VERSIONS file does not contain any information about the `collections.abc` submodule, + /// but *does* contain the line `collections: 3.10-`, + /// indicating that the entire `collections` package was added in Python 3.10. DoesNotExist, + + /// The module potentially exists in the stdlib and, if it does, + /// it definitely exists on the user-specified target version. + /// + /// This variant is only relevant for submodules, + /// for which the typeshed VERSIONS file does not provide comprehensive information. + /// (The VERSIONS file is guaranteed to provide information about all top-level stdlib modules and packages, + /// but not necessarily about all submodules within each top-level package.) + /// + /// For example: + /// - The target version is Python 3.8 + /// - We're querying whether the `asyncio.staggered` module exists in the stdlib + /// - The typeshed VERSIONS file contains the line `asyncio: 3.8`, + /// indicating that the `asyncio` package was added in Python 3.8, + /// but does not contain any explicit information about the `asyncio.staggered` submodule. MaybeExists, } From d4c11eeb0c680288224c3aa5fbb3d356b6bd92d9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:23:54 +0100 Subject: [PATCH 40/58] Docs for `query_module()` --- .../src/typeshed/versions.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 79d51985a9871..1b2e3ea2193ce 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -19,10 +19,23 @@ use crate::supported_py_version::{get_target_py_version, SupportedPyVersion}; pub(crate) struct LazyTypeshedVersions(OnceCell); impl LazyTypeshedVersions { + #[must_use] pub(crate) fn new() -> Self { Self(OnceCell::new()) } + /// Query whether a module exists at runtime in the stdlib on a certain Python version. + /// + /// Simply probing whether a file exists in typeshed is insufficient for this question, + /// as a module in the stdlib may have been added in Python 3.10, but the typeshed stub + /// will still be available (either in a custom typeshed dir or in our vendored copy) + /// even if the user specified Python 3.8 as the target version. + /// + /// For top-level modules and packages, the VERSIONS file can always provide an unambiguous answer + /// as to whether the module exists on the specified target version. However, VERSIONS does not + /// provide comprehensive information on all submodules, meaning that this method sometimes + /// returns [`TypeshedVersionsQueryResult::MaybeExists`]. + /// See [`TypeshedVersionsQueryResult`] for more details. #[must_use] pub(crate) fn query_module( &self, @@ -35,7 +48,7 @@ impl LazyTypeshedVersions { let Some(versions_file) = system_path_to_file(db.upcast(), &versions_path) else { todo!( "Still need to figure out how to handle VERSIONS files being deleted \ - from custom typeshed directories! Expected a file to exist at {versions_path:?}" + from custom typeshed directories! Expected a file to exist at {versions_path}" ) }; // TODO(Alex/Micha): If VERSIONS is invalid, From b0c49c25a59f9ffbde9ed866a6d2b3c3e10e50f5 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:28:29 +0100 Subject: [PATCH 41/58] Add a `create_resolver_test()` helper --- .../red_knot_module_resolver/src/resolver.rs | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index c6f32ab0966cb..6186bbba5a8a8 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -379,9 +379,13 @@ mod tests { use super::*; + fn setup_resolver_test() -> TestCase { + create_resolver_builder().unwrap().build() + } + #[test] fn first_party_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = setup_resolver_test(); let foo_module_name = ModuleName::new_static("foo").unwrap(); let foo_path = src.join("foo.py"); @@ -409,12 +413,12 @@ mod tests { } #[test] - fn stdlib() -> anyhow::Result<()> { + fn stdlib() { let TestCase { db, custom_typeshed, .. - } = create_resolver_builder()?.build(); + } = setup_resolver_test(); let stdlib_dir = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); @@ -438,8 +442,6 @@ mod tests { Some(functools_module), path_to_module(&db, &expected_functools_path) ); - - Ok(()) } fn create_module_names(raw_names: &[&str]) -> Vec { @@ -455,7 +457,7 @@ mod tests { db, custom_typeshed, .. - } = create_resolver_builder().unwrap().build(); + } = setup_resolver_test(); let existing_modules = create_module_names(&["asyncio", "functools", "xml.etree"]); for module_name in existing_modules { @@ -534,7 +536,7 @@ mod tests { #[test] fn first_party_precedence_over_stdlib() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = setup_resolver_test(); let first_party_functools_path = src.join("functools.py"); db.memory_file_system() @@ -564,7 +566,7 @@ mod tests { #[test] fn resolve_package() -> anyhow::Result<()> { - let TestCase { src, db, .. } = create_resolver_builder()?.build(); + let TestCase { src, db, .. } = setup_resolver_test(); let foo_dir = src.join("foo"); let foo_path = foo_dir.join("__init__.py"); @@ -591,7 +593,7 @@ mod tests { #[test] fn package_priority_over_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = setup_resolver_test(); let foo_dir = src.join("foo"); let foo_init = foo_dir.join("__init__.py"); @@ -620,7 +622,7 @@ mod tests { #[test] fn typing_stub_over_module() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = setup_resolver_test(); let foo_stub = src.join("foo.pyi"); let foo_py = src.join("foo.py"); @@ -643,7 +645,7 @@ mod tests { #[test] fn sub_packages() -> anyhow::Result<()> { - let TestCase { db, src, .. } = create_resolver_builder()?.build(); + let TestCase { db, src, .. } = setup_resolver_test(); let foo = src.join("foo"); let bar = foo.join("bar"); @@ -676,7 +678,7 @@ mod tests { src, site_packages, .. - } = create_resolver_builder()?.build(); + } = setup_resolver_test(); // From [PEP420](https://peps.python.org/pep-0420/#nested-namespace-packages). // But uses `src` for `project1` and `site_packages2` for `project2`. @@ -729,7 +731,7 @@ mod tests { src, site_packages, .. - } = create_resolver_builder()?.build(); + } = setup_resolver_test(); // Adopted test case from the [PEP420 examples](https://peps.python.org/pep-0420/#nested-namespace-packages). // The `src/parent/child` package is a regular package. Therefore, `site_packages/parent/child/two.py` should not be resolved. @@ -780,7 +782,7 @@ mod tests { src, site_packages, .. - } = create_resolver_builder()?.build(); + } = setup_resolver_test(); let foo_src = src.join("foo.py"); let foo_site_packages = site_packages.join("foo.py"); @@ -813,7 +815,7 @@ mod tests { src, site_packages, custom_typeshed, - } = create_resolver_builder()?.build(); + } = setup_resolver_test(); db.with_os_file_system(); @@ -878,7 +880,7 @@ mod tests { #[test] fn deleting_an_unrelated_file_doesnt_change_module_resolution() -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); + let TestCase { mut db, src, .. } = setup_resolver_test(); let foo_path = src.join("foo.py"); let bar_path = src.join("bar.py"); @@ -915,7 +917,7 @@ mod tests { #[test] fn adding_a_file_on_which_the_module_resolution_depends_on_invalidates_the_query( ) -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); + let TestCase { mut db, src, .. } = setup_resolver_test(); let foo_path = src.join("foo.py"); let foo_module_name = ModuleName::new_static("foo").unwrap(); @@ -935,7 +937,7 @@ mod tests { #[test] fn removing_a_file_that_the_module_resolution_depends_on_invalidates_the_query( ) -> anyhow::Result<()> { - let TestCase { mut db, src, .. } = create_resolver_builder()?.build(); + let TestCase { mut db, src, .. } = setup_resolver_test(); let foo_path = src.join("foo.py"); let foo_init_path = src.join("foo/__init__.py"); From 55538bb2ed2a7d509163eeb98b2a4c4f4acb9f04 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:33:57 +0100 Subject: [PATCH 42/58] Fix `Debug` implementations --- crates/red_knot_module_resolver/src/path.rs | 86 +++++++++++---------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 358abbdf693ff..78b33301c82dc 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -164,7 +164,7 @@ impl fmt::Debug for ModuleResolutionPathBuf { ModuleResolutionPathBufInner::SitePackages(path) => ("SitePackages", path), ModuleResolutionPathBufInner::StandardLibrary(path) => ("StandardLibrary", path), }; - f.debug_tuple(&format!("ModuleResolutionPath::{name}")) + f.debug_tuple(&format!("ModuleResolutionPathBuf::{name}")) .field(path) .finish() } @@ -349,7 +349,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct ModuleResolutionPathRef<'a>(ModuleResolutionPathRefInner<'a>); impl<'a> ModuleResolutionPathRef<'a> { @@ -458,6 +458,20 @@ impl<'a> ModuleResolutionPathRef<'a> { } } +impl fmt::Debug for ModuleResolutionPathRef<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (name, path) = match &self.0 { + ModuleResolutionPathRefInner::Extra(path) => ("Extra", path), + ModuleResolutionPathRefInner::FirstParty(path) => ("FirstParty", path), + ModuleResolutionPathRefInner::SitePackages(path) => ("SitePackages", path), + ModuleResolutionPathRefInner::StandardLibrary(path) => ("StandardLibrary", path), + }; + f.debug_tuple(&format!("ModuleResolutionPathRef::{name}")) + .field(path) + .finish() + } +} + impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { fn from(value: &'a ModuleResolutionPathBuf) -> Self { let inner = match &value.0 { @@ -574,7 +588,7 @@ mod tests { #[test] fn stdlib_path_no_extension() { assert_debug_snapshot!(stdlib_path_test_case("foo"), @r###" - ModuleResolutionPath::StandardLibrary( + ModuleResolutionPathBuf::StandardLibrary( "foo", ) "###); @@ -583,7 +597,7 @@ mod tests { #[test] fn stdlib_path_pyi_extension() { assert_debug_snapshot!(stdlib_path_test_case("foo.pyi"), @r###" - ModuleResolutionPath::StandardLibrary( + ModuleResolutionPathBuf::StandardLibrary( "foo.pyi", ) "###); @@ -592,7 +606,7 @@ mod tests { #[test] fn stdlib_path_dunder_init() { assert_debug_snapshot!(stdlib_path_test_case("foo/__init__.pyi"), @r###" - ModuleResolutionPath::StandardLibrary( + ModuleResolutionPathBuf::StandardLibrary( "foo/__init__.pyi", ) "###); @@ -608,7 +622,7 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::standard_library("foo").unwrap().with_pyi_extension(), @r###" - ModuleResolutionPath::StandardLibrary( + ModuleResolutionPathBuf::StandardLibrary( "foo.pyi", ) "### @@ -620,7 +634,7 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::first_party("foo").unwrap().with_py_extension().unwrap(), @r###" - ModuleResolutionPath::FirstParty( + ModuleResolutionPathBuf::FirstParty( "foo.py", ) "### @@ -632,7 +646,7 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::first_party("foo").unwrap().with_pyi_extension(), @r###" - ModuleResolutionPath::FirstParty( + ModuleResolutionPathBuf::FirstParty( "foo.pyi", ) "### @@ -797,10 +811,10 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), @r###" - ModuleResolutionPath::StandardLibrary( - "foo/bar", - ) - "### + ModuleResolutionPathBuf::StandardLibrary( + "foo/bar", + ) + "### ); }); } @@ -811,10 +825,10 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), @r###" - ModuleResolutionPath::SitePackages( - "foo/bar.pyi", - ) - "### + ModuleResolutionPathBuf::SitePackages( + "foo/bar.pyi", + ) + "### ); }); } @@ -825,10 +839,10 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), @r###" - ModuleResolutionPath::Extra( - "foo/bar.py", - ) - "### + ModuleResolutionPathBuf::Extra( + "foo/bar.py", + ) + "### ); }); } @@ -839,10 +853,10 @@ mod tests { assert_debug_snapshot!( ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), @r###" - ModuleResolutionPath::FirstParty( - "foo/bar/baz/eggs/__init__.py", - ) - "### + ModuleResolutionPathBuf::FirstParty( + "foo/bar/baz/eggs/__init__.py", + ) + "### ); }); } @@ -924,10 +938,8 @@ mod tests { .relativize_path("foo/baz/eggs/__init__.pyi") .unwrap(), @r###" - ModuleResolutionPathRef( - StandardLibrary( - "baz/eggs/__init__.pyi", - ), + ModuleResolutionPathRef::StandardLibrary( + "baz/eggs/__init__.pyi", ) "### ); @@ -943,10 +955,8 @@ mod tests { .relativize_path("foo/baz") .unwrap(), @r###" - ModuleResolutionPathRef( - SitePackages( - "baz", - ), + ModuleResolutionPathRef::SitePackages( + "baz", ) "### ); @@ -962,10 +972,8 @@ mod tests { .relativize_path("foo/baz/functools.py") .unwrap(), @r###" - ModuleResolutionPathRef( - Extra( - "functools.py", - ), + ModuleResolutionPathRef::Extra( + "functools.py", ) "### ); @@ -981,10 +989,8 @@ mod tests { .relativize_path("dev/src/package/bar/baz.pyi") .unwrap(), @r###" - ModuleResolutionPathRef( - SitePackages( - "package/bar/baz.pyi", - ), + ModuleResolutionPathRef::SitePackages( + "package/bar/baz.pyi", ) "### ); From a65c4dc410aba55e2a9b536cfd590815b7e06c16 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 14:39:19 +0100 Subject: [PATCH 43/58] Get rid of `stdlib_path_test_case()` helper --- crates/red_knot_module_resolver/src/path.rs | 62 +++++++++++++-------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 78b33301c82dc..ae2e339fc00e2 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -581,40 +581,50 @@ mod tests { ); } - fn stdlib_path_test_case(path: &str) -> ModuleResolutionPathBuf { - ModuleResolutionPathBuf::standard_library(path).unwrap() - } - #[test] fn stdlib_path_no_extension() { - assert_debug_snapshot!(stdlib_path_test_case("foo"), @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo", - ) - "###); + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo").unwrap(), + @r###" + ModuleResolutionPathBuf::StandardLibrary( + "foo", + ) + "### + ); } #[test] fn stdlib_path_pyi_extension() { - assert_debug_snapshot!(stdlib_path_test_case("foo.pyi"), @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo.pyi", - ) - "###); + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo.pyi").unwrap(), + @r###" + ModuleResolutionPathBuf::StandardLibrary( + "foo.pyi", + ) + "### + ); } #[test] fn stdlib_path_dunder_init() { - assert_debug_snapshot!(stdlib_path_test_case("foo/__init__.pyi"), @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo/__init__.pyi", - ) - "###); + assert_debug_snapshot!( + ModuleResolutionPathBuf::standard_library("foo/__init__.pyi").unwrap(), + @r###" + ModuleResolutionPathBuf::StandardLibrary( + "foo/__init__.pyi", + ) + "### + ); } #[test] fn stdlib_paths_can_only_be_pyi() { - assert_eq!(stdlib_path_test_case("foo").with_py_extension(), None); + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .with_py_extension(), + None + ); } #[test] @@ -864,13 +874,17 @@ mod tests { #[test] #[should_panic(expected = "Extension must be `pyi`; got `py`")] fn stdlib_path_invalid_join_py() { - stdlib_path_test_case("foo").push("bar.py"); + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .push("bar.py"); } #[test] #[should_panic(expected = "Extension must be `pyi`; got `rs`")] fn stdlib_path_invalid_join_rs() { - stdlib_path_test_case("foo").push("bar.rs"); + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .push("bar.rs"); } #[test] @@ -884,7 +898,9 @@ mod tests { #[test] #[should_panic(expected = "Component can have at most one '.'")] fn invalid_stdlib_join_too_many_extensions() { - stdlib_path_test_case("foo").push("bar.py.pyi"); + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .push("bar.py.pyi"); } #[test] From 884c57b6aba76836be40e2e79d1f3d4369271053 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 15:05:15 +0100 Subject: [PATCH 44/58] Share more code between some methods in `path.rs` --- crates/red_knot_module_resolver/src/path.rs | 67 ++++++++++----------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index ae2e339fc00e2..ccb2323c8a728 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -180,14 +180,20 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] - #[inline] - fn absolute_path_to_module_name( - absolute_path: &FileSystemPath, - stdlib_root: ModuleResolutionPathRef, - ) -> Option { - stdlib_root - .relativize_path(absolute_path) + fn query_stdlib_version( + module_path: &FileSystemPath, + typeshed_versions: &LazyTypeshedVersions, + stdlib_search_path: ModuleResolutionPathRef<'a>, + stdlib_root: &FileSystemPath, + db: &dyn Db, + ) -> TypeshedVersionsQueryResult { + let Some(module_name) = stdlib_search_path + .relativize_path(module_path) .and_then(ModuleResolutionPathRef::to_module_name) + else { + return TypeshedVersionsQueryResult::DoesNotExist; + }; + typeshed_versions.query_module(&module_name, db, stdlib_root) } #[must_use] @@ -202,15 +208,10 @@ impl<'a> ModuleResolutionPathRefInner<'a> { (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { - return false; - }; - match typeshed_versions.query_module(&module_name, db, stdlib_root) { - TypeshedVersionsQueryResult::Exists - | TypeshedVersionsQueryResult::MaybeExists => { - db.file_system().is_directory(path) - } + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { TypeshedVersionsQueryResult::DoesNotExist => false, + TypeshedVersionsQueryResult::Exists => db.file_system().is_directory(path), + TypeshedVersionsQueryResult::MaybeExists => db.file_system().is_directory(path), } } (path, root) => unreachable!( @@ -226,27 +227,24 @@ impl<'a> ModuleResolutionPathRefInner<'a> { search_path: ModuleResolutionPathRef<'a>, typeshed_versions: &LazyTypeshedVersions, ) -> bool { + fn is_non_stdlib_pkg(path: &FileSystemPath, db: &dyn Db) -> bool { + let file_system = db.file_system(); + file_system.exists(&path.join("__init__.py")) + || file_system.exists(&path.join("__init__.pyi")) + } + match (self, search_path.0) { - (Self::Extra(path), Self::Extra(_)) - | (Self::FirstParty(path), Self::FirstParty(_)) - | (Self::SitePackages(path), Self::SitePackages(_)) => { - let file_system = db.file_system(); - file_system.exists(&path.join("__init__.py")) - || file_system.exists(&path.join("__init__.pyi")) - } + (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(path, db), + (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(path, db), + (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(path, db), // Unlike the other variants: // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - let Some(module_name) = Self::absolute_path_to_module_name(path, search_path) else { - return false; - }; - match typeshed_versions.query_module(&module_name, db, stdlib_root) { - TypeshedVersionsQueryResult::Exists - | TypeshedVersionsQueryResult::MaybeExists => { - db.file_system().exists(&path.join("__init__.pyi")) - } + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { TypeshedVersionsQueryResult::DoesNotExist => false, + TypeshedVersionsQueryResult::Exists => db.file_system().exists(&path.join("__init__.pyi")), + TypeshedVersionsQueryResult::MaybeExists => db.file_system().exists(&path.join("__init__.pyi")), } } (path, root) => unreachable!( @@ -268,13 +266,10 @@ impl<'a> ModuleResolutionPathRefInner<'a> { system_path_to_file(db.upcast(), path) } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - let module_name = Self::absolute_path_to_module_name(path, search_path)?; - match typeshed_versions.query_module(&module_name, db, stdlib_root) { - TypeshedVersionsQueryResult::Exists - | TypeshedVersionsQueryResult::MaybeExists => { - system_path_to_file(db.upcast(), path) - } + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { TypeshedVersionsQueryResult::DoesNotExist => None, + TypeshedVersionsQueryResult::Exists => system_path_to_file(db.upcast(), path), + TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(db.upcast(), path) } } (path, root) => unreachable!( From 13e741e2af61f1d4af860ea29ea920d9ce9bd3f0 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 15:09:50 +0100 Subject: [PATCH 45/58] Split up some tests in `resolver.rs` --- crates/red_knot_module_resolver/src/resolver.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 6186bbba5a8a8..5300dc0495dbb 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -452,7 +452,7 @@ mod tests { } #[test] - fn stdlib_resolution_respects_versions_file_py38() { + fn stdlib_resolution_respects_versions_file_py38_existing_modules() { let TestCase { db, custom_typeshed, @@ -475,7 +475,11 @@ mod tests { "Expected a stdlib search path, but got {search_path:?}" ); } + } + #[test] + fn stdlib_resolution_respects_versions_file_py38_nonexisting_modules() { + let TestCase { db, .. } = setup_resolver_test(); let nonexisting_modules = create_module_names(&[ "collections", "importlib", @@ -492,7 +496,7 @@ mod tests { } #[test] - fn stdlib_resolution_respects_versions_file_py39() { + fn stdlib_resolution_respects_versions_file_py39_existing_modules() { let TestCase { db, custom_typeshed, @@ -524,6 +528,13 @@ mod tests { "Expected a stdlib search path, but got {search_path:?}" ); } + } + #[test] + fn stdlib_resolution_respects_versions_file_py39_nonexisting_modules() { + let TestCase { db, .. } = create_resolver_builder() + .unwrap() + .with_target_version(SupportedPyVersion::Py39) + .build(); let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]); for module_name in nonexisting_modules { From 70cd43b63822dcf9ba1fe7bc71165d1a6a08c5cc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 15:21:06 +0100 Subject: [PATCH 46/58] Reduce diff in `resolver.rs` tests --- crates/red_knot_module_resolver/src/resolver.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 5300dc0495dbb..54137053552ac 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -435,7 +435,6 @@ mod tests { let expected_functools_path = VfsPath::FileSystem(custom_typeshed.join("stdlib/functools.pyi")); - assert_eq!(&expected_functools_path, functools_module.file().path(&db)); assert_eq!( @@ -847,10 +846,6 @@ mod tests { std::fs::write(foo.as_std_path(), "")?; std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?; - let src = src.to_path_buf(); - let site_packages = site_packages.to_path_buf(); - let custom_typeshed = custom_typeshed.to_path_buf(); - let settings = ModuleResolutionSettings { target_version: SupportedPyVersion::Py38, extra_paths: vec![], From 82a99743fce944b47b0c8dc497df8efe5f89e7d6 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 16:18:03 +0100 Subject: [PATCH 47/58] Split up a big test in `versions.rs` --- .../src/typeshed/versions.rs | 76 ++++++++++++------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 1b2e3ea2193ce..1a006b7305672 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -358,7 +358,7 @@ impl fmt::Display for PyVersionRange { } #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub(crate) struct PyVersion { +struct PyVersion { major: u8, minor: u8, } @@ -574,40 +574,52 @@ foo: 3.8- # trailing comment foo: 3.8- "### ); + } - let foo = ModuleName::new_static("foo").unwrap(); + #[test] + fn version_within_range_parsed_correctly() { + let parsed_versions = TypeshedVersions::from_str("bar: 2.7-3.10").unwrap(); let bar = ModuleName::new_static("bar").unwrap(); - let bar_baz = ModuleName::new_static("bar.baz").unwrap(); - let bar_eggs = ModuleName::new_static("bar.eggs").unwrap(); - let spam = ModuleName::new_static("spam").unwrap(); - assert!(parsed_versions.contains_exact(&foo)); + assert!(parsed_versions.contains_exact(&bar)); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py37.into()), - TypeshedVersionsQueryResult::DoesNotExist + parsed_versions.query_module(&bar, SupportedPyVersion::Py37.into()), + TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py38.into()), + parsed_versions.query_module(&bar, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py311.into()), - TypeshedVersionsQueryResult::Exists + parsed_versions.query_module(&bar, SupportedPyVersion::Py311.into()), + TypeshedVersionsQueryResult::DoesNotExist ); + } - assert!(parsed_versions.contains_exact(&bar)); + #[test] + fn version_from_range_parsed_correctly() { + let parsed_versions = TypeshedVersions::from_str("foo: 3.8-").unwrap(); + let foo = ModuleName::new_static("foo").unwrap(); + + assert!(parsed_versions.contains_exact(&foo)); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py37.into()), - TypeshedVersionsQueryResult::Exists + parsed_versions.query_module(&foo, SupportedPyVersion::Py37.into()), + TypeshedVersionsQueryResult::DoesNotExist ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py310.into()), + parsed_versions.query_module(&foo, SupportedPyVersion::Py38.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py311.into()), - TypeshedVersionsQueryResult::DoesNotExist + parsed_versions.query_module(&foo, SupportedPyVersion::Py311.into()), + TypeshedVersionsQueryResult::Exists ); + } + + #[test] + fn explicit_submodule_parsed_correctly() { + let parsed_versions = TypeshedVersions::from_str("bar.baz: 3.1-3.9").unwrap(); + let bar_baz = ModuleName::new_static("bar.baz").unwrap(); assert!(parsed_versions.contains_exact(&bar_baz)); assert_eq!( @@ -622,16 +634,12 @@ foo: 3.8- # trailing comment parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py310.into()), TypeshedVersionsQueryResult::DoesNotExist ); + } - assert!(!parsed_versions.contains_exact(&spam)); - assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py37.into()), - TypeshedVersionsQueryResult::DoesNotExist - ); - assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py313.into()), - TypeshedVersionsQueryResult::DoesNotExist - ); + #[test] + fn implicit_submodule_queried_correctly() { + let parsed_versions = TypeshedVersions::from_str("bar: 2.7-3.10").unwrap(); + let bar_eggs = ModuleName::new_static("bar.eggs").unwrap(); assert!(!parsed_versions.contains_exact(&bar_eggs)); assert_eq!( @@ -648,6 +656,22 @@ foo: 3.8- # trailing comment ); } + #[test] + fn nonexistent_module_queried_correctly() { + let parsed_versions = TypeshedVersions::from_str("eggs: 3.8-").unwrap(); + let spam = ModuleName::new_static("spam").unwrap(); + + assert!(!parsed_versions.contains_exact(&spam)); + assert_eq!( + parsed_versions.query_module(&spam, SupportedPyVersion::Py37.into()), + TypeshedVersionsQueryResult::DoesNotExist + ); + assert_eq!( + parsed_versions.query_module(&spam, SupportedPyVersion::Py313.into()), + TypeshedVersionsQueryResult::DoesNotExist + ); + } + #[test] fn invalid_empty_versions_file() { assert_eq!( From 5f5ad7011335cc5a07ab3cc047d0e471149b9605 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 16:34:30 +0100 Subject: [PATCH 48/58] Simplify tests in `path.rs` --- crates/red_knot_module_resolver/Cargo.toml | 2 +- crates/red_knot_module_resolver/src/path.rs | 459 ++++++-------------- 2 files changed, 126 insertions(+), 335 deletions(-) diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index 121b75d15dee8..a6761665d6116 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -29,7 +29,7 @@ zip = { workspace = true } [dev-dependencies] anyhow = { workspace = true } -insta = { workspace = true, features = ["filters"] } +insta = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index ccb2323c8a728..fce21e8dc04bb 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -518,9 +518,6 @@ mod tests { use super::*; - // Replace windows paths - static WINDOWS_PATH_FILTER: [(&str, &str); 1] = [(r"\\\\", "/")]; - impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn join(&self, component: &str) -> Self { @@ -558,6 +555,7 @@ mod tests { ModuleResolutionPathBuf(inner) } + #[must_use] pub(crate) const fn is_stdlib_search_path(&self) -> bool { matches!(&self.0, ModuleResolutionPathRefInner::StandardLibrary(_)) } @@ -577,293 +575,157 @@ mod tests { } #[test] - fn stdlib_path_no_extension() { - assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo").unwrap(), - @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo", - ) - "### - ); - } - - #[test] - fn stdlib_path_pyi_extension() { + fn path_buf_debug_impl() { assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo.pyi").unwrap(), + ModuleResolutionPathBuf::standard_library("foo/bar.pyi").unwrap(), @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo.pyi", - ) - "### + ModuleResolutionPathBuf::StandardLibrary( + "foo/bar.pyi", + ) + "### ); } #[test] - fn stdlib_path_dunder_init() { + fn path_ref_debug_impl() { assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo/__init__.pyi").unwrap(), + ModuleResolutionPathRef::extra("foo/bar.py").unwrap(), @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo/__init__.pyi", - ) - "### + ModuleResolutionPathRef::Extra( + "foo/bar.py", + ) + "### ); } #[test] - fn stdlib_paths_can_only_be_pyi() { + fn with_extension_methods() { assert_eq!( ModuleResolutionPathBuf::standard_library("foo") .unwrap() .with_py_extension(), None ); - } - #[test] - fn stdlib_path_with_pyi_extension() { - assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo").unwrap().with_pyi_extension(), - @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo.pyi", - ) - "### + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .with_pyi_extension(), + ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( + FileSystemPathBuf::from("foo.pyi") + )) ); - } - #[test] - fn non_stdlib_path_with_py_extension() { - assert_debug_snapshot!( - ModuleResolutionPathBuf::first_party("foo").unwrap().with_py_extension().unwrap(), - @r###" - ModuleResolutionPathBuf::FirstParty( - "foo.py", - ) - "### + assert_eq!( + ModuleResolutionPathBuf::first_party("foo/bar") + .unwrap() + .with_py_extension() + .unwrap(), + ModuleResolutionPathBuf(ModuleResolutionPathBufInner::FirstParty( + FileSystemPathBuf::from("foo/bar.py") + )) ); } #[test] - fn non_stdlib_path_with_pyi_extension() { - assert_debug_snapshot!( - ModuleResolutionPathBuf::first_party("foo").unwrap().with_pyi_extension(), - @r###" - ModuleResolutionPathBuf::FirstParty( - "foo.pyi", - ) - "### + fn module_name_1_part() { + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::Extra(FileSystemPath::new( + "foo" + ))) + .to_module_name(), + ModuleName::new_static("foo") ); - } - - fn non_stdlib_module_name_test_case(path: &str) -> ModuleName { - let variants = [ - ModuleResolutionPathRef::extra, - ModuleResolutionPathRef::first_party, - ModuleResolutionPathRef::site_packages, - ]; - let results: Vec> = variants - .into_iter() - .map(|variant| variant(path).unwrap().to_module_name()) - .collect(); - assert!(results - .iter() - .zip(results.iter().take(1)) - .all(|(this, next)| this == next)); - results.into_iter().next().unwrap().unwrap() - } - - #[test] - fn module_name_1_part_no_extension() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo"), @r###" - ModuleName( - "foo", - ) - "###); - } - - #[test] - fn module_name_one_part_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo.pyi"), @r###" - ModuleName( - "foo", - ) - "###); - } - - #[test] - fn module_name_one_part_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo.py"), @r###" - ModuleName( - "foo", - ) - "###); - } - - #[test] - fn module_name_2_parts_dunder_init_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/__init__.py"), @r###" - ModuleName( - "foo", - ) - "###); - } - - #[test] - fn module_name_2_parts_dunder_init_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/__init__.pyi"), @r###" - ModuleName( - "foo", - ) - "###); - } - - #[test] - fn module_name_2_parts_no_extension() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar"), @r###" - ModuleName( - "foo.bar", - ) - "###); - } - - #[test] - fn module_name_2_parts_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar.pyi"), @r###" - ModuleName( - "foo.bar", - ) - "###); - } - - #[test] - fn module_name_2_parts_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar.py"), @r###" - ModuleName( - "foo.bar", - ) - "###); - } - - #[test] - fn module_name_3_parts_dunder_init_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/__init__.pyi"), @r###" - ModuleName( - "foo.bar", - ) - "###); - } - - #[test] - fn module_name_3_parts_dunder_init_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/__init__.py"), @r###" - ModuleName( - "foo.bar", - ) - "###); - } - #[test] - fn module_name_3_parts_no_extension() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz"), @r###" - ModuleName( - "foo.bar.baz", - ) - "###); - } - - #[test] - fn module_name_3_parts_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz.pyi"), @r###" - ModuleName( - "foo.bar.baz", - ) - "###); - } + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( + FileSystemPath::new("foo.pyi") + )) + .to_module_name(), + ModuleName::new_static("foo") + ); - #[test] - fn module_name_3_parts_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz.py"), @r###" - ModuleName( - "foo.bar.baz", - ) - "###); + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::FirstParty( + FileSystemPath::new("foo/__init__.py") + )) + .to_module_name(), + ModuleName::new_static("foo") + ); } #[test] - fn module_name_4_parts_dunder_init_pyi() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz/__init__.pyi"), @r###" - ModuleName( - "foo.bar.baz", - ) - "###); - } + fn module_name_2_parts() { + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( + FileSystemPath::new("foo/bar") + )) + .to_module_name(), + ModuleName::new_static("foo.bar") + ); - #[test] - fn module_name_4_parts_dunder_init_py() { - assert_debug_snapshot!(non_stdlib_module_name_test_case("foo/bar/baz/__init__.py"), @r###" - ModuleName( - "foo.bar.baz", - ) - "###); - } + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::Extra(FileSystemPath::new( + "foo/bar.pyi" + ))) + .to_module_name(), + ModuleName::new_static("foo.bar") + ); - #[test] - fn join_1() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo").unwrap().join("bar"), - @r###" - ModuleResolutionPathBuf::StandardLibrary( - "foo/bar", - ) - "### - ); - }); + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::SitePackages( + FileSystemPath::new("foo/bar/__init__.pyi") + )) + .to_module_name(), + ModuleName::new_static("foo.bar") + ); } #[test] - fn join_2() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::site_packages("foo").unwrap().join("bar.pyi"), - @r###" - ModuleResolutionPathBuf::SitePackages( - "foo/bar.pyi", - ) - "### - ); - }); - } + fn module_name_3_parts() { + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::SitePackages( + FileSystemPath::new("foo/bar/__init__.pyi") + )) + .to_module_name(), + ModuleName::new_static("foo.bar") + ); - #[test] - fn join_3() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::extra("foo").unwrap().join("bar.py"), - @r###" - ModuleResolutionPathBuf::Extra( - "foo/bar.py", - ) - "### - ); - }); + assert_eq!( + ModuleResolutionPathRef(ModuleResolutionPathRefInner::SitePackages( + FileSystemPath::new("foo/bar/baz") + )) + .to_module_name(), + ModuleName::new_static("foo.bar.baz") + ); } #[test] - fn join_4() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::first_party("foo").unwrap().join("bar/baz/eggs/__init__.py"), - @r###" - ModuleResolutionPathBuf::FirstParty( - "foo/bar/baz/eggs/__init__.py", - ) - "### - ); - }); + fn join() { + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .join("bar"), + ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( + FileSystemPathBuf::from("foo/bar") + )) + ); + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo") + .unwrap() + .join("bar.pyi"), + ModuleResolutionPathBuf(ModuleResolutionPathBufInner::StandardLibrary( + FileSystemPathBuf::from("foo/bar.pyi") + )) + ); + assert_eq!( + ModuleResolutionPathBuf::extra("foo") + .unwrap() + .join("bar.py"), + ModuleResolutionPathBuf(ModuleResolutionPathBufInner::Extra( + FileSystemPathBuf::from("foo/bar.py") + )) + ); } #[test] @@ -913,10 +775,9 @@ mod tests { assert_eq!(root.relativize_path(third_bad_absolute_path), None); } - fn non_stdlib_relativize_tester( - variant: impl FnOnce(&'static str) -> Option, - ) { - let root = variant("foo").unwrap(); + #[test] + fn relativize_non_stdlib_path_errors() { + let root = ModuleResolutionPathBuf::extra("foo/stdlib").unwrap(); // Must have a `.py` extension, a `.pyi` extension, or no extension: let bad_absolute_path = FileSystemPath::new("foo/stdlib/x.rs"); assert_eq!(root.relativize_path(bad_absolute_path), None); @@ -926,86 +787,16 @@ mod tests { } #[test] - fn relativize_site_packages_errors() { - non_stdlib_relativize_tester(ModuleResolutionPathBuf::site_packages); - } - - #[test] - fn relativize_extra_errors() { - non_stdlib_relativize_tester(ModuleResolutionPathBuf::extra); - } - - #[test] - fn relativize_first_party_errors() { - non_stdlib_relativize_tester(ModuleResolutionPathBuf::first_party); - } - - #[test] - fn relativize_stdlib_path() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::standard_library("foo") - .unwrap() - .relativize_path("foo/baz/eggs/__init__.pyi") - .unwrap(), - @r###" - ModuleResolutionPathRef::StandardLibrary( - "baz/eggs/__init__.pyi", - ) - "### - ); - }); - } - - #[test] - fn relativize_site_packages_path() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::site_packages("foo") - .unwrap() - .relativize_path("foo/baz") - .unwrap(), - @r###" - ModuleResolutionPathRef::SitePackages( - "baz", - ) - "### - ); - }); - } - - #[test] - fn relativize_extra_path() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::extra("foo/baz") - .unwrap() - .relativize_path("foo/baz/functools.py") - .unwrap(), - @r###" - ModuleResolutionPathRef::Extra( - "functools.py", - ) - "### - ); - }); - } - - #[test] - fn relativize_first_party_path() { - insta::with_settings!({filters => WINDOWS_PATH_FILTER.to_vec()}, { - assert_debug_snapshot!( - ModuleResolutionPathBuf::site_packages("dev/src") - .unwrap() - .relativize_path("dev/src/package/bar/baz.pyi") - .unwrap(), - @r###" - ModuleResolutionPathRef::SitePackages( - "package/bar/baz.pyi", - ) - "### - ); - }); + fn relativize_path() { + assert_eq!( + ModuleResolutionPathBuf::standard_library("foo/baz") + .unwrap() + .relativize_path("foo/baz/eggs/__init__.pyi") + .unwrap(), + ModuleResolutionPathRef(ModuleResolutionPathRefInner::StandardLibrary( + FileSystemPath::new("eggs/__init__.pyi") + )) + ); } fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { From 640d2fa0cc913e81ae36273c39ce4a37c556a6a0 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 16:55:40 +0100 Subject: [PATCH 49/58] Small cleanup in `path.rs` --- crates/red_knot_module_resolver/src/path.rs | 131 ++++++++------------ 1 file changed, 50 insertions(+), 81 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index fce21e8dc04bb..9515daa133a87 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -123,8 +123,7 @@ impl ModuleResolutionPathBuf { search_path: &Self, typeshed_versions: &LazyTypeshedVersions, ) -> bool { - let as_ref = ModuleResolutionPathRef::from(self); - as_ref.is_directory(db, search_path, typeshed_versions) + ModuleResolutionPathRef::from(self).is_directory(db, search_path, typeshed_versions) } #[must_use] @@ -181,15 +180,15 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn query_stdlib_version( - module_path: &FileSystemPath, + module_path: &'a FileSystemPath, typeshed_versions: &LazyTypeshedVersions, - stdlib_search_path: ModuleResolutionPathRef<'a>, + stdlib_search_path: Self, stdlib_root: &FileSystemPath, db: &dyn Db, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_search_path .relativize_path(module_path) - .and_then(ModuleResolutionPathRef::to_module_name) + .and_then(Self::to_module_name) else { return TypeshedVersionsQueryResult::DoesNotExist; }; @@ -200,10 +199,10 @@ impl<'a> ModuleResolutionPathRefInner<'a> { fn is_directory( &self, db: &dyn Db, - search_path: ModuleResolutionPathRef<'a>, + search_path: Self, typeshed_versions: &LazyTypeshedVersions, ) -> bool { - match (self, search_path.0) { + match (self, search_path) { (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), @@ -224,7 +223,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { fn is_regular_package( &self, db: &dyn Db, - search_path: ModuleResolutionPathRef<'a>, + search_path: Self, typeshed_versions: &LazyTypeshedVersions, ) -> bool { fn is_non_stdlib_pkg(path: &FileSystemPath, db: &dyn Db) -> bool { @@ -233,7 +232,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { || file_system.exists(&path.join("__init__.pyi")) } - match (self, search_path.0) { + match (self, search_path) { (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(path, db), (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(path, db), (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(path, db), @@ -256,10 +255,10 @@ impl<'a> ModuleResolutionPathRefInner<'a> { fn to_vfs_file( self, db: &dyn Db, - search_path: ModuleResolutionPathRef<'a>, + search_path: Self, typeshed_versions: &LazyTypeshedVersions, ) -> Option { - match (self, search_path.0) { + match (self, search_path) { (Self::Extra(path), Self::Extra(_)) => system_path_to_file(db.upcast(), path), (Self::FirstParty(path), Self::FirstParty(_)) => system_path_to_file(db.upcast(), path), (Self::SitePackages(path), Self::SitePackages(_)) => { @@ -279,7 +278,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - pub(crate) fn to_module_name(self) -> Option { + fn to_module_name(self) -> Option { let (fs_path, skip_final_part) = match self { Self::Extra(path) | Self::FirstParty(path) | Self::SitePackages(path) => ( path, @@ -300,17 +299,6 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } - #[must_use] - #[inline] - fn as_file_system_path(self) -> &'a FileSystemPath { - match self { - Self::Extra(path) => path, - Self::FirstParty(path) => path, - Self::StandardLibrary(path) => path, - Self::SitePackages(path) => path, - } - } - #[must_use] fn with_pyi_extension(&self) -> ModuleResolutionPathBufInner { match self { @@ -342,47 +330,38 @@ impl<'a> ModuleResolutionPathRefInner<'a> { )), } } + + #[must_use] + fn relativize_path(&self, absolute_path: &'a FileSystemPath) -> Option { + match self { + Self::Extra(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "py" | "pyi")) + .then_some(Self::Extra(path)) + }), + Self::FirstParty(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self::FirstParty(path)) + }), + Self::StandardLibrary(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| ext == "pyi") + .then_some(Self::StandardLibrary(path)) + }), + Self::SitePackages(root) => absolute_path.strip_prefix(root).ok().and_then(|path| { + path.extension() + .map_or(true, |ext| matches!(ext, "pyi" | "py")) + .then_some(Self::SitePackages(path)) + }), + } + } } #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) struct ModuleResolutionPathRef<'a>(ModuleResolutionPathRefInner<'a>); impl<'a> ModuleResolutionPathRef<'a> { - #[must_use] - pub(crate) fn extra(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - path.extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - .then_some(Self(ModuleResolutionPathRefInner::Extra(path))) - } - - #[must_use] - pub(crate) fn first_party(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - path.extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - .then_some(Self(ModuleResolutionPathRefInner::FirstParty(path))) - } - - #[must_use] - pub(crate) fn standard_library( - path: &'a (impl AsRef + ?Sized), - ) -> Option { - let path = path.as_ref(); - // Unlike other variants, only `.pyi` extensions are permitted - path.extension() - .map_or(true, |ext| ext == "pyi") - .then_some(Self(ModuleResolutionPathRefInner::StandardLibrary(path))) - } - - #[must_use] - pub(crate) fn site_packages(path: &'a (impl AsRef + ?Sized)) -> Option { - let path = path.as_ref(); - path.extension() - .map_or(true, |ext| matches!(ext, "pyi" | "py")) - .then_some(Self(ModuleResolutionPathRefInner::SitePackages(path))) - } - #[must_use] pub(crate) fn is_directory( &self, @@ -391,7 +370,7 @@ impl<'a> ModuleResolutionPathRef<'a> { typeshed_versions: &LazyTypeshedVersions, ) -> bool { self.0 - .is_directory(db, search_path.into(), typeshed_versions) + .is_directory(db, search_path.into().0, typeshed_versions) } #[must_use] @@ -402,7 +381,7 @@ impl<'a> ModuleResolutionPathRef<'a> { typeshed_versions: &LazyTypeshedVersions, ) -> bool { self.0 - .is_regular_package(db, search_path.into(), typeshed_versions) + .is_regular_package(db, search_path.into().0, typeshed_versions) } #[must_use] @@ -413,7 +392,7 @@ impl<'a> ModuleResolutionPathRef<'a> { typeshed_versions: &LazyTypeshedVersions, ) -> Option { self.0 - .to_vfs_file(db, search_path.into(), typeshed_versions) + .to_vfs_file(db, search_path.into().0, typeshed_versions) } #[must_use] @@ -433,23 +412,7 @@ impl<'a> ModuleResolutionPathRef<'a> { #[must_use] pub(crate) fn relativize_path(&self, absolute_path: &'a FileSystemPath) -> Option { - match self.0 { - ModuleResolutionPathRefInner::Extra(root) => { - absolute_path.strip_prefix(root).ok().and_then(Self::extra) - } - ModuleResolutionPathRefInner::FirstParty(root) => absolute_path - .strip_prefix(root) - .ok() - .and_then(Self::first_party), - ModuleResolutionPathRefInner::StandardLibrary(root) => absolute_path - .strip_prefix(root) - .ok() - .and_then(Self::standard_library), - ModuleResolutionPathRefInner::SitePackages(root) => absolute_path - .strip_prefix(root) - .ok() - .and_then(Self::site_packages), - } + self.0.relativize_path(absolute_path).map(Self) } } @@ -487,13 +450,19 @@ impl<'a> From<&'a ModuleResolutionPathBuf> for ModuleResolutionPathRef<'a> { impl PartialEq for ModuleResolutionPathRef<'_> { fn eq(&self, other: &FileSystemPath) -> bool { - self.0.as_file_system_path() == other + let fs_path = match self.0 { + ModuleResolutionPathRefInner::Extra(path) => path, + ModuleResolutionPathRefInner::FirstParty(path) => path, + ModuleResolutionPathRefInner::SitePackages(path) => path, + ModuleResolutionPathRefInner::StandardLibrary(path) => path, + }; + fs_path == other } } impl PartialEq> for FileSystemPath { fn eq(&self, other: &ModuleResolutionPathRef) -> bool { - self == other.0.as_file_system_path() + other == self } } @@ -589,7 +558,7 @@ mod tests { #[test] fn path_ref_debug_impl() { assert_debug_snapshot!( - ModuleResolutionPathRef::extra("foo/bar.py").unwrap(), + ModuleResolutionPathRef(ModuleResolutionPathRefInner::Extra(FileSystemPath::new("foo/bar.py"))), @r###" ModuleResolutionPathRef::Extra( "foo/bar.py", From 4c7a1061a9e11a1b404e1c5cf6c334a2890a73c1 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 17:42:38 +0100 Subject: [PATCH 50/58] Getting the target version is no longer a separate Salsa query --- crates/red_knot/src/main.rs | 4 +- crates/red_knot_module_resolver/src/db.rs | 12 +- crates/red_knot_module_resolver/src/lib.rs | 2 +- crates/red_knot_module_resolver/src/path.rs | 280 ++++++++++++++---- .../red_knot_module_resolver/src/resolver.rs | 104 ++++--- .../src/supported_py_version.rs | 30 +- .../src/typeshed/versions.rs | 62 ++-- .../src/semantic_model.rs | 4 +- crates/red_knot_python_semantic/src/types.rs | 4 +- .../src/types/infer.rs | 4 +- crates/ruff_benchmark/benches/red_knot.rs | 4 +- 11 files changed, 343 insertions(+), 167 deletions(-) diff --git a/crates/red_knot/src/main.rs b/crates/red_knot/src/main.rs index e22178ce64fa6..5b9bccb6dd8dc 100644 --- a/crates/red_knot/src/main.rs +++ b/crates/red_knot/src/main.rs @@ -13,7 +13,7 @@ use red_knot::program::{FileWatcherChange, Program}; use red_knot::watch::FileWatcher; use red_knot::Workspace; use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::{FileSystem, FileSystemPath, OsFileSystem}; use ruff_db::vfs::system_path_to_file; @@ -64,7 +64,7 @@ pub fn main() -> anyhow::Result<()> { workspace_root: workspace_search_path, site_packages: None, custom_typeshed: None, - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, }, ); diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 5785fe8cf9e6c..8bf63a77a6674 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -2,17 +2,15 @@ use ruff_db::Upcast; use crate::resolver::{ file_to_module, - internal::{ModuleNameIngredient, ModuleResolverSearchPaths}, + internal::{ModuleNameIngredient, ModuleResolverSettings}, resolve_module_query, }; -use crate::supported_py_version::TargetPyVersion; use crate::typeshed::parse_typeshed_versions; #[salsa::jar(db=Db)] pub struct Jar( ModuleNameIngredient<'_>, - ModuleResolverSearchPaths, - TargetPyVersion, + ModuleResolverSettings, resolve_module_query, file_to_module, parse_typeshed_versions, @@ -30,7 +28,7 @@ pub(crate) mod tests { use ruff_db::vfs::Vfs; use crate::resolver::{set_module_resolution_settings, ModuleResolutionSettings}; - use crate::supported_py_version::SupportedPyVersion; + use crate::supported_py_version::TargetVersion; use super::*; @@ -162,12 +160,12 @@ pub(crate) mod tests { src: FileSystemPathBuf, custom_typeshed: FileSystemPathBuf, site_packages: FileSystemPathBuf, - target_version: Option, + target_version: Option, } impl TestCaseBuilder { #[must_use] - pub(crate) fn with_target_version(mut self, target_version: SupportedPyVersion) -> Self { + pub(crate) fn with_target_version(mut self, target_version: TargetVersion) -> Self { self.target_version = Some(target_version); self } diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index 11f2a6a4b5b07..98c7e13552a47 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -10,5 +10,5 @@ pub use db::{Db, Jar}; pub use module::{Module, ModuleKind}; pub use module_name::ModuleName; pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; -pub use supported_py_version::SupportedPyVersion; +pub use supported_py_version::TargetVersion; pub use typeshed::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 9515daa133a87..8fb03a482e571 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -6,6 +6,7 @@ use ruff_db::vfs::{system_path_to_file, VfsFile}; use crate::db::Db; use crate::module_name::ModuleName; use crate::typeshed::{LazyTypeshedVersions, TypeshedVersionsQueryResult}; +use crate::TargetVersion; /// Enumeration of the different kinds of search paths type checkers are expected to support. /// @@ -112,8 +113,14 @@ impl ModuleResolutionPathBuf { db: &dyn Db, search_path: &Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { - ModuleResolutionPathRef::from(self).is_regular_package(db, search_path, typeshed_versions) + ModuleResolutionPathRef::from(self).is_regular_package( + db, + search_path, + typeshed_versions, + target_version, + ) } #[must_use] @@ -122,8 +129,14 @@ impl ModuleResolutionPathBuf { db: &dyn Db, search_path: &Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { - ModuleResolutionPathRef::from(self).is_directory(db, search_path, typeshed_versions) + ModuleResolutionPathRef::from(self).is_directory( + db, + search_path, + typeshed_versions, + target_version, + ) } #[must_use] @@ -150,8 +163,14 @@ impl ModuleResolutionPathBuf { db: &dyn Db, search_path: &Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> Option { - ModuleResolutionPathRef::from(self).to_vfs_file(db, search_path, typeshed_versions) + ModuleResolutionPathRef::from(self).to_vfs_file( + db, + search_path, + typeshed_versions, + target_version, + ) } } @@ -185,6 +204,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { stdlib_search_path: Self, stdlib_root: &FileSystemPath, db: &dyn Db, + target_version: TargetVersion, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_search_path .relativize_path(module_path) @@ -192,7 +212,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { else { return TypeshedVersionsQueryResult::DoesNotExist; }; - typeshed_versions.query_module(&module_name, db, stdlib_root) + typeshed_versions.query_module(&module_name, db, stdlib_root, target_version) } #[must_use] @@ -201,13 +221,14 @@ impl<'a> ModuleResolutionPathRefInner<'a> { db: &dyn Db, search_path: Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { match (self, search_path) { (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists => db.file_system().is_directory(path), TypeshedVersionsQueryResult::MaybeExists => db.file_system().is_directory(path), @@ -225,6 +246,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { db: &dyn Db, search_path: Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { fn is_non_stdlib_pkg(path: &FileSystemPath, db: &dyn Db) -> bool { let file_system = db.file_system(); @@ -240,7 +262,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists => db.file_system().exists(&path.join("__init__.pyi")), TypeshedVersionsQueryResult::MaybeExists => db.file_system().exists(&path.join("__init__.pyi")), @@ -257,6 +279,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { db: &dyn Db, search_path: Self, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> Option { match (self, search_path) { (Self::Extra(path), Self::Extra(_)) => system_path_to_file(db.upcast(), path), @@ -265,7 +288,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { system_path_to_file(db.upcast(), path) } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db) { + match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { TypeshedVersionsQueryResult::DoesNotExist => None, TypeshedVersionsQueryResult::Exists => system_path_to_file(db.upcast(), path), TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(db.upcast(), path) @@ -368,9 +391,10 @@ impl<'a> ModuleResolutionPathRef<'a> { db: &dyn Db, search_path: impl Into, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { self.0 - .is_directory(db, search_path.into().0, typeshed_versions) + .is_directory(db, search_path.into().0, typeshed_versions, target_version) } #[must_use] @@ -379,9 +403,10 @@ impl<'a> ModuleResolutionPathRef<'a> { db: &dyn Db, search_path: impl Into, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> bool { self.0 - .is_regular_package(db, search_path.into().0, typeshed_versions) + .is_regular_package(db, search_path.into().0, typeshed_versions, target_version) } #[must_use] @@ -390,9 +415,10 @@ impl<'a> ModuleResolutionPathRef<'a> { db: &dyn Db, search_path: impl Into, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> Option { self.0 - .to_vfs_file(db, search_path.into().0, typeshed_versions) + .to_vfs_file(db, search_path.into().0, typeshed_versions, target_version) } #[must_use] @@ -483,7 +509,7 @@ mod tests { use insta::assert_debug_snapshot; use crate::db::tests::{create_resolver_builder, TestCase, TestDb}; - use crate::supported_py_version::SupportedPyVersion; + use crate::supported_py_version::TargetVersion; use super::*; @@ -784,27 +810,47 @@ mod tests { let (db, stdlib_path, versions) = py38_stdlib_test_case(); let asyncio_regular_package = stdlib_path.join("asyncio"); - assert!(asyncio_regular_package.is_directory(&db, &stdlib_path, &versions)); - assert!(asyncio_regular_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(asyncio_regular_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); + assert!(asyncio_regular_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); // Paths to directories don't resolve to VfsFiles assert_eq!( - asyncio_regular_package.to_vfs_file(&db, &stdlib_path, &versions), + asyncio_regular_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), None ); assert!(asyncio_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) .is_some()); // The `asyncio` package exists on Python 3.8, but the `asyncio.tasks` submodule does not, // according to the `VERSIONS` file in our typeshed mock: let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); assert_eq!( - asyncio_tasks_module.to_vfs_file(&db, &stdlib_path, &versions), + asyncio_tasks_module.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), None ); - assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path, &versions)); - assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!asyncio_tasks_module.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); + assert!(!asyncio_tasks_module.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); } #[test] @@ -812,20 +858,30 @@ mod tests { let (db, stdlib_path, versions) = py38_stdlib_test_case(); let xml_namespace_package = stdlib_path.join("xml"); - assert!(xml_namespace_package.is_directory(&db, &stdlib_path, &versions)); + assert!(xml_namespace_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); // Paths to directories don't resolve to VfsFiles assert_eq!( - xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), None ); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!xml_namespace_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); let xml_etree = stdlib_path.join("xml/etree.pyi"); - assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions)); + assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); assert!(xml_etree - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) .is_some()); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions, TargetVersion::Py38)); } #[test] @@ -834,10 +890,15 @@ mod tests { let functools_module = stdlib_path.join("functools.pyi"); assert!(functools_module - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) .is_some()); - assert!(!functools_module.is_directory(&db, &stdlib_path, &versions)); - assert!(!functools_module.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!functools_module.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); + assert!(!functools_module.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); } #[test] @@ -846,11 +907,26 @@ mod tests { let collections_regular_package = stdlib_path.join("collections"); assert_eq!( - collections_regular_package.to_vfs_file(&db, &stdlib_path, &versions), + collections_regular_package.to_vfs_file( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + ), None ); - assert!(!collections_regular_package.is_directory(&db, &stdlib_path, &versions)); - assert!(!collections_regular_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!collections_regular_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); + assert!(!collections_regular_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); } #[test] @@ -859,19 +935,39 @@ mod tests { let importlib_namespace_package = stdlib_path.join("importlib"); assert_eq!( - importlib_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + importlib_namespace_package.to_vfs_file( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + ), None ); - assert!(!importlib_namespace_package.is_directory(&db, &stdlib_path, &versions)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!importlib_namespace_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); + assert!(!importlib_namespace_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); let importlib_abc = stdlib_path.join("importlib/abc.pyi"); assert_eq!( - importlib_abc.to_vfs_file(&db, &stdlib_path, &versions), + importlib_abc.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), None ); - assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions)); - assert!(!importlib_abc.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); + assert!(!importlib_abc.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); } #[test] @@ -879,9 +975,17 @@ mod tests { let (db, stdlib_path, versions) = py38_stdlib_test_case(); let non_existent = stdlib_path.join("doesnt_even_exist"); - assert_eq!(non_existent.to_vfs_file(&db, &stdlib_path, &versions), None); - assert!(!non_existent.is_directory(&db, &stdlib_path, &versions)); - assert!(!non_existent.is_regular_package(&db, &stdlib_path, &versions)); + assert_eq!( + non_existent.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), + None + ); + assert!(!non_existent.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); + assert!(!non_existent.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py38 + )); } fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { @@ -891,7 +995,7 @@ mod tests { .. } = create_resolver_builder() .unwrap() - .with_target_version(SupportedPyVersion::Py39) + .with_target_version(TargetVersion::Py39) .build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); @@ -905,25 +1009,50 @@ mod tests { // Since we've set the target version to Py39, // `collections` should now exist as a directory, according to VERSIONS... let collections_regular_package = stdlib_path.join("collections"); - assert!(collections_regular_package.is_directory(&db, &stdlib_path, &versions)); - assert!(collections_regular_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(collections_regular_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); + assert!(collections_regular_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - collections_regular_package.to_vfs_file(&db, &stdlib_path, &versions), + collections_regular_package.to_vfs_file( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + ), None ); assert!(collections_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) .is_some()); // ...and so should the `asyncio.tasks` submodule (though it's still not a directory): let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); assert!(asyncio_tasks_module - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) .is_some()); - assert!(!asyncio_tasks_module.is_directory(&db, &stdlib_path, &versions)); - assert!(!asyncio_tasks_module.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!asyncio_tasks_module.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); + assert!(!asyncio_tasks_module.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); } #[test] @@ -932,20 +1061,40 @@ mod tests { // The `importlib` directory now also exists... let importlib_namespace_package = stdlib_path.join("importlib"); - assert!(importlib_namespace_package.is_directory(&db, &stdlib_path, &versions)); - assert!(!importlib_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(importlib_namespace_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); + assert!(!importlib_namespace_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - importlib_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + importlib_namespace_package.to_vfs_file( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + ), None ); // ...As do submodules in the `importlib` namespace package: let importlib_abc = importlib_namespace_package.join("abc.pyi"); - assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions)); - assert!(!importlib_abc.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py39)); + assert!(!importlib_abc.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); assert!(importlib_abc - .to_vfs_file(&db, &stdlib_path, &versions) + .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) .is_some()); } @@ -956,15 +1105,28 @@ mod tests { // The `xml` package no longer exists on py39: let xml_namespace_package = stdlib_path.join("xml"); assert_eq!( - xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions), + xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39), None ); - assert!(!xml_namespace_package.is_directory(&db, &stdlib_path, &versions)); - assert!(!xml_namespace_package.is_regular_package(&db, &stdlib_path, &versions)); + assert!(!xml_namespace_package.is_directory( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); + assert!(!xml_namespace_package.is_regular_package( + &db, + &stdlib_path, + &versions, + TargetVersion::Py39 + )); let xml_etree = xml_namespace_package.join("etree.pyi"); - assert_eq!(xml_etree.to_vfs_file(&db, &stdlib_path, &versions), None); - assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions)); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions)); + assert_eq!( + xml_etree.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39), + None + ); + assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py39)); + assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions, TargetVersion::Py39)); } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 54137053552ac..4532207a75a09 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -8,8 +8,8 @@ use crate::db::Db; use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; use crate::path::ModuleResolutionPathBuf; -use crate::resolver::internal::ModuleResolverSearchPaths; -use crate::supported_py_version::{set_target_py_version, SupportedPyVersion}; +use crate::resolver::internal::ModuleResolverSettings; +use crate::supported_py_version::TargetVersion; use crate::typeshed::LazyTypeshedVersions; /// Configures the module resolver settings. @@ -19,13 +19,12 @@ pub fn set_module_resolution_settings(db: &mut dyn Db, config: ModuleResolutionS // There's no concurrency issue here because we hold a `&mut dyn Db` reference. No other // thread can mutate the `Db` while we're in this call, so using `try_get` to test if // the settings have already been set is safe. - let (target_version, search_paths) = config.into_ordered_search_paths(); - if let Some(existing) = ModuleResolverSearchPaths::try_get(db) { - existing.set_search_paths(db).to(search_paths); + let resolved_settings = config.into_resolved_settings(); + if let Some(existing) = ModuleResolverSettings::try_get(db) { + existing.set_settings(db).to(resolved_settings); } else { - ModuleResolverSearchPaths::new(db, search_paths); + ModuleResolverSettings::new(db, resolved_settings); } - set_target_py_version(db, target_version); } /// Resolves a module name to a module. @@ -83,9 +82,10 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { todo!("VendoredPaths are not yet supported") }; - let search_paths = module_search_paths(db); + let resolver_settings = module_resolver_settings(db); - let relative_path = search_paths + let relative_path = resolver_settings + .search_paths() .iter() .find_map(|root| root.relativize_path(path))?; @@ -115,7 +115,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { #[derive(Eq, PartialEq, Debug)] pub struct ModuleResolutionSettings { /// The target Python version the user has specified - pub target_version: SupportedPyVersion, + pub target_version: TargetVersion, /// List of user-provided paths that should take first priority in the module resolution. /// Examples in other type checkers are mypy's MYPYPATH environment variable, @@ -144,7 +144,7 @@ impl ModuleResolutionSettings { /// Rather than panicking if a path fails to validate, we should display an error message to the user /// and exit the process with a nonzero exit code. /// This validation should probably be done outside of Salsa? - fn into_ordered_search_paths(self) -> (SupportedPyVersion, OrderedSearchPaths) { + fn into_resolved_settings(self) -> ResolvedModuleResolutionSettings { let ModuleResolutionSettings { target_version, extra_paths, @@ -171,10 +171,10 @@ impl ModuleResolutionSettings { paths.push(ModuleResolutionPathBuf::site_packages(site_packages).unwrap()); } - ( + ResolvedModuleResolutionSettings { target_version, - OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()), - ) + search_paths: OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()), + } } } @@ -191,6 +191,22 @@ impl Deref for OrderedSearchPaths { } } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ResolvedModuleResolutionSettings { + search_paths: OrderedSearchPaths, + target_version: TargetVersion, +} + +impl ResolvedModuleResolutionSettings { + pub(crate) fn search_paths(&self) -> &[Arc] { + &self.search_paths + } + + pub(crate) fn target_version(&self) -> TargetVersion { + self.target_version + } +} + // The singleton methods generated by salsa are all `pub` instead of `pub(crate)` which triggers // `unreachable_pub`. Work around this by creating a module and allow `unreachable_pub` for it. // Salsa also generates uses to `_db` variables for `interned` which triggers `clippy::used_underscore_binding`. Suppress that too @@ -198,12 +214,12 @@ impl Deref for OrderedSearchPaths { #[allow(unreachable_pub, clippy::used_underscore_binding)] pub(crate) mod internal { use crate::module_name::ModuleName; - use crate::resolver::OrderedSearchPaths; + use crate::resolver::ResolvedModuleResolutionSettings; #[salsa::input(singleton)] - pub(crate) struct ModuleResolverSearchPaths { + pub(crate) struct ModuleResolverSettings { #[return_ref] - pub(super) search_paths: OrderedSearchPaths, + pub(super) settings: ResolvedModuleResolutionSettings, } /// A thin wrapper around `ModuleName` to make it a Salsa ingredient. @@ -216,8 +232,8 @@ pub(crate) mod internal { } } -fn module_search_paths(db: &dyn Db) -> &[Arc] { - ModuleResolverSearchPaths::get(db).search_paths(db) +fn module_resolver_settings(db: &dyn Db) -> &ResolvedModuleResolutionSettings { + ModuleResolverSettings::get(db).settings(db) } /// Given a module name and a list of search paths in which to lookup modules, @@ -226,21 +242,33 @@ fn resolve_name( db: &dyn Db, name: &ModuleName, ) -> Option<(Arc, VfsFile, ModuleKind)> { - let search_paths = module_search_paths(db); + let resolver_settings = module_resolver_settings(db); + let target_version = resolver_settings.target_version(); let typeshed_versions = LazyTypeshedVersions::new(); - for search_path in search_paths { + for search_path in resolver_settings.search_paths() { let mut components = name.components(); let module_name = components.next_back()?; - match resolve_package(db, search_path, components, &typeshed_versions) { + match resolve_package( + db, + search_path, + components, + &typeshed_versions, + target_version, + ) { Ok(resolved_package) => { let mut package_path = resolved_package.path; package_path.push(module_name); // Must be a `__init__.pyi` or `__init__.py` or it isn't a package. - let kind = if package_path.is_directory(db, search_path, &typeshed_versions) { + let kind = if package_path.is_directory( + db, + search_path, + &typeshed_versions, + target_version, + ) { package_path.push("__init__"); ModuleKind::Package } else { @@ -252,14 +280,14 @@ fn resolve_name( db, search_path, &typeshed_versions, + target_version, ) { return Some((search_path.clone(), stub, kind)); } - if let Some(module) = package_path - .with_py_extension() - .and_then(|path| path.to_vfs_file(db, search_path, &typeshed_versions)) - { + if let Some(module) = package_path.with_py_extension().and_then(|path| { + path.to_vfs_file(db, search_path, &typeshed_versions, target_version) + }) { return Some((search_path.clone(), module, kind)); } @@ -286,6 +314,7 @@ fn resolve_package<'a, I>( module_search_path: &ModuleResolutionPathBuf, components: I, typeshed_versions: &LazyTypeshedVersions, + target_version: TargetVersion, ) -> Result where I: Iterator, @@ -304,12 +333,21 @@ where for folder in components { package_path.push(folder); - let is_regular_package = - package_path.is_regular_package(db, module_search_path, typeshed_versions); + let is_regular_package = package_path.is_regular_package( + db, + module_search_path, + typeshed_versions, + target_version, + ); if is_regular_package { in_namespace_package = false; - } else if package_path.is_directory(db, module_search_path, typeshed_versions) { + } else if package_path.is_directory( + db, + module_search_path, + typeshed_versions, + target_version, + ) { // A directory without an `__init__.py` is a namespace package, continue with the next folder. in_namespace_package = true; } else if in_namespace_package { @@ -502,7 +540,7 @@ mod tests { .. } = create_resolver_builder() .unwrap() - .with_target_version(SupportedPyVersion::Py39) + .with_target_version(TargetVersion::Py39) .build(); let existing_modules = create_module_names(&[ @@ -532,7 +570,7 @@ mod tests { fn stdlib_resolution_respects_versions_file_py39_nonexisting_modules() { let TestCase { db, .. } = create_resolver_builder() .unwrap() - .with_target_version(SupportedPyVersion::Py39) + .with_target_version(TargetVersion::Py39) .build(); let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]); @@ -847,7 +885,7 @@ mod tests { std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?; let settings = ModuleResolutionSettings { - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, extra_paths: vec![], workspace_root: src.clone(), site_packages: Some(site_packages.clone()), diff --git a/crates/red_knot_module_resolver/src/supported_py_version.rs b/crates/red_knot_module_resolver/src/supported_py_version.rs index 96d3946694b61..466aae6b03055 100644 --- a/crates/red_knot_module_resolver/src/supported_py_version.rs +++ b/crates/red_knot_module_resolver/src/supported_py_version.rs @@ -1,13 +1,8 @@ -#![allow(clippy::used_underscore_binding)] // necessary for Salsa inputs -#![allow(unreachable_pub)] -#![allow(clippy::needless_lifetimes)] -#![allow(clippy::clone_on_copy)] - -use crate::db::Db; - -// TODO: unify with the PythonVersion enum in the linter/formatter crates? +/// Enumeration of all supported Python versions +/// +/// TODO: unify with the `PythonVersion` enum in the linter/formatter crates? #[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default)] -pub enum SupportedPyVersion { +pub enum TargetVersion { Py37, #[default] Py38, @@ -17,20 +12,3 @@ pub enum SupportedPyVersion { Py312, Py313, } - -#[salsa::input(singleton)] -pub(crate) struct TargetPyVersion { - pub(crate) target_py_version: SupportedPyVersion, -} - -pub(crate) fn set_target_py_version(db: &mut dyn Db, target_version: SupportedPyVersion) { - if let Some(existing) = TargetPyVersion::try_get(db) { - existing.set_target_py_version(db).to(target_version); - } else { - TargetPyVersion::new(db, target_version); - } -} - -pub(crate) fn get_target_py_version(db: &dyn Db) -> SupportedPyVersion { - TargetPyVersion::get(db).target_py_version(db) -} diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 1a006b7305672..8535e56d28aff 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -14,7 +14,7 @@ use rustc_hash::FxHashMap; use crate::db::Db; use crate::module_name::ModuleName; -use crate::supported_py_version::{get_target_py_version, SupportedPyVersion}; +use crate::supported_py_version::TargetVersion; pub(crate) struct LazyTypeshedVersions(OnceCell); @@ -42,6 +42,7 @@ impl LazyTypeshedVersions { module: &ModuleName, db: &dyn Db, stdlib_root: &FileSystemPath, + target_version: TargetVersion, ) -> TypeshedVersionsQueryResult { let versions = self.0.get_or_init(|| { let versions_path = stdlib_root.join("VERSIONS"); @@ -60,8 +61,7 @@ impl LazyTypeshedVersions { .unwrap() .clone() }); - let target_version = PyVersion::from(get_target_py_version(db)); - versions.query_module(module, target_version) + versions.query_module(module, PyVersion::from(target_version)) } } @@ -402,25 +402,25 @@ impl fmt::Display for PyVersion { } } -impl From for PyVersion { - fn from(value: SupportedPyVersion) -> Self { +impl From for PyVersion { + fn from(value: TargetVersion) -> Self { match value { - SupportedPyVersion::Py37 => PyVersion { major: 3, minor: 7 }, - SupportedPyVersion::Py38 => PyVersion { major: 3, minor: 8 }, - SupportedPyVersion::Py39 => PyVersion { major: 3, minor: 9 }, - SupportedPyVersion::Py310 => PyVersion { + TargetVersion::Py37 => PyVersion { major: 3, minor: 7 }, + TargetVersion::Py38 => PyVersion { major: 3, minor: 8 }, + TargetVersion::Py39 => PyVersion { major: 3, minor: 9 }, + TargetVersion::Py310 => PyVersion { major: 3, minor: 10, }, - SupportedPyVersion::Py311 => PyVersion { + TargetVersion::Py311 => PyVersion { major: 3, minor: 11, }, - SupportedPyVersion::Py312 => PyVersion { + TargetVersion::Py312 => PyVersion { major: 3, minor: 12, }, - SupportedPyVersion::Py313 => PyVersion { + TargetVersion::Py313 => PyVersion { major: 3, minor: 13, }, @@ -471,27 +471,27 @@ mod tests { assert!(versions.contains_exact(&asyncio)); assert_eq!( - versions.query_module(&asyncio, SupportedPyVersion::Py310.into()), + versions.query_module(&asyncio, TargetVersion::Py310.into()), TypeshedVersionsQueryResult::Exists ); assert!(versions.contains_exact(&asyncio_staggered)); assert_eq!( - versions.query_module(&asyncio_staggered, SupportedPyVersion::Py38.into()), + versions.query_module(&asyncio_staggered, TargetVersion::Py38.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - versions.query_module(&asyncio_staggered, SupportedPyVersion::Py37.into()), + versions.query_module(&asyncio_staggered, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert!(versions.contains_exact(&audioop)); assert_eq!( - versions.query_module(&audioop, SupportedPyVersion::Py312.into()), + versions.query_module(&audioop, TargetVersion::Py312.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - versions.query_module(&audioop, SupportedPyVersion::Py313.into()), + versions.query_module(&audioop, TargetVersion::Py313.into()), TypeshedVersionsQueryResult::DoesNotExist ); } @@ -583,15 +583,15 @@ foo: 3.8- # trailing comment assert!(parsed_versions.contains_exact(&bar)); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py37.into()), + parsed_versions.query_module(&bar, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py310.into()), + parsed_versions.query_module(&bar, TargetVersion::Py310.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar, SupportedPyVersion::Py311.into()), + parsed_versions.query_module(&bar, TargetVersion::Py311.into()), TypeshedVersionsQueryResult::DoesNotExist ); } @@ -603,15 +603,15 @@ foo: 3.8- # trailing comment assert!(parsed_versions.contains_exact(&foo)); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py37.into()), + parsed_versions.query_module(&foo, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py38.into()), + parsed_versions.query_module(&foo, TargetVersion::Py38.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&foo, SupportedPyVersion::Py311.into()), + parsed_versions.query_module(&foo, TargetVersion::Py311.into()), TypeshedVersionsQueryResult::Exists ); } @@ -623,15 +623,15 @@ foo: 3.8- # trailing comment assert!(parsed_versions.contains_exact(&bar_baz)); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py37.into()), + parsed_versions.query_module(&bar_baz, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py39.into()), + parsed_versions.query_module(&bar_baz, TargetVersion::Py39.into()), TypeshedVersionsQueryResult::Exists ); assert_eq!( - parsed_versions.query_module(&bar_baz, SupportedPyVersion::Py310.into()), + parsed_versions.query_module(&bar_baz, TargetVersion::Py310.into()), TypeshedVersionsQueryResult::DoesNotExist ); } @@ -643,15 +643,15 @@ foo: 3.8- # trailing comment assert!(!parsed_versions.contains_exact(&bar_eggs)); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py37.into()), + parsed_versions.query_module(&bar_eggs, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::MaybeExists ); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py310.into()), + parsed_versions.query_module(&bar_eggs, TargetVersion::Py310.into()), TypeshedVersionsQueryResult::MaybeExists ); assert_eq!( - parsed_versions.query_module(&bar_eggs, SupportedPyVersion::Py311.into()), + parsed_versions.query_module(&bar_eggs, TargetVersion::Py311.into()), TypeshedVersionsQueryResult::DoesNotExist ); } @@ -663,11 +663,11 @@ foo: 3.8- # trailing comment assert!(!parsed_versions.contains_exact(&spam)); assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py37.into()), + parsed_versions.query_module(&spam, TargetVersion::Py37.into()), TypeshedVersionsQueryResult::DoesNotExist ); assert_eq!( - parsed_versions.query_module(&spam, SupportedPyVersion::Py313.into()), + parsed_versions.query_module(&spam, TargetVersion::Py313.into()), TypeshedVersionsQueryResult::DoesNotExist ); } diff --git a/crates/red_knot_python_semantic/src/semantic_model.rs b/crates/red_knot_python_semantic/src/semantic_model.rs index 62fca8250dad2..440d247994934 100644 --- a/crates/red_knot_python_semantic/src/semantic_model.rs +++ b/crates/red_knot_python_semantic/src/semantic_model.rs @@ -178,7 +178,7 @@ impl HasTy for ast::Alias { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::parsed::parsed_module; @@ -197,7 +197,7 @@ mod tests { workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, custom_typeshed: None, - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, }, ); diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 31d4e3ece9701..7a0fc3e6b963d 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -500,7 +500,7 @@ impl<'db, 'inference> TypingContext<'db, 'inference> { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::parsed::parsed_module; @@ -518,7 +518,7 @@ mod tests { set_module_resolution_settings( &mut db, ModuleResolutionSettings { - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, extra_paths: vec![], workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index fd3cc1b528359..b2711408eb872 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -672,7 +672,7 @@ impl<'db> TypeInferenceBuilder<'db> { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::system_path_to_file; @@ -687,7 +687,7 @@ mod tests { set_module_resolution_settings( &mut db, ModuleResolutionSettings { - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, extra_paths: Vec::new(), workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs index 5db6c937888cf..65f8a56dfca4d 100644 --- a/crates/ruff_benchmark/benches/red_knot.rs +++ b/crates/ruff_benchmark/benches/red_knot.rs @@ -3,7 +3,7 @@ use red_knot::program::Program; use red_knot::Workspace; use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, SupportedPyVersion, + set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, }; use ruff_benchmark::criterion::{ criterion_group, criterion_main, BatchSize, Criterion, Throughput, @@ -77,7 +77,7 @@ fn setup_case() -> Case { workspace_root: workspace_root.to_path_buf(), site_packages: None, custom_typeshed: None, - target_version: SupportedPyVersion::Py38, + target_version: TargetVersion::Py38, }, ); From 777ec7dc3fa556a88ee5f16eedc20debe744ed35 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:04:08 +0100 Subject: [PATCH 51/58] `db` is now always the first argument --- crates/red_knot_module_resolver/src/path.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 8fb03a482e571..22a66d86e6b79 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -199,11 +199,11 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn query_stdlib_version( + db: &dyn Db, module_path: &'a FileSystemPath, typeshed_versions: &LazyTypeshedVersions, stdlib_search_path: Self, stdlib_root: &FileSystemPath, - db: &dyn Db, target_version: TargetVersion, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_search_path @@ -228,7 +228,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { + match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists => db.file_system().is_directory(path), TypeshedVersionsQueryResult::MaybeExists => db.file_system().is_directory(path), @@ -248,21 +248,21 @@ impl<'a> ModuleResolutionPathRefInner<'a> { typeshed_versions: &LazyTypeshedVersions, target_version: TargetVersion, ) -> bool { - fn is_non_stdlib_pkg(path: &FileSystemPath, db: &dyn Db) -> bool { + fn is_non_stdlib_pkg(db: &dyn Db, path: &FileSystemPath) -> bool { let file_system = db.file_system(); file_system.exists(&path.join("__init__.py")) || file_system.exists(&path.join("__init__.pyi")) } match (self, search_path) { - (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(path, db), - (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(path, db), - (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(path, db), + (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(db, path), + (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(db, path), + (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(db, path), // Unlike the other variants: // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { + match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { TypeshedVersionsQueryResult::DoesNotExist => false, TypeshedVersionsQueryResult::Exists => db.file_system().exists(&path.join("__init__.pyi")), TypeshedVersionsQueryResult::MaybeExists => db.file_system().exists(&path.join("__init__.pyi")), @@ -288,7 +288,7 @@ impl<'a> ModuleResolutionPathRefInner<'a> { system_path_to_file(db.upcast(), path) } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(path, typeshed_versions, search_path, stdlib_root, db, target_version) { + match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { TypeshedVersionsQueryResult::DoesNotExist => None, TypeshedVersionsQueryResult::Exists => system_path_to_file(db.upcast(), path), TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(db.upcast(), path) From f8e900e691e63b640163bef5a7969c6e5df47e14 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:06:43 +0100 Subject: [PATCH 52/58] Get rid of unneeded new dependency --- Cargo.lock | 1 - crates/red_knot_module_resolver/Cargo.toml | 1 - crates/red_knot_module_resolver/src/typeshed/versions.rs | 3 +-- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e7f401dd23e31..063413da01138 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1885,7 +1885,6 @@ dependencies = [ "camino", "compact_str", "insta", - "once_cell", "path-slash", "ruff_db", "ruff_python_stdlib", diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index a6761665d6116..99e69f35cc27f 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -16,7 +16,6 @@ ruff_python_stdlib = { workspace = true } compact_str = { workspace = true } camino = { workspace = true } -once_cell = { workspace = true } rustc-hash = { workspace = true } salsa = { workspace = true } tracing = { workspace = true } diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index 8535e56d28aff..f74a2f616ddca 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -1,3 +1,4 @@ +use std::cell::OnceCell; use std::collections::BTreeMap; use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; @@ -5,8 +6,6 @@ use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; use std::sync::Arc; -use once_cell::sync::OnceCell; - use ruff_db::file_system::FileSystemPath; use ruff_db::source::source_text; use ruff_db::vfs::{system_path_to_file, VfsFile}; From 8fc3a7d6353bcb781713a345a01a7fe7cb46acd7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:13:34 +0100 Subject: [PATCH 53/58] So many lifetimes --- crates/red_knot_module_resolver/src/path.rs | 66 ++++++++++--------- .../red_knot_module_resolver/src/resolver.rs | 6 +- .../src/typeshed/versions.rs | 20 +++--- 3 files changed, 45 insertions(+), 47 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 22a66d86e6b79..87a284631dba8 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -108,11 +108,11 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn is_regular_package( + pub(crate) fn is_regular_package<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: &Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { ModuleResolutionPathRef::from(self).is_regular_package( @@ -124,11 +124,11 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn is_directory( + pub(crate) fn is_directory<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: &Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { ModuleResolutionPathRef::from(self).is_directory( @@ -158,11 +158,11 @@ impl ModuleResolutionPathBuf { } /// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory. - pub(crate) fn to_vfs_file( + pub(crate) fn to_vfs_file<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: &Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> Option { ModuleResolutionPathRef::from(self).to_vfs_file( @@ -198,10 +198,10 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] - fn query_stdlib_version( - db: &dyn Db, + fn query_stdlib_version<'db>( + db: &'db dyn Db, module_path: &'a FileSystemPath, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, stdlib_search_path: Self, stdlib_root: &FileSystemPath, target_version: TargetVersion, @@ -216,11 +216,11 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_directory( + fn is_directory<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { match (self, search_path) { @@ -241,11 +241,11 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_regular_package( + fn is_regular_package<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { fn is_non_stdlib_pkg(db: &dyn Db, path: &FileSystemPath) -> bool { @@ -274,11 +274,11 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } - fn to_vfs_file( + fn to_vfs_file<'db>( self, - db: &dyn Db, + db: &'db dyn Db, search_path: Self, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> Option { match (self, search_path) { @@ -386,11 +386,11 @@ pub(crate) struct ModuleResolutionPathRef<'a>(ModuleResolutionPathRefInner<'a>); impl<'a> ModuleResolutionPathRef<'a> { #[must_use] - pub(crate) fn is_directory( + pub(crate) fn is_directory<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { self.0 @@ -398,11 +398,11 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn is_regular_package( + pub(crate) fn is_regular_package<'db>( &self, - db: &dyn Db, + db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> bool { self.0 @@ -410,11 +410,11 @@ impl<'a> ModuleResolutionPathRef<'a> { } #[must_use] - pub(crate) fn to_vfs_file( + pub(crate) fn to_vfs_file<'db>( self, - db: &dyn Db, + db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> Option { self.0 @@ -794,7 +794,8 @@ mod tests { ); } - fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { + fn py38_stdlib_test_case<'db>() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions<'db>) + { let TestCase { db, custom_typeshed, @@ -988,7 +989,8 @@ mod tests { )); } - fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions) { + fn py39_stdlib_test_case<'db>() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions<'db>) + { let TestCase { db, custom_typeshed, diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 4532207a75a09..82a656e6f2d83 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -309,11 +309,11 @@ fn resolve_name( None } -fn resolve_package<'a, I>( - db: &dyn Db, +fn resolve_package<'a, 'db, I>( + db: &'db dyn Db, module_search_path: &ModuleResolutionPathBuf, components: I, - typeshed_versions: &LazyTypeshedVersions, + typeshed_versions: &LazyTypeshedVersions<'db>, target_version: TargetVersion, ) -> Result where diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index f74a2f616ddca..e320b09ce96a5 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -4,7 +4,6 @@ use std::fmt; use std::num::{NonZeroU16, NonZeroUsize}; use std::ops::{RangeFrom, RangeInclusive}; use std::str::FromStr; -use std::sync::Arc; use ruff_db::file_system::FileSystemPath; use ruff_db::source::source_text; @@ -15,9 +14,9 @@ use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::TargetVersion; -pub(crate) struct LazyTypeshedVersions(OnceCell); +pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>); -impl LazyTypeshedVersions { +impl<'db> LazyTypeshedVersions<'db> { #[must_use] pub(crate) fn new() -> Self { Self(OnceCell::new()) @@ -39,7 +38,7 @@ impl LazyTypeshedVersions { pub(crate) fn query_module( &self, module: &ModuleName, - db: &dyn Db, + db: &'db dyn Db, stdlib_root: &FileSystemPath, target_version: TargetVersion, ) -> TypeshedVersionsQueryResult { @@ -55,16 +54,13 @@ impl LazyTypeshedVersions { // this should invalidate not just the specific module resolution we're currently attempting, // but all type inference that depends on any standard-library types. // Unwrapping here is not correct... - parse_typeshed_versions(db, versions_file) - .as_ref() - .unwrap() - .clone() + parse_typeshed_versions(db, versions_file).as_ref().unwrap() }); versions.query_module(module, PyVersion::from(target_version)) } } -#[salsa::tracked] +#[salsa::tracked(return_ref)] pub(crate) fn parse_typeshed_versions( db: &dyn Db, versions_file: VfsFile, @@ -151,8 +147,8 @@ impl fmt::Display for TypeshedVersionsParseErrorKind { } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TypeshedVersions(Arc>); +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct TypeshedVersions(FxHashMap); impl TypeshedVersions { #[must_use] @@ -299,7 +295,7 @@ impl FromStr for TypeshedVersions { reason: TypeshedVersionsParseErrorKind::EmptyVersionsFile, }) } else { - Ok(Self(Arc::new(map))) + Ok(Self(map)) } } } From b84841562a8614ac8ddd24fb41a790b9ebee6e2c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:17:26 +0100 Subject: [PATCH 54/58] Rename some things --- crates/red_knot/src/main.rs | 4 +-- crates/red_knot_module_resolver/src/db.rs | 4 +-- crates/red_knot_module_resolver/src/lib.rs | 2 +- .../red_knot_module_resolver/src/resolver.rs | 28 +++++++++---------- .../src/semantic_model.rs | 4 +-- crates/red_knot_python_semantic/src/types.rs | 4 +-- .../src/types/infer.rs | 4 +-- crates/ruff_benchmark/benches/red_knot.rs | 4 +-- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/crates/red_knot/src/main.rs b/crates/red_knot/src/main.rs index 5b9bccb6dd8dc..85d26458c3919 100644 --- a/crates/red_knot/src/main.rs +++ b/crates/red_knot/src/main.rs @@ -13,7 +13,7 @@ use red_knot::program::{FileWatcherChange, Program}; use red_knot::watch::FileWatcher; use red_knot::Workspace; use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, + set_module_resolution_settings, RawModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::{FileSystem, FileSystemPath, OsFileSystem}; use ruff_db::vfs::system_path_to_file; @@ -59,7 +59,7 @@ pub fn main() -> anyhow::Result<()> { set_module_resolution_settings( &mut program, - ModuleResolutionSettings { + RawModuleResolutionSettings { extra_paths: vec![], workspace_root: workspace_search_path, site_packages: None, diff --git a/crates/red_knot_module_resolver/src/db.rs b/crates/red_knot_module_resolver/src/db.rs index 8bf63a77a6674..3d64ee76f4cc0 100644 --- a/crates/red_knot_module_resolver/src/db.rs +++ b/crates/red_knot_module_resolver/src/db.rs @@ -27,7 +27,7 @@ pub(crate) mod tests { use ruff_db::file_system::{FileSystem, FileSystemPathBuf, MemoryFileSystem, OsFileSystem}; use ruff_db::vfs::Vfs; - use crate::resolver::{set_module_resolution_settings, ModuleResolutionSettings}; + use crate::resolver::{set_module_resolution_settings, RawModuleResolutionSettings}; use crate::supported_py_version::TargetVersion; use super::*; @@ -179,7 +179,7 @@ pub(crate) mod tests { target_version, } = self; - let settings = ModuleResolutionSettings { + let settings = RawModuleResolutionSettings { target_version: target_version.unwrap_or_default(), extra_paths: vec![], workspace_root: src.clone(), diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index 98c7e13552a47..71e8674fbd870 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -9,6 +9,6 @@ mod typeshed; pub use db::{Db, Jar}; pub use module::{Module, ModuleKind}; pub use module_name::ModuleName; -pub use resolver::{resolve_module, set_module_resolution_settings, ModuleResolutionSettings}; +pub use resolver::{resolve_module, set_module_resolution_settings, RawModuleResolutionSettings}; pub use supported_py_version::TargetVersion; pub use typeshed::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind}; diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 82a656e6f2d83..bda1ebedf056e 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -15,11 +15,11 @@ use crate::typeshed::LazyTypeshedVersions; /// Configures the module resolver settings. /// /// Must be called before calling any other module resolution functions. -pub fn set_module_resolution_settings(db: &mut dyn Db, config: ModuleResolutionSettings) { +pub fn set_module_resolution_settings(db: &mut dyn Db, config: RawModuleResolutionSettings) { // There's no concurrency issue here because we hold a `&mut dyn Db` reference. No other // thread can mutate the `Db` while we're in this call, so using `try_get` to test if // the settings have already been set is safe. - let resolved_settings = config.into_resolved_settings(); + let resolved_settings = config.into_configuration_settings(); if let Some(existing) = ModuleResolverSettings::try_get(db) { existing.set_settings(db).to(resolved_settings); } else { @@ -111,9 +111,9 @@ pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { } } -/// Configures the search paths that are used to resolve modules. +/// "Raw" configuration settings for module resolution: unvalidated, unnormalized #[derive(Eq, PartialEq, Debug)] -pub struct ModuleResolutionSettings { +pub struct RawModuleResolutionSettings { /// The target Python version the user has specified pub target_version: TargetVersion, @@ -134,7 +134,7 @@ pub struct ModuleResolutionSettings { pub site_packages: Option, } -impl ModuleResolutionSettings { +impl RawModuleResolutionSettings { /// Implementation of PEP 561's module resolution order /// (with some small, deliberate, differences) /// @@ -144,8 +144,8 @@ impl ModuleResolutionSettings { /// Rather than panicking if a path fails to validate, we should display an error message to the user /// and exit the process with a nonzero exit code. /// This validation should probably be done outside of Salsa? - fn into_resolved_settings(self) -> ResolvedModuleResolutionSettings { - let ModuleResolutionSettings { + fn into_configuration_settings(self) -> ModuleResolutionSettings { + let RawModuleResolutionSettings { target_version, extra_paths, workspace_root, @@ -171,7 +171,7 @@ impl ModuleResolutionSettings { paths.push(ModuleResolutionPathBuf::site_packages(site_packages).unwrap()); } - ResolvedModuleResolutionSettings { + ModuleResolutionSettings { target_version, search_paths: OrderedSearchPaths(paths.into_iter().map(Arc::new).collect()), } @@ -192,12 +192,12 @@ impl Deref for OrderedSearchPaths { } #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ResolvedModuleResolutionSettings { +pub(crate) struct ModuleResolutionSettings { search_paths: OrderedSearchPaths, target_version: TargetVersion, } -impl ResolvedModuleResolutionSettings { +impl ModuleResolutionSettings { pub(crate) fn search_paths(&self) -> &[Arc] { &self.search_paths } @@ -214,12 +214,12 @@ impl ResolvedModuleResolutionSettings { #[allow(unreachable_pub, clippy::used_underscore_binding)] pub(crate) mod internal { use crate::module_name::ModuleName; - use crate::resolver::ResolvedModuleResolutionSettings; + use crate::resolver::ModuleResolutionSettings; #[salsa::input(singleton)] pub(crate) struct ModuleResolverSettings { #[return_ref] - pub(super) settings: ResolvedModuleResolutionSettings, + pub(super) settings: ModuleResolutionSettings, } /// A thin wrapper around `ModuleName` to make it a Salsa ingredient. @@ -232,7 +232,7 @@ pub(crate) mod internal { } } -fn module_resolver_settings(db: &dyn Db) -> &ResolvedModuleResolutionSettings { +fn module_resolver_settings(db: &dyn Db) -> &ModuleResolutionSettings { ModuleResolverSettings::get(db).settings(db) } @@ -884,7 +884,7 @@ mod tests { std::fs::write(foo.as_std_path(), "")?; std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?; - let settings = ModuleResolutionSettings { + let settings = RawModuleResolutionSettings { target_version: TargetVersion::Py38, extra_paths: vec![], workspace_root: src.clone(), diff --git a/crates/red_knot_python_semantic/src/semantic_model.rs b/crates/red_knot_python_semantic/src/semantic_model.rs index 440d247994934..1c2c72cc3cc76 100644 --- a/crates/red_knot_python_semantic/src/semantic_model.rs +++ b/crates/red_knot_python_semantic/src/semantic_model.rs @@ -178,7 +178,7 @@ impl HasTy for ast::Alias { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, + set_module_resolution_settings, RawModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::parsed::parsed_module; @@ -192,7 +192,7 @@ mod tests { let mut db = TestDb::new(); set_module_resolution_settings( &mut db, - ModuleResolutionSettings { + RawModuleResolutionSettings { extra_paths: vec![], workspace_root: FileSystemPathBuf::from("/src"), site_packages: None, diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 7a0fc3e6b963d..d351438dec51e 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -500,7 +500,7 @@ impl<'db, 'inference> TypingContext<'db, 'inference> { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, + set_module_resolution_settings, RawModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::parsed::parsed_module; @@ -517,7 +517,7 @@ mod tests { let mut db = TestDb::new(); set_module_resolution_settings( &mut db, - ModuleResolutionSettings { + RawModuleResolutionSettings { target_version: TargetVersion::Py38, extra_paths: vec![], workspace_root: FileSystemPathBuf::from("/src"), diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index b2711408eb872..8d680f712d26f 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -672,7 +672,7 @@ impl<'db> TypeInferenceBuilder<'db> { #[cfg(test)] mod tests { use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, + set_module_resolution_settings, RawModuleResolutionSettings, TargetVersion, }; use ruff_db::file_system::FileSystemPathBuf; use ruff_db::vfs::system_path_to_file; @@ -686,7 +686,7 @@ mod tests { set_module_resolution_settings( &mut db, - ModuleResolutionSettings { + RawModuleResolutionSettings { target_version: TargetVersion::Py38, extra_paths: Vec::new(), workspace_root: FileSystemPathBuf::from("/src"), diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs index 65f8a56dfca4d..800d2f05e507b 100644 --- a/crates/ruff_benchmark/benches/red_knot.rs +++ b/crates/ruff_benchmark/benches/red_knot.rs @@ -3,7 +3,7 @@ use red_knot::program::Program; use red_knot::Workspace; use red_knot_module_resolver::{ - set_module_resolution_settings, ModuleResolutionSettings, TargetVersion, + set_module_resolution_settings, RawModuleResolutionSettings, TargetVersion, }; use ruff_benchmark::criterion::{ criterion_group, criterion_main, BatchSize, Criterion, Throughput, @@ -72,7 +72,7 @@ fn setup_case() -> Case { set_module_resolution_settings( &mut program, - ModuleResolutionSettings { + RawModuleResolutionSettings { extra_paths: vec![], workspace_root: workspace_root.to_path_buf(), site_packages: None, From daf64b45cbc11e268016796f4d4fb28752da683e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:24:43 +0100 Subject: [PATCH 55/58] Audit docs in `resolver.rs` --- crates/red_knot_module_resolver/src/resolver.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index bda1ebedf056e..97bef6a96244d 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -56,7 +56,7 @@ pub(crate) fn resolve_module_query<'db>( /// Resolves the module for the given path. /// -/// Returns `None` if the path is not a module locatable via `sys.path`. +/// Returns `None` if the path is not a module locatable via any of the known search paths. #[allow(unused)] pub(crate) fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option { // It's not entirely clear on first sight why this method calls `file_to_module` instead of @@ -73,7 +73,7 @@ pub(crate) fn path_to_module(db: &dyn Db, path: &VfsPath) -> Option { /// Resolves the module for the file with the given id. /// -/// Returns `None` if the file is not a module locatable via any of the PEP-561 search paths +/// Returns `None` if the file is not a module locatable via any of the known search paths. #[salsa::tracked] pub(crate) fn file_to_module(db: &dyn Db, file: VfsFile) -> Option { let _span = tracing::trace_span!("file_to_module", ?file).entered(); @@ -135,8 +135,7 @@ pub struct RawModuleResolutionSettings { } impl RawModuleResolutionSettings { - /// Implementation of PEP 561's module resolution order - /// (with some small, deliberate, differences) + /// Implementation of the typing spec's [module resolution order] /// /// TODO(Alex): this method does multiple `.unwrap()` calls when it should really return an error. /// Each `.unwrap()` call is a point where we're validating a setting that the user would pass @@ -144,6 +143,8 @@ impl RawModuleResolutionSettings { /// Rather than panicking if a path fails to validate, we should display an error message to the user /// and exit the process with a nonzero exit code. /// This validation should probably be done outside of Salsa? + /// + /// [module resolution order]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering fn into_configuration_settings(self) -> ModuleResolutionSettings { let RawModuleResolutionSettings { target_version, @@ -178,8 +179,9 @@ impl RawModuleResolutionSettings { } } -/// A resolved module resolution order, implementing PEP 561 -/// (with some small, deliberate differences) +/// A resolved module resolution order as per the [typing spec] +/// +/// [typing spec]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct OrderedSearchPaths(Vec>); From 47c346753c82905fadf3012c9cce3e033888f459 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 22:56:51 +0100 Subject: [PATCH 56/58] Make assertions in `push()` make a little more sense --- crates/red_knot_module_resolver/src/path.rs | 64 ++++++++++++--------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index 87a284631dba8..f3708f2e430de 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -25,30 +25,49 @@ enum ModuleResolutionPathBufInner { impl ModuleResolutionPathBufInner { fn push(&mut self, component: &str) { - if let Some(extension) = camino::Utf8Path::new(component).extension() { - match self { - Self::Extra(_) | Self::FirstParty(_) | Self::SitePackages(_) => assert!( - matches!(extension, "pyi" | "py"), - "Extension must be `py` or `pyi`; got `{extension}`" - ), - Self::StandardLibrary(_) => { + let extension = camino::Utf8Path::new(component).extension(); + let inner = match self { + Self::Extra(ref mut path) => { + if let Some(extension) = extension { + assert!( + matches!(extension, "pyi" | "py"), + "Extension must be `py` or `pyi`; got `{extension}`" + ); + } + path + } + Self::FirstParty(ref mut path) => { + if let Some(extension) = extension { assert!( - matches!(component.matches('.').count(), 0 | 1), - "Component can have at most one '.'; got {component}" + matches!(extension, "pyi" | "py"), + "Extension must be `py` or `pyi`; got `{extension}`" ); + } + path + } + Self::StandardLibrary(ref mut path) => { + if let Some(extension) = extension { assert_eq!( extension, "pyi", "Extension must be `pyi`; got `{extension}`" ); } - }; - } - let inner = match self { - Self::Extra(ref mut path) => path, - Self::FirstParty(ref mut path) => path, - Self::StandardLibrary(ref mut path) => path, - Self::SitePackages(ref mut path) => path, + path + } + Self::SitePackages(ref mut path) => { + if let Some(extension) = extension { + assert!( + matches!(extension, "pyi" | "py"), + "Extension must be `py` or `pyi`; got `{extension}`" + ); + } + path + } }; + assert!( + inner.extension().is_none(), + "Cannot push part {component} to {inner}, which already has an extension" + ); inner.push(component); } } @@ -86,9 +105,6 @@ impl ModuleResolutionPathBuf { #[must_use] pub(crate) fn standard_library(path: impl Into) -> Option { let path = path.into(); - if path.file_stem().is_some_and(|stem| stem.contains('.')) { - return None; - } path.extension() .map_or(true, |ext| ext == "pyi") .then_some(Self(ModuleResolutionPathBufInner::StandardLibrary(path))) @@ -563,10 +579,6 @@ mod tests { ModuleResolutionPathBuf::standard_library("foo/__init__.py"), None ); - assert_eq!( - ModuleResolutionPathBuf::standard_library("foo.py.pyi"), - None - ); } #[test] @@ -748,11 +760,11 @@ mod tests { } #[test] - #[should_panic(expected = "Component can have at most one '.'")] + #[should_panic(expected = "already has an extension")] fn invalid_stdlib_join_too_many_extensions() { - ModuleResolutionPathBuf::standard_library("foo") + ModuleResolutionPathBuf::standard_library("foo.pyi") .unwrap() - .push("bar.py.pyi"); + .push("bar.pyi"); } #[test] From 401779bff5aaceef17f4f6e699807b910332e93c Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 23:32:03 +0100 Subject: [PATCH 57/58] Encapsulate all state passed around in a single struct --- crates/red_knot_module_resolver/src/lib.rs | 1 + crates/red_knot_module_resolver/src/path.rs | 461 ++++++------------ .../red_knot_module_resolver/src/resolver.rs | 56 +-- crates/red_knot_module_resolver/src/state.rs | 25 + .../src/typeshed/versions.rs | 1 + 5 files changed, 197 insertions(+), 347 deletions(-) create mode 100644 crates/red_knot_module_resolver/src/state.rs diff --git a/crates/red_knot_module_resolver/src/lib.rs b/crates/red_knot_module_resolver/src/lib.rs index 71e8674fbd870..d6ec501ccb799 100644 --- a/crates/red_knot_module_resolver/src/lib.rs +++ b/crates/red_knot_module_resolver/src/lib.rs @@ -3,6 +3,7 @@ mod module; mod module_name; mod path; mod resolver; +mod state; mod supported_py_version; mod typeshed; diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index f3708f2e430de..f3cb0391d59ea 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -3,10 +3,9 @@ use std::fmt; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf}; use ruff_db::vfs::{system_path_to_file, VfsFile}; -use crate::db::Db; use crate::module_name::ModuleName; -use crate::typeshed::{LazyTypeshedVersions, TypeshedVersionsQueryResult}; -use crate::TargetVersion; +use crate::state::ResolverState; +use crate::typeshed::TypeshedVersionsQueryResult; /// Enumeration of the different kinds of search paths type checkers are expected to support. /// @@ -124,35 +123,13 @@ impl ModuleResolutionPathBuf { } #[must_use] - pub(crate) fn is_regular_package<'db>( - &self, - db: &'db dyn Db, - search_path: &Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, - ) -> bool { - ModuleResolutionPathRef::from(self).is_regular_package( - db, - search_path, - typeshed_versions, - target_version, - ) + pub(crate) fn is_regular_package(&self, search_path: &Self, resolver: &ResolverState) -> bool { + ModuleResolutionPathRef::from(self).is_regular_package(search_path, resolver) } #[must_use] - pub(crate) fn is_directory<'db>( - &self, - db: &'db dyn Db, - search_path: &Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, - ) -> bool { - ModuleResolutionPathRef::from(self).is_directory( - db, - search_path, - typeshed_versions, - target_version, - ) + pub(crate) fn is_directory(&self, search_path: &Self, resolver: &ResolverState) -> bool { + ModuleResolutionPathRef::from(self).is_directory(search_path, resolver) } #[must_use] @@ -174,19 +151,12 @@ impl ModuleResolutionPathBuf { } /// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory. - pub(crate) fn to_vfs_file<'db>( + pub(crate) fn to_vfs_file( &self, - db: &'db dyn Db, search_path: &Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, + resolver: &ResolverState, ) -> Option { - ModuleResolutionPathRef::from(self).to_vfs_file( - db, - search_path, - typeshed_versions, - target_version, - ) + ModuleResolutionPathRef::from(self).to_vfs_file(search_path, resolver) } } @@ -215,12 +185,10 @@ enum ModuleResolutionPathRefInner<'a> { impl<'a> ModuleResolutionPathRefInner<'a> { #[must_use] fn query_stdlib_version<'db>( - db: &'db dyn Db, module_path: &'a FileSystemPath, - typeshed_versions: &LazyTypeshedVersions<'db>, stdlib_search_path: Self, stdlib_root: &FileSystemPath, - target_version: TargetVersion, + resolver_state: &ResolverState<'db>, ) -> TypeshedVersionsQueryResult { let Some(module_name) = stdlib_search_path .relativize_path(module_path) @@ -228,26 +196,25 @@ impl<'a> ModuleResolutionPathRefInner<'a> { else { return TypeshedVersionsQueryResult::DoesNotExist; }; - typeshed_versions.query_module(&module_name, db, stdlib_root, target_version) + let ResolverState { + db, + typeshed_versions, + target_version, + } = resolver_state; + typeshed_versions.query_module(&module_name, *db, stdlib_root, *target_version) } #[must_use] - fn is_directory<'db>( - &self, - db: &'db dyn Db, - search_path: Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, - ) -> bool { + fn is_directory(&self, search_path: Self, resolver: &ResolverState) -> bool { match (self, search_path) { - (Self::Extra(path), Self::Extra(_)) => db.file_system().is_directory(path), - (Self::FirstParty(path), Self::FirstParty(_)) => db.file_system().is_directory(path), - (Self::SitePackages(path), Self::SitePackages(_)) => db.file_system().is_directory(path), + (Self::Extra(path), Self::Extra(_)) => resolver.file_system().is_directory(path), + (Self::FirstParty(path), Self::FirstParty(_)) => resolver.file_system().is_directory(path), + (Self::SitePackages(path), Self::SitePackages(_)) => resolver.file_system().is_directory(path), (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { + match Self::query_stdlib_version( path, search_path, stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, - TypeshedVersionsQueryResult::Exists => db.file_system().is_directory(path), - TypeshedVersionsQueryResult::MaybeExists => db.file_system().is_directory(path), + TypeshedVersionsQueryResult::Exists => resolver.file_system().is_directory(path), + TypeshedVersionsQueryResult::MaybeExists => resolver.file_system().is_directory(path), } } (path, root) => unreachable!( @@ -257,31 +224,25 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } #[must_use] - fn is_regular_package<'db>( - &self, - db: &'db dyn Db, - search_path: Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, - ) -> bool { - fn is_non_stdlib_pkg(db: &dyn Db, path: &FileSystemPath) -> bool { - let file_system = db.file_system(); + fn is_regular_package(&self, search_path: Self, resolver: &ResolverState) -> bool { + fn is_non_stdlib_pkg(state: &ResolverState, path: &FileSystemPath) -> bool { + let file_system = state.file_system(); file_system.exists(&path.join("__init__.py")) || file_system.exists(&path.join("__init__.pyi")) } match (self, search_path) { - (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(db, path), - (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(db, path), - (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(db, path), + (Self::Extra(path), Self::Extra(_)) => is_non_stdlib_pkg(resolver, path), + (Self::FirstParty(path), Self::FirstParty(_)) => is_non_stdlib_pkg(resolver, path), + (Self::SitePackages(path), Self::SitePackages(_)) => is_non_stdlib_pkg(resolver, path), // Unlike the other variants: // (1) Account for VERSIONS // (2) Only test for `__init__.pyi`, not `__init__.py` (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { + match Self::query_stdlib_version( path, search_path, stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => false, - TypeshedVersionsQueryResult::Exists => db.file_system().exists(&path.join("__init__.pyi")), - TypeshedVersionsQueryResult::MaybeExists => db.file_system().exists(&path.join("__init__.pyi")), + TypeshedVersionsQueryResult::Exists => resolver.db.file_system().exists(&path.join("__init__.pyi")), + TypeshedVersionsQueryResult::MaybeExists => resolver.db.file_system().exists(&path.join("__init__.pyi")), } } (path, root) => unreachable!( @@ -290,24 +251,18 @@ impl<'a> ModuleResolutionPathRefInner<'a> { } } - fn to_vfs_file<'db>( - self, - db: &'db dyn Db, - search_path: Self, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, - ) -> Option { + fn to_vfs_file(self, search_path: Self, resolver: &ResolverState) -> Option { match (self, search_path) { - (Self::Extra(path), Self::Extra(_)) => system_path_to_file(db.upcast(), path), - (Self::FirstParty(path), Self::FirstParty(_)) => system_path_to_file(db.upcast(), path), + (Self::Extra(path), Self::Extra(_)) => system_path_to_file(resolver.db.upcast(), path), + (Self::FirstParty(path), Self::FirstParty(_)) => system_path_to_file(resolver.db.upcast(), path), (Self::SitePackages(path), Self::SitePackages(_)) => { - system_path_to_file(db.upcast(), path) + system_path_to_file(resolver.db.upcast(), path) } (Self::StandardLibrary(path), Self::StandardLibrary(stdlib_root)) => { - match Self::query_stdlib_version(db, path, typeshed_versions, search_path, stdlib_root, target_version) { + match Self::query_stdlib_version(path, search_path, stdlib_root, resolver) { TypeshedVersionsQueryResult::DoesNotExist => None, - TypeshedVersionsQueryResult::Exists => system_path_to_file(db.upcast(), path), - TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(db.upcast(), path) + TypeshedVersionsQueryResult::Exists => system_path_to_file(resolver.db.upcast(), path), + TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(resolver.db.upcast(), path) } } (path, root) => unreachable!( @@ -402,39 +357,30 @@ pub(crate) struct ModuleResolutionPathRef<'a>(ModuleResolutionPathRefInner<'a>); impl<'a> ModuleResolutionPathRef<'a> { #[must_use] - pub(crate) fn is_directory<'db>( + pub(crate) fn is_directory( &self, - db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, + resolver: &ResolverState, ) -> bool { - self.0 - .is_directory(db, search_path.into().0, typeshed_versions, target_version) + self.0.is_directory(search_path.into().0, resolver) } #[must_use] - pub(crate) fn is_regular_package<'db>( + pub(crate) fn is_regular_package( &self, - db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, + resolver: &ResolverState, ) -> bool { - self.0 - .is_regular_package(db, search_path.into().0, typeshed_versions, target_version) + self.0.is_regular_package(search_path.into().0, resolver) } #[must_use] - pub(crate) fn to_vfs_file<'db>( + pub(crate) fn to_vfs_file( self, - db: &'db dyn Db, search_path: impl Into, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, + resolver: &ResolverState, ) -> Option { - self.0 - .to_vfs_file(db, search_path.into().0, typeshed_versions, target_version) + self.0.to_vfs_file(search_path.into().0, resolver) } #[must_use] @@ -526,6 +472,7 @@ mod tests { use crate::db::tests::{create_resolver_builder, TestCase, TestDb}; use crate::supported_py_version::TargetVersion; + use crate::typeshed::LazyTypeshedVersions; use super::*; @@ -806,8 +753,7 @@ mod tests { ); } - fn py38_stdlib_test_case<'db>() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions<'db>) - { + fn py38_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { let TestCase { db, custom_typeshed, @@ -815,194 +761,140 @@ mod tests { } = create_resolver_builder().unwrap().build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); - (db, stdlib_module_path, LazyTypeshedVersions::new()) + (db, stdlib_module_path) } #[test] fn mocked_typeshed_existing_regular_stdlib_pkg_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let asyncio_regular_package = stdlib_path.join("asyncio"); - assert!(asyncio_regular_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); - assert!(asyncio_regular_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(asyncio_regular_package.is_directory(&stdlib_path, &resolver)); + assert!(asyncio_regular_package.is_regular_package(&stdlib_path, &resolver)); // Paths to directories don't resolve to VfsFiles assert_eq!( - asyncio_regular_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), + asyncio_regular_package.to_vfs_file(&stdlib_path, &resolver), None ); assert!(asyncio_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) + .to_vfs_file(&stdlib_path, &resolver) .is_some()); // The `asyncio` package exists on Python 3.8, but the `asyncio.tasks` submodule does not, // according to the `VERSIONS` file in our typeshed mock: let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); assert_eq!( - asyncio_tasks_module.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), + asyncio_tasks_module.to_vfs_file(&stdlib_path, &resolver), None ); - assert!(!asyncio_tasks_module.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); - assert!(!asyncio_tasks_module.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(!asyncio_tasks_module.is_directory(&stdlib_path, &resolver)); + assert!(!asyncio_tasks_module.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_existing_namespace_stdlib_pkg_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let xml_namespace_package = stdlib_path.join("xml"); - assert!(xml_namespace_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(xml_namespace_package.is_directory(&stdlib_path, &resolver)); // Paths to directories don't resolve to VfsFiles assert_eq!( - xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), + xml_namespace_package.to_vfs_file(&stdlib_path, &resolver), None ); - assert!(!xml_namespace_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(!xml_namespace_package.is_regular_package(&stdlib_path, &resolver)); let xml_etree = stdlib_path.join("xml/etree.pyi"); - assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); - assert!(xml_etree - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) - .is_some()); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions, TargetVersion::Py38)); + assert!(!xml_etree.is_directory(&stdlib_path, &resolver)); + assert!(xml_etree.to_vfs_file(&stdlib_path, &resolver).is_some()); + assert!(!xml_etree.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_single_file_stdlib_module_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let functools_module = stdlib_path.join("functools.pyi"); assert!(functools_module - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38) + .to_vfs_file(&stdlib_path, &resolver) .is_some()); - assert!(!functools_module.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); - assert!(!functools_module.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(!functools_module.is_directory(&stdlib_path, &resolver)); + assert!(!functools_module.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_nonexistent_regular_stdlib_pkg_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let collections_regular_package = stdlib_path.join("collections"); assert_eq!( - collections_regular_package.to_vfs_file( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - ), + collections_regular_package.to_vfs_file(&stdlib_path, &resolver), None ); - assert!(!collections_regular_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); - assert!(!collections_regular_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(!collections_regular_package.is_directory(&stdlib_path, &resolver)); + assert!(!collections_regular_package.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let importlib_namespace_package = stdlib_path.join("importlib"); assert_eq!( - importlib_namespace_package.to_vfs_file( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - ), + importlib_namespace_package.to_vfs_file(&stdlib_path, &resolver), None ); - assert!(!importlib_namespace_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); - assert!(!importlib_namespace_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert!(!importlib_namespace_package.is_directory(&stdlib_path, &resolver)); + assert!(!importlib_namespace_package.is_regular_package(&stdlib_path, &resolver)); let importlib_abc = stdlib_path.join("importlib/abc.pyi"); - assert_eq!( - importlib_abc.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), - None - ); - assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); - assert!(!importlib_abc.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert_eq!(importlib_abc.to_vfs_file(&stdlib_path, &resolver), None); + assert!(!importlib_abc.is_directory(&stdlib_path, &resolver)); + assert!(!importlib_abc.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_nonexistent_single_file_module_py38() { - let (db, stdlib_path, versions) = py38_stdlib_test_case(); + let (db, stdlib_path) = py38_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py38, + }; let non_existent = stdlib_path.join("doesnt_even_exist"); - assert_eq!( - non_existent.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py38), - None - ); - assert!(!non_existent.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py38)); - assert!(!non_existent.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py38 - )); + assert_eq!(non_existent.to_vfs_file(&stdlib_path, &resolver), None); + assert!(!non_existent.is_directory(&stdlib_path, &resolver)); + assert!(!non_existent.is_regular_package(&stdlib_path, &resolver)); } - fn py39_stdlib_test_case<'db>() -> (TestDb, ModuleResolutionPathBuf, LazyTypeshedVersions<'db>) - { + fn py39_stdlib_test_case() -> (TestDb, ModuleResolutionPathBuf) { let TestCase { db, custom_typeshed, @@ -1013,134 +905,89 @@ mod tests { .build(); let stdlib_module_path = ModuleResolutionPathBuf::stdlib_from_typeshed_root(&custom_typeshed).unwrap(); - (db, stdlib_module_path, LazyTypeshedVersions::new()) + (db, stdlib_module_path) } #[test] fn mocked_typeshed_existing_regular_stdlib_pkgs_py39() { - let (db, stdlib_path, versions) = py39_stdlib_test_case(); + let (db, stdlib_path) = py39_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py39, + }; // Since we've set the target version to Py39, // `collections` should now exist as a directory, according to VERSIONS... let collections_regular_package = stdlib_path.join("collections"); - assert!(collections_regular_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); - assert!(collections_regular_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); + assert!(collections_regular_package.is_directory(&stdlib_path, &resolver)); + assert!(collections_regular_package.is_regular_package(&stdlib_path, &resolver)); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - collections_regular_package.to_vfs_file( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - ), + collections_regular_package.to_vfs_file(&stdlib_path, &resolver), None ); assert!(collections_regular_package .join("__init__.pyi") - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) + .to_vfs_file(&stdlib_path, &resolver) .is_some()); // ...and so should the `asyncio.tasks` submodule (though it's still not a directory): let asyncio_tasks_module = stdlib_path.join("asyncio/tasks.pyi"); assert!(asyncio_tasks_module - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) + .to_vfs_file(&stdlib_path, &resolver) .is_some()); - assert!(!asyncio_tasks_module.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); - assert!(!asyncio_tasks_module.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); + assert!(!asyncio_tasks_module.is_directory(&stdlib_path, &resolver)); + assert!(!asyncio_tasks_module.is_regular_package(&stdlib_path, &resolver)); } #[test] fn mocked_typeshed_existing_namespace_stdlib_pkg_py39() { - let (db, stdlib_path, versions) = py39_stdlib_test_case(); + let (db, stdlib_path) = py39_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py39, + }; // The `importlib` directory now also exists... let importlib_namespace_package = stdlib_path.join("importlib"); - assert!(importlib_namespace_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); - assert!(!importlib_namespace_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); + assert!(importlib_namespace_package.is_directory(&stdlib_path, &resolver)); + assert!(!importlib_namespace_package.is_regular_package(&stdlib_path, &resolver)); // (This is still `None`, as directories don't resolve to `Vfs` files) assert_eq!( - importlib_namespace_package.to_vfs_file( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - ), + importlib_namespace_package.to_vfs_file(&stdlib_path, &resolver), None ); // ...As do submodules in the `importlib` namespace package: let importlib_abc = importlib_namespace_package.join("abc.pyi"); - assert!(!importlib_abc.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py39)); - assert!(!importlib_abc.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); - assert!(importlib_abc - .to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39) - .is_some()); + assert!(!importlib_abc.is_directory(&stdlib_path, &resolver)); + assert!(!importlib_abc.is_regular_package(&stdlib_path, &resolver)); + assert!(importlib_abc.to_vfs_file(&stdlib_path, &resolver).is_some()); } #[test] fn mocked_typeshed_nonexistent_namespace_stdlib_pkg_py39() { - let (db, stdlib_path, versions) = py39_stdlib_test_case(); + let (db, stdlib_path) = py39_stdlib_test_case(); + let resolver = ResolverState { + db: &db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version: TargetVersion::Py39, + }; // The `xml` package no longer exists on py39: let xml_namespace_package = stdlib_path.join("xml"); assert_eq!( - xml_namespace_package.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39), + xml_namespace_package.to_vfs_file(&stdlib_path, &resolver), None ); - assert!(!xml_namespace_package.is_directory( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); - assert!(!xml_namespace_package.is_regular_package( - &db, - &stdlib_path, - &versions, - TargetVersion::Py39 - )); + assert!(!xml_namespace_package.is_directory(&stdlib_path, &resolver)); + assert!(!xml_namespace_package.is_regular_package(&stdlib_path, &resolver)); let xml_etree = xml_namespace_package.join("etree.pyi"); - assert_eq!( - xml_etree.to_vfs_file(&db, &stdlib_path, &versions, TargetVersion::Py39), - None - ); - assert!(!xml_etree.is_directory(&db, &stdlib_path, &versions, TargetVersion::Py39)); - assert!(!xml_etree.is_regular_package(&db, &stdlib_path, &versions, TargetVersion::Py39)); + assert_eq!(xml_etree.to_vfs_file(&stdlib_path, &resolver), None); + assert!(!xml_etree.is_directory(&stdlib_path, &resolver)); + assert!(!xml_etree.is_regular_package(&stdlib_path, &resolver)); } } diff --git a/crates/red_knot_module_resolver/src/resolver.rs b/crates/red_knot_module_resolver/src/resolver.rs index 97bef6a96244d..08438472cfadb 100644 --- a/crates/red_knot_module_resolver/src/resolver.rs +++ b/crates/red_knot_module_resolver/src/resolver.rs @@ -9,8 +9,8 @@ use crate::module::{Module, ModuleKind}; use crate::module_name::ModuleName; use crate::path::ModuleResolutionPathBuf; use crate::resolver::internal::ModuleResolverSettings; +use crate::state::ResolverState; use crate::supported_py_version::TargetVersion; -use crate::typeshed::LazyTypeshedVersions; /// Configures the module resolver settings. /// @@ -245,32 +245,20 @@ fn resolve_name( name: &ModuleName, ) -> Option<(Arc, VfsFile, ModuleKind)> { let resolver_settings = module_resolver_settings(db); - let target_version = resolver_settings.target_version(); - let typeshed_versions = LazyTypeshedVersions::new(); + let resolver_state = ResolverState::new(db, resolver_settings.target_version()); for search_path in resolver_settings.search_paths() { let mut components = name.components(); let module_name = components.next_back()?; - match resolve_package( - db, - search_path, - components, - &typeshed_versions, - target_version, - ) { + match resolve_package(search_path, components, &resolver_state) { Ok(resolved_package) => { let mut package_path = resolved_package.path; package_path.push(module_name); // Must be a `__init__.pyi` or `__init__.py` or it isn't a package. - let kind = if package_path.is_directory( - db, - search_path, - &typeshed_versions, - target_version, - ) { + let kind = if package_path.is_directory(search_path, &resolver_state) { package_path.push("__init__"); ModuleKind::Package } else { @@ -278,18 +266,17 @@ fn resolve_name( }; // TODO Implement full https://peps.python.org/pep-0561/#type-checker-module-resolution-order resolution - if let Some(stub) = package_path.with_pyi_extension().to_vfs_file( - db, - search_path, - &typeshed_versions, - target_version, - ) { + if let Some(stub) = package_path + .with_pyi_extension() + .to_vfs_file(search_path, &resolver_state) + { return Some((search_path.clone(), stub, kind)); } - if let Some(module) = package_path.with_py_extension().and_then(|path| { - path.to_vfs_file(db, search_path, &typeshed_versions, target_version) - }) { + if let Some(module) = package_path + .with_py_extension() + .and_then(|path| path.to_vfs_file(search_path, &resolver_state)) + { return Some((search_path.clone(), module, kind)); } @@ -312,11 +299,9 @@ fn resolve_name( } fn resolve_package<'a, 'db, I>( - db: &'db dyn Db, module_search_path: &ModuleResolutionPathBuf, components: I, - typeshed_versions: &LazyTypeshedVersions<'db>, - target_version: TargetVersion, + resolver_state: &ResolverState<'db>, ) -> Result where I: Iterator, @@ -335,21 +320,12 @@ where for folder in components { package_path.push(folder); - let is_regular_package = package_path.is_regular_package( - db, - module_search_path, - typeshed_versions, - target_version, - ); + let is_regular_package = + package_path.is_regular_package(module_search_path, resolver_state); if is_regular_package { in_namespace_package = false; - } else if package_path.is_directory( - db, - module_search_path, - typeshed_versions, - target_version, - ) { + } else if package_path.is_directory(module_search_path, resolver_state) { // A directory without an `__init__.py` is a namespace package, continue with the next folder. in_namespace_package = true; } else if in_namespace_package { diff --git a/crates/red_knot_module_resolver/src/state.rs b/crates/red_knot_module_resolver/src/state.rs new file mode 100644 index 0000000000000..ad9a7329a89ba --- /dev/null +++ b/crates/red_knot_module_resolver/src/state.rs @@ -0,0 +1,25 @@ +use ruff_db::file_system::FileSystem; + +use crate::db::Db; +use crate::supported_py_version::TargetVersion; +use crate::typeshed::LazyTypeshedVersions; + +pub(crate) struct ResolverState<'db> { + pub(crate) db: &'db dyn Db, + pub(crate) typeshed_versions: LazyTypeshedVersions<'db>, + pub(crate) target_version: TargetVersion, +} + +impl<'db> ResolverState<'db> { + pub(crate) fn new(db: &'db dyn Db, target_version: TargetVersion) -> Self { + Self { + db, + typeshed_versions: LazyTypeshedVersions::new(), + target_version, + } + } + + pub(crate) fn file_system(&self) -> &dyn FileSystem { + self.db.file_system() + } +} diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index e320b09ce96a5..61ef0249cfecb 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -14,6 +14,7 @@ use crate::db::Db; use crate::module_name::ModuleName; use crate::supported_py_version::TargetVersion; +#[derive(Debug)] pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>); impl<'db> LazyTypeshedVersions<'db> { From 38128e036fd9d6a1887683c8f7d03fdc4da69a28 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 5 Jul 2024 23:39:27 +0100 Subject: [PATCH 58/58] Add a TODO for better distinctions between absolute and relative paths --- crates/red_knot_module_resolver/src/path.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/red_knot_module_resolver/src/path.rs b/crates/red_knot_module_resolver/src/path.rs index f3cb0391d59ea..70a8ea483297c 100644 --- a/crates/red_knot_module_resolver/src/path.rs +++ b/crates/red_knot_module_resolver/src/path.rs @@ -1,3 +1,7 @@ +/// Internal abstractions for differentiating between different kinds of search paths. +/// +/// TODO(Alex): Should we use different types for absolute vs relative paths? +/// use std::fmt; use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};