diff --git a/Cargo.lock b/Cargo.lock index 657728c040f3..8a10094e3873 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.18", "serde", ] @@ -241,14 +241,14 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab69130804d941f8075cfd713bf8848a2c3b3f201a9457a11e6f87e1ab62305" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.8", + "regex-automata 0.4.18", "regex-syntax 0.8.5", ] @@ -273,7 +273,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.18", "same-file", "walkdir", "winapi-util", @@ -281,7 +281,7 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.24" +version = "0.4.33" dependencies = [ "bstr", "crossbeam-channel", @@ -290,7 +290,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.18", "same-file", "walkdir", "winapi-util", @@ -517,7 +517,7 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.18", "regex-syntax 0.8.5", ] @@ -532,9 +532,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -646,7 +646,7 @@ dependencies = [ "dunce", "fast-glob", "globwalk", - "ignore 0.4.24", + "ignore 0.4.33", "log", "pretty_assertions", "rayon", diff --git a/crates/ignore/Cargo.toml b/crates/ignore/Cargo.toml index bd0352d15fe3..d0af38ba94b6 100644 --- a/crates/ignore/Cargo.toml +++ b/crates/ignore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ignore" -version = "0.4.24" #:version +version = "0.4.33" #:version authors = ["Andrew Gallant "] description = """ A fast library for efficiently matching ignore files such as `.gitignore` @@ -12,7 +12,10 @@ repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore" readme = "README.md" keywords = ["glob", "ignore", "gitignore", "pattern", "file"] license = "Unlicense OR MIT" +# CHANGED: Use an explicit edition instead of `edition.workspace = true` since this crate is +# vendored into the Tailwind CSS workspace. edition = "2024" +rust-version = "1.88" [lib] name = "ignore" @@ -20,15 +23,17 @@ bench = false [dependencies] crossbeam-deque = "0.8.3" -globset = "0.4.17" +# CHANGED: Use the published globset crate instead of a path dependency. +globset = "0.4.20" log = "0.4.20" memchr = "2.6.3" same-file = "1.0.6" walkdir = "2.4.0" +# CHANGED: Added `dunce` to canonicalize paths without UNC prefixes on Windows. dunce = "1.0.5" [dependencies.regex-automata] -version = "0.4.0" +version = "0.4.18" default-features = false features = ["std", "perf", "syntax", "meta", "nfa", "hybrid", "dfa-onepass"] diff --git a/crates/ignore/examples/walk.rs b/crates/ignore/examples/walk.rs index 9c627dc3e55b..c61d0515e892 100644 --- a/crates/ignore/examples/walk.rs +++ b/crates/ignore/examples/walk.rs @@ -18,9 +18,7 @@ fn main() { let stdout_thread = std::thread::spawn(move || { let mut stdout = std::io::BufWriter::new(std::io::stdout()); for dent in rx { - stdout - .write_all(&Vec::from_path_lossy(dent.path())) - .unwrap(); + stdout.write_all(&Vec::from_path_lossy(dent.path())).unwrap(); stdout.write_all(b"\n").unwrap(); } }); diff --git a/crates/ignore/src/default_types.rs b/crates/ignore/src/default_types.rs index 4e060b76ae83..6b5bba0f018e 100644 --- a/crates/ignore/src/default_types.rs +++ b/crates/ignore/src/default_types.rs @@ -47,6 +47,7 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ (&["cml"], &["*.cml"]), (&["coffeescript"], &["*.coffee"]), (&["config"], &["*.cfg", "*.conf", "*.config", "*.ini"]), + (&["container"], &["*Containerfile*", "*Dockerfile*"]), (&["coq"], &["*.v"]), (&["cpp"], &[ "*.[ChH]", "*.cc", "*.[ch]pp", "*.[ch]xx", "*.hh", "*.inl", @@ -109,6 +110,7 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ (&["hbs"], &["*.hbs"]), (&["hs"], &["*.hs", "*.lhs"]), (&["html"], &["*.htm", "*.html", "*.ejs"]), + (&["hurl"], &["*.hurl"]), (&["hy"], &["*.hy"]), (&["idris"], &["*.idr", "*.lidr"]), (&["janet"], &["*.janet"]), @@ -185,6 +187,7 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ (&["mint"], &["*.mint"]), (&["mk"], &["mkfile"]), (&["ml"], &["*.ml"]), + (&["mojo"], &["*.mojo"]), (&["motoko"], &["*.mo"]), (&["msbuild"], &[ "*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets", @@ -206,11 +209,12 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ "*.php", "*.php3", "*.php4", "*.php5", "*.php7", "*.php8", "*.pht", "*.phtml" ]), + (&["pkgbuild"], &["PKGBUILD"]), (&["po"], &["*.po"]), (&["pod"], &["*.pod"]), (&["postscript"], &["*.eps", "*.ps"]), (&["prolog"], &["*.pl", "*.pro", "*.prolog", "*.P"]), - (&["protobuf"], &["*.proto"]), + (&["proto", "protobuf"], &["*.proto"]), (&["ps"], &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]), (&["puppet"], &["*.epp", "*.erb", "*.pp", "*.rb"]), (&["purs"], &["*.purs"]), @@ -231,6 +235,7 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ (&["red"], &["*.r", "*.red", "*.reds"]), (&["rescript"], &["*.res", "*.resi"]), (&["robot"], &["*.robot"]), + (&["rocq"], &["*.v"]), (&["rst"], &["*.rst"]), (&["ruby"], &[ // Idiomatic files @@ -274,6 +279,7 @@ pub(crate) const DEFAULT_TYPES: &[(&[&str], &[&str])] = &[ (&["spark"], &["*.spark"]), (&["spec"], &["*.spec"]), (&["sql"], &["*.sql", "*.psql"]), + (&["ssa"], &["*.ssa"]), (&["stylus"], &["*.styl"]), (&["sv"], &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]), (&["svelte"], &["*.svelte", "*.svelte.ts"]), @@ -359,4 +365,14 @@ mod tests { previous_name = name; } } + + #[test] + fn default_types_aliases_are_sorted() { + for (aliases, _) in DEFAULT_TYPES.iter() { + assert!( + aliases.is_sorted(), + "this alias list is not sorted: {aliases:?}", + ); + } + } } diff --git a/crates/ignore/src/dir.rs b/crates/ignore/src/dir.rs index 11b58f8ca525..6bee724c8fee 100644 --- a/crates/ignore/src/dir.rs +++ b/crates/ignore/src/dir.rs @@ -16,7 +16,7 @@ use std::{ collections::HashMap, ffi::{OsStr, OsString}, - fs::{File, FileType}, + fs::{self, File, FileType}, io::{self, BufRead}, path::{Path, PathBuf}, sync::{Arc, RwLock, Weak}, @@ -25,7 +25,7 @@ use std::{ use crate::{ gitignore::{self, Gitignore, GitignoreBuilder}, overrides::{self, Override}, - pathutil::{is_hidden, strip_prefix}, + pathutil::{is_hidden_entry, strip_prefix}, types::{self, Types}, walk::DirEntry, {Error, Match, PartialErrorBuilder}, @@ -91,7 +91,25 @@ struct IgnoreOptions { /// Ignore is a matcher useful for recursively walking one or more directories. #[derive(Clone, Debug)] -pub(crate) struct Ignore(Arc); +pub(crate) struct Ignore { + inner: Arc, + // Parent matchers are cached independently of the path being walked, but + // matching them still needs the canonicalized path originally passed to + // `add_parents`. For example, when walking `/tmp/project/src`, parent + // matchers use `/tmp/project/src` to rewrite `/tmp/project/src/foo.py` + // before matching it against ignore files from `/tmp/project` and its + // ancestors. + // + // For ripgrep itself, this means that `rg pat src tests` must rewrite + // `src/foo` relative to `.../src`, and not whatever root was prepared + // first. + // + // See: https://github.com/BurntSushi/ripgrep/pull/3420 + // See: https://github.com/BurntSushi/ripgrep/issues/3376 + // See: https://github.com/BurntSushi/ripgrep/issues/3419 + // See: https://github.com/BurntSushi/ripgrep/issues/3320 + absolute_base: Option>, +} #[derive(Clone, Debug)] struct IgnoreInner { @@ -112,12 +130,9 @@ struct IgnoreInner { /// /// If this is the root directory or there are otherwise no more /// directories to match, then `parent` is `None`. - parent: Option, + parent: Option>, /// Whether this is an absolute parent matcher, as added by add_parent. is_absolute_parent: bool, - /// The absolute base path of this matcher. Populated only if parent - /// directories are added. - absolute_base: Option>, /// The directory that gitignores should be interpreted relative to. /// /// Usually this is the directory containing the gitignore file. But in @@ -152,34 +167,36 @@ struct IgnoreInner { impl Ignore { /// Return the directory path of this matcher. + #[cfg(test)] pub(crate) fn path(&self) -> &Path { - &self.0.dir + &self.inner.dir } /// Return true if this matcher has no parent. pub(crate) fn is_root(&self) -> bool { - self.0.parent.is_none() - } - - /// Returns true if this matcher was added via the `add_parents` method. - pub(crate) fn is_absolute_parent(&self) -> bool { - self.0.is_absolute_parent + self.inner.parent.is_none() } /// Return this matcher's parent, if one exists. pub(crate) fn parent(&self) -> Option { - self.0.parent.clone() + self.inner.parent.as_ref().map(|parent| Ignore { + inner: parent.clone(), + absolute_base: self.absolute_base.clone(), + }) } /// Create a new `Ignore` matcher with the parent directories of `dir`. /// /// Note that this can only be called on an `Ignore` matcher with no /// parents (i.e., `is_root` returns `true`). This will panic otherwise. - pub(crate) fn add_parents>(&self, path: P) -> (Ignore, Option) { - if !self.0.opts.parents - && !self.0.opts.git_ignore - && !self.0.opts.git_exclude - && !self.0.opts.git_global + pub(crate) fn add_parents>( + &self, + path: P, + ) -> (Ignore, Option) { + if !self.inner.opts.parents + && !self.inner.opts.git_ignore + && !self.inner.opts.git_exclude + && !self.inner.opts.git_global { // If we never need info from parent directories, then don't do // anything. @@ -209,25 +226,34 @@ impl Ignore { let mut errs = PartialErrorBuilder::default(); let mut ig = self.clone(); for parent in parents.into_iter().rev() { - let mut compiled = self.0.compiled.write().unwrap(); + let mut compiled = self.inner.compiled.write().unwrap(); if let Some(weak) = compiled.get(parent.as_os_str()) { if let Some(prebuilt) = weak.upgrade() { - ig = Ignore(prebuilt); + ig = Ignore { + inner: prebuilt, + absolute_base: Some(absolute_base.clone()), + }; continue; } } let (mut igtmp, err) = ig.add_child_path(parent); errs.maybe_push(err); igtmp.is_absolute_parent = true; - igtmp.absolute_base = Some(absolute_base.clone()); - igtmp.has_git = if self.0.opts.require_git && self.0.opts.git_ignore { - parent.join(".git").exists() || parent.join(".jj").exists() - } else { - false - }; + igtmp.has_git = + if self.inner.opts.require_git && self.inner.opts.git_ignore { + parent.join(".git").exists() || parent.join(".jj").exists() + } else { + false + }; let ig_arc = Arc::new(igtmp); - ig = Ignore(ig_arc.clone()); - compiled.insert(parent.as_os_str().to_os_string(), Arc::downgrade(&ig_arc)); + ig = Ignore { + inner: ig_arc.clone(), + absolute_base: Some(absolute_base.clone()), + }; + compiled.insert( + parent.as_os_str().to_os_string(), + Arc::downgrade(&ig_arc), + ); } (ig, errs.into_error_option()) } @@ -240,60 +266,161 @@ impl Ignore { /// returned if it exists. /// /// Note that all I/O errors are completely ignored. - pub(crate) fn add_child>(&self, dir: P) -> (Ignore, Option) { + pub(crate) fn add_child>( + &self, + dir: P, + ) -> (Ignore, Option) { let (ig, err) = self.add_child_path(dir.as_ref()); - (Ignore(Arc::new(ig)), err) + ( + Ignore { + inner: Arc::new(ig), + absolute_base: self.absolute_base.clone(), + }, + err, + ) + } + + /// Like add_child, but uses successful read_dir entries to reduce + /// probing when discovering ignore files. + pub(crate) fn add_child_with_entries>( + &self, + dir: P, + entries: &[fs::DirEntry], + ) -> (Ignore, Option) { + let files = self.collect_ignore_files(entries); + let (ig, err) = self.add_child_path_with_found_ignore_files( + dir.as_ref(), + Some(&files), + ); + ( + Ignore { + inner: Arc::new(ig), + absolute_base: self.absolute_base.clone(), + }, + err, + ) } /// Like add_child, but takes a full path and returns an IgnoreInner. fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option) { - let git_type = - if self.0.opts.require_git && (self.0.opts.git_ignore || self.0.opts.git_exclude) { - dir.join(".git").metadata().ok().map(|md| md.file_type()) - } else { - None - }; - let has_git = git_type.is_some() || dir.join(".jj").exists(); + self.add_child_path_with_found_ignore_files(dir, None) + } + + fn collect_ignore_files( + &self, + entries: &[fs::DirEntry], + ) -> IgnoreFilesFound { + let custom_ignore_filenames = &self.inner.custom_ignore_filenames; + let mut files = IgnoreFilesFound { + has_ignore: false, + has_git_ignore: false, + has_git_dir: false, + has_jj_dir: false, + custom_ignore_files: vec![false; custom_ignore_filenames.len()], + }; + for entry in entries { + let file_name = entry.file_name(); + if file_name == OsStr::new(".ignore") { + files.has_ignore = true; + } else if file_name == OsStr::new(".gitignore") { + files.has_git_ignore = true; + } else if file_name == OsStr::new(".git") { + files.has_git_dir = true; + } else if file_name == OsStr::new(".jj") { + files.has_jj_dir = true; + } + for (i, name) in custom_ignore_filenames.iter().enumerate() { + if file_name == name.as_os_str() { + files.custom_ignore_files[i] = true; + } + } + } + files + } + + fn add_child_path_with_found_ignore_files( + &self, + dir: &Path, + ignore_files_list: Option<&IgnoreFilesFound>, + ) -> (IgnoreInner, Option) { + let check_vcs_dir = self.inner.opts.require_git + && (self.inner.opts.git_ignore || self.inner.opts.git_exclude); + let git_type = if check_vcs_dir + && ignore_files_list.is_none_or(|i| i.has_git_dir) + { + dir.join(".git").metadata().ok().map(|md| md.file_type()) + } else { + None + }; + let has_jj = check_vcs_dir + && ignore_files_list.is_none_or(|i| i.has_jj_dir) + && dir.join(".jj").exists(); + let has_git = check_vcs_dir && (git_type.is_some() || has_jj); let mut errs = PartialErrorBuilder::default(); - let custom_ig_matcher = if self.0.custom_ignore_filenames.is_empty() { + let custom_ig_matcher = if self + .inner + .custom_ignore_filenames + .is_empty() + { Gitignore::empty() } else { - let (m, err) = create_gitignore( - &dir, - &dir, - &self.0.custom_ignore_filenames, - self.0.opts.ignore_case_insensitive, - ); - errs.maybe_push(err); - m + let custom_ignore_names: Vec<&OsString> = match ignore_files_list { + None => self.inner.custom_ignore_filenames.iter().collect(), + Some(m) => self + .inner + .custom_ignore_filenames + .iter() + .zip(m.custom_ignore_files.iter()) + .filter(|&(_, &matched)| matched) + .map(|(name, _)| name) + .collect(), + }; + if custom_ignore_names.is_empty() { + Gitignore::empty() + } else { + let (m, err) = create_gitignore( + &dir, + &dir, + &custom_ignore_names, + self.inner.opts.ignore_case_insensitive, + ); + errs.maybe_push(err); + m + } }; - let ig_matcher = if !self.0.opts.ignore { + let ig_matcher = if !self.inner.opts.ignore + || !ignore_files_list.is_none_or(|i| i.has_ignore) + { Gitignore::empty() } else { let (m, err) = create_gitignore( &dir, &dir, &[".ignore"], - self.0.opts.ignore_case_insensitive, + self.inner.opts.ignore_case_insensitive, ); errs.maybe_push(err); m }; - let gi_matcher = if !self.0.opts.git_ignore { + let gi_matcher = if !self.inner.opts.git_ignore + || !ignore_files_list.is_none_or(|i| i.has_git_ignore) + { Gitignore::empty() } else { let (m, err) = create_gitignore( &dir, &dir, &[".gitignore"], - self.0.opts.ignore_case_insensitive, + self.inner.opts.ignore_case_insensitive, ); errs.maybe_push(err); m }; - let gi_exclude_matcher = if !self.0.opts.git_exclude { + let gi_exclude_matcher = if !self.inner.opts.git_exclude + || !ignore_files_list.is_none_or(|i| i.has_git_dir) + { Gitignore::empty() } else { match resolve_git_commondir(dir, git_type) { @@ -302,7 +429,7 @@ impl Ignore { &dir, &git_dir, &["info/exclude"], - self.0.opts.ignore_case_insensitive, + self.inner.opts.ignore_case_insensitive, ); errs.maybe_push(err); m @@ -314,32 +441,38 @@ impl Ignore { } }; let ig = IgnoreInner { - compiled: self.0.compiled.clone(), + compiled: self.inner.compiled.clone(), dir: dir.to_path_buf(), - overrides: self.0.overrides.clone(), - types: self.0.types.clone(), - parent: Some(self.clone()), + overrides: self.inner.overrides.clone(), + types: self.inner.types.clone(), + parent: Some(self.inner.clone()), is_absolute_parent: false, - absolute_base: self.0.absolute_base.clone(), - global_gitignores_relative_to: self.0.global_gitignores_relative_to.clone(), - explicit_ignores: self.0.explicit_ignores.clone(), - custom_ignore_filenames: self.0.custom_ignore_filenames.clone(), + global_gitignores_relative_to: self + .inner + .global_gitignores_relative_to + .clone(), + explicit_ignores: self.inner.explicit_ignores.clone(), + custom_ignore_filenames: self + .inner + .custom_ignore_filenames + .clone(), custom_ignore_matcher: custom_ig_matcher, ignore_matcher: ig_matcher, - git_global_matcher: self.0.git_global_matcher.clone(), + git_global_matcher: self.inner.git_global_matcher.clone(), git_ignore_matcher: gi_matcher, git_exclude_matcher: gi_exclude_matcher, has_git, - opts: self.0.opts, + opts: self.inner.opts, }; (ig, errs.into_error_option()) } /// Returns true if at least one type of ignore rule should be matched. fn has_any_ignore_rules(&self) -> bool { - let opts = self.0.opts; - let has_custom_ignore_files = !self.0.custom_ignore_filenames.is_empty(); - let has_explicit_ignores = !self.0.explicit_ignores.is_empty(); + let opts = self.inner.opts; + let has_custom_ignore_files = + !self.inner.custom_ignore_filenames.is_empty(); + let has_explicit_ignores = !self.inner.explicit_ignores.is_empty(); opts.ignore || opts.git_global @@ -350,9 +483,12 @@ impl Ignore { } /// Like `matched`, but works with a directory entry instead. - pub(crate) fn matched_dir_entry<'a>(&'a self, dent: &DirEntry) -> Match> { + pub(crate) fn matched_dir_entry<'a>( + &'a self, + dent: &DirEntry, + ) -> Match> { let m = self.matched(dent.path(), dent.is_dir()); - if m.is_none() && self.0.opts.hidden && is_hidden(dent) { + if m.is_none() && self.inner.opts.hidden && is_hidden_entry(dent) { return Match::Ignore(IgnoreMatch::hidden()); } m @@ -362,7 +498,11 @@ impl Ignore { /// ignored or not. /// /// The match contains information about its origin. - fn matched<'a, P: AsRef>(&'a self, path: P, is_dir: bool) -> Match> { + pub(crate) fn matched<'a, P: AsRef>( + &'a self, + path: P, + is_dir: bool, + ) -> Match> { // We need to be careful with our path. If it has a leading ./, then // strip it because it causes nothing but trouble. let mut path = path.as_ref(); @@ -373,9 +513,9 @@ impl Ignore { // regardless of whether it's whitelist/ignore, then we quit and // return that result immediately. Overrides have the highest // precedence. - if !self.0.overrides.is_empty() { + if !self.inner.overrides.is_empty() { let mat = self - .0 + .inner .overrides .matched(path, is_dir) .map(IgnoreMatch::overrides); @@ -392,8 +532,9 @@ impl Ignore { whitelisted = mat; } } - if !self.0.types.is_empty() { - let mat = self.0.types.matched(path, is_dir).map(IgnoreMatch::types); + if !self.inner.types.is_empty() { + let mat = + self.inner.types.matched(path, is_dir).map(IgnoreMatch::types); if mat.is_ignore() { return mat; } else if mat.is_whitelist() { @@ -405,98 +546,143 @@ impl Ignore { /// Performs matching only on the ignore files for this directory and /// all parent directories. - fn matched_ignore<'a>(&'a self, path: &Path, is_dir: bool) -> Match> { - let (mut m_custom_ignore, mut m_ignore, mut m_gi, mut m_gi_exclude, mut m_explicit) = ( - Match::None, - Match::None, - Match::None, - Match::None, - Match::None, - ); - let any_git = !self.0.opts.require_git || self.parents().any(|ig| ig.0.has_git); + pub(crate) fn matched_ignore<'a>( + &'a self, + path: &Path, + is_dir: bool, + ) -> Match> { + let ( + mut m_custom_ignore, + mut m_ignore, + mut m_gi, + mut m_gi_exclude, + mut m_explicit, + ) = (Match::None, Match::None, Match::None, Match::None, Match::None); + let any_git = !self.inner.opts.require_git + || self.parents().any(|ig| ig.inner.has_git); let mut saw_git = false; - for ig in self.parents().take_while(|ig| !ig.0.is_absolute_parent) { + for ig in self.parents().take_while(|ig| !ig.inner.is_absolute_parent) + { if m_custom_ignore.is_none() { - m_custom_ignore = - ig.0.custom_ignore_matcher - .matched(path, is_dir) - .map(IgnoreMatch::gitignore); + m_custom_ignore = ig + .inner + .custom_ignore_matcher + .matched(path, is_dir) + .map(IgnoreMatch::gitignore); } if m_ignore.is_none() { - m_ignore = - ig.0.ignore_matcher - .matched(path, is_dir) - .map(IgnoreMatch::gitignore); + m_ignore = ig + .inner + .ignore_matcher + .matched(path, is_dir) + .map(IgnoreMatch::gitignore); } if any_git && !saw_git && m_gi.is_none() { - m_gi = - ig.0.git_ignore_matcher - .matched(path, is_dir) - .map(IgnoreMatch::gitignore); + m_gi = ig + .inner + .git_ignore_matcher + .matched(path, is_dir) + .map(IgnoreMatch::gitignore); } if any_git && !saw_git && m_gi_exclude.is_none() { - m_gi_exclude = - ig.0.git_exclude_matcher - .matched(path, is_dir) - .map(IgnoreMatch::gitignore); + m_gi_exclude = ig + .inner + .git_exclude_matcher + .matched(path, is_dir) + .map(IgnoreMatch::gitignore); } - saw_git = saw_git || ig.0.has_git; + saw_git = saw_git || ig.inner.has_git; } - if self.0.opts.parents { - if let Some(_) = self.absolute_base() { - // CHANGED: We removed a code path that rewrote the `path` to be relative to - // `self.absolute_base()` because it assumed that the every path is inside the base - // which is not the case for us as we use `WalkBuilder#add` to add roots outside of the - // base. - for ig in self.parents().skip_while(|ig| !ig.0.is_absolute_parent) { + if self.inner.opts.parents { + if let Some(abs_parent_path) = self.absolute_base() { + // What we want to do here is take the absolute base path of + // this directory and join it with the path we're searching. + // The main issue we want to avoid is accidentally duplicating + // directory components, so we try to strip any common prefix + // off of `path`. Overall, this seems a little ham-fisted, but + // it does fix a nasty bug. It should do fine until we overhaul + // this crate. + let path = abs_parent_path.join( + self.parents() + .take_while(|ig| !ig.inner.is_absolute_parent) + .last() + .map_or(path, |ig| { + // This is a weird special case when ripgrep users + // search with just a `.`, as some tools do + // automatically (like consult). In this case, if + // we don't bail out now, the code below will strip + // a leading `.` from `path`, which might mangle + // a hidden file name! + if ig.inner.dir.as_path() == Path::new(".") { + return path; + } + let without_dot_slash = strip_if_is_prefix( + "./", + ig.inner.dir.as_path(), + ); + let relative_base = + strip_if_is_prefix(without_dot_slash, path); + strip_if_is_prefix("/", relative_base) + }), + ); + + for ig in self + .parents() + .skip_while(|ig| !ig.inner.is_absolute_parent) + { if m_custom_ignore.is_none() { - m_custom_ignore = - ig.0.custom_ignore_matcher - .matched(&path, is_dir) - .map(IgnoreMatch::gitignore); + m_custom_ignore = ig + .inner + .custom_ignore_matcher + .matched(&path, is_dir) + .map(IgnoreMatch::gitignore); } if m_ignore.is_none() { - m_ignore = - ig.0.ignore_matcher - .matched(&path, is_dir) - .map(IgnoreMatch::gitignore); + m_ignore = ig + .inner + .ignore_matcher + .matched(&path, is_dir) + .map(IgnoreMatch::gitignore); } if any_git && !saw_git && m_gi.is_none() { - m_gi = - ig.0.git_ignore_matcher - .matched(&path, is_dir) - .map(IgnoreMatch::gitignore); + m_gi = ig + .inner + .git_ignore_matcher + .matched(&path, is_dir) + .map(IgnoreMatch::gitignore); } if any_git && !saw_git && m_gi_exclude.is_none() { - m_gi_exclude = - ig.0.git_exclude_matcher - .matched(&path, is_dir) - .map(IgnoreMatch::gitignore); + m_gi_exclude = ig + .inner + .git_exclude_matcher + .matched(&path, is_dir) + .map(IgnoreMatch::gitignore); } - saw_git = saw_git || ig.0.has_git; + saw_git = saw_git || ig.inner.has_git; } } } - for gi in self.0.explicit_ignores.iter().rev() { - // CHANGED: We need to make sure that the explicit gitignore rules apply to the path - // - // path = Is the current file/folder we are traversing - // gi.path() = Is the path of the custom gitignore file + for gi in self.inner.explicit_ignores.iter().rev() { + if !m_explicit.is_none() { + break; + } + // CHANGED: We need to make sure that the explicit gitignore rules + // apply to the path // - // E.g.: If we have a custom rule for `/src/utils` with `**/*`, and we are looking at - // just `/src`, then the `**/*` rules do not apply to this folder, so we can - // ignore the current custom gitignore file. + // path = Is the current file/folder we are traversing + // gi.path() = Is the path of the custom gitignore file // + // E.g.: If we have a custom rule for `/src/utils` with `**/*`, and + // we are looking at just `/src`, then the `**/*` rules do + // not apply to this folder, so we can ignore the current + // custom gitignore file. if !path.starts_with(gi.path()) { continue; } - if !m_explicit.is_none() { - break; - } m_explicit = gi.matched(&path, is_dir).map(IgnoreMatch::gitignore); } let m_global = if any_git { - self.0 + self.inner .git_global_matcher .matched(&path, is_dir) .map(IgnoreMatch::gitignore) @@ -504,59 +690,90 @@ impl Ignore { Match::None }; - // CHANGED: We added logic to configure an order in which the ignore files are respected and - // allowed a whitelist in a later file to overrule a block on an earlier file. + // CHANGED: We added logic to configure an order in which the ignore + // files are respected. Explicitly added ignores (via + // `WalkBuilder::add_gitignore`) take precedence over all ignore files + // found on disk, and the first source with a definitive answer wins. let order = [ // Manually added ignores - &m_explicit, + m_explicit, // .custom-ignore - &m_custom_ignore, + m_custom_ignore, // .ignore - &m_ignore, + m_ignore, // .gitignore - &m_gi, + m_gi, // .git/info/exclude - &m_gi_exclude, + m_gi_exclude, // Global gitignore - &m_global, + m_global, ]; - for check in order.into_iter() { - if check.is_none() { - continue; + if !check.is_none() { + return check; } - - return check.clone(); } - - m_explicit + Match::None } /// Returns an iterator over parent ignore matchers, including this one. pub(crate) fn parents(&self) -> Parents<'_> { - Parents(Some(self)) + Parents(Some(IgnoreRef { inner: &self.inner })) } /// Returns the first absolute path of the first absolute parent, if /// one exists. fn absolute_base(&self) -> Option<&Path> { - self.0.absolute_base.as_ref().map(|p| &***p) + self.absolute_base.as_ref().map(|p| &***p) } } -/// An iterator over all parents of an ignore matcher, including itself. +/// State for tracking what kinds of files ripgrep is interested in for a +/// given directory. /// -/// The lifetime `'a` refers to the lifetime of the initial `Ignore` matcher. -pub(crate) struct Parents<'a>(Option<&'a Ignore>); +/// This is computed over the entire set of files in a directory instead of +/// trying to stat each file individually. If a file is present, it's only then +/// that we stat it for more information, instead of relying on the stat to +/// determine its existence. +#[derive(Debug)] +struct IgnoreFilesFound { + has_ignore: bool, + has_git_ignore: bool, + has_git_dir: bool, + has_jj_dir: bool, + custom_ignore_files: Vec, +} + +#[derive(Clone, Copy)] +pub(crate) struct IgnoreRef<'a> { + inner: &'a IgnoreInner, +} + +impl IgnoreRef<'_> { + pub(crate) fn path(&self) -> &Path { + &self.inner.dir + } + + pub(crate) fn is_absolute_parent(&self) -> bool { + self.inner.is_absolute_parent + } +} + +/// An iterator over all parents of an ignore matcher, including itself. +pub(crate) struct Parents<'a>(Option>); impl<'a> Iterator for Parents<'a> { - type Item = &'a Ignore; + type Item = IgnoreRef<'a>; - fn next(&mut self) -> Option<&'a Ignore> { + fn next(&mut self) -> Option> { match self.0.take() { None => None, Some(ig) => { - self.0 = ig.0.parent.as_ref(); + self.0 = ig + .inner + .parent + .as_deref() + .map(|inner| IgnoreRef { inner }); Some(ig) } } @@ -645,33 +862,42 @@ impl IgnoreBuilder { } gi } else { - log::debug!("ignoring global gitignore file because CWD is not known"); + log::debug!( + "ignoring global gitignore file because CWD is not known" + ); Gitignore::empty() }; - Ignore(Arc::new(IgnoreInner { - compiled: Arc::new(RwLock::new(HashMap::new())), - dir: self.dir.clone(), - overrides: self.overrides.clone(), - types: self.types.clone(), - parent: None, - is_absolute_parent: true, + Ignore { + inner: Arc::new(IgnoreInner { + compiled: Arc::new(RwLock::new(HashMap::new())), + dir: self.dir.clone(), + overrides: self.overrides.clone(), + types: self.types.clone(), + parent: None, + is_absolute_parent: true, + global_gitignores_relative_to, + explicit_ignores: Arc::new(self.explicit_ignores.clone()), + custom_ignore_filenames: Arc::new( + self.custom_ignore_filenames.clone(), + ), + custom_ignore_matcher: Gitignore::empty(), + ignore_matcher: Gitignore::empty(), + git_global_matcher: Arc::new(git_global_matcher), + git_ignore_matcher: Gitignore::empty(), + git_exclude_matcher: Gitignore::empty(), + has_git: false, + opts: self.opts, + }), absolute_base: None, - global_gitignores_relative_to, - explicit_ignores: Arc::new(self.explicit_ignores.clone()), - custom_ignore_filenames: Arc::new(self.custom_ignore_filenames.clone()), - custom_ignore_matcher: Gitignore::empty(), - ignore_matcher: Gitignore::empty(), - git_global_matcher: Arc::new(git_global_matcher), - git_ignore_matcher: Gitignore::empty(), - git_exclude_matcher: Gitignore::empty(), - has_git: false, - opts: self.opts, - })) + } } /// Set the current directory used for matching global gitignores. - pub(crate) fn current_dir(&mut self, cwd: impl Into) -> &mut IgnoreBuilder { + pub(crate) fn current_dir( + &mut self, + cwd: impl Into, + ) -> &mut IgnoreBuilder { self.global_gitignores_relative_to = Some(cwd.into()); self } @@ -681,7 +907,10 @@ impl IgnoreBuilder { /// By default, no override matcher is used. /// /// This overrides any previous setting. - pub(crate) fn overrides(&mut self, overrides: Override) -> &mut IgnoreBuilder { + pub(crate) fn overrides( + &mut self, + overrides: Override, + ) -> &mut IgnoreBuilder { self.overrides = Arc::new(overrides); self } @@ -712,8 +941,7 @@ impl IgnoreBuilder { &mut self, file_name: S, ) -> &mut IgnoreBuilder { - self.custom_ignore_filenames - .push(file_name.as_ref().to_os_string()); + self.custom_ignore_filenames.push(file_name.as_ref().to_os_string()); self } @@ -725,6 +953,11 @@ impl IgnoreBuilder { self } + /// Whether ignoring hidden files is enabled or not. + pub(crate) fn is_hidden(&self) -> bool { + self.opts.hidden + } + /// Enables reading `.ignore` files. /// /// `.ignore` files have the same semantics as `gitignore` files and are @@ -795,7 +1028,10 @@ impl IgnoreBuilder { /// Process ignore files case insensitively /// /// This is disabled by default. - pub(crate) fn ignore_case_insensitive(&mut self, yes: bool) -> &mut IgnoreBuilder { + pub(crate) fn ignore_case_insensitive( + &mut self, + yes: bool, + ) -> &mut IgnoreBuilder { self.opts.ignore_case_insensitive = yes; self } @@ -854,7 +1090,10 @@ pub(crate) fn create_gitignore>( /// them when multiple repositories are searched. /// /// Some I/O errors are ignored. -fn resolve_git_commondir(dir: &Path, git_type: Option) -> Result> { +fn resolve_git_commondir( + dir: &Path, + git_type: Option, +) -> Result> { let git_dir_path = || dir.join(".git"); let git_dir = git_dir_path(); if !git_type.map_or(false, |ft| ft.is_file()) { @@ -899,15 +1138,20 @@ fn resolve_git_commondir(dir: &Path, git_type: Option) -> Result + ?Sized>(prefix: &'a P, path: &'a Path) -> &'a Path { +fn strip_if_is_prefix<'a, P: AsRef + ?Sized>( + prefix: &'a P, + path: &'a Path, +) -> &'a Path { strip_prefix(prefix, path).map_or(path, |p| p) } #[cfg(test)] mod tests { - use std::{io::Write, path::Path}; + use std::{io::Write, path::Path, sync::Arc}; - use crate::{Error, dir::IgnoreBuilder, gitignore::Gitignore, tests::TempDir}; + use crate::{ + Error, dir::IgnoreBuilder, gitignore::Gitignore, tests::TempDir, + }; fn wfile>(path: P, contents: &str) { let mut file = std::fs::File::create(path).unwrap(); @@ -936,11 +1180,11 @@ mod tests { let (gi, err) = Gitignore::new(td.path().join("not-an-ignore")); assert!(err.is_none()); - let (ig, err) = IgnoreBuilder::new() - .add_ignore(gi) - .build() - .add_child(td.path()); + let (ig, err) = + IgnoreBuilder::new().add_ignore(gi).build().add_child(td.path()); assert!(err.is_none()); + // CHANGED: Explicit ignores only apply to paths inside the directory + // of the ignore file, so we have to match against full paths. assert!(ig.matched(td.path().join("foo"), false).is_ignore()); assert!(ig.matched(td.path().join("bar"), false).is_whitelist()); assert!(ig.matched(td.path().join("baz"), false).is_none()); @@ -1211,15 +1455,152 @@ mod tests { let (ig2, err) = ig1.add_child("src"); assert!(err.is_none()); - // CHANGED: These test cases do not make sense for us as we never call the Ignore with - // relative paths. - assert!(ig1.matched("llvm", true).is_ignore()); - assert!(ig2.matched("llvm", true).is_ignore()); + assert!(ig1.matched("llvm", true).is_none()); + assert!(ig2.matched("llvm", true).is_none()); assert!(ig2.matched("src/llvm", true).is_none()); assert!(ig2.matched("foo", false).is_ignore()); assert!(ig2.matched("src/foo", false).is_ignore()); } + #[test] + fn absolute_parent_matchers_are_cached_across_roots() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join("src/build")); + mkdirp(td.path().join("tests/build")); + wfile(td.path().join(".gitignore"), "tests/**/build/\n"); + + let ig0 = IgnoreBuilder::new().build(); + let (src_parents, err) = ig0.add_parents(td.path().join("src")); + assert!(err.is_none()); + let (src, err) = src_parents.add_child(td.path().join("src")); + assert!(err.is_none()); + let (tests_parents, err) = ig0.add_parents(td.path().join("tests")); + assert!(err.is_none()); + let (tests, err) = tests_parents.add_child(td.path().join("tests")); + assert!(err.is_none()); + + assert!(Arc::ptr_eq(&src_parents.inner, &tests_parents.inner)); + assert!(src.matched("build", true).is_none()); + assert!(tests.matched("build", true).is_ignore()); + } + + /// Parent matchers are shared across search roots, but path rewriting for + /// absolute parents must use each root's own base path. Otherwise a rule + /// like `src/invalid` is matched against the wrong absolute path when + /// `src` is searched before a sibling root (e.g. `tests`). + /// + /// Paths passed to `matched` use the same relative layout as `Walk` when + /// roots are given as relative directory names. + /// + /// Regression for: https://github.com/BurntSushi/ripgrep/issues/3376 + /// and https://github.com/BurntSushi/ripgrep/issues/3419 + #[test] + fn multi_root_gitignore_order_independent() { + let td = tmpdir(); + let cwd = std::env::current_dir().unwrap(); + // Use paths relative to CWD like the CLI walk does for `rg pat src tests`. + let root = td.path().strip_prefix(&cwd).unwrap_or(td.path()); + let src_root = root.join("src"); + let tests_root = root.join("tests"); + + mkdirp(td.path().join(".git")); + mkdirp(td.path().join("src")); + mkdirp(td.path().join("tests")); + wfile(td.path().join(".gitignore"), "src/invalid\n"); + wfile(td.path().join("src/invalid"), "x"); + wfile(td.path().join("src/valid"), "x"); + wfile(td.path().join("tests/valid"), "x"); + + let ig0 = IgnoreBuilder::new().build(); + + // Historically buggy order: search `src` first, then `tests`. + let (src_parents, err) = ig0.add_parents(&src_root); + assert!(err.is_none()); + let (src, err) = src_parents.add_child(&src_root); + assert!(err.is_none()); + let (tests_parents, err) = ig0.add_parents(&tests_root); + assert!(err.is_none()); + let (tests, err) = tests_parents.add_child(&tests_root); + assert!(err.is_none()); + + assert!(Arc::ptr_eq(&src_parents.inner, &tests_parents.inner)); + // Each root must carry its own absolute_base even though inners are shared. + assert_ne!( + src.absolute_base.as_ref().unwrap().as_path(), + tests.absolute_base.as_ref().unwrap().as_path() + ); + assert!( + src.matched(src_root.join("invalid"), false).is_ignore(), + "parent .gitignore must apply for the src root even when \ + another root was prepared in the same process" + ); + assert!(src.matched(src_root.join("valid"), false).is_none()); + assert!(tests.matched(tests_root.join("valid"), false).is_none()); + + // Reverse order should behave the same way. + let ig0 = IgnoreBuilder::new().build(); + let (tests_parents, err) = ig0.add_parents(&tests_root); + assert!(err.is_none()); + let (tests, err) = tests_parents.add_child(&tests_root); + assert!(err.is_none()); + let (src_parents, err) = ig0.add_parents(&src_root); + assert!(err.is_none()); + let (src, err) = src_parents.add_child(&src_root); + assert!(err.is_none()); + + assert!(src.matched(src_root.join("invalid"), false).is_ignore()); + assert!(src.matched(src_root.join("valid"), false).is_none()); + assert!(tests.matched(tests_root.join("valid"), false).is_none()); + } + + /// Same multi-root / order issue for non-git ignore files (e.g. `.rgignore` + /// via custom ignore names). + /// + /// Regression for: https://github.com/BurntSushi/ripgrep/issues/3320 + #[test] + fn multi_root_custom_ignore_order_independent() { + let td = tmpdir(); + let cwd = std::env::current_dir().unwrap(); + let root = td.path().strip_prefix(&cwd).unwrap_or(td.path()); + let alpha_root = root.join("alpha"); + let beta_root = root.join("beta"); + + mkdirp(td.path().join("alpha")); + mkdirp(td.path().join("beta")); + wfile(td.path().join(".rgignore"), "beta/**/*.svg\n"); + wfile(td.path().join("alpha/a.txt"), "x"); + wfile(td.path().join("beta/x.svg"), "x"); + + let ig0 = IgnoreBuilder::new() + .add_custom_ignore_filename(".rgignore") + .ignore(false) + .git_ignore(false) + .git_global(false) + .git_exclude(false) + .build(); + + let (alpha_parents, err) = ig0.add_parents(&alpha_root); + assert!(err.is_none()); + let (alpha, err) = alpha_parents.add_child(&alpha_root); + assert!(err.is_none()); + let (beta_parents, err) = ig0.add_parents(&beta_root); + assert!(err.is_none()); + let (beta, err) = beta_parents.add_child(&beta_root); + assert!(err.is_none()); + + assert_ne!( + alpha.absolute_base.as_ref().unwrap().as_path(), + beta.absolute_base.as_ref().unwrap().as_path() + ); + assert!(alpha.matched(alpha_root.join("a.txt"), false).is_none()); + assert!( + beta.matched(beta_root.join("x.svg"), false).is_ignore(), + "parent .rgignore must apply for the beta root regardless of \ + which root was set up first" + ); + } + #[test] fn git_info_exclude_in_linked_worktree() { let td = tmpdir(); @@ -1227,16 +1608,14 @@ mod tests { mkdirp(git_dir.join("info")); wfile(git_dir.join("info/exclude"), "ignore_me"); mkdirp(git_dir.join("worktrees/linked-worktree")); - let commondir_path = || git_dir.join("worktrees/linked-worktree/commondir"); + let commondir_path = + || git_dir.join("worktrees/linked-worktree/commondir"); mkdirp(td.path().join("linked-worktree")); let worktree_git_dir_abs = format!( "gitdir: {}", git_dir.join("worktrees/linked-worktree").to_str().unwrap(), ); - wfile( - td.path().join("linked-worktree/.git"), - &worktree_git_dir_abs, - ); + wfile(td.path().join("linked-worktree/.git"), &worktree_git_dir_abs); // relative commondir wfile(commondir_path(), "../.."); diff --git a/crates/ignore/src/gitignore.rs b/crates/ignore/src/gitignore.rs index f822d8390d20..8824139b5c28 100644 --- a/crates/ignore/src/gitignore.rs +++ b/crates/ignore/src/gitignore.rs @@ -102,7 +102,9 @@ impl Gitignore { /// /// Note that I/O errors are ignored. For more granular control over /// errors, use `GitignoreBuilder`. - pub fn new>(gitignore_path: P) -> (Gitignore, Option) { + pub fn new>( + gitignore_path: P, + ) -> (Gitignore, Option) { let path = gitignore_path.as_ref(); let parent = path.parent().unwrap_or(Path::new("/")); let mut builder = GitignoreBuilder::new(parent); @@ -123,6 +125,17 @@ impl Gitignore { /// The global config file path is specified by git's `core.excludesFile` /// config option. /// + /// # Behavior + /// + /// This routine does its best to discover any global git exclude files. + /// This will try to parse out the `excludesFile` config option in your + /// global git configuration, if necessary. + /// + /// The specific things this routine tries (which are subject to change + /// based on how git behaves) are: + /// + /// + /// /// Git's config file location is `$HOME/.gitconfig`. If `$HOME/.gitconfig` /// does not exist or does not specify `core.excludesFile`, then /// `$XDG_CONFIG_HOME/git/ignore` is read. If `$XDG_CONFIG_HOME` is not @@ -145,7 +158,8 @@ impl Gitignore { num_ignores: 0, num_whitelists: 0, matches: None, - // CHANGED: Add a flag to have Gitignore rules that apply only to files. + // CHANGED: Add a flag to have Gitignore rules that apply only to + // files. only_on_files: false, } } @@ -190,7 +204,11 @@ impl Gitignore { /// determined by a common suffix of the directory containing this /// gitignore) is stripped. If there is no common suffix/prefix overlap, /// then `path` is assumed to be relative to this matcher. - pub fn matched>(&self, path: P, is_dir: bool) -> Match<&Glob> { + pub fn matched>( + &self, + path: P, + is_dir: bool, + ) -> Match<&Glob> { if self.is_empty() { return Match::None; } @@ -243,11 +261,16 @@ impl Gitignore { } /// Like matched, but takes a path that has already been stripped. - fn matched_stripped>(&self, path: P, is_dir: bool) -> Match<&Glob> { + fn matched_stripped>( + &self, + path: P, + is_dir: bool, + ) -> Match<&Glob> { if self.is_empty() { return Match::None; } - // CHANGED: Rules marked as only_on_files can not match against directories. + // CHANGED: Rules marked as only_on_files can not match against + // directories. if self.only_on_files && is_dir { return Match::None; } @@ -270,7 +293,10 @@ impl Gitignore { /// Strips the given path such that it's suitable for matching with this /// gitignore matcher. - fn strip<'a, P: 'a + AsRef + ?Sized>(&'a self, path: &'a P) -> &'a Path { + fn strip<'a, P: 'a + AsRef + ?Sized>( + &'a self, + path: &'a P, + ) -> &'a Path { let mut path = path.as_ref(); // A leading ./ is completely superfluous. We also strip it from // our gitignore root path, so we need to strip it from our candidate @@ -326,7 +352,8 @@ impl GitignoreBuilder { globs: vec![], case_insensitive: false, allow_unclosed_class: true, - // CHANGED: Add a flag to have Gitignore rules that apply only to files. + // CHANGED: Add a flag to have Gitignore rules that apply only to + // files. only_on_files: false, } } @@ -337,18 +364,21 @@ impl GitignoreBuilder { pub fn build(&self) -> Result { let nignore = self.globs.iter().filter(|g| !g.is_whitelist()).count(); let nwhite = self.globs.iter().filter(|g| g.is_whitelist()).count(); - let set = self.builder.build().map_err(|err| Error::Glob { - glob: None, - err: err.to_string(), - })?; + let set = self + .builder + .build() + .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?; Ok(Gitignore { set, root: self.root.clone(), globs: self.globs.clone(), num_ignores: nignore as u64, num_whitelists: nwhite as u64, - matches: Some(Arc::new(Pool::new(|| vec![]))), - // CHANGED: Add a flag to have Gitignore rules that apply only to files. + matches: Some(Arc::new( + Pool::with_available_parallelism_capacity(|| vec![]), + )), + // CHANGED: Add a flag to have Gitignore rules that apply only to + // files. only_on_files: self.only_on_files, }) } @@ -411,11 +441,8 @@ impl GitignoreBuilder { // Match Git's handling of .gitignore files that begin with the Unicode BOM const UTF8_BOM: &str = "\u{feff}"; - let line = if i == 0 { - line.trim_start_matches(UTF8_BOM) - } else { - &line - }; + let line = + if i == 0 { line.trim_start_matches(UTF8_BOM) } else { &line }; if let Err(err) = self.add_line(Some(path.to_path_buf()), &line) { errs.push(err.tagged(path, lineno)); @@ -537,7 +564,10 @@ impl GitignoreBuilder { /// affected. /// /// This is disabled by default. - pub fn case_insensitive(&mut self, yes: bool) -> Result<&mut GitignoreBuilder, Error> { + pub fn case_insensitive( + &mut self, + yes: bool, + ) -> Result<&mut GitignoreBuilder, Error> { // TODO: This should not return a `Result`. Fix this in the next semver // release. self.case_insensitive = yes; @@ -556,7 +586,10 @@ impl GitignoreBuilder { /// modes since the glob parser becomes more permissive. You might want to /// enable this when compatibility (e.g., with POSIX glob implementations) /// is more important than good error messages. - pub fn allow_unclosed_class(&mut self, yes: bool) -> &mut GitignoreBuilder { + pub fn allow_unclosed_class( + &mut self, + yes: bool, + ) -> &mut GitignoreBuilder { self.allow_unclosed_class = yes; self } @@ -576,32 +609,56 @@ impl GitignoreBuilder { /// /// Note that the file path returned may not exist. pub fn gitconfig_excludes_path() -> Option { - // git supports $HOME/.gitconfig and $XDG_CONFIG_HOME/git/config. Notably, - // both can be active at the same time, where $HOME/.gitconfig takes - // precedent. So if $HOME/.gitconfig defines a `core.excludesFile`, then - // we're done. - match gitconfig_home_contents().and_then(|x| parse_excludes_file(&x)) { - Some(path) => return Some(path), - None => {} - } - match gitconfig_xdg_contents().and_then(|x| parse_excludes_file(&x)) { - Some(path) => return Some(path), - None => {} - } - excludes_file_default() + // When GIT_CONFIG_GLOBAL is set, it replaces both $HOME/.gitconfig and + // $XDG_CONFIG_HOME/git/config (per git 2.32+). Otherwise, git supports + // $HOME/.gitconfig and $XDG_CONFIG_HOME/git/config simultaneously, where + // $HOME/.gitconfig takes precedent. + gitconfig_global_env_contents() + .and_then(|x| parse_excludes_file(&x)) + .or_else(|| { + gitconfig_home_contents().and_then(|x| parse_excludes_file(&x)) + }) + .or_else(|| { + gitconfig_xdg_contents().and_then(|x| parse_excludes_file(&x)) + }) + // System-level config has the lowest priority for core.excludesFile. + // GIT_CONFIG_SYSTEM overrides the default /etc/gitconfig path. + .or_else(|| { + gitconfig_system_contents().and_then(|x| parse_excludes_file(&x)) + }) + .or_else(excludes_file_default) +} + +/// Returns the file contents of git's global config file from the path +/// specified by the `GIT_CONFIG_GLOBAL` environment variable. +fn gitconfig_global_env_contents() -> Option> { + let path = std::env::var_os("GIT_CONFIG_GLOBAL").map(PathBuf::from)?; + if path.as_os_str().is_empty() { + return None; + } + let mut file = BufReader::new(File::open(path).ok()?); + let mut contents = vec![]; + file.read_to_end(&mut contents).ok().map(|_| contents) +} + +/// Returns the file contents of git's system-level config file. +/// +/// Checks `GIT_CONFIG_SYSTEM` first, then falls back to `/etc/gitconfig`. +fn gitconfig_system_contents() -> Option> { + let path = std::env::var_os("GIT_CONFIG_SYSTEM") + .map(PathBuf::from) + .filter(|x| !x.as_os_str().is_empty()) + .unwrap_or_else(|| PathBuf::from("/etc/gitconfig")); + let mut file = BufReader::new(File::open(path).ok()?); + let mut contents = vec![]; + file.read_to_end(&mut contents).ok().map(|_| contents) } /// Returns the file contents of git's global config file, if one exists, in /// the user's home directory. fn gitconfig_home_contents() -> Option> { - let home = match home_dir() { - None => return None, - Some(home) => home, - }; - let mut file = match File::open(home.join(".gitconfig")) { - Err(_) => return None, - Ok(file) => BufReader::new(file), - }; + let home = home_dir()?; + let mut file = BufReader::new(File::open(home.join(".gitconfig")).ok()?); let mut contents = vec![]; file.read_to_end(&mut contents).ok().map(|_| contents) } @@ -610,19 +667,11 @@ fn gitconfig_home_contents() -> Option> { /// the user's XDG_CONFIG_HOME directory. fn gitconfig_xdg_contents() -> Option> { let path = std::env::var_os("XDG_CONFIG_HOME") - .and_then(|x| { - if x.is_empty() { - None - } else { - Some(PathBuf::from(x)) - } - }) + .map(PathBuf::from) + .filter(|x| !x.as_os_str().is_empty()) .or_else(|| home_dir().map(|p| p.join(".config"))) - .map(|x| x.join("git/config")); - let mut file = match path.and_then(|p| File::open(p).ok()) { - None => return None, - Some(file) => BufReader::new(file), - }; + .map(|x| x.join("git/config"))?; + let mut file = BufReader::new(File::open(path).ok()?); let mut contents = vec![]; file.read_to_end(&mut contents).ok().map(|_| contents) } @@ -632,13 +681,8 @@ fn gitconfig_xdg_contents() -> Option> { /// Specifically, this respects XDG_CONFIG_HOME. fn excludes_file_default() -> Option { std::env::var_os("XDG_CONFIG_HOME") - .and_then(|x| { - if x.is_empty() { - None - } else { - Some(PathBuf::from(x)) - } - }) + .map(PathBuf::from) + .filter(|x| !x.as_os_str().is_empty()) .or_else(|| home_dir().map(|p| p.join(".config"))) .map(|x| x.join("git/ignore")) } @@ -667,9 +711,7 @@ fn parse_excludes_file(data: &[u8]) -> Option { re.captures(data, &mut caps); let span = caps.get_group(1)?; let candidate = &data[span]; - std::str::from_utf8(candidate) - .ok() - .map(|s| PathBuf::from(expand_tilde(s))) + std::str::from_utf8(candidate).ok().map(|s| PathBuf::from(expand_tilde(s))) } /// Expands ~ in file paths to the value of $HOME. @@ -831,7 +873,10 @@ mod tests { fn parse_excludes_file4() { let data = bytes("[core]\nexcludesFile = \"~/foo/bar\""); let got = super::parse_excludes_file(&data); - assert_eq!(path_string(got.unwrap()), super::expand_tilde("~/foo/bar")); + assert_eq!( + path_string(got.unwrap()), + super::expand_tilde("~/foo/bar") + ); } #[test] diff --git a/crates/ignore/src/incremental.rs b/crates/ignore/src/incremental.rs new file mode 100644 index 000000000000..5030b0d00988 --- /dev/null +++ b/crates/ignore/src/incremental.rs @@ -0,0 +1,1286 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::OnceLock, +}; + +use crate::{ + Error, Match, PartialErrorBuilder, dir::Ignore, pathutil::is_hidden_path, +}; + +/// A cached matcher for checking paths against hierarchical ignore files. +/// +/// An `IncrementalIgnore` is built from a [`crate::WalkBuilder`]. Unlike a +/// recursive walk, it can check individual paths while still respecting the +/// ignore files in every relevant parent directory. Matchers for directories +/// are compiled on first use and then retained for later queries. +/// Each matcher corresponds to exactly one root configured on the builder, +/// and paths passed to it are interpreted relative to that root. +/// A matcher for the special `-` root representing standard input is inert +/// and always returns a non-match. +/// +/// The matcher checks path-based filters in the same precedence order as +/// a traversal. This includes glob overrides, `.ignore`, `.gitignore`, +/// `.git/info/exclude`, global and explicitly added ignore files, custom +/// ignore file names and file type selections. It does not apply filters that +/// require a directory entry or other traversal state, such as custom entry +/// predicates. Hidden-file detection, minimum and maximum depth limits and the +/// maximum file size are applied. +/// +/// A matcher is a snapshot at directory granularity. Once the ignore files in +/// a directory have been loaded, edits to those files are not observed. Build +/// a new matcher to reload them. +/// +/// # Warning +/// +/// The incremental path checking here necessarily needs to do a lot more work +/// per path matched. Callers should _not_ use this to run directory traversal. +/// This is intended to avoid the work of re-traversing an entire directory +/// tree when only a few changes are detected. (For example, in response to +/// file additions or deletions.) +/// +/// # Example +/// +/// ```rust,no_run +/// use ignore::WalkBuilder; +/// +/// let mut builder = WalkBuilder::new("."); +/// builder.add_custom_ignore_filename(".rgignore"); +/// let mut matchers = builder.build_matchers(); +/// let matcher = &mut matchers[0]; +/// +/// if matcher.matched("src/generated.rs", false).is_ignore() { +/// println!("ignored"); +/// } +/// ``` +#[derive(Clone, Debug)] +pub struct IncrementalIgnore { + /// The root exactly as it was given to `WalkBuilder`. + root: PathBuf, + /// The normalized root used only by the opt-in normalization routine. + normalized_root: OnceLock>, + /// The matcher for the configured root directory, loaded on first use. + ignore: RootIgnore, + /// Directory paths relative to `root`, excluding the root itself. + dirs: HashMap, + /// Options for additional filtering beyond gitignore. + options: IncrementalIgnoreOptions, +} + +/// The options for a matcher, mostly meant to duplicate as much as we can from +/// `WalkParallel`. +#[derive(Clone, Debug)] +pub(crate) struct IncrementalIgnoreOptions { + pub(crate) min_depth: Option, + pub(crate) max_depth: Option, + pub(crate) max_filesize: Option, + pub(crate) hidden: bool, + pub(crate) follow_links: bool, +} + +#[derive(Clone, Debug)] +enum RootIgnore { + Unloaded(Ignore), + Loaded(Ignore), + NotDirectory, + Stdin, +} + +/// Cached traversal state for a directory relative to the configured root. +/// +/// The presence of an entry means that the directory and every ancestor +/// between it and the root have already been checked. +#[derive(Clone, Debug)] +enum CachedDir { + /// The directory may be descended into. The matcher includes the ignore + /// rules loaded through this directory and is therefore the matcher to use + /// for its children. + Allowed(Ignore), + /// The directory may not be descended into because it was ignored by a + /// path rule or hidden-file filtering. Every descendant is consequently + /// ignored, and ignore files inside this directory are not loaded. + Ignored, +} + +impl IncrementalIgnore { + pub(crate) fn new( + root: PathBuf, + ignore: Ignore, + options: IncrementalIgnoreOptions, + ) -> IncrementalIgnore { + // File traversal special cases `-` to search stdin, so we recognize + // it here for completeness too. In particular, we really want + // `WalkBuilder::build_matchers` to return a matcher for every root, + // even when it's a simple file (handled automatically) or when it's + // stdin (necessarily special cased). + // + // If callers need to search a file or directory named `-`, then they + // can use `./-`. As is the case for file traversal too. + let ignore = if root == Path::new("-") { + RootIgnore::Stdin + } else { + RootIgnore::Unloaded(ignore) + }; + IncrementalIgnore { + root, + normalized_root: OnceLock::new(), + ignore, + dirs: HashMap::new(), + options, + } + } + + /// Return the root that paths matched by this matcher are relative to. + pub fn root(&self) -> &Path { + &self.root + } + + /// Normalize `path` and return it relative to this matcher's root. + /// + /// This returns `None` when `path` cannot be made absolute or + /// when it is known to be outside this matcher's root. Unlike + /// [`IncrementalIgnore::matched`], this performs absolute path conversion, + /// lexical normalization and allocation. It is intended as an opt-in + /// convenience for callers that do not already have root-relative paths. + /// + /// Note that `.` is interpreted relative to the process level current + /// working directory. It is _not_ interpreted relative to the root of + /// this matcher. + /// + /// Note also that this may reject paths that only differ in casing. For + /// example, if the root path for this matcher is `/FOO` but the provided + /// path is `/foo/bar`, then this may return `None`. Callers must ensure + /// casing is consistent between the path provided and the root path for + /// this matcher. + pub fn normalize>(&self, path: P) -> Option { + if matches!(self.ignore, RootIgnore::Stdin) { + return None; + } + let path = normalize_absolute(path.as_ref())?; + let root = self + .normalized_root + .get_or_init(|| normalize_absolute(&self.root)) + .as_ref()?; + path.strip_prefix(root).ok().map(Path::to_path_buf) + } + + /// Match a root-relative path against ignore files in its directory and + /// all relevant parent directories. + /// + /// `is_dir` should be true when `path` should be matched as a directory. + /// + /// For the return value, use [`IncrementalMatch::is_ignore`], + /// [`IncrementalMatch::is_whitelist`] or [`IncrementalMatch::is_none`] to + /// inspect it. + /// + /// Matchers for previously unseen directories are loaded and cached during + /// this call. Errors encountered while loading ignore files are logged. To + /// receive those errors, use [`IncrementalIgnore::matched_with_errors`]. + /// + /// `path` must be relative to this matcher's root and must not contain a + /// parent directory (`..`) component. Behavior is unspecified when these + /// preconditions are violated. Callers with an absolute path or with a + /// path containing `.` or `..` may use [`IncrementalIgnore::normalize`] + /// to get a path satisfying these preconditions. In all cases, a relative + /// path is *assumed* to be relative to the root of this ignore matcher. + /// + /// In general, it is intended that callers doing recursive directory + /// traversal on the root of this matcher may provide relative paths to + /// this routine *without* calling [`IncrementalIgnore::normalize`]. + /// + /// The empty path represents the explicitly configured root and also + /// returns non-match, consistent with recursive traversal where a root is + /// always treated as being at depth zero. + pub fn matched>( + &mut self, + path: P, + is_dir: bool, + ) -> IncrementalMatch { + let (matched, err) = self.matched_with_errors(path, is_dir); + if let Some(err) = err { + log::debug!("error while loading ignore files: {err}"); + } + matched + } + + /// Match a root-relative path and return errors encountered while loading + /// ignore files. + /// + /// This is equivalent to [`IncrementalIgnore::matched`], except that + /// it returns any errors from newly loaded ignore files. Loading can + /// partially succeed, so valid rules are always applied to the returned + /// match even when an error is present. + pub fn matched_with_errors>( + &mut self, + path: P, + is_dir: bool, + ) -> (IncrementalMatch, Option) { + let mut errs = PartialErrorBuilder::default(); + let matched = + self.matched_with_errors_impl(path.as_ref(), is_dir, &mut errs); + (matched, errs.into_error_option()) + } + + fn matched_with_errors_impl( + &mut self, + relative: &Path, + is_dir: bool, + errs: &mut PartialErrorBuilder, + ) -> IncrementalMatch { + // We short-circuit here when our matcher corresponds to `Stdin` in + // order to always return a non-match. This is somewhat redundant with + // `root_ignore()` which does this too, but we do it here so that it + // always happens, e.g., before depth filtering. + if relative.is_absolute() || matches!(self.ignore, RootIgnore::Stdin) { + return IncrementalMatch::none(is_dir); + } + + let (mut satisfies_min, mut edge_max) = (true, false); + if self.options.min_depth.is_some() || self.options.max_depth.is_some() + { + let depth = relative + .components() + .filter_map(|component| match component { + std::path::Component::CurDir => None, + component => Some(component), + }) + .count(); + satisfies_min = + self.options.min_depth.is_none_or(|min| depth >= min); + let satisfies_max = + self.options.max_depth.is_none_or(|max| depth <= max); + edge_max = self.options.max_depth.is_some_and(|max| depth == max); + // When we have a file that isn't past our min depth, we can give + // up right away. + if !is_dir && !satisfies_min { + return IncrementalMatch::ignore().not_within_depth(); + } + // Same for *anything* that exceeds our max depth. + if !satisfies_max { + return IncrementalMatch::ignore().not_within_depth(); + } + } + + // When the path is invalid in some way, we bail out early to avoid + // potentially doing a stat call below. + let (mut mat, valid) = self + .matched_with_errors_ignore(relative, is_dir, errs) + .map(|mat| (mat, true)) + .unwrap_or_else(|| (IncrementalMatch::none(is_dir), false)); + + if !is_dir + && !mat.is_ignore() + && valid + && let Some(max_filesize) = self.options.max_filesize + { + let path = self.root().join(relative); + let result = if self.options.follow_links { + path.metadata() + } else { + path.symlink_metadata() + }; + match result { + Ok(md) if md.len() > max_filesize => { + return IncrementalMatch::ignore(); + } + Ok(_) => {} + Err(err) => { + // Record the error but otherwise fall through + let err = Error::from(err); + errs.push(err.with_path(path)); + } + } + } + + // We still need to tag our `mat` if it doesn't pass the depth filter, + // or if it's a directory on the edge of a depth filter. This can only + // happen when the path is reported as a directory. Otherwise regular + // files are always handled above. + if is_dir { + if !satisfies_min { + mat = mat.not_within_depth(); + } + if edge_max { + mat = mat.no_descent(); + } + } + mat + } + + fn matched_with_errors_ignore( + &mut self, + relative: &Path, + is_dir: bool, + errs: &mut PartialErrorBuilder, + ) -> Option { + let mut components = relative + .components() + .filter_map(|component| match component { + std::path::Component::CurDir => None, + component => Some(component), + }) + .peekable(); + components.peek()?; + + // If the exact parent is cached, then all of its ancestors have + // already been checked. An allowed cache entry is the matcher to use + // for children of that directory, while an ignored cache entry is + // terminal for every descendant. In the usual case, this avoids both + // walking every component and allocating a relative directory path. + // + // Only try the fast path when the final component is a normal path + // component. Invalid paths are handled by the component walk below. + let has_normal_final_component = + relative.components().next_back().is_some_and(|component| { + matches!(component, std::path::Component::Normal(_)) + }); + if has_normal_final_component && let Some(parent) = relative.parent() { + match self.dirs.get(parent) { + Some(CachedDir::Allowed(ignore)) => { + return Some(self.match_path(ignore, relative, is_dir)); + } + Some(CachedDir::Ignored) => { + return Some(IncrementalMatch::ignore()); + } + None => {} + } + } + + let mut ignore = self.root_ignore(errs)?; + let mut dir = PathBuf::new(); + while let Some(component) = components.next() { + match component { + std::path::Component::ParentDir + | std::path::Component::RootDir + | std::path::Component::Prefix(_) => { + return None; + } + std::path::Component::CurDir => continue, + std::path::Component::Normal(_) => {} + } + if components.peek().is_none() { + break; + } + dir.push(component.as_os_str()); + match self.dirs.get(&dir) { + Some(CachedDir::Allowed(cached)) => { + ignore = cached.clone(); + continue; + } + Some(CachedDir::Ignored) => { + return Some(IncrementalMatch::ignore()); + } + None => {} + } + + let path = self.root.join(&dir); + let mat = ignore.matched(&path, true); + let is_hidden = + self.options.hidden && mat.is_none() && is_hidden_path(&path); + if mat.is_ignore() || is_hidden { + self.dirs.insert(dir.clone(), CachedDir::Ignored); + return Some(IncrementalMatch::ignore()); + } + let (child, err) = ignore.add_child(&path); + errs.maybe_push(err); + self.dirs.insert(dir.clone(), CachedDir::Allowed(child.clone())); + ignore = child; + } + + Some(self.match_path(&ignore, relative, is_dir)) + } + + fn match_path( + &self, + ignore: &Ignore, + relative: &Path, + is_dir: bool, + ) -> IncrementalMatch { + let path = self.root.join(relative); + let mut mat = IncrementalMatch::from_match( + ignore.matched(&path, is_dir).map(|_| ()), + is_dir, + ); + // Whether a file is hidden or not has low precedence in filtering. We + // only check it if we haven't matched anything above. This permits + // callers to whitelist hidden files or directories. + if self.options.hidden && mat.is_none() && is_hidden_path(&path) { + mat = IncrementalMatch::ignore(); + } + mat + } + + fn root_ignore( + &mut self, + errs: &mut PartialErrorBuilder, + ) -> Option { + let ignore = match self.ignore { + RootIgnore::Unloaded(ref ignore) => ignore.clone(), + RootIgnore::Loaded(ref ignore) => return Some(ignore.clone()), + RootIgnore::NotDirectory | RootIgnore::Stdin => return None, + }; + if !self.root.is_dir() { + self.ignore = RootIgnore::NotDirectory; + return None; + } + + let (parents, err) = ignore.add_parents(&self.root); + errs.maybe_push(err); + let (root, err) = parents.add_child(&self.root); + errs.maybe_push(err); + self.ignore = RootIgnore::Loaded(root.clone()); + Some(root) + } +} + +/// The result of an incremental match. +/// +/// This is similar to [`Match`] in that it reports whether a file path should +/// be ignored, whitelisted or didn't match anything at all. It also has extra +/// data, such as whether a directory matched but should not be descended into. +/// +/// Generally speaking, callers that only care about specific files can stick +/// to the `is_none()`, `is_ignore()` or `is_whitelist()` predicates. Directory +/// entries are more complicated because we sometimes want to yield directories +/// to descend into, but not actually visit (e.g., for the minimum depth +/// filter). Or, we may want to yield a directory to visit but not descend into +/// (e.g., for the maximum depth filter). +#[derive(Clone, Debug)] +pub struct IncrementalMatch { + mat: Match<()>, + should_descend: bool, + is_within_depth: bool, +} + +impl IncrementalMatch { + fn none(is_dir: bool) -> IncrementalMatch { + IncrementalMatch { + mat: Match::None, + should_descend: is_dir, + is_within_depth: true, + } + } + + fn ignore() -> IncrementalMatch { + IncrementalMatch { + mat: Match::Ignore(()), + should_descend: false, + is_within_depth: true, + } + } + + fn from_match(mat: Match<()>, is_dir: bool) -> IncrementalMatch { + let should_descend = is_dir && !mat.is_ignore(); + IncrementalMatch { mat, should_descend, is_within_depth: true } + } + + fn no_descent(self) -> IncrementalMatch { + IncrementalMatch { should_descend: false, ..self } + } + + fn not_within_depth(self) -> IncrementalMatch { + IncrementalMatch { is_within_depth: false, ..self } + } + + /// Returns true if the match result didn't match anything. + pub fn is_none(&self) -> bool { + self.mat.is_none() + } + + /// Returns true if the match result implies the path should be ignored. + pub fn is_ignore(&self) -> bool { + self.mat.is_ignore() + } + + /// Returns true if the match result implies the path should be + /// whitelisted. + pub fn is_whitelist(&self) -> bool { + self.mat.is_whitelist() + } + + /// Returns true only when this match corresponds to a directory *and* + /// whether the caller should look inside this directory for additional + /// results. + /// + /// It is possible for a match to report `false` for + /// [`IncrementalMatch::is_ignore`] _and_ `false` for + /// [`IncrementalMatch::should_descend`]. This can occur, for example, when + /// a maximum depth setting allows a directory through, but where none of + /// its children entries should be visited. + /// + /// This is always `false` for a file path that does *not* correspond to a + /// directory. + pub fn should_descend(&self) -> bool { + self.should_descend + } + + /// Returns true only when this result corresponds to an entry that is + /// within the depth filter. + /// + /// This is `false` when a path corresponds to a directory and is less than + /// the minimum depth. In this case, callers should continue looking inside + /// that directory. + /// + /// This is always `true` for a match corresponding to a file path that + /// isn't a directory. + pub fn is_within_depth(&self) -> bool { + self.is_within_depth + } + + /// Inverts the match so that `Ignore` becomes `Whitelist` and + /// `Whitelist` becomes `Ignore`. A non-match remains the same. + pub fn invert(self) -> IncrementalMatch { + IncrementalMatch { mat: self.mat.invert(), ..self } + } +} + +/// Return a lexically normalized absolute representation of `path`. +/// +/// This collapses `.` and `..`, but intentionally does not canonicalize or +/// resolve symlinks. Ignore rules apply to the lexical path, and resolving a +/// symlink below a root could move the result outside of that root. +fn normalize_absolute(path: &Path) -> Option { + let absolute = std::path::absolute(path).ok()?; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + normalized.pop(); + } + _ => normalized.push(component.as_os_str()), + } + } + Some(normalized) +} + +#[cfg(test)] +mod tests { + use std::{ + fs::{self, File}, + io::Write, + path::{Path, PathBuf}, + }; + + use crate::{ + IncrementalIgnore, IncrementalMatch, WalkBuilder, + overrides::OverrideBuilder, tests::TempDir, types::TypesBuilder, + }; + + use super::CachedDir; + + fn wfile>(path: P, contents: &str) { + let mut file = File::create(path).unwrap(); + file.write_all(contents.as_bytes()).unwrap(); + } + + fn mkdirp>(path: P) { + fs::create_dir_all(path).unwrap(); + } + + fn tmpdir() -> TempDir { + TempDir::new().unwrap() + } + + fn builder>(path: P) -> WalkBuilder { + let mut builder = WalkBuilder::new(path); + builder.git_global(false); + builder + } + + fn one_matcher(builder: &WalkBuilder) -> IncrementalIgnore { + let mut matchers = builder.build_matchers(); + assert_eq!(matchers.len(), 1); + matchers.pop().unwrap() + } + + fn builders, P: AsRef>( + paths: I, + ) -> WalkBuilder { + let mut builder = WalkBuilder::from_iter(paths); + builder.git_global(false); + builder + } + + fn matchers(builder: &WalkBuilder) -> Vec { + builder.build_matchers() + } + + fn matchedf>( + matcher: &mut IncrementalIgnore, + path: P, + ) -> IncrementalMatch { + let (matched, err) = matcher.matched_with_errors(path, false); + assert!(err.is_none(), "unexpected matcher error: {err:?}"); + matched + } + + fn matchedd>( + matcher: &mut IncrementalIgnore, + path: P, + ) -> IncrementalMatch { + let (matched, err) = matcher.matched_with_errors(path, true); + assert!(err.is_none(), "unexpected matcher error: {err:?}"); + matched + } + + // Test that multiple parent ignore files, when nested, are respected. + #[test] + fn nested_parent_gitignores() { + let td = tmpdir(); + let root = td.path().join("project/work"); + mkdirp(td.path().join(".git")); + mkdirp(root.join("src")); + wfile(td.path().join(".gitignore"), "*.tmp\n"); + wfile(td.path().join("project/.gitignore"), "!keep.tmp\nnested.log\n"); + + let mut m = one_matcher(&builder(&root)); + assert_eq!(m.root(), root); + assert!(matchedf(&mut m, "src/drop.tmp").is_ignore()); + assert!(matchedf(&mut m, "src/keep.tmp").is_whitelist()); + assert!(matchedf(&mut m, "src/nested.log").is_ignore()); + assert!(matchedf(&mut m, "src/ok.rs").is_none()); + } + + // Test that an anchored rule in a child directory is matched relative to + // that directory, not relative to the configured root. + #[test] + fn anchored_child_rule_uses_child_root() { + let td = tmpdir(); + let root = td.path().join("root"); + mkdirp(root.join("a")); + wfile(root.join("a/.ignore"), "/foo\n"); + + let mut m = one_matcher(&builder(&root)); + assert!(matchedf(&mut m, "a/foo").is_ignore()); + assert!(matchedf(&mut m, "a/b/foo").is_none()); + } + + // Test that a leading `./` impacts how the rules are matched. + #[cfg(not(windows))] + #[test] + fn leading_dot_slash_impacts_matching() { + let td = tmpdir(); + let root = td.path().join("root"); + mkdirp(root.join("a")); + wfile(root.join(".ignore"), "/foo\n"); + wfile(root.join("a/.ignore"), "/foo\n"); + + let mut m = one_matcher(&builder(&root)); + assert!(matchedf(&mut m, "foo").is_ignore()); + assert!(matchedf(&mut m, "./foo").is_none()); + assert!(matchedf(&mut m, "a/allowed").is_none()); + assert!(matchedf(&mut m, "a/foo").is_ignore()); + assert!(matchedf(&mut m, "./a/foo").is_none()); + } + + // Test that custom ignore files are respected. + #[test] + fn parent_ignore_and_custom_ignore() { + let td = tmpdir(); + let root = td.path().join("project/work"); + mkdirp(root.join("src")); + wfile(td.path().join(".ignore"), "*.cache\n"); + wfile(td.path().join(".rgignore"), "*.svg\n"); + wfile(td.path().join("project/.ignore"), "!keep.cache\n"); + wfile(td.path().join("project/.rgignore"), "!keep.svg\n"); + + let mut builder = builder(&root); + builder.add_custom_ignore_filename(".rgignore"); + let mut m = one_matcher(&builder); + assert!(matchedf(&mut m, "src/drop.cache").is_ignore()); + assert!(matchedf(&mut m, "src/keep.cache").is_whitelist()); + assert!(matchedf(&mut m, "src/drop.svg").is_ignore()); + assert!(matchedf(&mut m, "src/keep.svg").is_whitelist()); + } + + // Test that glob overrides take precedence over ignore files. + #[test] + fn glob_overrides_are_applied() { + let td = tmpdir(); + wfile(td.path().join(".ignore"), "keep.rs\n!drop.rs\n"); + + let mut overrides = OverrideBuilder::new(td.path()); + overrides.add("keep.rs").unwrap(); + overrides.add("!drop.rs").unwrap(); + let mut b = builder(td.path()); + b.overrides(overrides.build().unwrap()); + let mut m = one_matcher(&b); + + assert!(matchedf(&mut m, "keep.rs").is_whitelist()); + assert!(matchedf(&mut m, "drop.rs").is_ignore()); + } + + // Test that an override-ignored directory prevents matching rules below + // it, just as it prevents a traversal from descending into the directory. + #[test] + fn glob_overrides_apply_to_ancestors() { + let td = tmpdir(); + mkdirp(td.path().join("blocked")); + wfile(td.path().join("blocked/.ignore"), "!keep.rs\n"); + + let mut overrides = OverrideBuilder::new(td.path()); + overrides.add("!blocked/").unwrap(); + let mut b = builder(td.path()); + b.overrides(overrides.build().unwrap()); + let mut m = one_matcher(&b); + + assert!(matchedf(&mut m, "blocked/keep.rs").is_ignore()); + } + + // Test that file type selections are applied to files, but not + // directories. + #[test] + fn file_types_are_applied() { + let td = tmpdir(); + mkdirp(td.path().join("src")); + + let mut types = TypesBuilder::new(); + types.add("rust", "*.rs").unwrap(); + types.select("rust"); + let mut b = builder(td.path()); + b.types(types.build().unwrap()); + let mut m = one_matcher(&b); + + assert!(matchedd(&mut m, "src").is_none()); + assert!(matchedf(&mut m, "src/lib.rs").is_whitelist()); + assert!(matchedf(&mut m, "README.md").is_ignore()); + } + + #[test] + fn directly_ignored_directory_is_not_descended() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join("blocked")); + wfile(td.path().join(".gitignore"), "blocked/\n"); + + let mut m = one_matcher(&builder(td.path())); + let dir = matchedd(&mut m, "blocked"); + assert!(dir.is_ignore()); + assert!(!dir.should_descend()); + } + + // Test that when a directory is ignored, anything below it always ignored + // even when there are explicit whitelist rules. This matches directory + // traversal semantics. + #[test] + fn ignored_ancestor_wins() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join("blocked")); + mkdirp(td.path().join("open")); + // This is the key line: since the entire `blocked` directory is + // ignored, a proper file traversal won't ever descend into it. So + // `blocked/keep.rs` should be ignored even if there are ignore rules + // "beneath" it that whitelist it. + wfile(td.path().join(".gitignore"), "blocked/\n!blocked/keep.rs\n"); + wfile(td.path().join("blocked/.gitignore"), "!keep.rs\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "blocked/keep.rs").is_ignore()); + assert!(matchedf(&mut m, "blocked/other.rs").is_ignore()); + assert!(matchedf(&mut m, "open/keep.rs").is_none()); + } + + // Test that we respect git boundaries. And that we don't respect git + // boundaries when not configured to do so. + #[test] + fn respects_git_repository_boundaries() { + let td = tmpdir(); + let root = td.path().join("repo/src"); + mkdirp(td.path().join("repo/.git")); + mkdirp(&root); + wfile(td.path().join(".gitignore"), "outside-rule\n"); + wfile(td.path().join(".ignore"), "tool-rule\n"); + wfile(td.path().join("repo/.gitignore"), "inside-rule\n"); + + let mut m = one_matcher(&builder(&root)); + assert!(matchedf(&mut m, "inside-rule").is_ignore()); + assert!(matchedf(&mut m, "outside-rule").is_none()); + assert!(matchedf(&mut m, "tool-rule").is_ignore()); + + let mut no_git_required = builder(&root); + no_git_required.require_git(false); + let mut m = one_matcher(&no_git_required); + assert!(matchedf(&mut m, "outside-rule").is_ignore()); + } + + // Test that when a gitignore matcher in a parent directory is created, we + // reuse that matcher from memory even if it's changed on disk. + #[test] + fn compiled_matchers_are_reused() { + let td = tmpdir(); + mkdirp(td.path().join("a")); + wfile(td.path().join("a/.ignore"), "*.tmp\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "a/first.tmp").is_ignore()); + + // Below demonstrates that this new ignore file contents + // aren't actually picked up because it was already loaded. + wfile(td.path().join("a/.ignore"), "!*.tmp\n*.rs\n"); + assert!(matchedf(&mut m, "a/first.tmp").is_ignore()); + assert!(matchedf(&mut m, "a/second.tmp").is_ignore()); + assert!(matchedf(&mut m, "a/keep.rs").is_none()); + // To get the new ignore file, we need to rebuild the matcher. + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "a/first.tmp").is_whitelist()); + assert!(matchedf(&mut m, "a/second.tmp").is_whitelist()); + assert!(matchedf(&mut m, "a/keep.rs").is_ignore()); + } + + #[test] + fn cached_allowed_parent_matches_children() { + let td = tmpdir(); + mkdirp(td.path().join("a/b")); + wfile(td.path().join("a/b/.ignore"), "ignored\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "a/b/allowed").is_none()); + assert!(matches!( + m.dirs.get(Path::new("a/b")), + Some(CachedDir::Allowed(_)) + )); + assert!(matchedf(&mut m, "a/b/ignored").is_ignore()); + } + + #[test] + fn cached_ignored_parent_matches_children() { + let td = tmpdir(); + mkdirp(td.path().join("blocked")); + wfile(td.path().join(".ignore"), "blocked/\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "blocked/first").is_ignore()); + assert!(matches!( + m.dirs.get(Path::new("blocked")), + Some(CachedDir::Ignored) + )); + assert!(matchedf(&mut m, "blocked/second").is_ignore()); + } + + #[test] + fn cached_parent_matcher_is_used_for_directory() { + let td = tmpdir(); + mkdirp(td.path().join("a/b")); + wfile(td.path().join("a/b/.ignore"), "**\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, "a/b/file").is_ignore()); + assert!(matches!( + m.dirs.get(Path::new("a/b")), + Some(CachedDir::Allowed(_)) + )); + let dir = matchedd(&mut m, "a/b"); + assert!(dir.is_none()); + assert!(dir.should_descend()); + } + + // Like `compiled_matchers_are_reused`, but with multiple roots. + #[test] + fn compiled_multi_matchers_are_reused() { + let td = tmpdir(); + mkdirp(td.path().join("a/b/c")); + + wfile(td.path().join("a/.ignore"), "*.tmp\n"); + let mut mats = matchers(&builders([ + td.path().join("a/b"), + td.path().join("a/b/c"), + ])); + assert!(matchedf(&mut mats[0], "first.tmp").is_ignore()); + assert!(matchedf(&mut mats[0], "c/first.tmp").is_ignore()); + + // Even though we haven't used the second matcher, it should still + // reuse the `a/.ignore` above. If it didn't, then `a/b/c/first.tmp` + // below would be whitelisted. + wfile(td.path().join("a/.ignore"), "!*.tmp\n"); + assert!(matchedf(&mut mats[1], "first.tmp").is_ignore()); + } + + #[test] + fn compiled_multi_child_matchers_are_not_reused() { + let td = tmpdir(); + mkdirp(td.path().join("a/b/c/d/e")); + + wfile(td.path().join("a/b/c/d/e/.ignore"), "*.tmp\n"); + let mut mats = matchers(&builders([ + td.path().join("a/b"), + td.path().join("a/b/c"), + ])); + // Some sanity checking first. + assert!(matchedf(&mut mats[0], "c/first.tmp").is_none()); + assert!(matchedf(&mut mats[0], "c/d/first.tmp").is_none()); + assert!(matchedf(&mut mats[0], "c/d/e/first.tmp").is_ignore()); + + // Now write a new ignore file at the same location as above + // and check that the other matcher still uses the "stale" data. + wfile(td.path().join("a/b/c/d/e/.ignore"), "!*.tmp\n"); + assert!(matchedf(&mut mats[1], "first.tmp").is_none()); + assert!(matchedf(&mut mats[1], "d/first.tmp").is_none()); + // This is the punch line: because we didn't load `mats[1]` before + // changing the ignore file, it loads it here and thus this path gets + // whitelisted. + assert!(matchedf(&mut mats[1], "d/e/first.tmp").is_whitelist()); + // ... but `mats[0]` still uses the stale gitignore matcher cached in + // memory! + assert!(matchedf(&mut mats[0], "c/d/e/first.tmp").is_ignore()); + + // If we rebuilder the matcher... then we force reloading and they're + // now consistent with one another. + let mut mats = matchers(&builders([ + td.path().join("a/b"), + td.path().join("a/b/c"), + ])); + assert!(matchedf(&mut mats[0], "c/d/e/first.tmp").is_whitelist()); + assert!(matchedf(&mut mats[1], "d/e/first.tmp").is_whitelist()); + } + + // Tests that even when there is an error with a glob pattern, we still + // respect other glob patterns that are valid. + #[test] + fn partial_errors_keep_valid_rules() { + let td = tmpdir(); + let root = td.path().join("work"); + mkdirp(&root); + wfile(td.path().join(".ignore"), "{bad\n*.tmp\n"); + + let mut builder = builder(&root); + builder.git_ignore(false).git_exclude(false); + let mut m = one_matcher(&builder); + let (matched, err) = m.matched_with_errors("drop.tmp", false); + assert!(err.is_some()); + assert!(matched.is_ignore()); + } + + // Tests that parent rules are not respected if the matcher is configured + // not to do so. + #[test] + fn parent_loading_can_be_disabled() { + let td = tmpdir(); + let root = td.path().join("work"); + mkdirp(&root); + wfile(td.path().join(".ignore"), "parent-rule\n"); + wfile(root.join(".ignore"), "root-rule\n"); + + let mut b = builder(&root); + b.parents(false); + let mut m = one_matcher(&b); + assert!(matchedf(&mut m, "parent-rule").is_none()); + assert!(matchedf(&mut m, "root-rule").is_ignore()); + + // Sanity check that without `parents(false)`, the parent rule is + // respected. + let mut m = one_matcher(&builder(&root)); + assert!(matchedf(&mut m, "parent-rule").is_ignore()); + assert!(matchedf(&mut m, "root-rule").is_ignore()); + } + + // Tests that we can normalize a file path that isn't already in "normal" + // relative form, and then use that to match on ignore files. + #[test] + fn paths_are_relative_to_the_root() { + let td = tmpdir(); + let root = td.path().join("root"); + let outside = td.path().join("outside"); + mkdirp(&root); + mkdirp(&outside); + wfile(root.join(".ignore"), "file\n"); + + let mut m = one_matcher(&builder(&root)); + assert!(matchedd(&mut m, "").is_none()); + assert!(matchedd(&mut m, ".").is_none()); + assert!(matchedf(&mut m, "file").is_ignore()); + // Doesn't work because it isn't relative to the root. It's absolute. + assert!(matchedf(&mut m, root.join("file")).is_none()); + // Also doesn't work because while it's relative, it contains `..`. + assert!(matchedf(&mut m, "dir/../file").is_none()); + + let norm = m.normalize(root.join("dir/../file")).unwrap(); + assert_eq!(norm, Path::new("file")); + assert!(matchedf(&mut m, "file").is_ignore()); + + // Doesn't normalize because its an absolute path outside of our root. + assert_eq!(m.normalize(outside.join("file")), None); + } + + // Tests that two matchers in two different directories correctly interpret + // the same parent gitignore file. + #[test] + fn multiple_roots_keep_their_own_context() { + let td = tmpdir(); + let root_a = td.path().join("a"); + let root_b = td.path().join("b"); + mkdirp(td.path().join(".git")); + mkdirp(&root_a); + mkdirp(&root_b); + wfile(td.path().join(".gitignore"), "/a/*.tmp\n/b/*.log\n"); + + let mut builder = + builders([root_a.as_path(), Path::new("-"), root_b.as_path()]); + builder.git_global(false); + let mut ms = matchers(&builder); + assert_eq!(ms.len(), 3); + assert_eq!(ms[0].root(), root_a); + assert_eq!(ms[1].root(), Path::new("-")); + assert_eq!(ms[2].root(), root_b); + assert!(matchedf(&mut ms[0], "drop.tmp").is_ignore()); + assert!(matchedf(&mut ms[0], "keep.log").is_none()); + assert!(matchedf(&mut ms[1], "anything").is_none()); + assert_eq!(ms[1].normalize("anything"), None); + assert!(matchedf(&mut ms[2], "keep.tmp").is_none()); + assert!(matchedf(&mut ms[2], "drop.log").is_ignore()); + } + + // Tests that only the exact `-` root represents standard input. + #[test] + fn dot_dash_root_is_not_stdin() { + let m = one_matcher(&builder("./-")); + assert_eq!(m.root(), Path::new("./-")); + assert_eq!(m.normalize("./-/file"), Some(PathBuf::from("file"))); + } + + #[test] + fn stdin_is_inert_with_depth_limits() { + let mut b = builder("-"); + b.min_depth(Some(2)); + let mut m = one_matcher(&b); + let mat = matchedf(&mut m, "file"); + assert!(mat.is_none()); + assert!(mat.is_within_depth()); + + let mut b = builder("-"); + b.max_depth(Some(0)); + let mut m = one_matcher(&b); + let mat = matchedd(&mut m, "dir"); + assert!(mat.is_none()); + assert!(mat.is_within_depth()); + assert!(mat.should_descend()); + } + + // Tests that ignore matching works lexically, and doesn't accidentally + // resolve symbolic links. + #[cfg(unix)] + #[test] + fn symlink_path_stays_under_lexical_root() { + use std::os::unix::fs::symlink; + + let td = tmpdir(); + let root = td.path().join("root"); + let outside = td.path().join("outside"); + mkdirp(&root); + mkdirp(&outside); + wfile(root.join(".ignore"), "link\n"); + wfile(outside.join("target"), ""); + symlink(outside.join("target"), root.join("link")).unwrap(); + + let mut m = one_matcher(&builder(&root)); + assert!(matchedf(&mut m, "link").is_ignore()); + } + + #[test] + fn depth_limits() { + let td = tmpdir(); + mkdirp(td.path().join("a/b/c/d")); + + let mut b = builder(td.path()); + b.min_depth(Some(2)).max_depth(Some(3)); + let mut m = one_matcher(&b); + + let dir = matchedd(&mut m, "a"); + assert!(!dir.is_within_depth()); + assert!(dir.should_descend()); + assert!(matchedf(&mut m, "file").is_ignore()); + + let dir = matchedd(&mut m, "a/b"); + assert!(dir.is_within_depth()); + assert!(dir.should_descend()); + assert!(!matchedf(&mut m, "a/file").is_ignore()); + + let dir = matchedd(&mut m, "a/b/c"); + assert!(dir.is_within_depth()); + assert!(!dir.should_descend()); + assert!(!matchedf(&mut m, "a/b/file").is_ignore()); + assert!(matchedf(&mut m, "a/b/c/file").is_ignore()); + + let dir = matchedd(&mut m, "a/b/c/d"); + assert!(dir.is_ignore()); + assert!(!dir.is_within_depth()); + assert!(!dir.should_descend()); + assert!(matchedf(&mut m, "a/b/c/d/file").is_ignore()); + } + + #[test] + fn depth_limits_apply_to_root() { + let td = tmpdir(); + + let mut b = builder(td.path()); + b.min_depth(Some(1)); + let mut m = one_matcher(&b); + for path in ["", "."] { + let root = matchedd(&mut m, path); + assert!(root.is_none()); + assert!(!root.is_within_depth()); + assert!(root.should_descend()); + } + + let mut b = builder(td.path()); + b.max_depth(Some(0)); + let mut m = one_matcher(&b); + for path in ["", "."] { + let root = matchedd(&mut m, path); + assert!(root.is_none()); + assert!(root.is_within_depth()); + assert!(!root.should_descend()); + } + } + + #[test] + fn max_filesize() { + let td = tmpdir(); + mkdirp(td.path().join("dir")); + wfile(td.path().join("empty"), ""); + wfile(td.path().join("at-limit"), "12345"); + wfile(td.path().join("over-limit"), "123456"); + + let mut b = builder(td.path()); + b.max_filesize(Some(5)); + let mut m = one_matcher(&b); + + assert!(matchedf(&mut m, "empty").is_none()); + assert!(matchedf(&mut m, "at-limit").is_none()); + assert!(matchedf(&mut m, "over-limit").is_ignore()); + + let dir = matchedd(&mut m, "dir"); + assert!(dir.is_none()); + assert!(dir.should_descend()); + } + + #[test] + fn max_filesize_does_not_stat_ignored_file() { + let td = tmpdir(); + wfile(td.path().join(".ignore"), "ignored\n"); + + let mut b = builder(td.path()); + b.max_filesize(Some(0)); + let mut m = one_matcher(&b); + let (mat, err) = m.matched_with_errors("ignored", false); + assert!(mat.is_ignore()); + assert!(err.is_none(), "ignored missing file was statted: {err:?}"); + } + + #[cfg(unix)] + #[test] + fn max_filesize_respects_follow_links() { + use std::os::unix::fs::symlink; + + let td = tmpdir(); + wfile( + td.path().join("target"), + "target contents are much longer than the size limit", + ); + symlink("target", td.path().join("link")).unwrap(); + + let mut b = builder(td.path()); + b.max_filesize(Some(10)); + let mut m = one_matcher(&b); + assert!(matchedf(&mut m, "link").is_none()); + + b.follow_links(true); + let mut m = one_matcher(&b); + assert!(matchedf(&mut m, "link").is_ignore()); + } + + #[test] + fn hidden_files_and_directories() { + let td = tmpdir(); + mkdirp(td.path().join(".hidden-dir")); + mkdirp(td.path().join("visible-dir")); + wfile(td.path().join(".hidden-file"), ""); + wfile(td.path().join("visible-file"), ""); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, ".hidden-file").is_ignore()); + let dir = matchedd(&mut m, ".hidden-dir"); + assert!(dir.is_ignore()); + assert!(!dir.should_descend()); + assert!(matchedf(&mut m, "visible-file").is_none()); + let dir = matchedd(&mut m, "visible-dir"); + assert!(dir.is_none()); + assert!(dir.should_descend()); + + let mut b = builder(td.path()); + b.hidden(false); + let mut m = one_matcher(&b); + assert!(matchedf(&mut m, ".hidden-file").is_none()); + let dir = matchedd(&mut m, ".hidden-dir"); + assert!(dir.is_none()); + assert!(dir.should_descend()); + } + + #[test] + fn gitignore_whitelist_overrides_hidden_filter() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join(".hidden-dir")); + wfile(td.path().join(".hidden-file"), ""); + wfile(td.path().join(".gitignore"), "!.hidden-file\n!.hidden-dir/\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, ".hidden-file").is_whitelist()); + let dir = matchedd(&mut m, ".hidden-dir"); + assert!(dir.is_whitelist()); + assert!(dir.should_descend()); + } + + #[test] + fn descendants_of_hidden_directories_are_ignored() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join(".hidden/nested")); + mkdirp(td.path().join("visible/.hidden")); + wfile(td.path().join(".hidden/file"), ""); + wfile(td.path().join(".hidden/nested/file"), ""); + wfile(td.path().join("visible/.hidden/file"), ""); + wfile(td.path().join(".hidden/.gitignore"), "!file\n"); + + let mut m = one_matcher(&builder(td.path())); + assert!(matchedf(&mut m, ".hidden/file").is_ignore()); + assert!(matchedf(&mut m, ".hidden/nested/file").is_ignore()); + assert!(matchedf(&mut m, "visible/.hidden/file").is_ignore()); + } + + #[test] + fn whitelisted_hidden_directory_allows_descendants() { + let td = tmpdir(); + mkdirp(td.path().join(".git")); + mkdirp(td.path().join(".hidden")); + wfile(td.path().join(".hidden/file"), ""); + wfile(td.path().join(".gitignore"), "!.hidden/\n"); + + let mut m = one_matcher(&builder(td.path())); + let dir = matchedd(&mut m, ".hidden"); + assert!(dir.is_whitelist()); + assert!(dir.should_descend()); + assert!(matchedf(&mut m, ".hidden/file").is_none()); + } + + #[test] + fn hidden_directories_outside_depth_limits() { + let td = tmpdir(); + mkdirp(td.path().join("a/.hidden")); + + let mut b = builder(td.path()); + b.min_depth(Some(3)); + let mut m = one_matcher(&b); + let dir = matchedd(&mut m, "a/.hidden"); + assert!(dir.is_ignore()); + assert!(!dir.is_within_depth()); + + let mut b = builder(td.path()); + b.max_depth(Some(1)); + let mut m = one_matcher(&b); + let dir = matchedd(&mut m, "a/.hidden"); + assert!(dir.is_ignore()); + assert!(!dir.is_within_depth()); + } +} diff --git a/crates/ignore/src/lib.rs b/crates/ignore/src/lib.rs index 609004c4ea33..b9c9a3ece1a1 100644 --- a/crates/ignore/src/lib.rs +++ b/crates/ignore/src/lib.rs @@ -48,13 +48,16 @@ See the documentation for `WalkBuilder` for many other options. use std::path::{Path, PathBuf}; +pub use crate::incremental::{IncrementalIgnore, IncrementalMatch}; pub use crate::walk::{ - DirEntry, ParallelVisitor, ParallelVisitorBuilder, Walk, WalkBuilder, WalkParallel, WalkState, + DirEntry, ParallelVisitor, ParallelVisitorBuilder, Walk, WalkBuilder, + WalkParallel, WalkState, }; mod default_types; mod dir; pub mod gitignore; +mod incremental; pub mod overrides; mod pathutil; pub mod types; @@ -120,34 +123,31 @@ impl Clone for Error { fn clone(&self) -> Error { match *self { Error::Partial(ref errs) => Error::Partial(errs.clone()), - Error::WithLineNumber { line, ref err } => Error::WithLineNumber { - line, - err: err.clone(), - }, - Error::WithPath { ref path, ref err } => Error::WithPath { - path: path.clone(), - err: err.clone(), - }, - Error::WithDepth { depth, ref err } => Error::WithDepth { - depth, - err: err.clone(), - }, - Error::Loop { - ref ancestor, - ref child, - } => Error::Loop { + Error::WithLineNumber { line, ref err } => { + Error::WithLineNumber { line, err: err.clone() } + } + Error::WithPath { ref path, ref err } => { + Error::WithPath { path: path.clone(), err: err.clone() } + } + Error::WithDepth { depth, ref err } => { + Error::WithDepth { depth, err: err.clone() } + } + Error::Loop { ref ancestor, ref child } => Error::Loop { ancestor: ancestor.clone(), child: child.clone(), }, Error::Io(ref err) => match err.raw_os_error() { Some(e) => Error::Io(std::io::Error::from_raw_os_error(e)), - None => Error::Io(std::io::Error::new(err.kind(), err.to_string())), - }, - Error::Glob { ref glob, ref err } => Error::Glob { - glob: glob.clone(), - err: err.clone(), + None => { + Error::Io(std::io::Error::new(err.kind(), err.to_string())) + } }, - Error::UnrecognizedFileType(ref err) => Error::UnrecognizedFileType(err.clone()), + Error::Glob { ref glob, ref err } => { + Error::Glob { glob: glob.clone(), err: err.clone() } + } + Error::UnrecognizedFileType(ref err) => { + Error::UnrecognizedFileType(err.clone()) + } Error::InvalidDefinition => Error::InvalidDefinition, } } @@ -269,19 +269,14 @@ impl Error { /// Turn an error into a tagged error with the given depth. fn with_depth(self, depth: usize) -> Error { - Error::WithDepth { - depth, - err: Box::new(self), - } + Error::WithDepth { depth, err: Box::new(self) } } /// Turn an error into a tagged error with the given file path and line /// number. If path is empty, then it is omitted from the error. fn tagged>(self, path: P, lineno: u64) -> Error { - let errline = Error::WithLineNumber { - line: lineno, - err: Box::new(self), - }; + let errline = + Error::WithLineNumber { line: lineno, err: Box::new(self) }; if path.as_ref().as_os_str().is_empty() { return errline; } @@ -301,12 +296,12 @@ impl Error { }; } let path = err.path().map(|p| p.to_path_buf()); - let mut ig_err = Error::Io(std::io::Error::from(err)); + let mut ig_err = Error::WithDepth { + depth, + err: Box::new(Error::Io(std::io::Error::from(err))), + }; if let Some(path) = path { - ig_err = Error::WithPath { - path, - err: Box::new(ig_err), - }; + ig_err = Error::WithPath { path, err: Box::new(ig_err) }; } ig_err } @@ -333,7 +328,8 @@ impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match *self { Error::Partial(ref errs) => { - let msgs: Vec = errs.iter().map(|err| err.to_string()).collect(); + let msgs: Vec = + errs.iter().map(|err| err.to_string()).collect(); write!(f, "{}", msgs.join("\n")) } Error::WithLineNumber { line, ref err } => { @@ -343,10 +339,7 @@ impl std::fmt::Display for Error { write!(f, "{}: {}", path.display(), err) } Error::WithDepth { ref err, .. } => err.fmt(f), - Error::Loop { - ref ancestor, - ref child, - } => write!( + Error::Loop { ref ancestor, ref child } => write!( f, "File system loop found: \ {} points to an ancestor {}", @@ -354,14 +347,8 @@ impl std::fmt::Display for Error { ancestor.display() ), Error::Io(ref err) => err.fmt(f), - Error::Glob { - glob: None, - ref err, - } => write!(f, "{}", err), - Error::Glob { - glob: Some(ref glob), - ref err, - } => { + Error::Glob { glob: None, ref err } => write!(f, "{}", err), + Error::Glob { glob: Some(ref glob), ref err } => { write!(f, "error parsing glob '{}': {}", glob, err) } Error::UnrecognizedFileType(ref ty) => { @@ -507,7 +494,8 @@ mod tests { }; /// A convenient result type alias. - pub(crate) type Result = std::result::Result>; + pub(crate) type Result = + std::result::Result>; macro_rules! err { ($($tt:tt)*) => { @@ -545,8 +533,9 @@ mod tests { if path.is_dir() { continue; } - fs::create_dir_all(&path) - .map_err(|e| err!("failed to create {}: {}", path.display(), e))?; + fs::create_dir_all(&path).map_err(|e| { + err!("failed to create {}: {}", path.display(), e) + })?; return Ok(TempDir(path)); } Err(err!("failed to create temp dir after {} tries", TRIES)) diff --git a/crates/ignore/src/overrides.rs b/crates/ignore/src/overrides.rs index afb9f16ce2c6..005cae8f2d4a 100644 --- a/crates/ignore/src/overrides.rs +++ b/crates/ignore/src/overrides.rs @@ -94,7 +94,11 @@ impl Override { /// given) is stripped. If there is no common suffix/prefix overlap, then /// `path` is assumed to reside in the same directory as the root path for /// this set of overrides. - pub fn matched<'a, P: AsRef>(&'a self, path: P, is_dir: bool) -> Match> { + pub fn matched<'a, P: AsRef>( + &'a self, + path: P, + is_dir: bool, + ) -> Match> { if self.is_empty() { return Match::None; } @@ -146,7 +150,10 @@ impl OverrideBuilder { /// affected. /// /// This is disabled by default. - pub fn case_insensitive(&mut self, yes: bool) -> Result<&mut OverrideBuilder, Error> { + pub fn case_insensitive( + &mut self, + yes: bool, + ) -> Result<&mut OverrideBuilder, Error> { // TODO: This should not return a `Result`. Fix this in the next semver // release. self.builder.case_insensitive(yes)?; @@ -276,11 +283,8 @@ mod tests { #[test] fn default_case_sensitive() { - let ov = OverrideBuilder::new(ROOT) - .add("*.html") - .unwrap() - .build() - .unwrap(); + let ov = + OverrideBuilder::new(ROOT).add("*.html").unwrap().build().unwrap(); assert!(ov.matched("foo.html", false).is_whitelist()); assert!(ov.matched("foo.HTML", false).is_ignore()); assert!(ov.matched("foo.htm", false).is_ignore()); diff --git a/crates/ignore/src/pathutil.rs b/crates/ignore/src/pathutil.rs index 0ceb5a356c32..5de8c106c4ee 100644 --- a/crates/ignore/src/pathutil.rs +++ b/crates/ignore/src/pathutil.rs @@ -2,55 +2,89 @@ use std::{ffi::OsStr, path::Path}; use crate::walk::DirEntry; -/// Returns true if and only if this entry is considered to be hidden. +/// Returns true if and only if this path is considered to be hidden. /// -/// This only returns true if the base name of the path starts with a `.`. +/// # Platform behavior +/// +/// ## Windows +/// +/// This returns true if one of the following is true: +/// +/// * The base name of the path starts with a `.`. +/// * The file attributes have the `HIDDEN` property set. +/// +/// ## All other platforms /// -/// On Unix, this implements a more optimized check. -#[cfg(unix)] -pub(crate) fn is_hidden(dent: &DirEntry) -> bool { - use std::os::unix::ffi::OsStrExt; +/// This only returns true if the base name of the path starts with a `.`. +pub(crate) fn is_hidden_path(dent: &Path) -> bool { + #[cfg(not(windows))] + fn imp(path: &Path) -> bool { + is_hidden_path_only(path) + } - if let Some(name) = file_name(dent.path()) { - name.as_bytes().get(0) == Some(&b'.') - } else { - false + #[cfg(windows)] + fn imp(path: &Path) -> bool { + use std::os::windows::fs::MetadataExt; + use winapi_util::file; + + if let Ok(md) = path.metadata() { + if file::is_hidden(md.file_attributes() as u64) { + return true; + } + } + is_hidden_path_only(path) } + + imp(dent) } -/// Returns true if and only if this entry is considered to be hidden. +/// Returns true if and only if this directory entry is considered to be +/// hidden. +/// +/// # Platform behavior /// -/// On Windows, this returns true if one of the following is true: +/// ## Windows +/// +/// This returns true if one of the following is true: /// /// * The base name of the path starts with a `.`. /// * The file attributes have the `HIDDEN` property set. -#[cfg(windows)] -pub(crate) fn is_hidden(dent: &DirEntry) -> bool { - use std::os::windows::fs::MetadataExt; - use winapi_util::file; - - // This looks like we're doing an extra stat call, but on Windows, the - // directory traverser reuses the metadata retrieved from each directory - // entry and stores it on the DirEntry itself. So this is "free." - if let Ok(md) = dent.metadata() { - if file::is_hidden(md.file_attributes() as u64) { - return true; - } +/// +/// ## All other platforms +/// +/// This only returns true if the base name of the path starts with a `.`. +pub(crate) fn is_hidden_entry(dent: &DirEntry) -> bool { + #[cfg(not(windows))] + fn imp(dent: &DirEntry) -> bool { + is_hidden_path_only(dent.path()) } - if let Some(name) = file_name(dent.path()) { - name.to_str().map(|s| s.starts_with(".")).unwrap_or(false) - } else { - false + + #[cfg(windows)] + fn imp(dent: &DirEntry) -> bool { + use std::os::windows::fs::MetadataExt; + use winapi_util::file; + + // This looks like we're doing an extra stat call, but on Windows, the + // directory traverser reuses the metadata retrieved from each directory + // entry and stores it on the DirEntry itself. So this is "free." + if let Ok(md) = dent.metadata() { + if file::is_hidden(md.file_attributes() as u64) { + return true; + } + } + is_hidden_path_only(dent.path()) } + + imp(dent) } -/// Returns true if and only if this entry is considered to be hidden. +/// Returns true if and only if this path is considered to be hidden from only +/// the path itself. /// -/// This only returns true if the base name of the path starts with a `.`. -#[cfg(not(any(unix, windows)))] -pub(crate) fn is_hidden(dent: &DirEntry) -> bool { - if let Some(name) = file_name(dent.path()) { - name.to_str().map(|s| s.starts_with(".")).unwrap_or(false) +/// This has the same behavior on all platforms. +fn is_hidden_path_only(path: &Path) -> bool { + if let Some(name) = file_name(path) { + name.as_encoded_bytes().starts_with(b".") } else { false } @@ -59,83 +93,79 @@ pub(crate) fn is_hidden(dent: &DirEntry) -> bool { /// Strip `prefix` from the `path` and return the remainder. /// /// If `path` doesn't have a prefix `prefix`, then return `None`. -#[cfg(unix)] pub(crate) fn strip_prefix<'a, P: AsRef + ?Sized>( prefix: &'a P, path: &'a Path, ) -> Option<&'a Path> { - use std::os::unix::ffi::OsStrExt; - - let prefix = prefix.as_ref().as_os_str().as_bytes(); - let path = path.as_os_str().as_bytes(); - if prefix.len() > path.len() || prefix != &path[0..prefix.len()] { - None - } else { - Some(&Path::new(OsStr::from_bytes(&path[prefix.len()..]))) + #[cfg(unix)] + fn imp<'a>(prefix: &'a Path, path: &'a Path) -> Option<&'a Path> { + use std::os::unix::ffi::OsStrExt; + + let prefix = prefix.as_os_str().as_bytes(); + let path = path.as_os_str().as_bytes(); + if prefix.len() > path.len() || prefix != &path[0..prefix.len()] { + None + } else { + Some(&Path::new(OsStr::from_bytes(&path[prefix.len()..]))) + } } -} - -/// Strip `prefix` from the `path` and return the remainder. -/// -/// If `path` doesn't have a prefix `prefix`, then return `None`. -#[cfg(not(unix))] -pub(crate) fn strip_prefix<'a, P: AsRef + ?Sized>( - prefix: &'a P, - path: &'a Path, -) -> Option<&'a Path> { - path.strip_prefix(prefix).ok() -} -/// Returns true if this file path is just a file name. i.e., Its parent is -/// the empty string. -#[cfg(unix)] -pub(crate) fn is_file_name>(path: P) -> bool { - use std::os::unix::ffi::OsStrExt; - - use memchr::memchr; + #[cfg(not(unix))] + fn imp<'a>(prefix: &'a Path, path: &'a Path) -> Option<&'a Path> { + path.strip_prefix(prefix).ok() + } - let path = path.as_ref().as_os_str().as_bytes(); - memchr(b'/', path).is_none() + imp(prefix.as_ref(), path) } /// Returns true if this file path is just a file name. i.e., Its parent is /// the empty string. -#[cfg(not(unix))] pub(crate) fn is_file_name>(path: P) -> bool { - path.as_ref() - .parent() - .map(|p| p.as_os_str().is_empty()) - .unwrap_or(false) + #[cfg(unix)] + { + memchr::memchr(b'/', path.as_ref().as_os_str().as_encoded_bytes()) + .is_none() + } + #[cfg(not(unix))] + { + path.as_ref() + .parent() + .map(|p| p.as_os_str().is_empty()) + .unwrap_or(false) + } } /// The final component of the path, if it is a normal file. /// -/// If the path terminates in ., .., or consists solely of a root of prefix, -/// file_name will return None. -#[cfg(unix)] -pub(crate) fn file_name<'a, P: AsRef + ?Sized>(path: &'a P) -> Option<&'a OsStr> { - use memchr::memrchr; - use std::os::unix::ffi::OsStrExt; - - let path = path.as_ref().as_os_str().as_bytes(); - if path.is_empty() { - return None; - } else if path.len() == 1 && path[0] == b'.' { - return None; - } else if path.last() == Some(&b'.') { - return None; - } else if path.len() >= 2 && &path[path.len() - 2..] == &b".."[..] { - return None; +/// If the path terminates in `.`, `..`, or consists solely of a root of +/// prefix, this will return `None`. +pub(crate) fn file_name<'a, P: AsRef + ?Sized>( + path: &'a P, +) -> Option<&'a OsStr> { + #[cfg(unix)] + fn imp(path: &Path) -> Option<&OsStr> { + use std::os::unix::ffi::OsStrExt; + + use memchr::memrchr; + + let path = path.as_os_str().as_bytes(); + if path.is_empty() { + return None; + } else if path.len() == 1 && path[0] == b'.' { + return None; + } else if path.last() == Some(&b'.') { + return None; + } else if path.len() >= 2 && &path[path.len() - 2..] == &b".."[..] { + return None; + } + let last_slash = memrchr(b'/', path).map(|i| i + 1).unwrap_or(0); + Some(OsStr::from_bytes(&path[last_slash..])) } - let last_slash = memrchr(b'/', path).map(|i| i + 1).unwrap_or(0); - Some(OsStr::from_bytes(&path[last_slash..])) -} -/// The final component of the path, if it is a normal file. -/// -/// If the path terminates in ., .., or consists solely of a root of prefix, -/// file_name will return None. -#[cfg(not(unix))] -pub(crate) fn file_name<'a, P: AsRef + ?Sized>(path: &'a P) -> Option<&'a OsStr> { - path.as_ref().file_name() + #[cfg(not(unix))] + fn imp(path: &Path) -> Option<&OsStr> { + path.file_name() + } + + imp(path.as_ref()) } diff --git a/crates/ignore/src/types.rs b/crates/ignore/src/types.rs index aa23999c06fe..313cf5c0ec24 100644 --- a/crates/ignore/src/types.rs +++ b/crates/ignore/src/types.rs @@ -204,8 +204,12 @@ impl Selection { fn map U>(self, f: F) -> Selection { match self { - Selection::Select(name, inner) => Selection::Select(name, f(inner)), - Selection::Negate(name, inner) => Selection::Negate(name, f(inner)), + Selection::Select(name, inner) => { + Selection::Select(name, f(inner)) + } + Selection::Negate(name, inner) => { + Selection::Negate(name, f(inner)) + } } } @@ -227,7 +231,9 @@ impl Types { has_selected: false, glob_to_selection: vec![], set: GlobSetBuilder::new().build().unwrap(), - matches: Arc::new(Pool::new(|| vec![])), + matches: Arc::new(Pool::with_available_parallelism_capacity( + || vec![], + )), } } @@ -254,7 +260,11 @@ impl Types { /// The path is considered ignored if it matches a negated file type. /// If at least one file type is selected and `path` doesn't match, then /// the path is also considered ignored. - pub fn matched<'a, P: AsRef>(&'a self, path: P, is_dir: bool) -> Match> { + pub fn matched<'a, P: AsRef>( + &'a self, + path: P, + is_dir: bool, + ) -> Match> { // File types don't apply to directories, and we can't do anything // if our glob set is empty. if is_dir || self.set.is_empty() { @@ -306,10 +316,7 @@ impl TypesBuilder { /// of default type definitions can be added with `add_defaults`, and /// additional type definitions can be added with `select` and `negate`. pub fn new() -> TypesBuilder { - TypesBuilder { - types: HashMap::new(), - selections: vec![], - } + TypesBuilder { types: HashMap::new(), selections: vec![] } } /// Build the current set of file type definitions *and* selections into @@ -343,17 +350,18 @@ impl TypesBuilder { } selections.push(selection.clone().map(move |_| def)); } - let set = build_set.build().map_err(|err| Error::Glob { - glob: None, - err: err.to_string(), - })?; + let set = build_set + .build() + .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?; Ok(Types { defs, selections, has_selected, glob_to_selection, set, - matches: Arc::new(Pool::new(|| vec![])), + matches: Arc::new(Pool::with_available_parallelism_capacity( + || vec![], + )), }) } @@ -377,12 +385,10 @@ impl TypesBuilder { pub fn select(&mut self, name: &str) -> &mut TypesBuilder { if name == "all" { for name in self.types.keys() { - self.selections - .push(Selection::Select(name.to_string(), ())); + self.selections.push(Selection::Select(name.to_string(), ())); } } else { - self.selections - .push(Selection::Select(name.to_string(), ())); + self.selections.push(Selection::Select(name.to_string(), ())); } self } @@ -393,12 +399,10 @@ impl TypesBuilder { pub fn negate(&mut self, name: &str) -> &mut TypesBuilder { if name == "all" { for name in self.types.keys() { - self.selections - .push(Selection::Negate(name.to_string(), ())); + self.selections.push(Selection::Negate(name.to_string(), ())); } } else { - self.selections - .push(Selection::Negate(name.to_string(), ())); + self.selections.push(Selection::Negate(name.to_string(), ())); } self } @@ -453,7 +457,10 @@ impl TypesBuilder { 3 => { let name = parts[0]; let types_string = parts[2]; - if name.is_empty() || parts[1] != "include" || types_string.is_empty() { + if name.is_empty() + || parts[1] != "include" + || types_string.is_empty() + { return Err(Error::InvalidDefinition); } let types = types_string.split(','); @@ -463,7 +470,8 @@ impl TypesBuilder { return Err(Error::InvalidDefinition); } for type_name in types { - let globs = self.types.get(type_name).unwrap().globs.clone(); + let globs = + self.types.get(type_name).unwrap().globs.clone(); for glob in globs { self.add(name, &glob)?; } @@ -549,30 +557,9 @@ mod tests { matched!(not, matchnot1, types(), vec!["rust"], vec![], "index.html"); matched!(not, matchnot2, types(), vec![], vec!["rust"], "main.rs"); - matched!( - not, - matchnot3, - types(), - vec!["foo"], - vec!["rust"], - "main.rs" - ); - matched!( - not, - matchnot4, - types(), - vec!["rust"], - vec!["foo"], - "main.rs" - ); - matched!( - not, - matchnot5, - types(), - vec!["rust"], - vec!["foo"], - "main.foo" - ); + matched!(not, matchnot3, types(), vec!["foo"], vec!["rust"], "main.rs"); + matched!(not, matchnot4, types(), vec!["rust"], vec!["foo"], "main.rs"); + matched!(not, matchnot5, types(), vec!["rust"], vec!["foo"], "main.foo"); matched!(not, matchnot6, types(), vec!["combo"], vec![], "leftpad.js"); matched!(not, matchnot7, types(), vec!["py"], vec![], "index.html"); matched!(not, matchnot8, types(), vec!["python"], vec![], "doc.md"); diff --git a/crates/ignore/src/walk.rs b/crates/ignore/src/walk.rs index ebf20947a692..2dff78852f25 100644 --- a/crates/ignore/src/walk.rs +++ b/crates/ignore/src/walk.rs @@ -17,7 +17,9 @@ use { use crate::{ Error, PartialErrorBuilder, dir::{Ignore, IgnoreBuilder}, + // CHANGED: Also import `Gitignore` for `WalkBuilder::add_gitignore`. gitignore::{Gitignore, GitignoreBuilder}, + incremental::{IncrementalIgnore, IncrementalIgnoreOptions}, overrides::Override, types::Types, }; @@ -104,24 +106,15 @@ impl DirEntry { } fn new_stdin() -> DirEntry { - DirEntry { - dent: DirEntryInner::Stdin, - err: None, - } + DirEntry { dent: DirEntryInner::Stdin, err: None } } fn new_walkdir(dent: walkdir::DirEntry, err: Option) -> DirEntry { - DirEntry { - dent: DirEntryInner::Walkdir(dent), - err, - } + DirEntry { dent: DirEntryInner::Walkdir(dent), err } } fn new_raw(dent: DirEntryRaw, err: Option) -> DirEntry { - DirEntry { - dent: DirEntryInner::Raw(dent), - err, - } + DirEntry { dent: DirEntryInner::Raw(dent), err } } } @@ -187,9 +180,11 @@ impl DirEntryInner { )); Err(err.with_path("")) } - Walkdir(ref x) => x - .metadata() - .map_err(|err| Error::Io(io::Error::from(err)).with_path(x.path())), + Walkdir(ref x) => x.metadata().map_err(|err| { + Error::Io(io::Error::from(err)) + .with_depth(x.depth()) + .with_path(x.path()) + }), Raw(ref x) => x.metadata(), } } @@ -308,7 +303,9 @@ impl DirEntryRaw { } else { fs::symlink_metadata(&self.path) } - .map_err(|err| Error::Io(io::Error::from(err)).with_path(&self.path)) + .map_err(|err| { + Error::Io(err).with_depth(self.depth).with_path(&self.path) + }) } fn file_type(&self) -> FileType { @@ -316,9 +313,7 @@ impl DirEntryRaw { } fn file_name(&self) -> &OsStr { - self.path - .file_name() - .unwrap_or_else(|| self.path.as_os_str()) + self.path.file_name().unwrap_or_else(|| self.path.as_os_str()) } fn depth(&self) -> usize { @@ -330,13 +325,13 @@ impl DirEntryRaw { self.ino } - fn from_entry(depth: usize, ent: &fs::DirEntry) -> Result { + fn from_entry( + depth: usize, + ent: &fs::DirEntry, + ) -> Result { let ty = ent.file_type().map_err(|err| { - let err = Error::Io(io::Error::from(err)).with_path(ent.path()); - Error::WithDepth { - depth, - err: Box::new(err), - } + let err = Error::Io(err).with_depth(depth).with_path(ent.path()); + Error::WithDepth { depth, err: Box::new(err) } })?; DirEntryRaw::from_entry_os(depth, ent, ty) } @@ -348,11 +343,8 @@ impl DirEntryRaw { ty: fs::FileType, ) -> Result { let md = ent.metadata().map_err(|err| { - let err = Error::Io(io::Error::from(err)).with_path(ent.path()); - Error::WithDepth { - depth, - err: Box::new(err), - } + let err = Error::Io(err).with_depth(depth).with_path(ent.path()); + Error::WithDepth { depth, err: Box::new(err) } })?; Ok(DirEntryRaw { path: ent.path(), @@ -395,8 +387,13 @@ impl DirEntryRaw { } #[cfg(windows)] - fn from_path(depth: usize, pb: PathBuf, link: bool) -> Result { - let md = fs::metadata(&pb).map_err(|err| Error::Io(err).with_path(&pb))?; + fn from_path( + depth: usize, + pb: PathBuf, + link: bool, + ) -> Result { + let md = fs::metadata(&pb) + .map_err(|err| Error::Io(err).with_depth(depth).with_path(&pb))?; Ok(DirEntryRaw { path: pb, ty: md.file_type(), @@ -407,10 +404,15 @@ impl DirEntryRaw { } #[cfg(unix)] - fn from_path(depth: usize, pb: PathBuf, link: bool) -> Result { + fn from_path( + depth: usize, + pb: PathBuf, + link: bool, + ) -> Result { use std::os::unix::fs::MetadataExt; - let md = fs::metadata(&pb).map_err(|err| Error::Io(err).with_path(&pb))?; + let md = fs::metadata(&pb) + .map_err(|err| Error::Io(err).with_depth(depth).with_path(&pb))?; Ok(DirEntryRaw { path: pb, ty: md.file_type(), @@ -423,7 +425,11 @@ impl DirEntryRaw { // Placeholder implementation to allow compiling on non-standard platforms // (e.g. wasm32). #[cfg(not(any(windows, unix)))] - fn from_path(depth: usize, pb: PathBuf, link: bool) -> Result { + fn from_path( + depth: usize, + pb: PathBuf, + link: bool, + ) -> Result { Err(Error::Io(io::Error::new( io::ErrorKind::Other, "unsupported platform", @@ -502,7 +508,8 @@ pub struct WalkBuilder { /// /// When `None`, the CWD is fetched from `std::env::current_dir()`. If /// that fails, then global gitignores are ignored (an error is logged). - global_gitignores_relative_to: OnceLock>>, + global_gitignores_relative_to: + OnceLock>>, } #[derive(Clone)] @@ -544,8 +551,16 @@ impl WalkBuilder { /// is better to call `add` on this builder than to create multiple /// `Walk` values. pub fn new>(path: P) -> WalkBuilder { + WalkBuilder::from_iter([path]) + } + + /// Create an empty builder to which paths can be added. + /// + /// Note that if you call `build` on this instance before calling `add` + /// on it, it will return exactly zero items during iteration. + pub fn empty() -> WalkBuilder { WalkBuilder { - paths: vec![path.as_ref().to_path_buf()], + paths: vec![], ig_builder: IgnoreBuilder::new(), max_depth: None, min_depth: None, @@ -560,6 +575,21 @@ impl WalkBuilder { } } + /// Create a new builder for a recursive directory iterator from the + /// sequence of paths. + /// + /// Note that if the iterator is empty, this is the same as + /// `WalkBuilder::empty`. + pub fn from_iter, I: IntoIterator>( + paths: I, + ) -> WalkBuilder { + let mut builder = WalkBuilder::empty(); + for path in paths.into_iter() { + builder.add(path); + } + builder + } + /// Build a new `Walk` iterator. pub fn build(&self) -> Walk { let follow_links = self.follow_links; @@ -585,10 +615,14 @@ impl WalkBuilder { if let Some(ref sorter) = sorter { match sorter.clone() { Sorter::ByName(cmp) => { - wd = wd.sort_by(move |a, b| cmp(a.file_name(), b.file_name())); + wd = wd.sort_by(move |a, b| { + cmp(a.file_name(), b.file_name()) + }); } Sorter::ByPath(cmp) => { - wd = wd.sort_by(move |a, b| cmp(a.path(), b.path())); + wd = wd.sort_by(move |a, b| { + cmp(a.path(), b.path()) + }); } } } @@ -597,31 +631,76 @@ impl WalkBuilder { }) .collect::>() .into_iter(); - let ig_root = self - .get_or_set_current_dir() - .map(|cwd| self.ig_builder.build_with_cwd(Some(cwd.to_path_buf()))) - .unwrap_or_else(|| self.ig_builder.build()); + let ig_root = self.build_ignore(); Walk { its, it: None, ig_root: ig_root.clone(), ig: ig_root.clone(), + max_depth: self.max_depth, max_filesize: self.max_filesize, skip: self.skip.clone(), filter: self.filter.clone(), } } + /// Build matchers for checking paths against ignore files without + /// recursively walking the configured roots. + /// + /// The returned matchers use the path-based filtering configuration + /// on this builder, including glob overrides, file type selections, + /// parent ignore files, `.ignore`, `.gitignore`, global Git + /// ignore files, explicitly added ignore files and custom ignore + /// file names. For example, ripgrep configures `.rgignore` via + /// [`WalkBuilder::add_custom_ignore_filename`]. Minimum and maximum depth + /// limits, maximum file size and hidden-file filtering are also applied. + /// Other options that only control traversal or require a directory entry, + /// such as custom entry predicates, are not applied. + /// + /// One matcher is returned for each configured path, in the same order as + /// the paths were added to this builder. Each matcher accepts paths + /// relative to its own [`IncrementalIgnore::root`]. The matcher for the + /// special `-` path representing standard input always returns a non-match + /// for all inputs. + /// + /// Ignore matchers are loaded lazily and cached by directory. + /// Thus, the first query may read ignore files from the root and + /// its parents, while later queries reuse the compiled matchers. + /// Errors encountered while loading ignore files are returned by + /// [`IncrementalIgnore::matched_with_errors`]. Once an ignore file has + /// been loaded, changes to it are not observed. Build new matchers to + /// reload changed ignore files. + /// + /// Matchers built together share the builder's base ignore configuration + /// and compiled parent matchers. + pub fn build_matchers(&self) -> Vec { + let ignore = self.build_ignore(); + let options = IncrementalIgnoreOptions { + min_depth: self.min_depth, + max_depth: self.max_depth, + max_filesize: self.max_filesize, + hidden: self.ig_builder.is_hidden(), + follow_links: self.follow_links, + }; + self.paths + .iter() + .map(move |path| { + IncrementalIgnore::new( + path.clone(), + ignore.clone(), + options.clone(), + ) + }) + .collect() + } + /// Build a new `WalkParallel` iterator. /// /// Note that this *doesn't* return something that implements `Iterator`. /// Instead, the returned value must be run with a closure. e.g., /// `builder.build_parallel().run(|| |path| { println!("{path:?}"); WalkState::Continue })`. pub fn build_parallel(&self) -> WalkParallel { - let ig_root = self - .get_or_set_current_dir() - .map(|cwd| self.ig_builder.build_with_cwd(Some(cwd.to_path_buf()))) - .unwrap_or_else(|| self.ig_builder.build()); + let ig_root = self.build_ignore(); WalkParallel { paths: self.paths.clone().into_iter(), ig_root, @@ -651,7 +730,10 @@ impl WalkBuilder { /// The default, `None`, imposes no depth restriction. pub fn max_depth(&mut self, depth: Option) -> &mut WalkBuilder { self.max_depth = depth; - if self.min_depth.is_some() && self.max_depth.is_some() && self.max_depth < self.min_depth { + if self.min_depth.is_some() + && self.max_depth.is_some() + && self.max_depth < self.min_depth + { self.max_depth = self.min_depth; } self @@ -662,7 +744,10 @@ impl WalkBuilder { /// The default, `None`, imposes no minimum depth restriction. pub fn min_depth(&mut self, depth: Option) -> &mut WalkBuilder { self.min_depth = depth; - if self.max_depth.is_some() && self.min_depth.is_some() && self.min_depth > self.max_depth { + if self.max_depth.is_some() + && self.min_depth.is_some() + && self.min_depth > self.max_depth + { self.min_depth = self.max_depth; } self @@ -705,7 +790,12 @@ impl WalkBuilder { /// An error will also occur if this walker could not get the current /// working directory (and `WalkBuilder::current_dir` isn't set). pub fn add_ignore>(&mut self, path: P) -> Option { - // CHANGED: Dropped this code + // CHANGED: Root the ignore file at `""` instead of the current working + // directory. Explicit ignores are scoped to the directory of the + // ignore file (see `matched_ignore`), and a root of `""` makes the + // rules apply to every walked path regardless of the walk root. This + // also avoids depending on the current working directory entirely. + // // let path = path.as_ref(); // let Some(cwd) = self.get_or_set_current_dir() else { // let err = std::io::Error::other(format!( @@ -729,7 +819,11 @@ impl WalkBuilder { errs.into_error_option() } - /// CHANGED: Add a Gitignore to the builder. + /// CHANGED: Add a prebuilt Gitignore to the builder. + /// + /// Like the ignore file added via `add_ignore`, these rules are matched + /// against the full path of each walked entry, scoped to the `Gitignore`'s + /// root path. pub fn add_gitignore(&mut self, gi: Gitignore) { self.ig_builder.add_ignore(gi); } @@ -982,7 +1076,10 @@ impl WalkBuilder { /// /// Global gitignore files come from things like a user's git configuration /// or from gitignore files added via [`WalkBuilder::add_ignore`]. - pub fn current_dir(&mut self, cwd: impl Into) -> &mut WalkBuilder { + pub fn current_dir( + &mut self, + cwd: impl Into, + ) -> &mut WalkBuilder { let cwd = cwd.into(); self.ig_builder.current_dir(cwd.clone()); if let Err(cwd) = self.global_gitignores_relative_to.set(Ok(cwd)) { @@ -1002,7 +1099,10 @@ impl WalkBuilder { let result = std::env::current_dir().map_err(Arc::new); match result { Ok(ref path) => { - log::trace!("automatically discovered CWD: {}", path.display()); + log::trace!( + "automatically discovered CWD: {}", + path.display() + ); } Err(ref err) => { log::debug!( @@ -1016,6 +1116,13 @@ impl WalkBuilder { }); result.as_ref().ok().map(|path| &**path) } + + /// Build the root ignore matcher shared by all consumers of this builder. + fn build_ignore(&self) -> Ignore { + self.get_or_set_current_dir() + .map(|cwd| self.ig_builder.build_with_cwd(Some(cwd.to_path_buf()))) + .unwrap_or_else(|| self.ig_builder.build()) + } } /// Walk is a recursive directory iterator over file paths in one or more @@ -1029,6 +1136,7 @@ pub struct Walk { it: Option, ig_root: Ignore, ig: Ignore, + max_depth: Option, max_filesize: Option, skip: Option>, filter: Option, @@ -1044,6 +1152,17 @@ impl Walk { WalkBuilder::new(path).build() } + /// Create a new recursive directory iterator from the sequence of paths + /// given. + /// + /// Note that if the provided iterator is empty, then `Walk` is guaranteed + /// to yield zero entries. + pub fn from_iter, I: IntoIterator>( + paths: I, + ) -> Walk { + WalkBuilder::from_iter(paths).build() + } + fn skip_entry(&self, ent: &DirEntry) -> Result { if ent.depth() == 0 { return Ok(false); @@ -1128,12 +1247,17 @@ impl Iterator for Walk { self.it.as_mut().unwrap().it.skip_current_dir(); // Still need to push this on the stack because // we'll get a WalkEvent::Exit event for this dir. - // We don't care if it errors though. - let (igtmp, _) = self.ig.add_child(ent.path()); + // Its ignore files cannot apply to any visited entry. + let (igtmp, _) = + self.ig.add_child_with_entries(ent.path(), &[]); self.ig = igtmp; continue; } - let (igtmp, err) = self.ig.add_child(ent.path()); + let (igtmp, err) = if self.max_depth == Some(ent.depth()) { + self.ig.add_child_with_entries(ent.path(), &[]) + } else { + self.ig.add_child(ent.path()) + }; self.ig = igtmp; ent.err = err; return Some(Ok(ent)); @@ -1175,11 +1299,7 @@ enum WalkEvent { impl From for WalkEventIter { fn from(it: WalkDir) -> WalkEventIter { - WalkEventIter { - depth: 0, - it: it.into_iter(), - next: None, - } + WalkEventIter { depth: 0, it: it.into_iter(), next: None } } } @@ -1252,7 +1372,9 @@ pub trait ParallelVisitorBuilder<'s> { fn build(&mut self) -> Box; } -impl<'a, 's, P: ParallelVisitorBuilder<'s>> ParallelVisitorBuilder<'s> for &'a mut P { +impl<'a, 's, P: ParallelVisitorBuilder<'s>> ParallelVisitorBuilder<'s> + for &'a mut P +{ fn build(&mut self) -> Box { (**self).build() } @@ -1273,14 +1395,17 @@ struct FnBuilder { builder: F, } -impl<'s, F: FnMut() -> FnVisitor<'s>> ParallelVisitorBuilder<'s> for FnBuilder { +impl<'s, F: FnMut() -> FnVisitor<'s>> ParallelVisitorBuilder<'s> + for FnBuilder +{ fn build(&mut self) -> Box { let visitor = (self.builder)(); Box::new(FnVisitorImp { visitor }) } } -type FnVisitor<'s> = Box) -> WalkState + Send + 's>; +type FnVisitor<'s> = + Box) -> WalkState + Send + 's>; struct FnVisitorImp<'s> { visitor: FnVisitor<'s>, @@ -1370,7 +1495,9 @@ impl WalkParallel { } }; match DirEntryRaw::from_path(0, path, false) { - Ok(dent) => (DirEntry::new_raw(dent, None), root_device), + Ok(dent) => { + (DirEntry::new_raw(dent, None), root_device) + } Err(err) => { if visitor.visit(Err(err)).is_quit() { return; @@ -1394,21 +1521,28 @@ impl WalkParallel { let quit_now = Arc::new(AtomicBool::new(false)); let active_workers = Arc::new(AtomicUsize::new(threads)); let stacks = Stack::new_for_each_thread(threads, stack); + // Collect all of the workers first. In the case that + // `builder.build()` panics, we want that to happen and + // propagate before we actually start to run any of the + // workers. + let workers: Vec<_> = stacks + .into_iter() + .map(|stack| Worker { + visitor: builder.build(), + stack, + quit_now: quit_now.clone(), + active_workers: active_workers.clone(), + max_depth: self.max_depth, + min_depth: self.min_depth, + max_filesize: self.max_filesize, + follow_links: self.follow_links, + skip: self.skip.clone(), + filter: self.filter.clone(), + }) + .collect(); std::thread::scope(|s| { - let handles: Vec<_> = stacks + let handles: Vec<_> = workers .into_iter() - .map(|stack| Worker { - visitor: builder.build(), - stack, - quit_now: quit_now.clone(), - active_workers: active_workers.clone(), - max_depth: self.max_depth, - min_depth: self.min_depth, - max_filesize: self.max_filesize, - follow_links: self.follow_links, - skip: self.skip.clone(), - filter: self.filter.clone(), - }) .map(|worker| s.spawn(|| worker.run())) .collect(); for handle in handles { @@ -1419,9 +1553,7 @@ impl WalkParallel { fn threads(&self) -> usize { if self.threads == 0 { - std::thread::available_parallelism() - .map_or(1, |n| n.get()) - .min(12) + std::thread::available_parallelism().map_or(1, |n| n.get()).min(12) } else { self.threads } @@ -1452,6 +1584,12 @@ struct Work { root_device: Option, } +#[derive(Default)] +struct ReadDirResult { + entries: Vec, + errors: Vec, +} + impl Work { /// Returns true if and only if this work item is a directory. fn is_dir(&self) -> bool { @@ -1478,6 +1616,13 @@ impl Work { err } + /// Adds ignore rules for this directory without reading its contents. + fn add_ignore(&mut self) { + let (ig, err) = self.ignore.add_child(self.dent.path()); + self.ignore = ig; + self.dent.err = err; + } + /// Reads the directory contents of this work item and adds ignore /// rules for this directory. /// @@ -1485,7 +1630,7 @@ impl Work { /// an error is returned. If there was a problem reading the ignore /// rules for this directory, then the error is attached to this /// work item's directory entry. - fn read_dir(&mut self) -> Result { + fn read_dir(&mut self) -> Result { let readdir = match fs::read_dir(self.dent.path()) { Ok(readdir) => readdir, Err(err) => { @@ -1495,10 +1640,24 @@ impl Work { return Err(err); } }; - let (ig, err) = self.ignore.add_child(self.dent.path()); + // Actually descend into the directory and read its contents + let mut result = ReadDirResult::default(); + for entry in readdir { + match entry { + Ok(entry) => result.entries.push(entry), + Err(err) => result.errors.push( + Error::from(err) + .with_path(self.dent.path()) + .with_depth(self.dent.depth() + 1), + ), + } + } + let (ig, err) = self + .ignore + .add_child_with_entries(self.dent.path(), &result.entries); self.ignore = ig; self.dent.err = err; - Ok(readdir) + Ok(result) } } @@ -1522,11 +1681,11 @@ impl Stack { // breadth-first. We do depth-first because a breadth first traversal // on wide directories with a lot of gitignores is disastrous (for // example, searching a directory tree containing all of crates.io). - let deques: Vec> = std::iter::repeat_with(Deque::new_lifo) - .take(threads) - .collect(); - let stealers = - Arc::<[Stealer]>::from(deques.iter().map(Deque::stealer).collect::>()); + let deques: Vec> = + std::iter::repeat_with(Deque::new_lifo).take(threads).collect(); + let stealers = Arc::<[Stealer]>::from( + deques.iter().map(Deque::stealer).collect::>(), + ); let stacks: Vec = deques .into_iter() .enumerate() @@ -1668,8 +1827,13 @@ impl<'s> Worker<'s> { // have sufficient read permissions to list the directory. // In that case we still want to provide the closure with a valid // entry before passing the error value. - let readdir = work.read_dir(); let depth = work.dent.depth(); + let readdir = if descend && self.max_depth.is_none_or(|m| depth < m) { + Some(work.read_dir()) + } else { + work.add_ignore(); + None + }; if should_visit { let state = self.visitor.visit(Ok(work.dent)); if !state.is_continue() { @@ -1680,6 +1844,10 @@ impl<'s> Worker<'s> { return WalkState::Skip; } + let readdir = match readdir { + Some(readdir) => readdir, + None => return WalkState::Skip, + }; let readdir = match readdir { Ok(readdir) => readdir, Err(err) => { @@ -1687,11 +1855,19 @@ impl<'s> Worker<'s> { } }; - if self.max_depth.map_or(false, |max| depth >= max) { - return WalkState::Skip; + for result in readdir.entries { + let state = self.generate_work( + &work.ignore, + depth + 1, + work.root_device, + result, + ); + if state.is_quit() { + return state; + } } - for result in readdir { - let state = self.generate_work(&work.ignore, depth + 1, work.root_device, result); + for err in readdir.errors { + let state = self.visitor.visit(Err(err)); if state.is_quit() { return state; } @@ -1717,14 +1893,8 @@ impl<'s> Worker<'s> { ig: &Ignore, depth: usize, root_device: Option, - result: Result, + fs_dent: fs::DirEntry, ) -> WalkState { - let fs_dent = match result { - Ok(fs_dent) => fs_dent, - Err(err) => { - return self.visitor.visit(Err(Error::from(err).with_depth(depth))); - } - }; let mut dent = match DirEntryRaw::from_entry(depth, &fs_dent) { Ok(dent) => DirEntry::new_raw(dent, None), Err(err) => { @@ -1760,26 +1930,24 @@ impl<'s> Worker<'s> { return WalkState::Continue; } } - let should_skip_filesize = if self.max_filesize.is_some() && !dent.is_dir() { - skip_filesize( - self.max_filesize.unwrap(), - dent.path(), - &dent.metadata().ok(), - ) - } else { - false - }; - let should_skip_filtered = if let Some(Filter(predicate)) = &self.filter { - !predicate(&dent) - } else { - false - }; + let should_skip_filesize = + if self.max_filesize.is_some() && !dent.is_dir() { + skip_filesize( + self.max_filesize.unwrap(), + dent.path(), + &dent.metadata().ok(), + ) + } else { + false + }; + let should_skip_filtered = + if let Some(Filter(predicate)) = &self.filter { + !predicate(&dent) + } else { + false + }; if !should_skip_filesize && !should_skip_filtered { - self.send(Work { - dent, - ignore: ig.clone(), - root_device, - }); + self.send(Work { dent, ignore: ig.clone(), root_device }); } WalkState::Continue } @@ -1820,6 +1988,9 @@ impl<'s> Worker<'s> { } // Wait for next `Work` or `Quit` message. loop { + if self.is_quit_now() { + return None; + } if let Some(v) = self.recv() { self.activate_worker(); value = Some(v); @@ -1873,24 +2044,25 @@ impl<'s> Worker<'s> { } } +impl<'s> Drop for Worker<'s> { + fn drop(&mut self) { + if std::thread::panicking() { + self.quit_now(); + } + } +} + fn check_symlink_loop( ig_parent: &Ignore, child_path: &Path, child_depth: usize, ) -> Result<(), Error> { let hchild = Handle::from_path(child_path).map_err(|err| { - Error::from(err) - .with_path(child_path) - .with_depth(child_depth) + Error::from(err).with_path(child_path).with_depth(child_depth) })?; - for ig in ig_parent - .parents() - .take_while(|ig| !ig.is_absolute_parent()) - { + for ig in ig_parent.parents().take_while(|ig| !ig.is_absolute_parent()) { let h = Handle::from_path(ig.path()).map_err(|err| { - Error::from(err) - .with_path(child_path) - .with_depth(child_depth) + Error::from(err).with_path(child_path).with_depth(child_depth) })?; if hchild == h { return Err(Error::Loop { @@ -1905,7 +2077,11 @@ fn check_symlink_loop( // Before calling this function, make sure that you ensure that is really // necessary as the arguments imply a file stat. -fn skip_filesize(max_filesize: u64, path: &Path, ent: &Option) -> bool { +fn skip_filesize( + max_filesize: u64, + path: &Path, + ent: &Option, +) -> bool { let filesize = match *ent { Some(ref md) => Some(md.len()), None => None, @@ -1977,9 +2153,9 @@ fn path_equals(dent: &DirEntry, handle: &Handle) -> Result { if dent.is_stdin() || never_equal(dent, handle) { return Ok(false); } - Handle::from_path(dent.path()) - .map(|h| &h == handle) - .map_err(|err| Error::Io(err).with_path(dent.path())) + Handle::from_path(dent.path()).map(|h| &h == handle).map_err(|err| { + Error::Io(err).with_depth(dent.depth()).with_path(dent.path()) + }) } /// Returns true if the given walkdir entry corresponds to a directory. @@ -1997,16 +2173,14 @@ fn walkdir_is_dir(dent: &walkdir::DirEntry) -> bool { if !dent.file_type().is_symlink() || dent.depth() > 0 { return false; } - dent.path() - .metadata() - .ok() - .map_or(false, |md| md.file_type().is_dir()) + dent.path().metadata().ok().map_or(false, |md| md.file_type().is_dir()) } /// Returns true if and only if the given path is on the same device as the /// given root device. fn is_same_file_system(root_device: u64, path: &Path) -> Result { - let dent_device = device_num(path).map_err(|err| Error::Io(err).with_path(path))?; + let dent_device = + device_num(path).map_err(|err| Error::Io(err).with_path(path))?; Ok(root_device == dent_device) } @@ -2065,11 +2239,7 @@ mod tests { } fn normal_path(unix: &str) -> String { - if cfg!(windows) { - unix.replace("\\", "/") - } else { - unix.to_string() - } + if cfg!(windows) { unix.replace("\\", "/") } else { unix.to_string() } } fn walk_collect(prefix: &Path, builder: &WalkBuilder) -> Vec { @@ -2089,7 +2259,10 @@ mod tests { paths } - fn walk_collect_parallel(prefix: &Path, builder: &WalkBuilder) -> Vec { + fn walk_collect_parallel( + prefix: &Path, + builder: &WalkBuilder, + ) -> Vec { let mut paths = vec![]; for dent in walk_collect_entries_parallel(builder) { let path = dent.path().strip_prefix(prefix).unwrap(); @@ -2278,6 +2451,27 @@ mod tests { ); } + #[test] + fn max_depth_does_not_load_unreachable_ignore_files() { + let td = tmpdir(); + let leaf = td.path().join("leaf"); + mkdirp(&leaf); + wfile(leaf.join(".ignore"), "{invalid\n"); + + let mut builder = WalkBuilder::new(td.path()); + builder.max_depth(Some(1)); + let entry = builder + .build() + .find_map(|result| { + let entry = result.unwrap(); + (entry.path() == leaf).then_some(entry) + }) + .unwrap(); + + assert!(entry.error().is_none()); + assert_paths(td.path(), &builder, &["leaf"]); + } + #[test] fn min_depth() { let td = tmpdir(); @@ -2388,7 +2582,9 @@ mod tests { assert_eq!(1, dents.len()); assert!(!dents[0].path_is_symlink()); - let dents = walk_collect_entries_parallel(&WalkBuilder::new(td.path().join("foo"))); + let dents = walk_collect_entries_parallel(&WalkBuilder::new( + td.path().join("foo"), + )); assert_eq!(1, dents.len()); assert!(!dents[0].path_is_symlink()); } @@ -2474,8 +2670,88 @@ mod tests { assert_paths( td.path(), - &WalkBuilder::new(td.path()).filter_entry(|entry| entry.file_name() != OsStr::new("a")), + &WalkBuilder::new(td.path()) + .filter_entry(|entry| entry.file_name() != OsStr::new("a")), &["x", "x/y", "x/y/foo"], ); } + + #[test] + fn empty() { + let td = tmpdir(); + assert_paths(td.path(), &WalkBuilder::empty(), &[]); + + let empty_paths: Vec<&OsStr> = Vec::new(); + assert_paths(td.path(), &WalkBuilder::from_iter(empty_paths), &[]); + } + + #[test] + fn from_iter() { + let td = tmpdir(); + mkdirp(td.path().join("a/b/c")); + mkdirp(td.path().join("d/e/f")); + mkdirp(td.path().join("x/y")); + wfile(td.path().join("a/b/foo"), ""); + wfile(td.path().join("d/e/f/foo"), ""); + wfile(td.path().join("x/y/foo"), ""); + + let paths = vec![ + td.path().join("a"), + td.path().join("d"), + td.path().join("x"), + ]; + + assert_paths( + td.path(), + &WalkBuilder::from_iter(paths), + &[ + "x", + "x/y", + "x/y/foo", + "d", + "d/e", + "d/e/f", + "d/e/f/foo", + "a", + "a/b", + "a/b/foo", + "a/b/c", + ], + ); + } + + // This should always panic and never hang. + // + // Ref: https://github.com/BurntSushi/ripgrep/issues/3009 + #[test] + #[should_panic] + fn panic_in_parallel() { + let td = tmpdir(); + wfile(td.path().join("foo.txt"), ""); + + WalkBuilder::new(td.path()) + .threads(40) + .build_parallel() + .run(|| Box::new(|_| panic!("oops!"))); + } + + // This should always panic and never hang. The first call to the visitor + // builder is used while processing the root paths. Previously, a panic on + // the third call occurred after the first worker had already been spawned, + // leaving it waiting indefinitely for workers that were never created. + #[test] + #[should_panic(expected = "builder panic")] + fn panic_in_parallel_builder() { + let td = tmpdir(); + wfile(td.path().join("foo.txt"), ""); + + let mut builds = 0; + WalkBuilder::new(td.path()).threads(2).build_parallel().run(|| { + builds += 1; + if builds == 3 { + panic!("builder panic"); + } + Box::new(|_| WalkState::Continue) + }); + } } diff --git a/crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs b/crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs index b7b7c6f95087..ecb7b47e3586 100644 --- a/crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs +++ b/crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs @@ -2,7 +2,8 @@ use std::path::Path; use ignore::gitignore::{Gitignore, GitignoreBuilder}; -const IGNORE_FILE: &'static str = "tests/gitignore_matched_path_or_any_parents_tests.gitignore"; +const IGNORE_FILE: &'static str = + "tests/gitignore_matched_path_or_any_parents_tests.gitignore"; fn get_gitignore() -> Gitignore { let mut builder = GitignoreBuilder::new("ROOT"); @@ -23,7 +24,9 @@ fn test_path_should_be_under_root() { #[test] fn test_files_in_root() { let gitignore = get_gitignore(); - let m = |path: &str| gitignore.matched_path_or_any_parents(Path::new(path), false); + let m = |path: &str| { + gitignore.matched_path_or_any_parents(Path::new(path), false) + }; // 0x assert!(m("ROOT/file_root_00").is_ignore()); @@ -53,7 +56,9 @@ fn test_files_in_root() { #[test] fn test_files_in_deep() { let gitignore = get_gitignore(); - let m = |path: &str| gitignore.matched_path_or_any_parents(Path::new(path), false); + let m = |path: &str| { + gitignore.matched_path_or_any_parents(Path::new(path), false) + }; // 0x assert!(m("ROOT/parent_dir/file_deep_00").is_ignore()); @@ -83,8 +88,9 @@ fn test_files_in_deep() { #[test] fn test_dirs_in_root() { let gitignore = get_gitignore(); - let m = - |path: &str, is_dir: bool| gitignore.matched_path_or_any_parents(Path::new(path), is_dir); + let m = |path: &str, is_dir: bool| { + gitignore.matched_path_or_any_parents(Path::new(path), is_dir) + }; // 00 assert!(m("ROOT/dir_root_00", true).is_ignore()); @@ -186,20 +192,25 @@ fn test_dirs_in_root() { #[test] fn test_dirs_in_deep() { let gitignore = get_gitignore(); - let m = - |path: &str, is_dir: bool| gitignore.matched_path_or_any_parents(Path::new(path), is_dir); + let m = |path: &str, is_dir: bool| { + gitignore.matched_path_or_any_parents(Path::new(path), is_dir) + }; // 00 assert!(m("ROOT/parent_dir/dir_deep_00", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_00/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_00/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore() + ); // 01 assert!(m("ROOT/parent_dir/dir_deep_01", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_01/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_01/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore() + ); // 02 assert!(m("ROOT/parent_dir/dir_deep_02", true).is_none()); @@ -241,51 +252,67 @@ fn test_dirs_in_deep() { assert!(m("ROOT/parent_dir/dir_deep_20", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_20/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_20/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore() + ); // 21 assert!(m("ROOT/parent_dir/dir_deep_21", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_21/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_21/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore() + ); // 22 // dir itself doesn't match assert!(m("ROOT/parent_dir/dir_deep_22", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_22/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_22/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore() + ); // 23 // dir itself doesn't match assert!(m("ROOT/parent_dir/dir_deep_23", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_23/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_23/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore() + ); // 30 assert!(m("ROOT/parent_dir/dir_deep_30", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_30/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_30/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore() + ); // 31 assert!(m("ROOT/parent_dir/dir_deep_31", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_31/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_31/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore() + ); // 32 // dir itself doesn't match assert!(m("ROOT/parent_dir/dir_deep_32", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_32/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_32/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore() + ); // 33 // dir itself doesn't match assert!(m("ROOT/parent_dir/dir_deep_33", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_33/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_33/child_dir", true).is_ignore()); - assert!(m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore()); + assert!( + m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore() + ); }