Skip to content

Commit 14e5415

Browse files
committed
Skip work on the error-free parse path
Return early from `prioritize_tokenizer_error` when there are no errors instead of re-lexing the source, and track seen string prefixes in a bitmask instead of allocating a `Vec` for every identifier. Assisted-by: Claude Code:claude-opus-5-5
1 parent c3c644c commit 14e5415

2 files changed

Lines changed: 11 additions & 5 deletions

File tree

‎crates/ruff_python_parser/src/lexer.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -780,24 +780,26 @@ impl<'src> Lexer<'src> {
780780
('f', 't'),
781781
];
782782

783-
let mut seen = Vec::with_capacity(PREFIXES.len());
783+
let bit = |prefix: char| PREFIXES.iter().position(|&c| c == prefix).map(|i| 1u8 << i);
784+
let mut seen = 0u8;
784785
let mut len = TextSize::new(0);
785786
let mut chars = std::iter::once(first).chain(self.cursor.rest().chars());
786787
loop {
787788
let c = chars.next()?;
788789
if is_quote(c) {
789790
break;
790791
}
791-
let prefix = c.to_ascii_lowercase();
792-
if !PREFIXES.contains(&prefix) || seen.contains(&prefix) {
792+
let prefix = bit(c.to_ascii_lowercase())?;
793+
if seen & prefix != 0 {
793794
return None;
794795
}
795-
seen.push(prefix);
796+
seen |= prefix;
796797
len += c.text_len();
797798
}
799+
let has = |prefix: char| bit(prefix).is_some_and(|prefix| seen & prefix != 0);
798800
let (first, second) = INCOMPATIBLE
799801
.into_iter()
800-
.find(|(first, second)| seen.contains(first) && seen.contains(second))?;
802+
.find(|(first, second)| has(*first) && has(*second))?;
801803
Some(LexicalError::new(
802804
LexicalErrorType::IncompatibleStringPrefixes { first, second },
803805
TextRange::at(self.token_range().start(), len),

‎crates/ruff_python_parser/src/parser/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1990,6 +1990,10 @@ fn prioritize_tokenizer_error(
19901990
start_offset: TextSize,
19911991
unclosed_bracket_recovery: Option<TextSize>,
19921992
) {
1993+
// The lexer reports every tokenizer error, so a source without errors has none to find.
1994+
if errors.is_empty() {
1995+
return;
1996+
}
19931997
let is_tokenizer_error = |error: &ParseError| matches!(&error.error, ParseErrorType::Lexical(lexical) if lexical.is_tokenizer_error());
19941998
let first = errors
19951999
.iter()

0 commit comments

Comments
 (0)