rustc_lexer/
lib.rs

1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use unicode_properties::UnicodeEmoji;
34pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
35
36use self::LiteralKind::*;
37use self::TokenKind::*;
38use crate::cursor::EOF_CHAR;
39pub use crate::cursor::{Cursor, FrontmatterAllowed};
40
41/// Parsed token.
42/// It doesn't contain information about data that has been parsed,
43/// only the type of the token and its size.
44#[derive(Debug)]
45pub struct Token {
46    pub kind: TokenKind,
47    pub len: u32,
48}
49
50impl Token {
51    fn new(kind: TokenKind, len: u32) -> Token {
52        Token { kind, len }
53    }
54}
55
56/// Enum representing common lexeme types.
57#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58pub enum TokenKind {
59    /// A line comment, e.g. `// comment`.
60    LineComment {
61        doc_style: Option<DocStyle>,
62    },
63
64    /// A block comment, e.g. `/* block comment */`.
65    ///
66    /// Block comments can be recursive, so a sequence like `/* /* */`
67    /// will not be considered terminated and will result in a parsing error.
68    BlockComment {
69        doc_style: Option<DocStyle>,
70        terminated: bool,
71    },
72
73    /// Any whitespace character sequence.
74    Whitespace,
75
76    Frontmatter {
77        has_invalid_preceding_whitespace: bool,
78        invalid_infostring: bool,
79    },
80
81    /// An identifier or keyword, e.g. `ident` or `continue`.
82    Ident,
83
84    /// An identifier that is invalid because it contains emoji.
85    InvalidIdent,
86
87    /// A raw identifier, e.g. "r#ident".
88    RawIdent,
89
90    /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
91    /// literal prefixes that contain emoji, which are considered "invalid".
92    ///
93    /// Note that only the
94    /// prefix (`foo`) is included in the token, not the separator (which is
95    /// lexed as its own distinct token). In Rust 2021 and later, reserved
96    /// prefixes are reported as errors; in earlier editions, they result in a
97    /// (allowed by default) lint, and are treated as regular identifier
98    /// tokens.
99    UnknownPrefix,
100
101    /// An unknown prefix in a lifetime, like `'foo#`.
102    ///
103    /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
104    /// and not the separator.
105    UnknownPrefixLifetime,
106
107    /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
108    /// several tokens: `'r` and `#` and `foo`.
109    RawLifetime,
110
111    /// Guarded string literal prefix: `#"` or `##`.
112    ///
113    /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
114    /// Split into the component tokens on older editions.
115    GuardedStrPrefix,
116
117    /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
118    /// suffix, but may be present here on string and float literals. Users of
119    /// this type will need to check for and reject that case.
120    ///
121    /// See [LiteralKind] for more details.
122    Literal {
123        kind: LiteralKind,
124        suffix_start: u32,
125    },
126
127    /// A lifetime, e.g. `'a`.
128    Lifetime {
129        starts_with_number: bool,
130    },
131
132    /// `;`
133    Semi,
134    /// `,`
135    Comma,
136    /// `.`
137    Dot,
138    /// `(`
139    OpenParen,
140    /// `)`
141    CloseParen,
142    /// `{`
143    OpenBrace,
144    /// `}`
145    CloseBrace,
146    /// `[`
147    OpenBracket,
148    /// `]`
149    CloseBracket,
150    /// `@`
151    At,
152    /// `#`
153    Pound,
154    /// `~`
155    Tilde,
156    /// `?`
157    Question,
158    /// `:`
159    Colon,
160    /// `$`
161    Dollar,
162    /// `=`
163    Eq,
164    /// `!`
165    Bang,
166    /// `<`
167    Lt,
168    /// `>`
169    Gt,
170    /// `-`
171    Minus,
172    /// `&`
173    And,
174    /// `|`
175    Or,
176    /// `+`
177    Plus,
178    /// `*`
179    Star,
180    /// `/`
181    Slash,
182    /// `^`
183    Caret,
184    /// `%`
185    Percent,
186
187    /// Unknown token, not expected by the lexer, e.g. "№"
188    Unknown,
189
190    /// End of input.
191    Eof,
192}
193
194#[derive(Clone, Copy, Debug, PartialEq, Eq)]
195pub enum DocStyle {
196    Outer,
197    Inner,
198}
199
200/// Enum representing the literal types supported by the lexer.
201///
202/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
203/// this type. This means that float literals like `1f32` are classified by this
204/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
205/// `rustc_ast::ast::LitKind`).
206#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
207pub enum LiteralKind {
208    /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
209    Int { base: Base, empty_int: bool },
210    /// `12.34f32`, `1e3`, but not `1f32`.
211    Float { base: Base, empty_exponent: bool },
212    /// `'a'`, `'\\'`, `'''`, `';`
213    Char { terminated: bool },
214    /// `b'a'`, `b'\\'`, `b'''`, `b';`
215    Byte { terminated: bool },
216    /// `"abc"`, `"abc`
217    Str { terminated: bool },
218    /// `b"abc"`, `b"abc`
219    ByteStr { terminated: bool },
220    /// `c"abc"`, `c"abc`
221    CStr { terminated: bool },
222    /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
223    /// an invalid literal.
224    RawStr { n_hashes: Option<u8> },
225    /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
226    /// indicates an invalid literal.
227    RawByteStr { n_hashes: Option<u8> },
228    /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
229    RawCStr { n_hashes: Option<u8> },
230}
231
232/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
233///
234/// Can capture fewer closing hashes than starting hashes,
235/// for more efficient lexing and better backwards diagnostics.
236#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
237pub struct GuardedStr {
238    pub n_hashes: u32,
239    pub terminated: bool,
240    pub token_len: u32,
241}
242
243#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
244pub enum RawStrError {
245    /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
246    InvalidStarter { bad_char: char },
247    /// The string was not terminated, e.g. `r###"abcde"##`.
248    /// `possible_terminator_offset` is the number of characters after `r` or
249    /// `br` where they may have intended to terminate it.
250    NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
251    /// More than 255 `#`s exist.
252    TooManyDelimiters { found: u32 },
253}
254
255/// Base of numeric literal encoding according to its prefix.
256#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
257pub enum Base {
258    /// Literal starts with "0b".
259    Binary = 2,
260    /// Literal starts with "0o".
261    Octal = 8,
262    /// Literal doesn't contain a prefix.
263    Decimal = 10,
264    /// Literal starts with "0x".
265    Hexadecimal = 16,
266}
267
268/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
269/// but shebang isn't a part of rust syntax.
270pub fn strip_shebang(input: &str) -> Option<usize> {
271    // Shebang must start with `#!` literally, without any preceding whitespace.
272    // For simplicity we consider any line starting with `#!` a shebang,
273    // regardless of restrictions put on shebangs by specific platforms.
274    if let Some(input_tail) = input.strip_prefix("#!") {
275        // Ok, this is a shebang but if the next non-whitespace token is `[`,
276        // then it may be valid Rust code, so consider it Rust code.
277        let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
278            !matches!(
279                tok,
280                TokenKind::Whitespace
281                    | TokenKind::LineComment { doc_style: None }
282                    | TokenKind::BlockComment { doc_style: None, .. }
283            )
284        });
285        if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
286            // No other choice than to consider this a shebang.
287            return Some(2 + input_tail.lines().next().unwrap_or_default().len());
288        }
289    }
290    None
291}
292
293/// Validates a raw string literal. Used for getting more information about a
294/// problem with a `RawStr`/`RawByteStr` with a `None` field.
295#[inline]
296pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
297    debug_assert!(!input.is_empty());
298    let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
299    // Move past the leading `r` or `br`.
300    for _ in 0..prefix_len {
301        cursor.bump().unwrap();
302    }
303    cursor.raw_double_quoted_string(prefix_len).map(|_| ())
304}
305
306/// Creates an iterator that produces tokens from the input string.
307pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
308    let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
309    std::iter::from_fn(move || {
310        let token = cursor.advance_token();
311        if token.kind != TokenKind::Eof { Some(token) } else { None }
312    })
313}
314
315/// True if `c` is considered a whitespace according to Rust language definition.
316/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
317/// for definitions of these classes.
318pub fn is_whitespace(c: char) -> bool {
319    // This is Pattern_White_Space.
320    //
321    // Note that this set is stable (ie, it doesn't change with different
322    // Unicode versions), so it's ok to just hard-code the values.
323
324    matches!(
325        c,
326        // Usual ASCII suspects
327        '\u{0009}'   // \t
328        | '\u{000A}' // \n
329        | '\u{000B}' // vertical tab
330        | '\u{000C}' // form feed
331        | '\u{000D}' // \r
332        | '\u{0020}' // space
333
334        // NEXT LINE from latin1
335        | '\u{0085}'
336
337        // Bidi markers
338        | '\u{200E}' // LEFT-TO-RIGHT MARK
339        | '\u{200F}' // RIGHT-TO-LEFT MARK
340
341        // Dedicated whitespace characters from Unicode
342        | '\u{2028}' // LINE SEPARATOR
343        | '\u{2029}' // PARAGRAPH SEPARATOR
344    )
345}
346
347/// True if `c` is valid as a first character of an identifier.
348/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
349/// a formal definition of valid identifier name.
350pub fn is_id_start(c: char) -> bool {
351    // This is XID_Start OR '_' (which formally is not a XID_Start).
352    c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
353}
354
355/// True if `c` is valid as a non-first character of an identifier.
356/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
357/// a formal definition of valid identifier name.
358pub fn is_id_continue(c: char) -> bool {
359    unicode_xid::UnicodeXID::is_xid_continue(c)
360}
361
362/// The passed string is lexically an identifier.
363pub fn is_ident(string: &str) -> bool {
364    let mut chars = string.chars();
365    if let Some(start) = chars.next() {
366        is_id_start(start) && chars.all(is_id_continue)
367    } else {
368        false
369    }
370}
371
372impl Cursor<'_> {
373    /// Parses a token from the input string.
374    pub fn advance_token(&mut self) -> Token {
375        let first_char = match self.bump() {
376            Some(c) => c,
377            None => return Token::new(TokenKind::Eof, 0),
378        };
379
380        let token_kind = match first_char {
381            c if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
382                && is_whitespace(c) =>
383            {
384                let mut last = first_char;
385                while is_whitespace(self.first()) {
386                    let Some(c) = self.bump() else {
387                        break;
388                    };
389                    last = c;
390                }
391                // invalid frontmatter opening as whitespace preceding it isn't newline.
392                // combine the whitespace and the frontmatter to a single token as we shall
393                // error later.
394                if last != '\n' && self.as_str().starts_with("---") {
395                    self.bump();
396                    self.frontmatter(true)
397                } else {
398                    Whitespace
399                }
400            }
401            '-' if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
402                && self.as_str().starts_with("--") =>
403            {
404                // happy path
405                self.frontmatter(false)
406            }
407            // Slash, comment or block comment.
408            '/' => match self.first() {
409                '/' => self.line_comment(),
410                '*' => self.block_comment(),
411                _ => Slash,
412            },
413
414            // Whitespace sequence.
415            c if is_whitespace(c) => self.whitespace(),
416
417            // Raw identifier, raw string literal or identifier.
418            'r' => match (self.first(), self.second()) {
419                ('#', c1) if is_id_start(c1) => self.raw_ident(),
420                ('#', _) | ('"', _) => {
421                    let res = self.raw_double_quoted_string(1);
422                    let suffix_start = self.pos_within_token();
423                    if res.is_ok() {
424                        self.eat_literal_suffix();
425                    }
426                    let kind = RawStr { n_hashes: res.ok() };
427                    Literal { kind, suffix_start }
428                }
429                _ => self.ident_or_unknown_prefix(),
430            },
431
432            // Byte literal, byte string literal, raw byte string literal or identifier.
433            'b' => self.c_or_byte_string(
434                |terminated| ByteStr { terminated },
435                |n_hashes| RawByteStr { n_hashes },
436                Some(|terminated| Byte { terminated }),
437            ),
438
439            // c-string literal, raw c-string literal or identifier.
440            'c' => self.c_or_byte_string(
441                |terminated| CStr { terminated },
442                |n_hashes| RawCStr { n_hashes },
443                None,
444            ),
445
446            // Identifier (this should be checked after other variant that can
447            // start as identifier).
448            c if is_id_start(c) => self.ident_or_unknown_prefix(),
449
450            // Numeric literal.
451            c @ '0'..='9' => {
452                let literal_kind = self.number(c);
453                let suffix_start = self.pos_within_token();
454                self.eat_literal_suffix();
455                TokenKind::Literal { kind: literal_kind, suffix_start }
456            }
457
458            // Guarded string literal prefix: `#"` or `##`
459            '#' if matches!(self.first(), '"' | '#') => {
460                self.bump();
461                TokenKind::GuardedStrPrefix
462            }
463
464            // One-symbol tokens.
465            ';' => Semi,
466            ',' => Comma,
467            '.' => Dot,
468            '(' => OpenParen,
469            ')' => CloseParen,
470            '{' => OpenBrace,
471            '}' => CloseBrace,
472            '[' => OpenBracket,
473            ']' => CloseBracket,
474            '@' => At,
475            '#' => Pound,
476            '~' => Tilde,
477            '?' => Question,
478            ':' => Colon,
479            '$' => Dollar,
480            '=' => Eq,
481            '!' => Bang,
482            '<' => Lt,
483            '>' => Gt,
484            '-' => Minus,
485            '&' => And,
486            '|' => Or,
487            '+' => Plus,
488            '*' => Star,
489            '^' => Caret,
490            '%' => Percent,
491
492            // Lifetime or character literal.
493            '\'' => self.lifetime_or_char(),
494
495            // String literal.
496            '"' => {
497                let terminated = self.double_quoted_string();
498                let suffix_start = self.pos_within_token();
499                if terminated {
500                    self.eat_literal_suffix();
501                }
502                let kind = Str { terminated };
503                Literal { kind, suffix_start }
504            }
505            // Identifier starting with an emoji. Only lexed for graceful error recovery.
506            c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
507            _ => Unknown,
508        };
509        if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
510            && !matches!(token_kind, Whitespace)
511        {
512            // stop allowing frontmatters after first non-whitespace token
513            self.frontmatter_allowed = FrontmatterAllowed::No;
514        }
515        let res = Token::new(token_kind, self.pos_within_token());
516        self.reset_pos_within_token();
517        res
518    }
519
520    /// Given that one `-` was eaten, eat the rest of the frontmatter.
521    fn frontmatter(&mut self, has_invalid_preceding_whitespace: bool) -> TokenKind {
522        debug_assert_eq!('-', self.prev());
523
524        let pos = self.pos_within_token();
525        self.eat_while(|c| c == '-');
526
527        // one `-` is eaten by the caller.
528        let length_opening = self.pos_within_token() - pos + 1;
529
530        // must be ensured by the caller
531        debug_assert!(length_opening >= 3);
532
533        // whitespace between the opening and the infostring.
534        self.eat_while(|ch| ch != '\n' && is_whitespace(ch));
535
536        // copied from `eat_identifier`, but allows `.` in infostring to allow something like
537        // `---Cargo.toml` as a valid opener
538        if is_id_start(self.first()) {
539            self.bump();
540            self.eat_while(|c| is_id_continue(c) || c == '.');
541        }
542
543        self.eat_while(|ch| ch != '\n' && is_whitespace(ch));
544        let invalid_infostring = self.first() != '\n';
545
546        let mut s = self.as_str();
547        let mut found = false;
548        let mut size = 0;
549        while let Some(closing) = s.find(&"-".repeat(length_opening as usize)) {
550            let preceding_chars_start = s[..closing].rfind("\n").map_or(0, |i| i + 1);
551            if s[preceding_chars_start..closing].chars().all(is_whitespace) {
552                // candidate found
553                self.bump_bytes(size + closing);
554                // in case like
555                // ---cargo
556                // --- blahblah
557                // or
558                // ---cargo
559                // ----
560                // combine those stuff into this frontmatter token such that it gets detected later.
561                self.eat_until(b'\n');
562                found = true;
563                break;
564            } else {
565                s = &s[closing + length_opening as usize..];
566                size += closing + length_opening as usize;
567            }
568        }
569
570        if !found {
571            // recovery strategy: a closing statement might have precending whitespace/newline
572            // but not have enough dashes to properly close. In this case, we eat until there,
573            // and report a mismatch in the parser.
574            let mut rest = self.as_str();
575            // We can look for a shorter closing (starting with four dashes but closing with three)
576            // and other indications that Rust has started and the infostring has ended.
577            let mut potential_closing = rest
578                .find("\n---")
579                // n.b. only in the case where there are dashes, we move the index to the line where
580                // the dashes start as we eat to include that line. For other cases those are Rust code
581                // and not included in the frontmatter.
582                .map(|x| x + 1)
583                .or_else(|| rest.find("\nuse "))
584                .or_else(|| rest.find("\n//!"))
585                .or_else(|| rest.find("\n#!["));
586
587            if potential_closing.is_none() {
588                // a less fortunate recovery if all else fails which finds any dashes preceded by whitespace
589                // on a standalone line. Might be wrong.
590                while let Some(closing) = rest.find("---") {
591                    let preceding_chars_start = rest[..closing].rfind("\n").map_or(0, |i| i + 1);
592                    if rest[preceding_chars_start..closing].chars().all(is_whitespace) {
593                        // candidate found
594                        potential_closing = Some(closing);
595                        break;
596                    } else {
597                        rest = &rest[closing + 3..];
598                    }
599                }
600            }
601
602            if let Some(potential_closing) = potential_closing {
603                // bump to the potential closing, and eat everything on that line.
604                self.bump_bytes(potential_closing);
605                self.eat_until(b'\n');
606            } else {
607                // eat everything. this will get reported as an unclosed frontmatter.
608                self.eat_while(|_| true);
609            }
610        }
611
612        Frontmatter { has_invalid_preceding_whitespace, invalid_infostring }
613    }
614
615    fn line_comment(&mut self) -> TokenKind {
616        debug_assert!(self.prev() == '/' && self.first() == '/');
617        self.bump();
618
619        let doc_style = match self.first() {
620            // `//!` is an inner line doc comment.
621            '!' => Some(DocStyle::Inner),
622            // `////` (more than 3 slashes) is not considered a doc comment.
623            '/' if self.second() != '/' => Some(DocStyle::Outer),
624            _ => None,
625        };
626
627        self.eat_until(b'\n');
628        LineComment { doc_style }
629    }
630
631    fn block_comment(&mut self) -> TokenKind {
632        debug_assert!(self.prev() == '/' && self.first() == '*');
633        self.bump();
634
635        let doc_style = match self.first() {
636            // `/*!` is an inner block doc comment.
637            '!' => Some(DocStyle::Inner),
638            // `/***` (more than 2 stars) is not considered a doc comment.
639            // `/**/` is not considered a doc comment.
640            '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
641            _ => None,
642        };
643
644        let mut depth = 1usize;
645        while let Some(c) = self.bump() {
646            match c {
647                '/' if self.first() == '*' => {
648                    self.bump();
649                    depth += 1;
650                }
651                '*' if self.first() == '/' => {
652                    self.bump();
653                    depth -= 1;
654                    if depth == 0 {
655                        // This block comment is closed, so for a construction like "/* */ */"
656                        // there will be a successfully parsed block comment "/* */"
657                        // and " */" will be processed separately.
658                        break;
659                    }
660                }
661                _ => (),
662            }
663        }
664
665        BlockComment { doc_style, terminated: depth == 0 }
666    }
667
668    fn whitespace(&mut self) -> TokenKind {
669        debug_assert!(is_whitespace(self.prev()));
670        self.eat_while(is_whitespace);
671        Whitespace
672    }
673
674    fn raw_ident(&mut self) -> TokenKind {
675        debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
676        // Eat "#" symbol.
677        self.bump();
678        // Eat the identifier part of RawIdent.
679        self.eat_identifier();
680        RawIdent
681    }
682
683    fn ident_or_unknown_prefix(&mut self) -> TokenKind {
684        debug_assert!(is_id_start(self.prev()));
685        // Start is already eaten, eat the rest of identifier.
686        self.eat_while(is_id_continue);
687        // Known prefixes must have been handled earlier. So if
688        // we see a prefix here, it is definitely an unknown prefix.
689        match self.first() {
690            '#' | '"' | '\'' => UnknownPrefix,
691            c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
692            _ => Ident,
693        }
694    }
695
696    fn invalid_ident(&mut self) -> TokenKind {
697        // Start is already eaten, eat the rest of identifier.
698        self.eat_while(|c| {
699            const ZERO_WIDTH_JOINER: char = '\u{200d}';
700            is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
701        });
702        // An invalid identifier followed by '#' or '"' or '\'' could be
703        // interpreted as an invalid literal prefix. We don't bother doing that
704        // because the treatment of invalid identifiers and invalid prefixes
705        // would be the same.
706        InvalidIdent
707    }
708
709    fn c_or_byte_string(
710        &mut self,
711        mk_kind: fn(bool) -> LiteralKind,
712        mk_kind_raw: fn(Option<u8>) -> LiteralKind,
713        single_quoted: Option<fn(bool) -> LiteralKind>,
714    ) -> TokenKind {
715        match (self.first(), self.second(), single_quoted) {
716            ('\'', _, Some(single_quoted)) => {
717                self.bump();
718                let terminated = self.single_quoted_string();
719                let suffix_start = self.pos_within_token();
720                if terminated {
721                    self.eat_literal_suffix();
722                }
723                let kind = single_quoted(terminated);
724                Literal { kind, suffix_start }
725            }
726            ('"', _, _) => {
727                self.bump();
728                let terminated = self.double_quoted_string();
729                let suffix_start = self.pos_within_token();
730                if terminated {
731                    self.eat_literal_suffix();
732                }
733                let kind = mk_kind(terminated);
734                Literal { kind, suffix_start }
735            }
736            ('r', '"', _) | ('r', '#', _) => {
737                self.bump();
738                let res = self.raw_double_quoted_string(2);
739                let suffix_start = self.pos_within_token();
740                if res.is_ok() {
741                    self.eat_literal_suffix();
742                }
743                let kind = mk_kind_raw(res.ok());
744                Literal { kind, suffix_start }
745            }
746            _ => self.ident_or_unknown_prefix(),
747        }
748    }
749
750    fn number(&mut self, first_digit: char) -> LiteralKind {
751        debug_assert!('0' <= self.prev() && self.prev() <= '9');
752        let mut base = Base::Decimal;
753        if first_digit == '0' {
754            // Attempt to parse encoding base.
755            match self.first() {
756                'b' => {
757                    base = Base::Binary;
758                    self.bump();
759                    if !self.eat_decimal_digits() {
760                        return Int { base, empty_int: true };
761                    }
762                }
763                'o' => {
764                    base = Base::Octal;
765                    self.bump();
766                    if !self.eat_decimal_digits() {
767                        return Int { base, empty_int: true };
768                    }
769                }
770                'x' => {
771                    base = Base::Hexadecimal;
772                    self.bump();
773                    if !self.eat_hexadecimal_digits() {
774                        return Int { base, empty_int: true };
775                    }
776                }
777                // Not a base prefix; consume additional digits.
778                '0'..='9' | '_' => {
779                    self.eat_decimal_digits();
780                }
781
782                // Also not a base prefix; nothing more to do here.
783                '.' | 'e' | 'E' => {}
784
785                // Just a 0.
786                _ => return Int { base, empty_int: false },
787            }
788        } else {
789            // No base prefix, parse number in the usual way.
790            self.eat_decimal_digits();
791        };
792
793        match self.first() {
794            // Don't be greedy if this is actually an
795            // integer literal followed by field/method access or a range pattern
796            // (`0..2` and `12.foo()`)
797            '.' if self.second() != '.' && !is_id_start(self.second()) => {
798                // might have stuff after the ., and if it does, it needs to start
799                // with a number
800                self.bump();
801                let mut empty_exponent = false;
802                if self.first().is_ascii_digit() {
803                    self.eat_decimal_digits();
804                    match self.first() {
805                        'e' | 'E' => {
806                            self.bump();
807                            empty_exponent = !self.eat_float_exponent();
808                        }
809                        _ => (),
810                    }
811                }
812                Float { base, empty_exponent }
813            }
814            'e' | 'E' => {
815                self.bump();
816                let empty_exponent = !self.eat_float_exponent();
817                Float { base, empty_exponent }
818            }
819            _ => Int { base, empty_int: false },
820        }
821    }
822
823    fn lifetime_or_char(&mut self) -> TokenKind {
824        debug_assert!(self.prev() == '\'');
825
826        let can_be_a_lifetime = if self.second() == '\'' {
827            // It's surely not a lifetime.
828            false
829        } else {
830            // If the first symbol is valid for identifier, it can be a lifetime.
831            // Also check if it's a number for a better error reporting (so '0 will
832            // be reported as invalid lifetime and not as unterminated char literal).
833            is_id_start(self.first()) || self.first().is_ascii_digit()
834        };
835
836        if !can_be_a_lifetime {
837            let terminated = self.single_quoted_string();
838            let suffix_start = self.pos_within_token();
839            if terminated {
840                self.eat_literal_suffix();
841            }
842            let kind = Char { terminated };
843            return Literal { kind, suffix_start };
844        }
845
846        if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
847            // Eat "r" and `#`, and identifier start characters.
848            self.bump();
849            self.bump();
850            self.bump();
851            self.eat_while(is_id_continue);
852            return RawLifetime;
853        }
854
855        // Either a lifetime or a character literal with
856        // length greater than 1.
857        let starts_with_number = self.first().is_ascii_digit();
858
859        // Skip the literal contents.
860        // First symbol can be a number (which isn't a valid identifier start),
861        // so skip it without any checks.
862        self.bump();
863        self.eat_while(is_id_continue);
864
865        match self.first() {
866            // Check if after skipping literal contents we've met a closing
867            // single quote (which means that user attempted to create a
868            // string with single quotes).
869            '\'' => {
870                self.bump();
871                let kind = Char { terminated: true };
872                Literal { kind, suffix_start: self.pos_within_token() }
873            }
874            '#' if !starts_with_number => UnknownPrefixLifetime,
875            _ => Lifetime { starts_with_number },
876        }
877    }
878
879    fn single_quoted_string(&mut self) -> bool {
880        debug_assert!(self.prev() == '\'');
881        // Check if it's a one-symbol literal.
882        if self.second() == '\'' && self.first() != '\\' {
883            self.bump();
884            self.bump();
885            return true;
886        }
887
888        // Literal has more than one symbol.
889
890        // Parse until either quotes are terminated or error is detected.
891        loop {
892            match self.first() {
893                // Quotes are terminated, finish parsing.
894                '\'' => {
895                    self.bump();
896                    return true;
897                }
898                // Probably beginning of the comment, which we don't want to include
899                // to the error report.
900                '/' => break,
901                // Newline without following '\'' means unclosed quote, stop parsing.
902                '\n' if self.second() != '\'' => break,
903                // End of file, stop parsing.
904                EOF_CHAR if self.is_eof() => break,
905                // Escaped slash is considered one character, so bump twice.
906                '\\' => {
907                    self.bump();
908                    self.bump();
909                }
910                // Skip the character.
911                _ => {
912                    self.bump();
913                }
914            }
915        }
916        // String was not terminated.
917        false
918    }
919
920    /// Eats double-quoted string and returns true
921    /// if string is terminated.
922    fn double_quoted_string(&mut self) -> bool {
923        debug_assert!(self.prev() == '"');
924        while let Some(c) = self.bump() {
925            match c {
926                '"' => {
927                    return true;
928                }
929                '\\' if self.first() == '\\' || self.first() == '"' => {
930                    // Bump again to skip escaped character.
931                    self.bump();
932                }
933                _ => (),
934            }
935        }
936        // End of file reached.
937        false
938    }
939
940    /// Attempt to lex for a guarded string literal.
941    ///
942    /// Used by `rustc_parse::lexer` to lex for guarded strings
943    /// conditionally based on edition.
944    ///
945    /// Note: this will not reset the `Cursor` when a
946    /// guarded string is not found. It is the caller's
947    /// responsibility to do so.
948    pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
949        debug_assert!(self.prev() != '#');
950
951        let mut n_start_hashes: u32 = 0;
952        while self.first() == '#' {
953            n_start_hashes += 1;
954            self.bump();
955        }
956
957        if self.first() != '"' {
958            return None;
959        }
960        self.bump();
961        debug_assert!(self.prev() == '"');
962
963        // Lex the string itself as a normal string literal
964        // so we can recover that for older editions later.
965        let terminated = self.double_quoted_string();
966        if !terminated {
967            let token_len = self.pos_within_token();
968            self.reset_pos_within_token();
969
970            return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
971        }
972
973        // Consume closing '#' symbols.
974        // Note that this will not consume extra trailing `#` characters:
975        // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
976        // followed by a `#` token.
977        let mut n_end_hashes = 0;
978        while self.first() == '#' && n_end_hashes < n_start_hashes {
979            n_end_hashes += 1;
980            self.bump();
981        }
982
983        // Reserved syntax, always an error, so it doesn't matter if
984        // `n_start_hashes != n_end_hashes`.
985
986        self.eat_literal_suffix();
987
988        let token_len = self.pos_within_token();
989        self.reset_pos_within_token();
990
991        Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
992    }
993
994    /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
995    fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
996        // Wrap the actual function to handle the error with too many hashes.
997        // This way, it eats the whole raw string.
998        let n_hashes = self.raw_string_unvalidated(prefix_len)?;
999        // Only up to 255 `#`s are allowed in raw strings
1000        match u8::try_from(n_hashes) {
1001            Ok(num) => Ok(num),
1002            Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
1003        }
1004    }
1005
1006    fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
1007        debug_assert!(self.prev() == 'r');
1008        let start_pos = self.pos_within_token();
1009        let mut possible_terminator_offset = None;
1010        let mut max_hashes = 0;
1011
1012        // Count opening '#' symbols.
1013        let mut eaten = 0;
1014        while self.first() == '#' {
1015            eaten += 1;
1016            self.bump();
1017        }
1018        let n_start_hashes = eaten;
1019
1020        // Check that string is started.
1021        match self.bump() {
1022            Some('"') => (),
1023            c => {
1024                let c = c.unwrap_or(EOF_CHAR);
1025                return Err(RawStrError::InvalidStarter { bad_char: c });
1026            }
1027        }
1028
1029        // Skip the string contents and on each '#' character met, check if this is
1030        // a raw string termination.
1031        loop {
1032            self.eat_until(b'"');
1033
1034            if self.is_eof() {
1035                return Err(RawStrError::NoTerminator {
1036                    expected: n_start_hashes,
1037                    found: max_hashes,
1038                    possible_terminator_offset,
1039                });
1040            }
1041
1042            // Eat closing double quote.
1043            self.bump();
1044
1045            // Check that amount of closing '#' symbols
1046            // is equal to the amount of opening ones.
1047            // Note that this will not consume extra trailing `#` characters:
1048            // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
1049            // followed by a `#` token.
1050            let mut n_end_hashes = 0;
1051            while self.first() == '#' && n_end_hashes < n_start_hashes {
1052                n_end_hashes += 1;
1053                self.bump();
1054            }
1055
1056            if n_end_hashes == n_start_hashes {
1057                return Ok(n_start_hashes);
1058            } else if n_end_hashes > max_hashes {
1059                // Keep track of possible terminators to give a hint about
1060                // where there might be a missing terminator
1061                possible_terminator_offset =
1062                    Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
1063                max_hashes = n_end_hashes;
1064            }
1065        }
1066    }
1067
1068    fn eat_decimal_digits(&mut self) -> bool {
1069        let mut has_digits = false;
1070        loop {
1071            match self.first() {
1072                '_' => {
1073                    self.bump();
1074                }
1075                '0'..='9' => {
1076                    has_digits = true;
1077                    self.bump();
1078                }
1079                _ => break,
1080            }
1081        }
1082        has_digits
1083    }
1084
1085    fn eat_hexadecimal_digits(&mut self) -> bool {
1086        let mut has_digits = false;
1087        loop {
1088            match self.first() {
1089                '_' => {
1090                    self.bump();
1091                }
1092                '0'..='9' | 'a'..='f' | 'A'..='F' => {
1093                    has_digits = true;
1094                    self.bump();
1095                }
1096                _ => break,
1097            }
1098        }
1099        has_digits
1100    }
1101
1102    /// Eats the float exponent. Returns true if at least one digit was met,
1103    /// and returns false otherwise.
1104    fn eat_float_exponent(&mut self) -> bool {
1105        debug_assert!(self.prev() == 'e' || self.prev() == 'E');
1106        if self.first() == '-' || self.first() == '+' {
1107            self.bump();
1108        }
1109        self.eat_decimal_digits()
1110    }
1111
1112    // Eats the suffix of the literal, e.g. "u8".
1113    fn eat_literal_suffix(&mut self) {
1114        self.eat_identifier();
1115    }
1116
1117    // Eats the identifier. Note: succeeds on `_`, which isn't a valid
1118    // identifier.
1119    fn eat_identifier(&mut self) {
1120        if !is_id_start(self.first()) {
1121            return;
1122        }
1123        self.bump();
1124
1125        self.eat_while(is_id_continue);
1126    }
1127}