rustc_lexer/lib.rs
1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use unicode_properties::UnicodeEmoji;
34pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
35
36use self::LiteralKind::*;
37use self::TokenKind::*;
38use crate::cursor::EOF_CHAR;
39pub use crate::cursor::{Cursor, FrontmatterAllowed};
40
41/// Parsed token.
42/// It doesn't contain information about data that has been parsed,
43/// only the type of the token and its size.
44#[derive(Debug)]
45pub struct Token {
46 pub kind: TokenKind,
47 pub len: u32,
48}
49
50impl Token {
51 fn new(kind: TokenKind, len: u32) -> Token {
52 Token { kind, len }
53 }
54}
55
56/// Enum representing common lexeme types.
57#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58pub enum TokenKind {
59 /// A line comment, e.g. `// comment`.
60 LineComment {
61 doc_style: Option<DocStyle>,
62 },
63
64 /// A block comment, e.g. `/* block comment */`.
65 ///
66 /// Block comments can be recursive, so a sequence like `/* /* */`
67 /// will not be considered terminated and will result in a parsing error.
68 BlockComment {
69 doc_style: Option<DocStyle>,
70 terminated: bool,
71 },
72
73 /// Any whitespace character sequence.
74 Whitespace,
75
76 Frontmatter {
77 has_invalid_preceding_whitespace: bool,
78 invalid_infostring: bool,
79 },
80
81 /// An identifier or keyword, e.g. `ident` or `continue`.
82 Ident,
83
84 /// An identifier that is invalid because it contains emoji.
85 InvalidIdent,
86
87 /// A raw identifier, e.g. "r#ident".
88 RawIdent,
89
90 /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
91 /// literal prefixes that contain emoji, which are considered "invalid".
92 ///
93 /// Note that only the
94 /// prefix (`foo`) is included in the token, not the separator (which is
95 /// lexed as its own distinct token). In Rust 2021 and later, reserved
96 /// prefixes are reported as errors; in earlier editions, they result in a
97 /// (allowed by default) lint, and are treated as regular identifier
98 /// tokens.
99 UnknownPrefix,
100
101 /// An unknown prefix in a lifetime, like `'foo#`.
102 ///
103 /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
104 /// and not the separator.
105 UnknownPrefixLifetime,
106
107 /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
108 /// several tokens: `'r` and `#` and `foo`.
109 RawLifetime,
110
111 /// Guarded string literal prefix: `#"` or `##`.
112 ///
113 /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
114 /// Split into the component tokens on older editions.
115 GuardedStrPrefix,
116
117 /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
118 /// suffix, but may be present here on string and float literals. Users of
119 /// this type will need to check for and reject that case.
120 ///
121 /// See [LiteralKind] for more details.
122 Literal {
123 kind: LiteralKind,
124 suffix_start: u32,
125 },
126
127 /// A lifetime, e.g. `'a`.
128 Lifetime {
129 starts_with_number: bool,
130 },
131
132 /// `;`
133 Semi,
134 /// `,`
135 Comma,
136 /// `.`
137 Dot,
138 /// `(`
139 OpenParen,
140 /// `)`
141 CloseParen,
142 /// `{`
143 OpenBrace,
144 /// `}`
145 CloseBrace,
146 /// `[`
147 OpenBracket,
148 /// `]`
149 CloseBracket,
150 /// `@`
151 At,
152 /// `#`
153 Pound,
154 /// `~`
155 Tilde,
156 /// `?`
157 Question,
158 /// `:`
159 Colon,
160 /// `$`
161 Dollar,
162 /// `=`
163 Eq,
164 /// `!`
165 Bang,
166 /// `<`
167 Lt,
168 /// `>`
169 Gt,
170 /// `-`
171 Minus,
172 /// `&`
173 And,
174 /// `|`
175 Or,
176 /// `+`
177 Plus,
178 /// `*`
179 Star,
180 /// `/`
181 Slash,
182 /// `^`
183 Caret,
184 /// `%`
185 Percent,
186
187 /// Unknown token, not expected by the lexer, e.g. "№"
188 Unknown,
189
190 /// End of input.
191 Eof,
192}
193
194#[derive(Clone, Copy, Debug, PartialEq, Eq)]
195pub enum DocStyle {
196 Outer,
197 Inner,
198}
199
200/// Enum representing the literal types supported by the lexer.
201///
202/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
203/// this type. This means that float literals like `1f32` are classified by this
204/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
205/// `rustc_ast::ast::LitKind`).
206#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
207pub enum LiteralKind {
208 /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
209 Int { base: Base, empty_int: bool },
210 /// `12.34f32`, `1e3`, but not `1f32`.
211 Float { base: Base, empty_exponent: bool },
212 /// `'a'`, `'\\'`, `'''`, `';`
213 Char { terminated: bool },
214 /// `b'a'`, `b'\\'`, `b'''`, `b';`
215 Byte { terminated: bool },
216 /// `"abc"`, `"abc`
217 Str { terminated: bool },
218 /// `b"abc"`, `b"abc`
219 ByteStr { terminated: bool },
220 /// `c"abc"`, `c"abc`
221 CStr { terminated: bool },
222 /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
223 /// an invalid literal.
224 RawStr { n_hashes: Option<u8> },
225 /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
226 /// indicates an invalid literal.
227 RawByteStr { n_hashes: Option<u8> },
228 /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
229 RawCStr { n_hashes: Option<u8> },
230}
231
232/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
233///
234/// Can capture fewer closing hashes than starting hashes,
235/// for more efficient lexing and better backwards diagnostics.
236#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
237pub struct GuardedStr {
238 pub n_hashes: u32,
239 pub terminated: bool,
240 pub token_len: u32,
241}
242
243#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
244pub enum RawStrError {
245 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
246 InvalidStarter { bad_char: char },
247 /// The string was not terminated, e.g. `r###"abcde"##`.
248 /// `possible_terminator_offset` is the number of characters after `r` or
249 /// `br` where they may have intended to terminate it.
250 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
251 /// More than 255 `#`s exist.
252 TooManyDelimiters { found: u32 },
253}
254
255/// Base of numeric literal encoding according to its prefix.
256#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
257pub enum Base {
258 /// Literal starts with "0b".
259 Binary = 2,
260 /// Literal starts with "0o".
261 Octal = 8,
262 /// Literal doesn't contain a prefix.
263 Decimal = 10,
264 /// Literal starts with "0x".
265 Hexadecimal = 16,
266}
267
268/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
269/// but shebang isn't a part of rust syntax.
270pub fn strip_shebang(input: &str) -> Option<usize> {
271 // Shebang must start with `#!` literally, without any preceding whitespace.
272 // For simplicity we consider any line starting with `#!` a shebang,
273 // regardless of restrictions put on shebangs by specific platforms.
274 if let Some(input_tail) = input.strip_prefix("#!") {
275 // Ok, this is a shebang but if the next non-whitespace token is `[`,
276 // then it may be valid Rust code, so consider it Rust code.
277 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
278 !matches!(
279 tok,
280 TokenKind::Whitespace
281 | TokenKind::LineComment { doc_style: None }
282 | TokenKind::BlockComment { doc_style: None, .. }
283 )
284 });
285 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
286 // No other choice than to consider this a shebang.
287 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
288 }
289 }
290 None
291}
292
293/// Validates a raw string literal. Used for getting more information about a
294/// problem with a `RawStr`/`RawByteStr` with a `None` field.
295#[inline]
296pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
297 debug_assert!(!input.is_empty());
298 let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
299 // Move past the leading `r` or `br`.
300 for _ in 0..prefix_len {
301 cursor.bump().unwrap();
302 }
303 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
304}
305
306/// Creates an iterator that produces tokens from the input string.
307pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
308 let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
309 std::iter::from_fn(move || {
310 let token = cursor.advance_token();
311 if token.kind != TokenKind::Eof { Some(token) } else { None }
312 })
313}
314
315/// True if `c` is considered a whitespace according to Rust language definition.
316/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
317/// for definitions of these classes.
318pub fn is_whitespace(c: char) -> bool {
319 // This is Pattern_White_Space.
320 //
321 // Note that this set is stable (ie, it doesn't change with different
322 // Unicode versions), so it's ok to just hard-code the values.
323
324 matches!(
325 c,
326 // Usual ASCII suspects
327 '\u{0009}' // \t
328 | '\u{000A}' // \n
329 | '\u{000B}' // vertical tab
330 | '\u{000C}' // form feed
331 | '\u{000D}' // \r
332 | '\u{0020}' // space
333
334 // NEXT LINE from latin1
335 | '\u{0085}'
336
337 // Bidi markers
338 | '\u{200E}' // LEFT-TO-RIGHT MARK
339 | '\u{200F}' // RIGHT-TO-LEFT MARK
340
341 // Dedicated whitespace characters from Unicode
342 | '\u{2028}' // LINE SEPARATOR
343 | '\u{2029}' // PARAGRAPH SEPARATOR
344 )
345}
346
347/// True if `c` is valid as a first character of an identifier.
348/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
349/// a formal definition of valid identifier name.
350pub fn is_id_start(c: char) -> bool {
351 // This is XID_Start OR '_' (which formally is not a XID_Start).
352 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
353}
354
355/// True if `c` is valid as a non-first character of an identifier.
356/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
357/// a formal definition of valid identifier name.
358pub fn is_id_continue(c: char) -> bool {
359 unicode_xid::UnicodeXID::is_xid_continue(c)
360}
361
362/// The passed string is lexically an identifier.
363pub fn is_ident(string: &str) -> bool {
364 let mut chars = string.chars();
365 if let Some(start) = chars.next() {
366 is_id_start(start) && chars.all(is_id_continue)
367 } else {
368 false
369 }
370}
371
372impl Cursor<'_> {
373 /// Parses a token from the input string.
374 pub fn advance_token(&mut self) -> Token {
375 let first_char = match self.bump() {
376 Some(c) => c,
377 None => return Token::new(TokenKind::Eof, 0),
378 };
379
380 let token_kind = match first_char {
381 c if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
382 && is_whitespace(c) =>
383 {
384 let mut last = first_char;
385 while is_whitespace(self.first()) {
386 let Some(c) = self.bump() else {
387 break;
388 };
389 last = c;
390 }
391 // invalid frontmatter opening as whitespace preceding it isn't newline.
392 // combine the whitespace and the frontmatter to a single token as we shall
393 // error later.
394 if last != '\n' && self.as_str().starts_with("---") {
395 self.bump();
396 self.frontmatter(true)
397 } else {
398 Whitespace
399 }
400 }
401 '-' if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
402 && self.as_str().starts_with("--") =>
403 {
404 // happy path
405 self.frontmatter(false)
406 }
407 // Slash, comment or block comment.
408 '/' => match self.first() {
409 '/' => self.line_comment(),
410 '*' => self.block_comment(),
411 _ => Slash,
412 },
413
414 // Whitespace sequence.
415 c if is_whitespace(c) => self.whitespace(),
416
417 // Raw identifier, raw string literal or identifier.
418 'r' => match (self.first(), self.second()) {
419 ('#', c1) if is_id_start(c1) => self.raw_ident(),
420 ('#', _) | ('"', _) => {
421 let res = self.raw_double_quoted_string(1);
422 let suffix_start = self.pos_within_token();
423 if res.is_ok() {
424 self.eat_literal_suffix();
425 }
426 let kind = RawStr { n_hashes: res.ok() };
427 Literal { kind, suffix_start }
428 }
429 _ => self.ident_or_unknown_prefix(),
430 },
431
432 // Byte literal, byte string literal, raw byte string literal or identifier.
433 'b' => self.c_or_byte_string(
434 |terminated| ByteStr { terminated },
435 |n_hashes| RawByteStr { n_hashes },
436 Some(|terminated| Byte { terminated }),
437 ),
438
439 // c-string literal, raw c-string literal or identifier.
440 'c' => self.c_or_byte_string(
441 |terminated| CStr { terminated },
442 |n_hashes| RawCStr { n_hashes },
443 None,
444 ),
445
446 // Identifier (this should be checked after other variant that can
447 // start as identifier).
448 c if is_id_start(c) => self.ident_or_unknown_prefix(),
449
450 // Numeric literal.
451 c @ '0'..='9' => {
452 let literal_kind = self.number(c);
453 let suffix_start = self.pos_within_token();
454 self.eat_literal_suffix();
455 TokenKind::Literal { kind: literal_kind, suffix_start }
456 }
457
458 // Guarded string literal prefix: `#"` or `##`
459 '#' if matches!(self.first(), '"' | '#') => {
460 self.bump();
461 TokenKind::GuardedStrPrefix
462 }
463
464 // One-symbol tokens.
465 ';' => Semi,
466 ',' => Comma,
467 '.' => Dot,
468 '(' => OpenParen,
469 ')' => CloseParen,
470 '{' => OpenBrace,
471 '}' => CloseBrace,
472 '[' => OpenBracket,
473 ']' => CloseBracket,
474 '@' => At,
475 '#' => Pound,
476 '~' => Tilde,
477 '?' => Question,
478 ':' => Colon,
479 '$' => Dollar,
480 '=' => Eq,
481 '!' => Bang,
482 '<' => Lt,
483 '>' => Gt,
484 '-' => Minus,
485 '&' => And,
486 '|' => Or,
487 '+' => Plus,
488 '*' => Star,
489 '^' => Caret,
490 '%' => Percent,
491
492 // Lifetime or character literal.
493 '\'' => self.lifetime_or_char(),
494
495 // String literal.
496 '"' => {
497 let terminated = self.double_quoted_string();
498 let suffix_start = self.pos_within_token();
499 if terminated {
500 self.eat_literal_suffix();
501 }
502 let kind = Str { terminated };
503 Literal { kind, suffix_start }
504 }
505 // Identifier starting with an emoji. Only lexed for graceful error recovery.
506 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
507 _ => Unknown,
508 };
509 if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
510 && !matches!(token_kind, Whitespace)
511 {
512 // stop allowing frontmatters after first non-whitespace token
513 self.frontmatter_allowed = FrontmatterAllowed::No;
514 }
515 let res = Token::new(token_kind, self.pos_within_token());
516 self.reset_pos_within_token();
517 res
518 }
519
520 /// Given that one `-` was eaten, eat the rest of the frontmatter.
521 fn frontmatter(&mut self, has_invalid_preceding_whitespace: bool) -> TokenKind {
522 debug_assert_eq!('-', self.prev());
523
524 let pos = self.pos_within_token();
525 self.eat_while(|c| c == '-');
526
527 // one `-` is eaten by the caller.
528 let length_opening = self.pos_within_token() - pos + 1;
529
530 // must be ensured by the caller
531 debug_assert!(length_opening >= 3);
532
533 // whitespace between the opening and the infostring.
534 self.eat_while(|ch| ch != '\n' && is_whitespace(ch));
535
536 // copied from `eat_identifier`, but allows `.` in infostring to allow something like
537 // `---Cargo.toml` as a valid opener
538 if is_id_start(self.first()) {
539 self.bump();
540 self.eat_while(|c| is_id_continue(c) || c == '.');
541 }
542
543 self.eat_while(|ch| ch != '\n' && is_whitespace(ch));
544 let invalid_infostring = self.first() != '\n';
545
546 let mut s = self.as_str();
547 let mut found = false;
548 let mut size = 0;
549 while let Some(closing) = s.find(&"-".repeat(length_opening as usize)) {
550 let preceding_chars_start = s[..closing].rfind("\n").map_or(0, |i| i + 1);
551 if s[preceding_chars_start..closing].chars().all(is_whitespace) {
552 // candidate found
553 self.bump_bytes(size + closing);
554 // in case like
555 // ---cargo
556 // --- blahblah
557 // or
558 // ---cargo
559 // ----
560 // combine those stuff into this frontmatter token such that it gets detected later.
561 self.eat_until(b'\n');
562 found = true;
563 break;
564 } else {
565 s = &s[closing + length_opening as usize..];
566 size += closing + length_opening as usize;
567 }
568 }
569
570 if !found {
571 // recovery strategy: a closing statement might have precending whitespace/newline
572 // but not have enough dashes to properly close. In this case, we eat until there,
573 // and report a mismatch in the parser.
574 let mut rest = self.as_str();
575 // We can look for a shorter closing (starting with four dashes but closing with three)
576 // and other indications that Rust has started and the infostring has ended.
577 let mut potential_closing = rest
578 .find("\n---")
579 // n.b. only in the case where there are dashes, we move the index to the line where
580 // the dashes start as we eat to include that line. For other cases those are Rust code
581 // and not included in the frontmatter.
582 .map(|x| x + 1)
583 .or_else(|| rest.find("\nuse "))
584 .or_else(|| rest.find("\n//!"))
585 .or_else(|| rest.find("\n#!["));
586
587 if potential_closing.is_none() {
588 // a less fortunate recovery if all else fails which finds any dashes preceded by whitespace
589 // on a standalone line. Might be wrong.
590 while let Some(closing) = rest.find("---") {
591 let preceding_chars_start = rest[..closing].rfind("\n").map_or(0, |i| i + 1);
592 if rest[preceding_chars_start..closing].chars().all(is_whitespace) {
593 // candidate found
594 potential_closing = Some(closing);
595 break;
596 } else {
597 rest = &rest[closing + 3..];
598 }
599 }
600 }
601
602 if let Some(potential_closing) = potential_closing {
603 // bump to the potential closing, and eat everything on that line.
604 self.bump_bytes(potential_closing);
605 self.eat_until(b'\n');
606 } else {
607 // eat everything. this will get reported as an unclosed frontmatter.
608 self.eat_while(|_| true);
609 }
610 }
611
612 Frontmatter { has_invalid_preceding_whitespace, invalid_infostring }
613 }
614
615 fn line_comment(&mut self) -> TokenKind {
616 debug_assert!(self.prev() == '/' && self.first() == '/');
617 self.bump();
618
619 let doc_style = match self.first() {
620 // `//!` is an inner line doc comment.
621 '!' => Some(DocStyle::Inner),
622 // `////` (more than 3 slashes) is not considered a doc comment.
623 '/' if self.second() != '/' => Some(DocStyle::Outer),
624 _ => None,
625 };
626
627 self.eat_until(b'\n');
628 LineComment { doc_style }
629 }
630
631 fn block_comment(&mut self) -> TokenKind {
632 debug_assert!(self.prev() == '/' && self.first() == '*');
633 self.bump();
634
635 let doc_style = match self.first() {
636 // `/*!` is an inner block doc comment.
637 '!' => Some(DocStyle::Inner),
638 // `/***` (more than 2 stars) is not considered a doc comment.
639 // `/**/` is not considered a doc comment.
640 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
641 _ => None,
642 };
643
644 let mut depth = 1usize;
645 while let Some(c) = self.bump() {
646 match c {
647 '/' if self.first() == '*' => {
648 self.bump();
649 depth += 1;
650 }
651 '*' if self.first() == '/' => {
652 self.bump();
653 depth -= 1;
654 if depth == 0 {
655 // This block comment is closed, so for a construction like "/* */ */"
656 // there will be a successfully parsed block comment "/* */"
657 // and " */" will be processed separately.
658 break;
659 }
660 }
661 _ => (),
662 }
663 }
664
665 BlockComment { doc_style, terminated: depth == 0 }
666 }
667
668 fn whitespace(&mut self) -> TokenKind {
669 debug_assert!(is_whitespace(self.prev()));
670 self.eat_while(is_whitespace);
671 Whitespace
672 }
673
674 fn raw_ident(&mut self) -> TokenKind {
675 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
676 // Eat "#" symbol.
677 self.bump();
678 // Eat the identifier part of RawIdent.
679 self.eat_identifier();
680 RawIdent
681 }
682
683 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
684 debug_assert!(is_id_start(self.prev()));
685 // Start is already eaten, eat the rest of identifier.
686 self.eat_while(is_id_continue);
687 // Known prefixes must have been handled earlier. So if
688 // we see a prefix here, it is definitely an unknown prefix.
689 match self.first() {
690 '#' | '"' | '\'' => UnknownPrefix,
691 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
692 _ => Ident,
693 }
694 }
695
696 fn invalid_ident(&mut self) -> TokenKind {
697 // Start is already eaten, eat the rest of identifier.
698 self.eat_while(|c| {
699 const ZERO_WIDTH_JOINER: char = '\u{200d}';
700 is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
701 });
702 // An invalid identifier followed by '#' or '"' or '\'' could be
703 // interpreted as an invalid literal prefix. We don't bother doing that
704 // because the treatment of invalid identifiers and invalid prefixes
705 // would be the same.
706 InvalidIdent
707 }
708
709 fn c_or_byte_string(
710 &mut self,
711 mk_kind: fn(bool) -> LiteralKind,
712 mk_kind_raw: fn(Option<u8>) -> LiteralKind,
713 single_quoted: Option<fn(bool) -> LiteralKind>,
714 ) -> TokenKind {
715 match (self.first(), self.second(), single_quoted) {
716 ('\'', _, Some(single_quoted)) => {
717 self.bump();
718 let terminated = self.single_quoted_string();
719 let suffix_start = self.pos_within_token();
720 if terminated {
721 self.eat_literal_suffix();
722 }
723 let kind = single_quoted(terminated);
724 Literal { kind, suffix_start }
725 }
726 ('"', _, _) => {
727 self.bump();
728 let terminated = self.double_quoted_string();
729 let suffix_start = self.pos_within_token();
730 if terminated {
731 self.eat_literal_suffix();
732 }
733 let kind = mk_kind(terminated);
734 Literal { kind, suffix_start }
735 }
736 ('r', '"', _) | ('r', '#', _) => {
737 self.bump();
738 let res = self.raw_double_quoted_string(2);
739 let suffix_start = self.pos_within_token();
740 if res.is_ok() {
741 self.eat_literal_suffix();
742 }
743 let kind = mk_kind_raw(res.ok());
744 Literal { kind, suffix_start }
745 }
746 _ => self.ident_or_unknown_prefix(),
747 }
748 }
749
750 fn number(&mut self, first_digit: char) -> LiteralKind {
751 debug_assert!('0' <= self.prev() && self.prev() <= '9');
752 let mut base = Base::Decimal;
753 if first_digit == '0' {
754 // Attempt to parse encoding base.
755 match self.first() {
756 'b' => {
757 base = Base::Binary;
758 self.bump();
759 if !self.eat_decimal_digits() {
760 return Int { base, empty_int: true };
761 }
762 }
763 'o' => {
764 base = Base::Octal;
765 self.bump();
766 if !self.eat_decimal_digits() {
767 return Int { base, empty_int: true };
768 }
769 }
770 'x' => {
771 base = Base::Hexadecimal;
772 self.bump();
773 if !self.eat_hexadecimal_digits() {
774 return Int { base, empty_int: true };
775 }
776 }
777 // Not a base prefix; consume additional digits.
778 '0'..='9' | '_' => {
779 self.eat_decimal_digits();
780 }
781
782 // Also not a base prefix; nothing more to do here.
783 '.' | 'e' | 'E' => {}
784
785 // Just a 0.
786 _ => return Int { base, empty_int: false },
787 }
788 } else {
789 // No base prefix, parse number in the usual way.
790 self.eat_decimal_digits();
791 };
792
793 match self.first() {
794 // Don't be greedy if this is actually an
795 // integer literal followed by field/method access or a range pattern
796 // (`0..2` and `12.foo()`)
797 '.' if self.second() != '.' && !is_id_start(self.second()) => {
798 // might have stuff after the ., and if it does, it needs to start
799 // with a number
800 self.bump();
801 let mut empty_exponent = false;
802 if self.first().is_ascii_digit() {
803 self.eat_decimal_digits();
804 match self.first() {
805 'e' | 'E' => {
806 self.bump();
807 empty_exponent = !self.eat_float_exponent();
808 }
809 _ => (),
810 }
811 }
812 Float { base, empty_exponent }
813 }
814 'e' | 'E' => {
815 self.bump();
816 let empty_exponent = !self.eat_float_exponent();
817 Float { base, empty_exponent }
818 }
819 _ => Int { base, empty_int: false },
820 }
821 }
822
823 fn lifetime_or_char(&mut self) -> TokenKind {
824 debug_assert!(self.prev() == '\'');
825
826 let can_be_a_lifetime = if self.second() == '\'' {
827 // It's surely not a lifetime.
828 false
829 } else {
830 // If the first symbol is valid for identifier, it can be a lifetime.
831 // Also check if it's a number for a better error reporting (so '0 will
832 // be reported as invalid lifetime and not as unterminated char literal).
833 is_id_start(self.first()) || self.first().is_ascii_digit()
834 };
835
836 if !can_be_a_lifetime {
837 let terminated = self.single_quoted_string();
838 let suffix_start = self.pos_within_token();
839 if terminated {
840 self.eat_literal_suffix();
841 }
842 let kind = Char { terminated };
843 return Literal { kind, suffix_start };
844 }
845
846 if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
847 // Eat "r" and `#`, and identifier start characters.
848 self.bump();
849 self.bump();
850 self.bump();
851 self.eat_while(is_id_continue);
852 return RawLifetime;
853 }
854
855 // Either a lifetime or a character literal with
856 // length greater than 1.
857 let starts_with_number = self.first().is_ascii_digit();
858
859 // Skip the literal contents.
860 // First symbol can be a number (which isn't a valid identifier start),
861 // so skip it without any checks.
862 self.bump();
863 self.eat_while(is_id_continue);
864
865 match self.first() {
866 // Check if after skipping literal contents we've met a closing
867 // single quote (which means that user attempted to create a
868 // string with single quotes).
869 '\'' => {
870 self.bump();
871 let kind = Char { terminated: true };
872 Literal { kind, suffix_start: self.pos_within_token() }
873 }
874 '#' if !starts_with_number => UnknownPrefixLifetime,
875 _ => Lifetime { starts_with_number },
876 }
877 }
878
879 fn single_quoted_string(&mut self) -> bool {
880 debug_assert!(self.prev() == '\'');
881 // Check if it's a one-symbol literal.
882 if self.second() == '\'' && self.first() != '\\' {
883 self.bump();
884 self.bump();
885 return true;
886 }
887
888 // Literal has more than one symbol.
889
890 // Parse until either quotes are terminated or error is detected.
891 loop {
892 match self.first() {
893 // Quotes are terminated, finish parsing.
894 '\'' => {
895 self.bump();
896 return true;
897 }
898 // Probably beginning of the comment, which we don't want to include
899 // to the error report.
900 '/' => break,
901 // Newline without following '\'' means unclosed quote, stop parsing.
902 '\n' if self.second() != '\'' => break,
903 // End of file, stop parsing.
904 EOF_CHAR if self.is_eof() => break,
905 // Escaped slash is considered one character, so bump twice.
906 '\\' => {
907 self.bump();
908 self.bump();
909 }
910 // Skip the character.
911 _ => {
912 self.bump();
913 }
914 }
915 }
916 // String was not terminated.
917 false
918 }
919
920 /// Eats double-quoted string and returns true
921 /// if string is terminated.
922 fn double_quoted_string(&mut self) -> bool {
923 debug_assert!(self.prev() == '"');
924 while let Some(c) = self.bump() {
925 match c {
926 '"' => {
927 return true;
928 }
929 '\\' if self.first() == '\\' || self.first() == '"' => {
930 // Bump again to skip escaped character.
931 self.bump();
932 }
933 _ => (),
934 }
935 }
936 // End of file reached.
937 false
938 }
939
940 /// Attempt to lex for a guarded string literal.
941 ///
942 /// Used by `rustc_parse::lexer` to lex for guarded strings
943 /// conditionally based on edition.
944 ///
945 /// Note: this will not reset the `Cursor` when a
946 /// guarded string is not found. It is the caller's
947 /// responsibility to do so.
948 pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
949 debug_assert!(self.prev() != '#');
950
951 let mut n_start_hashes: u32 = 0;
952 while self.first() == '#' {
953 n_start_hashes += 1;
954 self.bump();
955 }
956
957 if self.first() != '"' {
958 return None;
959 }
960 self.bump();
961 debug_assert!(self.prev() == '"');
962
963 // Lex the string itself as a normal string literal
964 // so we can recover that for older editions later.
965 let terminated = self.double_quoted_string();
966 if !terminated {
967 let token_len = self.pos_within_token();
968 self.reset_pos_within_token();
969
970 return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
971 }
972
973 // Consume closing '#' symbols.
974 // Note that this will not consume extra trailing `#` characters:
975 // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
976 // followed by a `#` token.
977 let mut n_end_hashes = 0;
978 while self.first() == '#' && n_end_hashes < n_start_hashes {
979 n_end_hashes += 1;
980 self.bump();
981 }
982
983 // Reserved syntax, always an error, so it doesn't matter if
984 // `n_start_hashes != n_end_hashes`.
985
986 self.eat_literal_suffix();
987
988 let token_len = self.pos_within_token();
989 self.reset_pos_within_token();
990
991 Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
992 }
993
994 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
995 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
996 // Wrap the actual function to handle the error with too many hashes.
997 // This way, it eats the whole raw string.
998 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
999 // Only up to 255 `#`s are allowed in raw strings
1000 match u8::try_from(n_hashes) {
1001 Ok(num) => Ok(num),
1002 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
1003 }
1004 }
1005
1006 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
1007 debug_assert!(self.prev() == 'r');
1008 let start_pos = self.pos_within_token();
1009 let mut possible_terminator_offset = None;
1010 let mut max_hashes = 0;
1011
1012 // Count opening '#' symbols.
1013 let mut eaten = 0;
1014 while self.first() == '#' {
1015 eaten += 1;
1016 self.bump();
1017 }
1018 let n_start_hashes = eaten;
1019
1020 // Check that string is started.
1021 match self.bump() {
1022 Some('"') => (),
1023 c => {
1024 let c = c.unwrap_or(EOF_CHAR);
1025 return Err(RawStrError::InvalidStarter { bad_char: c });
1026 }
1027 }
1028
1029 // Skip the string contents and on each '#' character met, check if this is
1030 // a raw string termination.
1031 loop {
1032 self.eat_until(b'"');
1033
1034 if self.is_eof() {
1035 return Err(RawStrError::NoTerminator {
1036 expected: n_start_hashes,
1037 found: max_hashes,
1038 possible_terminator_offset,
1039 });
1040 }
1041
1042 // Eat closing double quote.
1043 self.bump();
1044
1045 // Check that amount of closing '#' symbols
1046 // is equal to the amount of opening ones.
1047 // Note that this will not consume extra trailing `#` characters:
1048 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
1049 // followed by a `#` token.
1050 let mut n_end_hashes = 0;
1051 while self.first() == '#' && n_end_hashes < n_start_hashes {
1052 n_end_hashes += 1;
1053 self.bump();
1054 }
1055
1056 if n_end_hashes == n_start_hashes {
1057 return Ok(n_start_hashes);
1058 } else if n_end_hashes > max_hashes {
1059 // Keep track of possible terminators to give a hint about
1060 // where there might be a missing terminator
1061 possible_terminator_offset =
1062 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
1063 max_hashes = n_end_hashes;
1064 }
1065 }
1066 }
1067
1068 fn eat_decimal_digits(&mut self) -> bool {
1069 let mut has_digits = false;
1070 loop {
1071 match self.first() {
1072 '_' => {
1073 self.bump();
1074 }
1075 '0'..='9' => {
1076 has_digits = true;
1077 self.bump();
1078 }
1079 _ => break,
1080 }
1081 }
1082 has_digits
1083 }
1084
1085 fn eat_hexadecimal_digits(&mut self) -> bool {
1086 let mut has_digits = false;
1087 loop {
1088 match self.first() {
1089 '_' => {
1090 self.bump();
1091 }
1092 '0'..='9' | 'a'..='f' | 'A'..='F' => {
1093 has_digits = true;
1094 self.bump();
1095 }
1096 _ => break,
1097 }
1098 }
1099 has_digits
1100 }
1101
1102 /// Eats the float exponent. Returns true if at least one digit was met,
1103 /// and returns false otherwise.
1104 fn eat_float_exponent(&mut self) -> bool {
1105 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
1106 if self.first() == '-' || self.first() == '+' {
1107 self.bump();
1108 }
1109 self.eat_decimal_digits()
1110 }
1111
1112 // Eats the suffix of the literal, e.g. "u8".
1113 fn eat_literal_suffix(&mut self) {
1114 self.eat_identifier();
1115 }
1116
1117 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
1118 // identifier.
1119 fn eat_identifier(&mut self) {
1120 if !is_id_start(self.first()) {
1121 return;
1122 }
1123 self.bump();
1124
1125 self.eat_while(is_id_continue);
1126 }
1127}