[gcc/devel/rust/master] lex: Add source code token string representation

[gcc/devel/rust/master] lex: Add source code token string representation

[Thomas Schwinge]

https://gcc.gnu.org/g:c3a62ea7d7849efdee24808b5ebd4902af16d57e

commit c3a62ea7d7849efdee24808b5ebd4902af16d57e
Author: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>
Date:   Mon Mar 27 12:54:53 2023 +0200

    lex: Add source code token string representation
    
    Add a new representation for tokens which should reflect the string
    token as it could be found in the original source.
    
    gcc/rust/ChangeLog:
    
            * lex/rust-token.cc (Token::as_string): Add as_string
            implementation.
            * lex/rust-token.h: Add as_string prototype.
    
    Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

Diff:
---
 gcc/rust/lex/rust-token.cc | 13 +++++++++++++
 gcc/rust/lex/rust-token.h  |  4 ++++
 2 files changed, 17 insertions(+)

diff --git a/gcc/rust/lex/rust-token.cc b/gcc/rust/lex/rust-token.cc
index 5a7dad6c5f0..a6be8473fe6 100644
--- a/gcc/rust/lex/rust-token.cc
+++ b/gcc/rust/lex/rust-token.cc
@@ -131,4 +131,17 @@ Token::get_str () const
     }
   return *str;
 }
+
+std::string
+Token::as_string () const
+{
+  if (should_have_str ())
+    {
+      return get_str ();
+    }
+  else
+    {
+      return get_token_description ();
+    }
+}
 } // namespace Rust
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
index f00d9cf07dd..39c21cd15c4 100644
--- a/gcc/rust/lex/rust-token.h
+++ b/gcc/rust/lex/rust-token.h
@@ -443,6 +443,10 @@ return *str;
 
   // Returns whether the token is a pure decimal int literal
   bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
+
+  // Return the token representation as someone would find it in the original
+  // source code file.
+  std::string as_string () const;
 };
 } // namespace Rust