public inbox for gcc-rust@gcc.gnu.org
 help / color / mirror / Atom feed
From: Arthur Cohen <arthur.cohen@embecosm.com>
To: gcc-patches@gcc.gnu.org
Cc: gcc-rust@gcc.gnu.org, liushuyu <liushuyu011@gmail.com>
Subject: [COMMITTED] gccrs: expand: eager evaluate macros inside builtin macros
Date: Tue, 31 Jan 2023 14:27:03 +0100	[thread overview]
Message-ID: <20230131132703.663677-1-arthur.cohen@embecosm.com> (raw)

From: liushuyu <liushuyu011@gmail.com>

gcc/rust/ChangeLog:

	* ast/rust-ast.h (class MacroInvocData): Store expander as
	member of the class.
	(class Expr): Add `is_literal` virtual method
	* ast/rust-expr.h: Override `is_literal` for `LiteralExpr`s.
	* expand/rust-macro-builtins.cc (try_expand_macro_expression): New function.
	(try_extract_string_literal_from_fragment): Likewise.
	(try_expand_single_string_literal): Likewise.
	(try_expand_many_expr): Likewise.
	(parse_single_string_literal): Add macro expander as argument.
	(MacroBuiltin::include_bytes): Pass expander as argument to
	`parse_single_string_literal`.
	(MacroBuiltin::include_str): Likewise.
	(MacroBuiltin::compile_error): Likewise.
	(MacroBuiltin::include): Likewise.
	(MacroBuiltin::concat): Likewise and add better error handling.
	(MacroBuiltin::env): Likewise.
	* expand/rust-macro-expand.cc (MacroExpander::expand_invoc): Expand
	invocations recursively.

gcc/testsuite/ChangeLog:

	* rust/compile/builtin_macro_concat.rs: Fix test error messages.
	* rust/compile/builtin_macro_env.rs: Likewise.

Signed-off-by: Zixing Liu <liushuyu011@gmail.com>

Tested on x86_64-pc-linux-gnu, committed on master.

---
 gcc/rust/ast/rust-ast.h                       |  12 +
 gcc/rust/ast/rust-expr.h                      |   2 +
 gcc/rust/expand/rust-macro-builtins.cc        | 206 ++++++++++++++----
 gcc/rust/expand/rust-macro-expand.cc          |   1 +
 .../rust/compile/builtin_macro_concat.rs      |   8 +-
 .../rust/compile/builtin_macro_env.rs         |   4 +-
 6 files changed, 182 insertions(+), 51 deletions(-)

diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h
index 9e1b8b11373..ccabccd6aff 100644
--- a/gcc/rust/ast/rust-ast.h
+++ b/gcc/rust/ast/rust-ast.h
@@ -31,6 +31,7 @@ namespace Rust {
 typedef std::string Identifier;
 typedef int TupleIndex;
 struct Session;
+struct MacroExpander;
 
 namespace AST {
 // foward decl: ast visitor
@@ -951,6 +952,8 @@ public:
 
   virtual Location get_locus () const = 0;
 
+  virtual bool is_literal () const { return false; }
+
   // HACK: strictly not needed, but faster than full downcast clone
   virtual bool is_expr_without_block () const = 0;
 
@@ -1471,6 +1474,7 @@ private:
   // One way of parsing the macro. Probably not applicable for all macros.
   std::vector<std::unique_ptr<MetaItemInner> > parsed_items;
   bool parsed_to_meta_item = false;
+  MacroExpander *expander = nullptr;
 
 public:
   std::string as_string () const;
@@ -1495,6 +1499,7 @@ public:
     path = other.path;
     token_tree = other.token_tree;
     parsed_to_meta_item = other.parsed_to_meta_item;
+    expander = other.expander;
 
     parsed_items.reserve (other.parsed_items.size ());
     for (const auto &e : other.parsed_items)
@@ -1523,6 +1528,13 @@ public:
   SimplePath &get_path () { return path; }
   const SimplePath &get_path () const { return path; }
 
+  void set_expander (MacroExpander *new_expander) { expander = new_expander; }
+  MacroExpander *get_expander ()
+  {
+    rust_assert (expander);
+    return expander;
+  }
+
   void
   set_meta_item_output (std::vector<std::unique_ptr<MetaItemInner> > new_items)
   {
diff --git a/gcc/rust/ast/rust-expr.h b/gcc/rust/ast/rust-expr.h
index 1966a590c94..c764f9c4c66 100644
--- a/gcc/rust/ast/rust-expr.h
+++ b/gcc/rust/ast/rust-expr.h
@@ -67,6 +67,8 @@ public:
 
   Location get_locus () const override final { return locus; }
 
+  bool is_literal () const override final { return true; }
+
   Literal get_literal () const { return literal; }
 
   void accept_vis (ASTVisitor &vis) override;
diff --git a/gcc/rust/expand/rust-macro-builtins.cc b/gcc/rust/expand/rust-macro-builtins.cc
index f5e3e188423..606f33c65bc 100644
--- a/gcc/rust/expand/rust-macro-builtins.cc
+++ b/gcc/rust/expand/rust-macro-builtins.cc
@@ -17,12 +17,14 @@
 // <http://www.gnu.org/licenses/>.
 
 #include "rust-macro-builtins.h"
+#include "rust-ast.h"
 #include "rust-diagnostics.h"
 #include "rust-expr.h"
 #include "rust-session-manager.h"
 #include "rust-macro-invoc-lexer.h"
 #include "rust-lex.h"
 #include "rust-parse.h"
+#include "rust-attribute-visitor.h"
 
 namespace Rust {
 namespace {
@@ -61,13 +63,119 @@ macro_end_token (AST::DelimTokenTree &invoc_token_tree,
   return last_token_id;
 }
 
+/* Expand and extract an expression from the macro */
+
+static inline AST::ASTFragment
+try_expand_macro_expression (AST::Expr *expr, MacroExpander *expander)
+{
+  rust_assert (expander);
+
+  auto vis = Rust::AttrVisitor (*expander);
+  expr->accept_vis (vis);
+  return expander->take_expanded_fragment (vis);
+}
+
+/* Expand and then extract a string literal from the macro */
+
+static std::unique_ptr<AST::LiteralExpr>
+try_extract_string_literal_from_fragment (const Location &parent_locus,
+					  std::unique_ptr<AST::Expr> &node)
+{
+  auto maybe_lit = static_cast<AST::LiteralExpr *> (node.get ());
+  if (!node || !node->is_literal ()
+      || maybe_lit->get_lit_type () != AST::Literal::STRING)
+    {
+      rust_error_at (parent_locus, "argument must be a string literal");
+      if (node)
+	rust_inform (node->get_locus (), "expanded from here");
+      return nullptr;
+    }
+  return std::unique_ptr<AST::LiteralExpr> (
+    static_cast<AST::LiteralExpr *> (node->clone_expr ().release ()));
+}
+
+static std::unique_ptr<AST::LiteralExpr>
+try_expand_single_string_literal (AST::Expr *input_expr,
+				  const Location &invoc_locus,
+				  MacroExpander *expander)
+{
+  auto nodes = try_expand_macro_expression (input_expr, expander);
+  if (nodes.is_error () || nodes.is_expression_fragment ())
+    {
+      rust_error_at (input_expr->get_locus (),
+		     "argument must be a string literal");
+      return nullptr;
+    }
+  auto expr = nodes.take_expression_fragment ();
+  return try_extract_string_literal_from_fragment (input_expr->get_locus (),
+						   expr);
+}
+
+static std::vector<std::unique_ptr<AST::Expr>>
+try_expand_many_expr (Parser<MacroInvocLexer> &parser,
+		      const Location &invoc_locus, const TokenId last_token_id,
+		      MacroExpander *expander, bool &has_error)
+{
+  auto restrictions = Rust::ParseRestrictions ();
+  // stop parsing when encountered a braces/brackets
+  restrictions.expr_can_be_null = true;
+  // we can't use std::optional, so...
+  auto result = std::vector<std::unique_ptr<AST::Expr>> ();
+  auto empty_expr = std::vector<std::unique_ptr<AST::Expr>> ();
+
+  auto first_token = parser.peek_current_token ()->get_id ();
+  if (first_token == COMMA)
+    {
+      rust_error_at (parser.peek_current_token ()->get_locus (),
+		     "expected expression, found %<,%>");
+      has_error = true;
+      return empty_expr;
+    }
+
+  while (parser.peek_current_token ()->get_id () != last_token_id
+	 && parser.peek_current_token ()->get_id () != END_OF_FILE)
+    {
+      auto expr = parser.parse_expr (AST::AttrVec (), restrictions);
+      // something must be so wrong that the expression could not be parsed
+      rust_assert (expr);
+      auto nodes = try_expand_macro_expression (expr.get (), expander);
+      if (nodes.is_error ())
+	{
+	  // not macro
+	  result.push_back (std::move (expr));
+	}
+      else if (!nodes.is_expression_fragment ())
+	{
+	  rust_error_at (expr->get_locus (), "expected expression");
+	  has_error = true;
+	  return empty_expr;
+	}
+      else
+	{
+	  result.push_back (nodes.take_expression_fragment ());
+	}
+
+      auto next_token = parser.peek_current_token ();
+      if (!parser.skip_token (COMMA) && next_token->get_id () != last_token_id)
+	{
+	  rust_error_at (next_token->get_locus (), "expected token: %<,%>");
+	  // TODO: is this recoverable? to avoid crashing the parser in the next
+	  // fragment we have to exit early here
+	  has_error = true;
+	  return empty_expr;
+	}
+    }
+
+  return result;
+}
+
 /* Parse a single string literal from the given delimited token tree,
    and return the LiteralExpr for it. Allow for an optional trailing comma,
    but otherwise enforce that these are the only tokens.  */
 
 std::unique_ptr<AST::LiteralExpr>
 parse_single_string_literal (AST::DelimTokenTree &invoc_token_tree,
-			     Location invoc_locus)
+			     Location invoc_locus, MacroExpander *expander)
 {
   MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
   Parser<MacroInvocLexer> parser (lex);
@@ -89,7 +197,13 @@ parse_single_string_literal (AST::DelimTokenTree &invoc_token_tree,
   else if (parser.peek_current_token ()->get_id () == last_token_id)
     rust_error_at (invoc_locus, "macro takes 1 argument");
   else
-    rust_error_at (invoc_locus, "argument must be a string literal");
+    {
+      // when the expression does not seem to be a string literal, we then try
+      // to parse/expand it as macro to see if it expands to a string literal
+      auto expr = parser.parse_expr ();
+      lit_expr
+	= try_expand_single_string_literal (expr.get (), invoc_locus, expander);
+    }
 
   parser.skip_token (last_token_id);
 
@@ -188,7 +302,8 @@ MacroBuiltin::include_bytes (Location invoc_locus, AST::MacroInvocData &invoc)
   /* Get target filename from the macro invocation, which is treated as a path
      relative to the include!-ing file (currently being compiled).  */
   auto lit_expr
-    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus);
+    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus,
+				   invoc.get_expander ());
   if (lit_expr == nullptr)
     return AST::ASTFragment::create_error ();
 
@@ -230,7 +345,8 @@ MacroBuiltin::include_str (Location invoc_locus, AST::MacroInvocData &invoc)
   /* Get target filename from the macro invocation, which is treated as a path
      relative to the include!-ing file (currently being compiled).  */
   auto lit_expr
-    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus);
+    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus,
+				   invoc.get_expander ());
   if (lit_expr == nullptr)
     return AST::ASTFragment::create_error ();
 
@@ -252,7 +368,8 @@ AST::ASTFragment
 MacroBuiltin::compile_error (Location invoc_locus, AST::MacroInvocData &invoc)
 {
   auto lit_expr
-    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus);
+    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus,
+				   invoc.get_expander ());
   if (lit_expr == nullptr)
     return AST::ASTFragment::create_error ();
 
@@ -278,23 +395,30 @@ MacroBuiltin::concat (Location invoc_locus, AST::MacroInvocData &invoc)
   auto last_token_id = macro_end_token (invoc_token_tree, parser);
 
   /* NOTE: concat! could accept no argument, so we don't have any checks here */
-  while (parser.peek_current_token ()->get_id () != last_token_id)
+  auto expanded_expr = try_expand_many_expr (parser, invoc_locus, last_token_id,
+					     invoc.get_expander (), has_error);
+  for (auto &expr : expanded_expr)
     {
-      auto lit_expr = parser.parse_literal_expr ();
-      if (lit_expr)
+      if (!expr->is_literal ())
 	{
-	  str += lit_expr->as_string ();
+	  has_error = true;
+	  rust_error_at (expr->get_locus (), "expected a literal");
+	  // diagnostics copied from rustc
+	  rust_inform (expr->get_locus (),
+		       "only literals (like %<\"foo\"%>, %<42%> and "
+		       "%<3.14%>) can be passed to %<concat!()%>");
+	  continue;
 	}
-      else
+      auto *literal = static_cast<AST::LiteralExpr *> (expr.get ());
+      if (literal->get_lit_type () == AST::Literal::BYTE
+	  || literal->get_lit_type () == AST::Literal::BYTE_STRING)
 	{
-	  auto current_token = parser.peek_current_token ();
-	  rust_error_at (current_token->get_locus (),
-			 "argument must be a constant literal");
 	  has_error = true;
-	  // Just crash if the current token can't be skipped
-	  rust_assert (parser.skip_token (current_token->get_id ()));
+	  rust_error_at (expr->get_locus (),
+			 "cannot concatenate a byte string literal");
+	  continue;
 	}
-      parser.maybe_skip_token (COMMA);
+      str += literal->as_string ();
     }
 
   parser.skip_token (last_token_id);
@@ -317,45 +441,36 @@ MacroBuiltin::env (Location invoc_locus, AST::MacroInvocData &invoc)
   Parser<MacroInvocLexer> parser (lex);
 
   auto last_token_id = macro_end_token (invoc_token_tree, parser);
+  std::unique_ptr<AST::LiteralExpr> error_expr = nullptr;
+  std::unique_ptr<AST::LiteralExpr> lit_expr = nullptr;
+  bool has_error = false;
 
-  if (parser.peek_current_token ()->get_id () != STRING_LITERAL)
+  auto expanded_expr = try_expand_many_expr (parser, invoc_locus, last_token_id,
+					     invoc.get_expander (), has_error);
+  if (has_error)
+    return AST::ASTFragment::create_error ();
+  if (expanded_expr.size () < 1 || expanded_expr.size () > 2)
     {
-      if (parser.peek_current_token ()->get_id () == last_token_id)
-	rust_error_at (invoc_locus, "env! takes 1 or 2 arguments");
-      else
-	rust_error_at (parser.peek_current_token ()->get_locus (),
-		       "argument must be a string literal");
+      rust_error_at (invoc_locus, "env! takes 1 or 2 arguments");
       return AST::ASTFragment::create_error ();
     }
-
-  auto lit_expr = parser.parse_literal_expr ();
-  auto comma_skipped = parser.maybe_skip_token (COMMA);
-
-  std::unique_ptr<AST::LiteralExpr> error_expr = nullptr;
-
-  if (parser.peek_current_token ()->get_id () != last_token_id)
+  if (expanded_expr.size () > 0)
     {
-      if (!comma_skipped)
+      if (!(lit_expr
+	    = try_extract_string_literal_from_fragment (invoc_locus,
+							expanded_expr[0])))
 	{
-	  rust_error_at (parser.peek_current_token ()->get_locus (),
-			 "expected token: %<,%>");
 	  return AST::ASTFragment::create_error ();
 	}
-      if (parser.peek_current_token ()->get_id () != STRING_LITERAL)
+    }
+  if (expanded_expr.size () > 1)
+    {
+      if (!(error_expr
+	    = try_extract_string_literal_from_fragment (invoc_locus,
+							expanded_expr[1])))
 	{
-	  rust_error_at (parser.peek_current_token ()->get_locus (),
-			 "argument must be a string literal");
 	  return AST::ASTFragment::create_error ();
 	}
-
-      error_expr = parser.parse_literal_expr ();
-      parser.maybe_skip_token (COMMA);
-    }
-
-  if (parser.peek_current_token ()->get_id () != last_token_id)
-    {
-      rust_error_at (invoc_locus, "env! takes 1 or 2 arguments");
-      return AST::ASTFragment::create_error ();
     }
 
   parser.skip_token (last_token_id);
@@ -421,7 +536,8 @@ MacroBuiltin::include (Location invoc_locus, AST::MacroInvocData &invoc)
   /* Get target filename from the macro invocation, which is treated as a path
      relative to the include!-ing file (currently being compiled).  */
   auto lit_expr
-    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus);
+    = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus,
+				   invoc.get_expander ());
   if (lit_expr == nullptr)
     return AST::ASTFragment::create_error ();
 
diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc
index c68faba86ad..fbf33209c5f 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -163,6 +163,7 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc, bool has_semicolon)
   rust_assert (ok);
 
   auto fragment = AST::ASTFragment::create_error ();
+  invoc_data.set_expander (this);
 
   if (rules_def->is_builtin ())
     fragment
diff --git a/gcc/testsuite/rust/compile/builtin_macro_concat.rs b/gcc/testsuite/rust/compile/builtin_macro_concat.rs
index 9b878af764d..3b8eb2dcda7 100644
--- a/gcc/testsuite/rust/compile/builtin_macro_concat.rs
+++ b/gcc/testsuite/rust/compile/builtin_macro_concat.rs
@@ -6,12 +6,12 @@ macro_rules! concat {
 fn main() {
     let not_literal = "identifier";
     concat!();
-    concat! (,); // { dg-error "argument must be a constant literal" }
-    concat!(not_literal); // { dg-error "argument must be a constant literal" }
+    concat! (,); // { dg-error "expected expression, found .,." }
+    concat!(not_literal); // { dg-error "expected a literal" }
     concat!("message");
     concat!("message",);
     concat!("message", 1, true, false, 1.0, 10usize, 2000u64);
     concat!("message", 1, true, false, 1.0, 10usize, 2000u64,);
-    concat! ("m", not_literal); // { dg-error "argument must be a constant literal" }
-    concat!(not_literal invalid 'm' !!,); // { dg-error "argument must be a constant literal" }
+    concat! ("m", not_literal); // { dg-error "expected a literal" }
+    concat!(not_literal invalid 'm' !!,); // { dg-error "expected token: .,." }
 }
diff --git a/gcc/testsuite/rust/compile/builtin_macro_env.rs b/gcc/testsuite/rust/compile/builtin_macro_env.rs
index 289e6db2cf1..337f0ae4316 100644
--- a/gcc/testsuite/rust/compile/builtin_macro_env.rs
+++ b/gcc/testsuite/rust/compile/builtin_macro_env.rs
@@ -7,13 +7,13 @@ fn main () {
   let message = "error message";
   env! (message); // { dg-error "argument must be a string literal" "" }
   env! (); // { dg-error "env! takes 1 or 2 arguments" "" }
-  env! (,); // { dg-error "argument must be a string literal" "" }
+  env! (,); // { dg-error "expected expression, found .,." "" }
   env! (1); // { dg-error "argument must be a string literal" "" }
   env! ("NOT_DEFINED"); // { dg-error "environment variable 'NOT_DEFINED' not defined" "" }
   env! ("NOT_DEFINED",); // { dg-error "environment variable 'NOT_DEFINED' not defined" "" }
   env! ("NOT_DEFINED", 1); // { dg-error "argument must be a string literal" "" }
   env! ("NOT_DEFINED", "two", "three"); // { dg-error "env! takes 1 or 2 arguments" "" }
-  env! ("NOT_DEFINED" "expected error message"); // { dg-error "expected token: ','" "" }
+  env! ("NOT_DEFINED" "expected error message"); // { dg-error "expected token: .,." "" }
   env! ("NOT_DEFINED", "expected error message"); // { dg-error "expected error message" "" }
   env! ("NOT_DEFINED", "expected error message",); // { dg-error "expected error message" "" }
   env! (1, "two"); // { dg-error "argument must be a string literal" "" }
-- 
2.39.1


                 reply	other threads:[~2023-01-31 13:23 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230131132703.663677-1-arthur.cohen@embecosm.com \
    --to=arthur.cohen@embecosm.com \
    --cc=gcc-patches@gcc.gnu.org \
    --cc=gcc-rust@gcc.gnu.org \
    --cc=liushuyu011@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).