[gcc/devel/rust/master] macro: Allow builtin `MacroInvocation`s within the AST

[gcc/devel/rust/master] macro: Allow builtin `MacroInvocation`s within the AST

[Thomas Schwinge]

https://gcc.gnu.org/g:3630e0e2bf0fced043efb23f077adfe9b576dcad

commit 3630e0e2bf0fced043efb23f077adfe9b576dcad
Author: Arthur Cohen <arthur.cohen@embecosm.com>
Date:   Wed Jan 11 15:29:22 2023 +0100

    macro: Allow builtin `MacroInvocation`s within the AST
    
    gcc/rust/ChangeLog:
    
            * ast/rust-macro.h (enum class): Add `BuiltinMacro` enum class.
            * expand/rust-attribute-visitor.cc (AttrVisitor::visit): Mention
            switching on `macro.kind` once builtin macro invocations are properly
            handled.
            * parse/rust-parse-impl.h (Parser::parse_macro_invocation): Switch to new MacroInvocation
            API.
            (Parser::parse_type): Likewise.
            (Parser::parse_type_no_bounds): Likewise.

Diff:
---
 gcc/rust/ast/rust-macro.h                 | 110 +++++++++++++++++++++++++-----
 gcc/rust/expand/rust-attribute-visitor.cc |   2 +
 gcc/rust/parse/rust-parse-impl.h          |  70 +++++++++----------
 3 files changed, 126 insertions(+), 56 deletions(-)

diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index 33a250ed84a..48e4524c590 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -578,8 +578,30 @@ protected:
   }
 };
 
+/**
+ * All builtin macros possible
+ */
+enum class BuiltinMacro
+{
+  Assert,
+  File,
+  Line,
+  Column,
+  IncludeBytes,
+  IncludeStr,
+  CompileError,
+  Concat,
+  Env,
+  Cfg,
+  Include
+};
+
 /* AST node of a macro invocation, which is replaced by the macro result at
- * compile time */
+ * compile time. This is technically a sum-type/tagged-union, which represents
+ * both classic macro invocations and builtin macro invocations. Regular macro
+ * invocations are expanded lazily, but builtin macro invocations need to be
+ * expanded eagerly, hence the differentiation.
+ */
 class MacroInvocation : public TypeNoBounds,
 			public Pattern,
 			public Item,
@@ -589,26 +611,47 @@ class MacroInvocation : public TypeNoBounds,
 			public ExternalItem,
 			public ExprWithoutBlock
 {
-  std::vector<Attribute> outer_attrs;
-  MacroInvocData invoc_data;
-  Location locus;
-
-  // Important for when we actually expand the macro
-  bool is_semi_coloned;
-
-  NodeId node_id;
-
 public:
+  enum class InvocKind
+  {
+    Regular,
+    Builtin,
+  };
+
   std::string as_string () const override;
 
-  MacroInvocation (MacroInvocData invoc_data,
-		   std::vector<Attribute> outer_attrs, Location locus,
-		   bool is_semi_coloned = false)
-    : outer_attrs (std::move (outer_attrs)),
-      invoc_data (std::move (invoc_data)), locus (locus),
-      is_semi_coloned (is_semi_coloned),
-      node_id (Analysis::Mappings::get ()->get_next_node_id ())
-  {}
+  /**
+   * The default constructor you should use. Whenever we parse a macro call, we
+   * cannot possibly know whether or not this call refers to a builtin macro or
+   * a regular macro. With name resolution and scopes and nested macro calls,
+   * this is literally impossible. Hence, always start by creating a `Regular`
+   * MacroInvocation which will then (maybe!) become a `Builtin` macro
+   * invocation in the expander.
+   */
+  static std::unique_ptr<MacroInvocation>
+  Regular (MacroInvocData invoc_data, std::vector<Attribute> outer_attrs,
+	   Location locus, bool is_semi_coloned = false)
+  {
+    return std::unique_ptr<MacroInvocation> (
+      new MacroInvocation (InvocKind::Regular, Optional<BuiltinMacro>::none (),
+			   invoc_data, outer_attrs, locus, is_semi_coloned));
+  }
+
+  /**
+   * Create a builtin macro invocation. This can only be done after macro
+   * name-resolution and within the macro expander, so unless you're modifying
+   * these visitors, you probably do not want to use this function.
+   */
+  static std::unique_ptr<MacroInvocation>
+  Builtin (BuiltinMacro kind, MacroInvocData invoc_data,
+	   std::vector<Attribute> outer_attrs, Location locus,
+	   bool is_semi_coloned = false)
+  {
+    return std::unique_ptr<MacroInvocation> (
+      new MacroInvocation (InvocKind::Builtin,
+			   Optional<BuiltinMacro>::some (kind), invoc_data,
+			   outer_attrs, locus, is_semi_coloned));
+  }
 
   Location get_locus () const override final { return locus; }
 
@@ -642,6 +685,37 @@ public:
 
   bool has_semicolon () const { return is_semi_coloned; }
 
+  InvocKind get_kind () const { return kind; }
+  Optional<BuiltinMacro> get_builtin_kind () const { return builtin_kind; }
+
+private:
+  /* Full constructor */
+  MacroInvocation (InvocKind kind, Optional<BuiltinMacro> builtin_kind,
+		   MacroInvocData invoc_data,
+		   std::vector<Attribute> outer_attrs, Location locus,
+		   bool is_semi_coloned)
+    : outer_attrs (std::move (outer_attrs)), locus (locus),
+      node_id (Analysis::Mappings::get ()->get_next_node_id ()),
+      invoc_data (std::move (invoc_data)), is_semi_coloned (is_semi_coloned),
+      kind (kind), builtin_kind (builtin_kind)
+  {}
+
+  std::vector<Attribute> outer_attrs;
+  Location locus;
+  NodeId node_id;
+
+  /* The data given to the macro invocation */
+  MacroInvocData invoc_data;
+
+  /* Important for when we actually expand the macro */
+  bool is_semi_coloned;
+
+  /* Is this a builtin macro or a regular macro */
+  InvocKind kind;
+
+  /* If it is a builtin macro, which one */
+  Optional<BuiltinMacro> builtin_kind = Optional<BuiltinMacro>::none ();
+
 protected:
   /* Use covariance to implement clone function as returning this object rather
    * than base */
diff --git a/gcc/rust/expand/rust-attribute-visitor.cc b/gcc/rust/expand/rust-attribute-visitor.cc
index 5e4eac59970..82580740901 100644
--- a/gcc/rust/expand/rust-attribute-visitor.cc
+++ b/gcc/rust/expand/rust-attribute-visitor.cc
@@ -389,6 +389,8 @@ AttrVisitor::visit (AST::ConstGenericParam &)
 void
 AttrVisitor::visit (AST::MacroInvocation &macro_invoc)
 {
+  // FIXME: Probably need to check macro_invoc.kind
+
   // initial strip test based on outer attrs
   expander.expand_cfg_attrs (macro_invoc.get_outer_attrs ());
   if (expander.fails_cfg_with_expand (macro_invoc.get_outer_attrs ()))
diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h
index c19fe9b6ec4..7e4e7947e02 100644
--- a/gcc/rust/parse/rust-parse-impl.h
+++ b/gcc/rust/parse/rust-parse-impl.h
@@ -1752,10 +1752,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_semi (
 	    {
 	      // as this is the end, allow recovery (probably) - may change
 
-	      return std::unique_ptr<AST::MacroInvocation> (
-		new AST::MacroInvocation (std::move (invoc_data),
-					  std::move (outer_attrs), macro_locus,
-					  true));
+	      return AST::MacroInvocation::Regular (std::move (invoc_data),
+						    std::move (outer_attrs),
+						    macro_locus, true);
 	    }
 	}
 
@@ -1764,9 +1763,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_semi (
 		  t->get_token_description (),
 		  lexer.peek_token ()->get_token_description ());
 
-      return std::unique_ptr<AST::MacroInvocation> (
-	new AST::MacroInvocation (std::move (invoc_data),
-				  std::move (outer_attrs), macro_locus, true));
+      return AST::MacroInvocation::Regular (std::move (invoc_data),
+					    std::move (outer_attrs),
+					    macro_locus, true);
     }
   else
     {
@@ -1814,10 +1813,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation (AST::AttrVec outer_attrs)
 
   Location macro_locus = macro_path.get_locus ();
 
-  return std::unique_ptr<AST::MacroInvocation> (
-    new AST::MacroInvocation (AST::MacroInvocData (std::move (macro_path),
-						   std::move (delim_tok_tree)),
-			      std::move (outer_attrs), macro_locus));
+  return AST::MacroInvocation::Regular (
+    AST::MacroInvocData (std::move (macro_path), std::move (delim_tok_tree)),
+    std::move (outer_attrs), macro_locus);
 }
 
 // Parses a macro rule definition - does not parse semicolons.
@@ -9308,11 +9306,10 @@ Parser<ManagedTokenSource>::parse_type (bool save_errors)
 
 	      AST::DelimTokenTree tok_tree = parse_delim_token_tree ();
 
-	      return std::unique_ptr<AST::MacroInvocation> (
-		new AST::MacroInvocation (
-		  AST::MacroInvocData (std::move (macro_path),
-				       std::move (tok_tree)),
-		  {}, locus));
+	      return AST::MacroInvocation::Regular (
+		AST::MacroInvocData (std::move (macro_path),
+				     std::move (tok_tree)),
+		{}, locus);
 	    }
 	    case PLUS: {
 	      // type param bounds
@@ -10146,11 +10143,10 @@ Parser<ManagedTokenSource>::parse_type_no_bounds ()
 
 	      AST::DelimTokenTree tok_tree = parse_delim_token_tree ();
 
-	      return std::unique_ptr<AST::MacroInvocation> (
-		new AST::MacroInvocation (
-		  AST::MacroInvocData (std::move (macro_path),
-				       std::move (tok_tree)),
-		  {}, locus));
+	      return AST::MacroInvocation::Regular (
+		AST::MacroInvocData (std::move (macro_path),
+				     std::move (tok_tree)),
+		{}, locus);
 	    }
 	  default:
 	    // assume that this is a type path and not an error
@@ -12010,18 +12006,17 @@ Parser<ManagedTokenSource>::parse_path_based_stmt_or_expr (
 	      {
 		lexer.skip_token ();
 
-		std::unique_ptr<AST::MacroInvocation> stmt (
-		  new AST::MacroInvocation (std::move (invoc_data),
-					    std::move (outer_attrs),
-					    stmt_or_expr_loc, true));
+		auto stmt
+		  = AST::MacroInvocation::Regular (std::move (invoc_data),
+						   std::move (outer_attrs),
+						   stmt_or_expr_loc, true);
 		return ExprOrStmt (std::move (stmt));
 	      }
 
 	    // otherwise, create macro invocation
-	    std::unique_ptr<AST::MacroInvocation> expr (
-	      new AST::MacroInvocation (std::move (invoc_data),
-					std::move (outer_attrs),
-					stmt_or_expr_loc, false));
+	    auto expr = AST::MacroInvocation::Regular (std::move (invoc_data),
+						       std::move (outer_attrs),
+						       stmt_or_expr_loc, false);
 	    return ExprOrStmt (std::move (expr));
 	  }
 	else
@@ -12330,17 +12325,16 @@ Parser<ManagedTokenSource>::parse_macro_invocation_maybe_semi (
 	{
 	  lexer.skip_token ();
 
-	  std::unique_ptr<AST::MacroInvocation> stmt (
-	    new AST::MacroInvocation (std::move (invoc_data),
-				      std::move (outer_attrs), macro_locus,
-				      true));
+	  auto stmt = AST::MacroInvocation::Regular (std::move (invoc_data),
+						     std::move (outer_attrs),
+						     macro_locus, true);
 	  return ExprOrStmt (std::move (stmt));
 	}
 
       // otherwise, create macro invocation
-      std::unique_ptr<AST::MacroInvocation> expr (
-	new AST::MacroInvocation (std::move (invoc_data),
-				  std::move (outer_attrs), macro_locus));
+      auto expr
+	= AST::MacroInvocation::Regular (std::move (invoc_data),
+					 std::move (outer_attrs), macro_locus);
       return ExprOrStmt (std::move (expr));
     }
   else
@@ -14546,9 +14540,9 @@ Parser<ManagedTokenSource>::parse_macro_invocation_partial (
 
   Location macro_locus = converted_path.get_locus ();
 
-  return std::unique_ptr<AST::MacroInvocation> (new AST::MacroInvocation (
+  return AST::MacroInvocation::Regular (
     AST::MacroInvocData (std::move (converted_path), std::move (tok_tree)),
-    std::move (outer_attrs), macro_locus, restrictions.expr_can_be_stmt));
+    std::move (outer_attrs), macro_locus, restrictions.expr_can_be_stmt);
 }
 
 /* Parses a struct expr struct with a path in expression already parsed (but

[gcc/devel/rust/master] macro: Allow builtin `MacroInvocation`s within the AST

[Thomas Schwinge]

https://gcc.gnu.org/g:b9501cbe2624a80365e33550fc92035620a64e7b

commit b9501cbe2624a80365e33550fc92035620a64e7b
Author: Arthur Cohen <arthur.cohen@embecosm.com>
Date:   Wed Jan 18 12:23:03 2023 +0100

    macro: Allow builtin `MacroInvocation`s within the AST
    
    This commit turns AST::MacroInvocation into a sum type.
    The class can now represent a regular macro invocation (lazily expanded)
    or a builtin one (eagerly expanded)
    
    gcc/rust/ChangeLog:
    
            * expand/rust-macro-builtins.cc (make_macro_invocation): Add short hand
            function for returning fragments containing macro invocations.
            (MacroBuiltin::compile_error_handler): Add explanation for eager
            invocation

Diff:
---
 gcc/rust/expand/rust-macro-builtins.cc | 72 +++++++++++++++++++++++++++++++++-
 1 file changed, 71 insertions(+), 1 deletion(-)

diff --git a/gcc/rust/expand/rust-macro-builtins.cc b/gcc/rust/expand/rust-macro-builtins.cc
index f6ddb1a803e..11b5d5fbc9d 100644
--- a/gcc/rust/expand/rust-macro-builtins.cc
+++ b/gcc/rust/expand/rust-macro-builtins.cc
@@ -37,8 +37,39 @@ make_string (Location locus, std::string value)
 			  PrimitiveCoreType::CORETYPE_STR, {}, locus));
 }
 
-/* Match the end token of a macro given the start delimiter of the macro */
+// TODO: Is this correct?
+static std::unique_ptr<AST::Expr>
+make_macro_invocation (AST::BuiltinMacro kind, AST::DelimTokenTree arguments)
+{
+  std::string path_str;
+
+  switch (kind)
+    {
+    case AST::BuiltinMacro::Assert:
+    case AST::BuiltinMacro::File:
+    case AST::BuiltinMacro::Line:
+    case AST::BuiltinMacro::Column:
+    case AST::BuiltinMacro::IncludeBytes:
+    case AST::BuiltinMacro::IncludeStr:
+    case AST::BuiltinMacro::CompileError:
+    case AST::BuiltinMacro::Concat:
+      path_str = "concat";
+      break;
+    case AST::BuiltinMacro::Env:
+    case AST::BuiltinMacro::Cfg:
+    case AST::BuiltinMacro::Include:
+      break;
+    }
 
+  return AST::MacroInvocation::builtin (
+    kind,
+    AST::MacroInvocData (AST::SimplePath (
+			   {AST::SimplePathSegment (path_str, Location ())}),
+			 std::move (arguments)),
+    {}, Location ());
+}
+
+/* Match the end token of a macro given the start delimiter of the macro */
 static inline TokenId
 macro_end_token (AST::DelimTokenTree &invoc_token_tree,
 		 Parser<MacroInvocLexer> &parser)
@@ -386,6 +417,45 @@ MacroBuiltin::compile_error_handler (Location invoc_locus,
 /* Expand builtin macro concat!(), which joins all the literal parameters
    into a string with no delimiter. */
 
+// This is a weird one. We want to do something where, if something cannot be
+// expanded yet (i.e. macro invocation?) we return the whole MacroInvocation
+// node again but expanded as much as possible.
+// Is that possible? How do we do that?
+//
+// Let's take a few examples:
+//
+// 1. concat!(1, 2, true);
+// 2. concat!(a!(), 2, true);
+// 3. concat!(concat!(1, false), 2, true);
+// 4. concat!(concat!(1, a!()), 2, true);
+//
+// 1. We simply want to return the new fragment: "12true"
+// 2. We want to return `concat!(a_expanded, 2, true)` as a fragment
+// 3. We want to return `concat!(1, false, 2, true)`
+// 4. We want to return `concat!(concat!(1, a_expanded), 2, true);
+//
+// How do we do that?
+//
+// For each (un)expanded fragment: we check if it is expanded fully
+//
+// 1. What is expanded fully?
+// 2. How to check?
+//
+// If it is expanded fully and not a literal, then we error out.
+// Otherwise we simply emplace it back and keep going.
+//
+// In the second case, we must mark that this concat invocation still has some
+// expansion to do: This allows us to return a `MacroInvocation { ... }` as an
+// AST fragment, instead of a completed string.
+//
+// This means that we must change all the `try_expand_many_*` APIs and so on to
+// return some sort of index or way to signify that we might want to reuse some
+// bits and pieces of the original token tree.
+//
+// Now, before that: How do we resolve the names used in a builtin macro
+// invocation?
+// Do we split the two passes of parsing the token tree and then expanding it?
+// Can we do that easily?
 AST::Fragment
 MacroBuiltin::concat_handler (Location invoc_locus, AST::MacroInvocData &invoc)
 {