From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by sourceware.org (Postfix, from userid 1643) id ECBDB3858CDA; Tue, 27 Sep 2022 08:23:21 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org ECBDB3858CDA DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gcc.gnu.org; s=default; t=1664267001; bh=KsA/VYcrxvzhvXZCKvxO9JbsOdveA6AvvZo4GmKLMkg=; h=From:To:Subject:Date:From; b=x4EkopBIK3guh/kFW63qu9lm7qxyETjZyCVLrmZBkpgmGDx9a/w8hIkWCLMzhXgBX myxYl1FiqHpizP069tXqiAf6W3VmC5qhsYI4lit85DLz3+jkAxfAKL30SHrRgIwzHr LtYo71o0D2qQ+6Gn/kKpptSPAfcPwfnGhEeAkXzQ= Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: Thomas Schwinge To: gcc-cvs@gcc.gnu.org Subject: [gcc/devel/rust/master] expand: eager evaluate macros inside builtin macros X-Act-Checkin: gcc X-Git-Author: liushuyu X-Git-Refname: refs/heads/devel/rust/master X-Git-Oldrev: 8ed1bbaa40527c561b25b5dadb963ca404f2da37 X-Git-Newrev: a16c35340c833ed25035c664ee33551d17309601 Message-Id: <20220927082321.ECBDB3858CDA@sourceware.org> Date: Tue, 27 Sep 2022 08:23:21 +0000 (GMT) List-Id: https://gcc.gnu.org/g:a16c35340c833ed25035c664ee33551d17309601 commit a16c35340c833ed25035c664ee33551d17309601 Author: liushuyu Date: Fri Sep 2 16:08:39 2022 -0600 expand: eager evaluate macros inside builtin macros Diff: --- gcc/rust/ast/rust-ast.h | 12 ++ gcc/rust/ast/rust-expr.h | 2 + gcc/rust/expand/rust-macro-builtins.cc | 206 ++++++++++++++++----- gcc/rust/expand/rust-macro-expand.cc | 1 + gcc/testsuite/rust/compile/builtin_macro_concat.rs | 9 +- 5 files changed, 181 insertions(+), 49 deletions(-) diff --git a/gcc/rust/ast/rust-ast.h b/gcc/rust/ast/rust-ast.h index e9e16e77f65..492faea558a 100644 --- a/gcc/rust/ast/rust-ast.h +++ b/gcc/rust/ast/rust-ast.h @@ -31,6 +31,7 @@ namespace Rust { typedef std::string Identifier; typedef int TupleIndex; struct Session; +struct MacroExpander; namespace AST { // foward decl: ast visitor @@ -951,6 +952,8 @@ public: virtual Location get_locus () const = 0; + virtual bool is_literal () const { return false; } + // HACK: strictly not needed, but faster than full downcast clone virtual bool is_expr_without_block () const = 0; @@ -1471,6 +1474,7 @@ private: // One way of parsing the macro. Probably not applicable for all macros. std::vector > parsed_items; bool parsed_to_meta_item = false; + MacroExpander *expander = nullptr; public: std::string as_string () const; @@ -1495,6 +1499,7 @@ public: path = other.path; token_tree = other.token_tree; parsed_to_meta_item = other.parsed_to_meta_item; + expander = other.expander; parsed_items.reserve (other.parsed_items.size ()); for (const auto &e : other.parsed_items) @@ -1523,6 +1528,13 @@ public: SimplePath &get_path () { return path; } const SimplePath &get_path () const { return path; } + void set_expander (MacroExpander *new_expander) { expander = new_expander; } + MacroExpander *get_expander () + { + rust_assert (expander); + return expander; + } + void set_meta_item_output (std::vector > new_items) { diff --git a/gcc/rust/ast/rust-expr.h b/gcc/rust/ast/rust-expr.h index 1966a590c94..c764f9c4c66 100644 --- a/gcc/rust/ast/rust-expr.h +++ b/gcc/rust/ast/rust-expr.h @@ -67,6 +67,8 @@ public: Location get_locus () const override final { return locus; } + bool is_literal () const override final { return true; } + Literal get_literal () const { return literal; } void accept_vis (ASTVisitor &vis) override; diff --git a/gcc/rust/expand/rust-macro-builtins.cc b/gcc/rust/expand/rust-macro-builtins.cc index 5eace13d197..a843fe4c4a4 100644 --- a/gcc/rust/expand/rust-macro-builtins.cc +++ b/gcc/rust/expand/rust-macro-builtins.cc @@ -17,12 +17,14 @@ // . #include "rust-macro-builtins.h" +#include "rust-ast.h" #include "rust-diagnostics.h" #include "rust-expr.h" #include "rust-session-manager.h" #include "rust-macro-invoc-lexer.h" #include "rust-lex.h" #include "rust-parse.h" +#include "rust-attribute-visitor.h" namespace Rust { namespace { @@ -61,13 +63,119 @@ macro_end_token (AST::DelimTokenTree &invoc_token_tree, return last_token_id; } +/* Expand and extract an expression from the macro */ + +static inline AST::ASTFragment +try_expand_macro_expression (AST::Expr *expr, MacroExpander *expander) +{ + rust_assert (expander); + + auto vis = Rust::AttrVisitor (*expander); + expr->accept_vis (vis); + return expander->take_expanded_fragment (vis); +} + +/* Expand and then extract a string literal from the macro */ + +static std::unique_ptr +try_extract_string_literal_from_fragment (const Location &parent_locus, + std::unique_ptr &node) +{ + auto maybe_lit = static_cast (node.get ()); + if (!node || !node->is_literal () + || maybe_lit->get_lit_type () != AST::Literal::STRING) + { + rust_error_at (parent_locus, "argument must be a string literal"); + if (node) + rust_inform (node->get_locus (), "expanded from here"); + return nullptr; + } + return std::unique_ptr ( + static_cast (node->clone_expr ().release ())); +} + +static std::unique_ptr +try_expand_single_string_literal (AST::Expr *input_expr, + const Location &invoc_locus, + MacroExpander *expander) +{ + auto nodes = try_expand_macro_expression (input_expr, expander); + if (nodes.is_error () || nodes.is_expression_fragment ()) + { + rust_error_at (input_expr->get_locus (), + "argument must be a string literal"); + return nullptr; + } + auto expr = nodes.take_expression_fragment (); + return try_extract_string_literal_from_fragment (input_expr->get_locus (), + expr); +} + +static std::vector> +try_expand_many_expr (Parser &parser, + const Location &invoc_locus, const TokenId last_token_id, + MacroExpander *expander, bool &has_error) +{ + auto restrictions = Rust::ParseRestrictions (); + // stop parsing when encountered a braces/brackets + restrictions.expr_can_be_null = true; + // we can't use std::optional, so... + auto result = std::vector> (); + auto empty_expr = std::vector> (); + + auto first_token = parser.peek_current_token ()->get_id (); + if (first_token == COMMA) + { + rust_error_at (parser.peek_current_token ()->get_locus (), + "expected expression, found %<,%>"); + has_error = true; + return empty_expr; + } + + while (parser.peek_current_token ()->get_id () != last_token_id + && parser.peek_current_token ()->get_id () != END_OF_FILE) + { + auto expr = parser.parse_expr (AST::AttrVec (), restrictions); + // something must be so wrong that the expression could not be parsed + rust_assert (expr); + auto nodes = try_expand_macro_expression (expr.get (), expander); + if (nodes.is_error ()) + { + // not macro + result.push_back (std::move (expr)); + } + else if (!nodes.is_expression_fragment ()) + { + rust_error_at (expr->get_locus (), "expected expression"); + has_error = true; + return empty_expr; + } + else + { + result.push_back (nodes.take_expression_fragment ()); + } + + auto next_token = parser.peek_current_token (); + if (!parser.skip_token (COMMA) && next_token->get_id () != last_token_id) + { + rust_error_at (next_token->get_locus (), "expected token: %<,%>"); + // TODO: is this recoverable? to avoid crashing the parser in the next + // fragment we have to exit early here + has_error = true; + return empty_expr; + } + } + + return result; +} + /* Parse a single string literal from the given delimited token tree, and return the LiteralExpr for it. Allow for an optional trailing comma, but otherwise enforce that these are the only tokens. */ std::unique_ptr parse_single_string_literal (AST::DelimTokenTree &invoc_token_tree, - Location invoc_locus) + Location invoc_locus, MacroExpander *expander) { MacroInvocLexer lex (invoc_token_tree.to_token_stream ()); Parser parser (lex); @@ -89,7 +197,13 @@ parse_single_string_literal (AST::DelimTokenTree &invoc_token_tree, else if (parser.peek_current_token ()->get_id () == last_token_id) rust_error_at (invoc_locus, "macro takes 1 argument"); else - rust_error_at (invoc_locus, "argument must be a string literal"); + { + // when the expression does not seem to be a string literal, we then try + // to parse/expand it as macro to see if it expands to a string literal + auto expr = parser.parse_expr (); + lit_expr + = try_expand_single_string_literal (expr.get (), invoc_locus, expander); + } parser.skip_token (last_token_id); @@ -188,7 +302,8 @@ MacroBuiltin::include_bytes (Location invoc_locus, AST::MacroInvocData &invoc) /* Get target filename from the macro invocation, which is treated as a path relative to the include!-ing file (currently being compiled). */ auto lit_expr - = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus); + = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus, + invoc.get_expander ()); if (lit_expr == nullptr) return AST::ASTFragment::create_error (); @@ -230,7 +345,8 @@ MacroBuiltin::include_str (Location invoc_locus, AST::MacroInvocData &invoc) /* Get target filename from the macro invocation, which is treated as a path relative to the include!-ing file (currently being compiled). */ auto lit_expr - = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus); + = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus, + invoc.get_expander ()); if (lit_expr == nullptr) return AST::ASTFragment::create_error (); @@ -252,7 +368,8 @@ AST::ASTFragment MacroBuiltin::compile_error (Location invoc_locus, AST::MacroInvocData &invoc) { auto lit_expr - = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus); + = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus, + invoc.get_expander ()); if (lit_expr == nullptr) return AST::ASTFragment::create_error (); @@ -278,23 +395,30 @@ MacroBuiltin::concat (Location invoc_locus, AST::MacroInvocData &invoc) auto last_token_id = macro_end_token (invoc_token_tree, parser); /* NOTE: concat! could accept no argument, so we don't have any checks here */ - while (parser.peek_current_token ()->get_id () != last_token_id) + auto expanded_expr = try_expand_many_expr (parser, invoc_locus, last_token_id, + invoc.get_expander (), has_error); + for (auto &expr : expanded_expr) { - auto lit_expr = parser.parse_literal_expr (); - if (lit_expr) + if (!expr->is_literal ()) { - str += lit_expr->as_string (); + has_error = true; + rust_error_at (expr->get_locus (), "expected a literal"); + // diagnostics copied from rustc + rust_inform (expr->get_locus (), + "only literals (like %<\"foo\"%>, %<42%> and " + "%<3.14%>) can be passed to %"); + continue; } - else + auto *literal = static_cast (expr.get ()); + if (literal->get_lit_type () == AST::Literal::BYTE + || literal->get_lit_type () == AST::Literal::BYTE_STRING) { - auto current_token = parser.peek_current_token (); - rust_error_at (current_token->get_locus (), - "argument must be a constant literal"); has_error = true; - // Just crash if the current token can't be skipped - rust_assert (parser.skip_token (current_token->get_id ())); + rust_error_at (expr->get_locus (), + "cannot concatenate a byte string literal"); + continue; } - parser.maybe_skip_token (COMMA); + str += literal->as_string (); } parser.skip_token (last_token_id); @@ -317,45 +441,36 @@ MacroBuiltin::env (Location invoc_locus, AST::MacroInvocData &invoc) Parser parser (lex); auto last_token_id = macro_end_token (invoc_token_tree, parser); + std::unique_ptr error_expr = nullptr; + std::unique_ptr lit_expr = nullptr; + bool has_error = false; - if (parser.peek_current_token ()->get_id () != STRING_LITERAL) + auto expanded_expr = try_expand_many_expr (parser, invoc_locus, last_token_id, + invoc.get_expander (), has_error); + if (has_error) + return AST::ASTFragment::create_error (); + if (expanded_expr.size () < 1 || expanded_expr.size () > 2) { - if (parser.peek_current_token ()->get_id () == last_token_id) - rust_error_at (invoc_locus, "env! takes 1 or 2 arguments"); - else - rust_error_at (parser.peek_current_token ()->get_locus (), - "argument must be a string literal"); + rust_error_at (invoc_locus, "env! takes 1 or 2 arguments"); return AST::ASTFragment::create_error (); } - - auto lit_expr = parser.parse_literal_expr (); - auto comma_skipped = parser.maybe_skip_token (COMMA); - - std::unique_ptr error_expr = nullptr; - - if (parser.peek_current_token ()->get_id () != last_token_id) + if (expanded_expr.size () > 0) { - if (!comma_skipped) + if (!(lit_expr + = try_extract_string_literal_from_fragment (invoc_locus, + expanded_expr[0]))) { - rust_error_at (parser.peek_current_token ()->get_locus (), - "expected token: %<,%>"); return AST::ASTFragment::create_error (); } - if (parser.peek_current_token ()->get_id () != STRING_LITERAL) + } + if (expanded_expr.size () > 1) + { + if (!(error_expr + = try_extract_string_literal_from_fragment (invoc_locus, + expanded_expr[1]))) { - rust_error_at (parser.peek_current_token ()->get_locus (), - "argument must be a string literal"); return AST::ASTFragment::create_error (); } - - error_expr = parser.parse_literal_expr (); - parser.maybe_skip_token (COMMA); - } - - if (parser.peek_current_token ()->get_id () != last_token_id) - { - rust_error_at (invoc_locus, "env! takes 1 or 2 arguments"); - return AST::ASTFragment::create_error (); } parser.skip_token (last_token_id); @@ -421,7 +536,8 @@ MacroBuiltin::include (Location invoc_locus, AST::MacroInvocData &invoc) /* Get target filename from the macro invocation, which is treated as a path relative to the include!-ing file (currently being compiled). */ auto lit_expr - = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus); + = parse_single_string_literal (invoc.get_delim_tok_tree (), invoc_locus, + invoc.get_expander ()); if (lit_expr == nullptr) return AST::ASTFragment::create_error (); diff --git a/gcc/rust/expand/rust-macro-expand.cc b/gcc/rust/expand/rust-macro-expand.cc index dd8d468aad1..e7a7b846ed4 100644 --- a/gcc/rust/expand/rust-macro-expand.cc +++ b/gcc/rust/expand/rust-macro-expand.cc @@ -163,6 +163,7 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc, bool has_semicolon) rust_assert (ok); auto fragment = AST::ASTFragment::create_error (); + invoc_data.set_expander (this); if (rules_def->is_builtin ()) fragment diff --git a/gcc/testsuite/rust/compile/builtin_macro_concat.rs b/gcc/testsuite/rust/compile/builtin_macro_concat.rs index 9b878af764d..8a84934f949 100644 --- a/gcc/testsuite/rust/compile/builtin_macro_concat.rs +++ b/gcc/testsuite/rust/compile/builtin_macro_concat.rs @@ -6,12 +6,13 @@ macro_rules! concat { fn main() { let not_literal = "identifier"; concat!(); - concat! (,); // { dg-error "argument must be a constant literal" } - concat!(not_literal); // { dg-error "argument must be a constant literal" } + concat! (,); // { dg-error "expected expression, found .,." } + concat!(not_literal); // { dg-error "expected a literal" } concat!("message"); concat!("message",); concat!("message", 1, true, false, 1.0, 10usize, 2000u64); concat!("message", 1, true, false, 1.0, 10usize, 2000u64,); - concat! ("m", not_literal); // { dg-error "argument must be a constant literal" } - concat!(not_literal invalid 'm' !!,); // { dg-error "argument must be a constant literal" } + concat! ("m", not_literal); // { dg-error "expected a literal" } + concat!(not_literal invalid 'm' !!,); // { dg-error "expected token: .,." } + // { dg-error "expected a literal" "" { target *-*-* } .-1 } }