From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gnu.wildebeest.org (wildebeest.demon.nl [212.238.236.112]) by sourceware.org (Postfix) with ESMTPS id 4A756385C405 for ; Wed, 28 Jul 2021 22:13:48 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 4A756385C405 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=klomp.org Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=klomp.org Received: from reform (deer0x10.wildebeest.org [172.31.17.146]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by gnu.wildebeest.org (Postfix) with ESMTPSA id 0B27A30291A9; Thu, 29 Jul 2021 00:13:45 +0200 (CEST) Received: by reform (Postfix, from userid 1000) id 7D7262E8158A; Thu, 29 Jul 2021 00:13:45 +0200 (CEST) From: Mark Wielaard To: gcc-rust@gcc.gnu.org Cc: Mark Wielaard Subject: [PATCH] Pass pratt parsed token to expr parser functions to fix expr locus Date: Thu, 29 Jul 2021 00:13:42 +0200 Message-Id: <20210728221342.77649-1-mark@klomp.org> X-Mailer: git-send-email 2.32.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-11.1 required=5.0 tests=BAYES_00, GIT_PATCH_0, KAM_DMARC_STATUS, SPF_HELO_NONE, SPF_PASS, TXREP autolearn=ham autolearn_force=no version=3.4.4 X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on server2.sourceware.org X-BeenThere: gcc-rust@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: gcc-rust mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Jul 2021 22:13:51 -0000 The pratt parser skips the first token of an expression before invoking the actual expression parsing function. This makes getting the correct starting location of a pratt parsed expression hard. The "correction" of the location by subtracting an integer is often wrong (since there may be arbitrary whitespace or comments between tokens). Fix this by passing the token to the expression parsing functions (instead of just providing a pratt_parse boolean). Use this token to set the start of the expression. Before gccrs would generate the following error message: return.rs:3:22: error: cannot ‘break’ outside of a loop 3 | let x = 5 - break return (16 + 2); | ^~~~~~~~~~~~~~~~~ Now we get: return.rs:3:17: error: cannot ‘break’ outside of a loop 3 | let x = 5 - break return (16 + 2); | ^ --- gcc/rust/parse/rust-parse-impl.h | 180 +++++++++++++------------------ gcc/rust/parse/rust-parse.h | 58 +++++----- 2 files changed, 103 insertions(+), 135 deletions(-) diff --git a/gcc/rust/parse/rust-parse-impl.h b/gcc/rust/parse/rust-parse-impl.h index 340fea70201..6241a972ff7 100644 --- a/gcc/rust/parse/rust-parse-impl.h +++ b/gcc/rust/parse/rust-parse-impl.h @@ -6580,7 +6580,7 @@ Parser::parse_path_expr_segment () template AST::QualifiedPathInExpression Parser::parse_qualified_path_in_expression ( - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { /* Note: the Rust grammar is defined in such a way that it is impossible to * determine whether a prospective qualified path is a @@ -6595,7 +6595,7 @@ Parser::parse_qualified_path_in_expression ( // parse the qualified path type (required) AST::QualifiedPathType qual_path_type - = parse_qualified_path_type (pratt_parse); + = parse_qualified_path_type (pratt_parsed_token); if (qual_path_type.is_error ()) { // TODO: should this create a parse error? @@ -6661,12 +6661,13 @@ Parser::parse_qualified_path_in_expression ( // Parses the type syntactical construction at the start of a qualified path. template AST::QualifiedPathType -Parser::parse_qualified_path_type (bool pratt_parse) +Parser::parse_qualified_path_type ( + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); + Location locus; /* TODO: should this actually be error? is there anywhere where this could be * valid? */ - if (!pratt_parse) + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); if (!skip_token (LEFT_ANGLE)) @@ -6676,10 +6677,7 @@ Parser::parse_qualified_path_type (bool pratt_parse) } } else - { - // move back by 1 if pratt parsing due to skipping '<' - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); // parse type (required) std::unique_ptr type = parse_type (); @@ -7305,10 +7303,10 @@ Parser::parse_expr_without_block (AST::AttrVec outer_attrs) template std::unique_ptr Parser::parse_block_expr (AST::AttrVec outer_attrs, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); if (!skip_token (LEFT_CURLY)) @@ -7318,9 +7316,7 @@ Parser::parse_block_expr (AST::AttrVec outer_attrs, } } else - { - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); AST::AttrVec inner_attrs = parse_inner_attributes (); @@ -7611,22 +7607,17 @@ Parser::parse_literal_expr (AST::AttrVec outer_attrs) // Parses a return expression (including any expression to return). template std::unique_ptr -Parser::parse_return_expr (AST::AttrVec outer_attrs, - bool pratt_parse) +Parser::parse_return_expr ( + AST::AttrVec outer_attrs, const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (RETURN_TOK); } else - { - // minus 7 chars for 6 in return and a space - // or just TODO: pass in location data - locus = lexer.peek_token ()->get_locus () - 7; - } + locus = pratt_parsed_token->get_locus (); // parse expression to return, if it exists ParseRestrictions restrictions; @@ -7644,21 +7635,16 @@ Parser::parse_return_expr (AST::AttrVec outer_attrs, template std::unique_ptr Parser::parse_break_expr (AST::AttrVec outer_attrs, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (BREAK); } else - { - // minus 6 chars for 5 in return and a space - // or just TODO: pass in location data - locus = lexer.peek_token ()->get_locus () - 6; - } + locus = pratt_parsed_token->get_locus (); // parse label (lifetime) if it exists - create dummy first AST::Lifetime label = AST::Lifetime::error (); @@ -7681,22 +7667,17 @@ Parser::parse_break_expr (AST::AttrVec outer_attrs, // Parses a continue expression (including any label to continue from). template std::unique_ptr -Parser::parse_continue_expr (AST::AttrVec outer_attrs, - bool pratt_parse) +Parser::parse_continue_expr ( + AST::AttrVec outer_attrs, const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (CONTINUE); } else - { - // minus 9 chars for 8 in return and a space - // or just TODO: pass in location data - locus = lexer.peek_token ()->get_locus () - 9; - } + locus = pratt_parsed_token->get_locus (); // parse label (lifetime) if it exists - create dummy first AST::Lifetime label = AST::Lifetime::error (); @@ -7740,11 +7721,11 @@ Parser::parse_loop_label () template std::unique_ptr Parser::parse_if_expr (AST::AttrVec outer_attrs, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { // TODO: make having outer attributes an error? - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); if (!skip_token (IF)) @@ -7754,9 +7735,7 @@ Parser::parse_if_expr (AST::AttrVec outer_attrs, } } else - { - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); // detect accidental if let if (lexer.peek_token ()->get_id () == LET) @@ -7901,12 +7880,12 @@ Parser::parse_if_expr (AST::AttrVec outer_attrs, * expressions don't support them. */ template std::unique_ptr -Parser::parse_if_let_expr (AST::AttrVec outer_attrs, - bool pratt_parse) +Parser::parse_if_let_expr ( + AST::AttrVec outer_attrs, const_TokenPtr pratt_parsed_token) { // TODO: make having outer attributes an error? - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); if (!skip_token (IF)) @@ -7916,9 +7895,7 @@ Parser::parse_if_let_expr (AST::AttrVec outer_attrs, } } else - { - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); // detect accidental if expr parsed as if let expr if (lexer.peek_token ()->get_id () != LET) @@ -8094,10 +8071,10 @@ template std::unique_ptr Parser::parse_loop_expr (AST::AttrVec outer_attrs, AST::LoopLabel label, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { if (label.is_error ()) locus = lexer.peek_token ()->get_locus (); @@ -8113,7 +8090,7 @@ Parser::parse_loop_expr (AST::AttrVec outer_attrs, else { if (label.is_error ()) - locus = lexer.peek_token ()->get_locus () - 1; + locus = pratt_parsed_token->get_locus (); else locus = label.get_locus (); } @@ -8138,12 +8115,12 @@ Parser::parse_loop_expr (AST::AttrVec outer_attrs, * via parse_labelled_loop_expr, which would call this. */ template std::unique_ptr -Parser::parse_while_loop_expr (AST::AttrVec outer_attrs, - AST::LoopLabel label, - bool pratt_parse) +Parser::parse_while_loop_expr ( + AST::AttrVec outer_attrs, AST::LoopLabel label, + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { if (label.is_error ()) locus = lexer.peek_token ()->get_locus (); @@ -8159,7 +8136,7 @@ Parser::parse_while_loop_expr (AST::AttrVec outer_attrs, else { if (label.is_error ()) - locus = lexer.peek_token ()->get_locus () - 1; + locus = pratt_parsed_token->get_locus (); else locus = label.get_locus (); } @@ -8417,21 +8394,16 @@ Parser::parse_labelled_loop_expr (AST::AttrVec outer_attrs) template std::unique_ptr Parser::parse_match_expr (AST::AttrVec outer_attrs, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (MATCH_TOK); } else - { - // TODO: probably just pass in location data as param - // get current pos then move back 6 - 5 for match, 1 for space - locus = lexer.peek_token ()->get_locus () - 6; - } + locus = pratt_parsed_token->get_locus (); /* parse scrutinee expression, which is required (and HACK to prevent struct * expr) */ @@ -8704,17 +8676,17 @@ Parser::parse_async_block_expr (AST::AttrVec outer_attrs) // Parses an unsafe block expression. template std::unique_ptr -Parser::parse_unsafe_block_expr (AST::AttrVec outer_attrs, - bool pratt_parse) +Parser::parse_unsafe_block_expr ( + AST::AttrVec outer_attrs, const_TokenPtr pratt_parsed_token) { Location locus; - if (!pratt_parse) + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); skip_token (UNSAFE); } else - locus = lexer.peek_token ()->get_locus () - 1; + locus = pratt_parsed_token->get_locus (); // parse block expression (required) std::unique_ptr block_expr = parse_block_expr (); @@ -8738,19 +8710,16 @@ Parser::parse_unsafe_block_expr (AST::AttrVec outer_attrs, template std::unique_ptr Parser::parse_array_expr (AST::AttrVec outer_attrs, - bool pratt_parse) + const_TokenPtr pratt_parsed_token) { - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (LEFT_SQUARE); } else - { - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); // parse optional inner attributes AST::AttrVec inner_attrs = parse_inner_attributes (); @@ -8927,20 +8896,17 @@ Parser::parse_closure_param () template std::unique_ptr Parser::parse_grouped_or_tuple_expr ( - AST::AttrVec outer_attrs, bool pratt_parse) + AST::AttrVec outer_attrs, const_TokenPtr pratt_parsed_token) { // adjustment to allow Pratt parsing to reuse function without copy-paste - Location locus = Linemap::unknown_location (); - if (!pratt_parse) + Location locus; + if (pratt_parsed_token == nullptr) { locus = lexer.peek_token ()->get_locus (); - skip_token (LEFT_PAREN); } else - { - locus = lexer.peek_token ()->get_locus () - 1; - } + locus = pratt_parsed_token->get_locus (); // parse optional inner attributes AST::AttrVec inner_attrs = parse_inner_attributes (); @@ -12535,7 +12501,7 @@ Parser::null_denotation (const_TokenPtr tok, // qualified path // HACK: add outer attrs to path AST::QualifiedPathInExpression path - = parse_qualified_path_in_expression (true); + = parse_qualified_path_in_expression (tok); path.set_outer_attrs (std::move (outer_attrs)); return std::unique_ptr ( new AST::QualifiedPathInExpression (std::move (path))); @@ -12578,7 +12544,7 @@ Parser::null_denotation (const_TokenPtr tok, new AST::LiteralExpr ("false", AST::Literal::BOOL, tok->get_type_hint (), {}, tok->get_locus ())); case LEFT_PAREN: - return parse_grouped_or_tuple_expr (std::move (outer_attrs), true); + return parse_grouped_or_tuple_expr (std::move (outer_attrs), tok); /*case PLUS: { // unary plus operator // invoke parse_expr recursively with appropriate priority, etc. for @@ -12810,41 +12776,41 @@ Parser::null_denotation (const_TokenPtr tok, return parse_range_to_inclusive_expr (tok, std::move (outer_attrs)); case RETURN_TOK: // FIXME: is this really a null denotation expression? - return parse_return_expr (std::move (outer_attrs), true); + return parse_return_expr (std::move (outer_attrs), tok); case BREAK: // FIXME: is this really a null denotation expression? - return parse_break_expr (std::move (outer_attrs), true); + return parse_break_expr (std::move (outer_attrs), tok); case CONTINUE: - return parse_continue_expr (std::move (outer_attrs), true); + return parse_continue_expr (std::move (outer_attrs), tok); case LEFT_CURLY: // ok - this is an expression with block for once. - return parse_block_expr (std::move (outer_attrs), true); + return parse_block_expr (std::move (outer_attrs), tok); case IF: // if or if let, so more lookahead to find out if (lexer.peek_token (1)->get_id () == LET) { // if let expr - return parse_if_let_expr (std::move (outer_attrs), true); + return parse_if_let_expr (std::move (outer_attrs), tok); } else { // if expr - return parse_if_expr (std::move (outer_attrs), true); + return parse_if_expr (std::move (outer_attrs), tok); } case LOOP: return parse_loop_expr (std::move (outer_attrs), AST::LoopLabel::error (), - true); + tok); case WHILE: return parse_while_loop_expr (std::move (outer_attrs), - AST::LoopLabel::error (), true); + AST::LoopLabel::error (), tok); case MATCH_TOK: // also an expression with block - return parse_match_expr (std::move (outer_attrs), true); + return parse_match_expr (std::move (outer_attrs), tok); case LEFT_SQUARE: // array definition expr (not indexing) - return parse_array_expr (std::move (outer_attrs), true); + return parse_array_expr (std::move (outer_attrs), tok); case UNSAFE: - return parse_unsafe_block_expr (std::move (outer_attrs), true); + return parse_unsafe_block_expr (std::move (outer_attrs), tok); default: if (!restrictions.expr_can_be_null) add_error (Error (tok->get_locus (), diff --git a/gcc/rust/parse/rust-parse.h b/gcc/rust/parse/rust-parse.h index 1c7bd781b3f..78608839265 100644 --- a/gcc/rust/parse/rust-parse.h +++ b/gcc/rust/parse/rust-parse.h @@ -121,8 +121,10 @@ private: AST::PathInExpression parse_path_in_expression (); AST::PathExprSegment parse_path_expr_segment (); AST::QualifiedPathInExpression - parse_qualified_path_in_expression (bool pratt_parse = false); - AST::QualifiedPathType parse_qualified_path_type (bool pratt_parse = false); + parse_qualified_path_in_expression (const_TokenPtr pratt_parsed_token + = nullptr); + AST::QualifiedPathType + parse_qualified_path_type (const_TokenPtr pratt_parsed_token = nullptr); AST::QualifiedPathInType parse_qualified_path_in_type (); // Token tree or macro related @@ -470,32 +472,32 @@ private: parse_expr_with_block (AST::AttrVec outer_attrs); std::unique_ptr parse_expr_without_block (AST::AttrVec outer_attrs = AST::AttrVec ()); - std::unique_ptr parse_block_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); - std::unique_ptr parse_if_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); - std::unique_ptr parse_if_let_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); + std::unique_ptr + parse_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); + std::unique_ptr + parse_if_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); + std::unique_ptr + parse_if_let_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_loop_expr (AST::AttrVec outer_attrs = AST::AttrVec (), AST::LoopLabel label = AST::LoopLabel::error (), - bool pratt_parse = false); + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_while_loop_expr (AST::AttrVec outer_attrs = AST::AttrVec (), AST::LoopLabel label = AST::LoopLabel::error (), - bool pratt_parse = false); + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_while_let_loop_expr (AST::AttrVec outer_attrs = AST::AttrVec (), AST::LoopLabel label = AST::LoopLabel::error ()); std::unique_ptr parse_for_loop_expr (AST::AttrVec outer_attrs = AST::AttrVec (), AST::LoopLabel label = AST::LoopLabel::error ()); - std::unique_ptr parse_match_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); + std::unique_ptr + parse_match_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); AST::MatchArm parse_match_arm (); std::vector > parse_match_arm_patterns (TokenId end_token_id); @@ -511,24 +513,24 @@ private: AST::ClosureParam parse_closure_param (); std::unique_ptr parse_literal_expr (AST::AttrVec outer_attrs = AST::AttrVec ()); - std::unique_ptr parse_return_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); - std::unique_ptr parse_break_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); + std::unique_ptr + parse_return_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); + std::unique_ptr + parse_break_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_continue_expr (AST::AttrVec outer_attrs = AST::AttrVec (), - bool pratt_parse = false); + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_unsafe_block_expr (AST::AttrVec outer_attrs = AST::AttrVec (), - bool pratt_parse = false); - std::unique_ptr parse_array_expr (AST::AttrVec outer_attrs - = AST::AttrVec (), - bool pratt_parse = false); + const_TokenPtr pratt_parsed_token = nullptr); + std::unique_ptr + parse_array_expr (AST::AttrVec outer_attrs = AST::AttrVec (), + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_grouped_or_tuple_expr (AST::AttrVec outer_attrs = AST::AttrVec (), - bool pratt_parse = false); + const_TokenPtr pratt_parsed_token = nullptr); std::unique_ptr parse_struct_expr_field (); bool will_be_expr_with_block (); -- 2.32.0