Merge #996

996: Allow attributes on top level expressions r=matklad a=pcpthm

This PR modifies parser to allow outer attributes on top level expression. Here, top level expression means either
- Expression statement e.g. `foo();`
- Last expression in a block without semicolon `bar()` in `{ foo(); bar() }`.

Except for binary operation expressions and `if` expressions, which are errors (feature gated) in rustc.
Attributes on inner expressions like `foo(#[a] 1)` are not implemented.

I first tried to implement this by passing `Maker` to expression parsers. However, this implementation couldn't parse `#[attr] foo()` correctly as `CallExpr(Attr(..), PathExpr(..), ArgList(..))` and instead parsed incorrectly as `CallExpr(PathExpr(Attr(..), ..), ArgList(..))` due to the way left recursion is handled.
In the end, I introduce `undo_completion` method. Which is not the suggested approach, but it seems not very bad.

Fix #759.


Co-authored-by: pcpthm <[email protected]>

10 files changed, 305 insertions, 42 deletions

diff --git a/crates/ra_parser/src/event.rs b/crates/ra_parser/src/event.rs
index 6361d5d86..c1773e8e0 100644
--- a/crates/ra_parser/src/event.rs
+++ b/crates/ra_parser/src/event.rs
@@ -105,7 +105,9 @@ pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
                     // append `A`'s forward_parent `B`
                     fp = match mem::replace(&mut events[idx], Event::tombstone()) {
                         Event::Start { kind, forward_parent } => {
-                            forward_parents.push(kind);
+                            if kind != TOMBSTONE {
+                                forward_parents.push(kind);
+                            }
                             forward_parent
                         }
                         _ => unreachable!(),
diff --git a/crates/ra_parser/src/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs
index 83812e938..73e1acd5a 100644
--- a/crates/ra_parser/src/grammar/expressions.rs
+++ b/crates/ra_parser/src/grammar/expressions.rs
@@ -8,10 +8,10 @@ const EXPR_FIRST: TokenSet = LHS_FIRST;
 
 pub(super) fn expr(p: &mut Parser) -> BlockLike {
     let r = Restrictions { forbid_structs: false, prefer_stmt: false };
-    expr_bp(p, r, 1)
+    expr_bp(p, r, 1).1
 }
 
-pub(super) fn expr_stmt(p: &mut Parser) -> BlockLike {
+pub(super) fn expr_stmt(p: &mut Parser) -> (Option<CompletedMarker>, BlockLike) {
     let r = Restrictions { forbid_structs: false, prefer_stmt: true };
     expr_bp(p, r, 1)
 }
@@ -38,6 +38,13 @@ pub(crate) fn block(p: &mut Parser) {
     m.complete(p, BLOCK);
 }
 
+fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool {
+    match kind {
+        BIN_EXPR | RANGE_EXPR | IF_EXPR => false,
+        _ => true,
+    }
+}
+
 pub(crate) fn expr_block_contents(p: &mut Parser) {
     // This is checked by a validator
     attributes::inner_attributes(p);
@@ -55,6 +62,13 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
         // test block_items
         // fn a() { fn b() {} }
         let m = p.start();
+        // test attr_on_expr_stmt
+        // fn foo() {
+        //     #[A] foo();
+        //     #[B] bar!{}
+        //     #[C] #[D] {}
+        //     #[D] return ();
+        // }
         let has_attrs = p.at(POUND);
         attributes::outer_attributes(p);
         if p.at(LET_KW) {
@@ -67,35 +81,51 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
             Err(m) => m,
         };
 
-        if has_attrs {
-            m.abandon(p);
-            p.error("expected a let statement or an item after attributes in block");
-        } else {
-            let is_blocklike = expressions::expr_stmt(p) == BlockLike::Block;
-            if p.at(R_CURLY) {
+        let (cm, blocklike) = expr_stmt(p);
+        let kind = cm.as_ref().map(|cm| cm.kind()).unwrap_or(ERROR);
+
+        if has_attrs && !is_expr_stmt_attr_allowed(kind) {
+            // test_err attr_on_expr_not_allowed
+            // fn foo() {
+            //    #[A] 1 + 2;
+            //    #[B] if true {};
+            // }
+            p.error(format!("attributes are not allowed on {:?}", kind));
+        }
+
+        if p.at(R_CURLY) {
+            // test attr_on_last_expr_in_block
+            // fn foo() {
+            //     { #[A] bar!()? }
+            //     #[B] &()
+            // }
+            if let Some(cm) = cm {
+                cm.undo_completion(p).abandon(p);
+                m.complete(p, kind);
+            } else {
                 m.abandon(p);
+            }
+        } else {
+            // test no_semi_after_block
+            // fn foo() {
+            //     if true {}
+            //     loop {}
+            //     match () {}
+            //     while true {}
+            //     for _ in () {}
+            //     {}
+            //     {}
+            //     macro_rules! test {
+            //          () => {}
+            //     }
+            //     test!{}
+            // }
+            if blocklike.is_block() {
+                p.eat(SEMI);
             } else {
-                // test no_semi_after_block
-                // fn foo() {
-                //     if true {}
-                //     loop {}
-                //     match () {}
-                //     while true {}
-                //     for _ in () {}
-                //     {}
-                //     {}
-                //     macro_rules! test {
-                //          () => {}
-                //     }
-                //     test!{}
-                // }
-                if is_blocklike {
-                    p.eat(SEMI);
-                } else {
-                    p.expect(SEMI);
-                }
-                m.complete(p, EXPR_STMT);
+                p.expect(SEMI);
             }
+            m.complete(p, EXPR_STMT);
         }
     }
 
@@ -176,7 +206,7 @@ fn current_op(p: &Parser) -> (u8, Op) {
 }
 
 // Parses expression with binding power of at least bp.
-fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> BlockLike {
+fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option<CompletedMarker>, BlockLike) {
     let mut lhs = match lhs(p, r) {
         Some((lhs, blocklike)) => {
             // test stmt_bin_expr_ambiguity
@@ -185,11 +215,11 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> BlockLike {
             //     {1} &2;
             // }
             if r.prefer_stmt && blocklike.is_block() {
-                return BlockLike::Block;
+                return (Some(lhs), BlockLike::Block);
             }
             lhs
         }
-        None => return BlockLike::NotBlock,
+        None => return (None, BlockLike::NotBlock),
     };
 
     loop {
@@ -208,7 +238,7 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> BlockLike {
         expr_bp(p, r, op_bp + 1);
         lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR });
     }
-    BlockLike::NotBlock
+    (Some(lhs), BlockLike::NotBlock)
 }
 
 const LHS_FIRST: TokenSet =
diff --git a/crates/ra_parser/src/grammar/expressions/atom.rs b/crates/ra_parser/src/grammar/expressions/atom.rs
index d933288cd..a23977bfb 100644
--- a/crates/ra_parser/src/grammar/expressions/atom.rs
+++ b/crates/ra_parser/src/grammar/expressions/atom.rs
@@ -392,9 +392,9 @@ fn match_arm(p: &mut Parser) -> BlockLike {
         match_guard(p);
     }
     p.expect(FAT_ARROW);
-    let ret = expr_stmt(p);
+    let blocklike = expr_stmt(p).1;
     m.complete(p, MATCH_ARM);
-    ret
+    blocklike
 }
 
 // test match_guard
diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs
index a18458148..3c326452b 100644
--- a/crates/ra_parser/src/parser.rs
+++ b/crates/ra_parser/src/parser.rs
@@ -212,8 +212,9 @@ impl Marker {
             }
             _ => unreachable!(),
         }
+        let finish_pos = p.events.len() as u32;
         p.push_event(Event::Finish);
-        CompletedMarker::new(self.pos, kind)
+        CompletedMarker::new(self.pos, finish_pos, kind)
     }
 
     /// Abandons the syntax tree node. All its children
@@ -230,11 +231,15 @@ impl Marker {
     }
 }
 
-pub(crate) struct CompletedMarker(u32, SyntaxKind);
+pub(crate) struct CompletedMarker {
+    start_pos: u32,
+    finish_pos: u32,
+    kind: SyntaxKind,
+}
 
 impl CompletedMarker {
-    fn new(pos: u32, kind: SyntaxKind) -> Self {
-        CompletedMarker(pos, kind)
+    fn new(start_pos: u32, finish_pos: u32, kind: SyntaxKind) -> Self {
+        CompletedMarker { start_pos, finish_pos, kind }
     }
 
     /// This method allows to create a new node which starts
@@ -251,17 +256,32 @@ impl CompletedMarker {
     /// distance to `NEWSTART` into forward_parent(=2 in this case);
     pub(crate) fn precede(self, p: &mut Parser) -> Marker {
         let new_pos = p.start();
-        let idx = self.0 as usize;
+        let idx = self.start_pos as usize;
         match p.events[idx] {
             Event::Start { ref mut forward_parent, .. } => {
-                *forward_parent = Some(new_pos.pos - self.0);
+                *forward_parent = Some(new_pos.pos - self.start_pos);
             }
             _ => unreachable!(),
         }
         new_pos
     }
 
+    /// Undo this completion and turns into a `Marker`
+    pub(crate) fn undo_completion(self, p: &mut Parser) -> Marker {
+        let start_idx = self.start_pos as usize;
+        let finish_idx = self.finish_pos as usize;
+        match p.events[start_idx] {
+            Event::Start { ref mut kind, forward_parent: None } => *kind = TOMBSTONE,
+            _ => unreachable!(),
+        }
+        match p.events[finish_idx] {
+            ref mut slot @ Event::Finish => *slot = Event::tombstone(),
+            _ => unreachable!(),
+        }
+        Marker::new(self.start_pos)
+    }
+
     pub(crate) fn kind(&self) -> SyntaxKind {
-        self.1
+        self.kind
     }
 }
diff --git a/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.rs b/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.rs
new file mode 100644
index 000000000..d725a07ce
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.rs
@@ -0,0 +1,4 @@
+fn foo() {
+   #[A] 1 + 2;
+   #[B] if true {};
+}
diff --git a/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.txt b/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.txt
new file mode 100644
index 000000000..fdea1ec1e
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/err/0009_attr_on_expr_not_allowed.txt
@@ -0,0 +1,55 @@
+SOURCE_FILE@[0; 48)
+  FN_DEF@[0; 47)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 47)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 14)
+      EXPR_STMT@[14; 25)
+        ATTR@[14; 18)
+          POUND@[14; 15)
+          TOKEN_TREE@[15; 18)
+            L_BRACK@[15; 16)
+            IDENT@[16; 17) "A"
+            R_BRACK@[17; 18)
+        WHITESPACE@[18; 19)
+        BIN_EXPR@[19; 24)
+          LITERAL@[19; 20)
+            INT_NUMBER@[19; 20) "1"
+          WHITESPACE@[20; 21)
+          PLUS@[21; 22)
+          WHITESPACE@[22; 23)
+          LITERAL@[23; 24)
+            INT_NUMBER@[23; 24) "2"
+            err: `attributes are not allowed on BIN_EXPR`
+        SEMI@[24; 25)
+      WHITESPACE@[25; 29)
+      EXPR_STMT@[29; 45)
+        ATTR@[29; 33)
+          POUND@[29; 30)
+          TOKEN_TREE@[30; 33)
+            L_BRACK@[30; 31)
+            IDENT@[31; 32) "B"
+            R_BRACK@[32; 33)
+        WHITESPACE@[33; 34)
+        IF_EXPR@[34; 44)
+          IF_KW@[34; 36)
+          WHITESPACE@[36; 37)
+          CONDITION@[37; 41)
+            LITERAL@[37; 41)
+              TRUE_KW@[37; 41)
+          WHITESPACE@[41; 42)
+          BLOCK@[42; 44)
+            L_CURLY@[42; 43)
+            R_CURLY@[43; 44)
+            err: `attributes are not allowed on IF_EXPR`
+        SEMI@[44; 45)
+      WHITESPACE@[45; 46)
+      R_CURLY@[46; 47)
+  WHITESPACE@[47; 48)
diff --git a/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.rs b/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.rs
new file mode 100644
index 000000000..b28c078f9
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.rs
@@ -0,0 +1,6 @@
+fn foo() {
+    #[A] foo();
+    #[B] bar!{}
+    #[C] #[D] {}
+    #[D] return ();
+}
diff --git a/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.txt b/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.txt
new file mode 100644
index 000000000..7cd525cc7
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/ok/0126_attr_on_expr_stmt.txt
@@ -0,0 +1,88 @@
+SOURCE_FILE@[0; 82)
+  FN_DEF@[0; 81)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 81)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 15)
+      EXPR_STMT@[15; 26)
+        ATTR@[15; 19)
+          POUND@[15; 16)
+          TOKEN_TREE@[16; 19)
+            L_BRACK@[16; 17)
+            IDENT@[17; 18) "A"
+            R_BRACK@[18; 19)
+        WHITESPACE@[19; 20)
+        CALL_EXPR@[20; 25)
+          PATH_EXPR@[20; 23)
+            PATH@[20; 23)
+              PATH_SEGMENT@[20; 23)
+                NAME_REF@[20; 23)
+                  IDENT@[20; 23) "foo"
+          ARG_LIST@[23; 25)
+            L_PAREN@[23; 24)
+            R_PAREN@[24; 25)
+        SEMI@[25; 26)
+      WHITESPACE@[26; 31)
+      EXPR_STMT@[31; 42)
+        ATTR@[31; 35)
+          POUND@[31; 32)
+          TOKEN_TREE@[32; 35)
+            L_BRACK@[32; 33)
+            IDENT@[33; 34) "B"
+            R_BRACK@[34; 35)
+        WHITESPACE@[35; 36)
+        MACRO_CALL@[36; 42)
+          PATH@[36; 39)
+            PATH_SEGMENT@[36; 39)
+              NAME_REF@[36; 39)
+                IDENT@[36; 39) "bar"
+          EXCL@[39; 40)
+          TOKEN_TREE@[40; 42)
+            L_CURLY@[40; 41)
+            R_CURLY@[41; 42)
+      WHITESPACE@[42; 47)
+      EXPR_STMT@[47; 59)
+        ATTR@[47; 51)
+          POUND@[47; 48)
+          TOKEN_TREE@[48; 51)
+            L_BRACK@[48; 49)
+            IDENT@[49; 50) "C"
+            R_BRACK@[50; 51)
+        WHITESPACE@[51; 52)
+        ATTR@[52; 56)
+          POUND@[52; 53)
+          TOKEN_TREE@[53; 56)
+            L_BRACK@[53; 54)
+            IDENT@[54; 55) "D"
+            R_BRACK@[55; 56)
+        WHITESPACE@[56; 57)
+        BLOCK_EXPR@[57; 59)
+          BLOCK@[57; 59)
+            L_CURLY@[57; 58)
+            R_CURLY@[58; 59)
+      WHITESPACE@[59; 64)
+      EXPR_STMT@[64; 79)
+        ATTR@[64; 68)
+          POUND@[64; 65)
+          TOKEN_TREE@[65; 68)
+            L_BRACK@[65; 66)
+            IDENT@[66; 67) "D"
+            R_BRACK@[67; 68)
+        WHITESPACE@[68; 69)
+        RETURN_EXPR@[69; 78)
+          RETURN_KW@[69; 75)
+          WHITESPACE@[75; 76)
+          TUPLE_EXPR@[76; 78)
+            L_PAREN@[76; 77)
+            R_PAREN@[77; 78)
+        SEMI@[78; 79)
+      WHITESPACE@[79; 80)
+      R_CURLY@[80; 81)
+  WHITESPACE@[81; 82)
diff --git a/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.rs b/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.rs
new file mode 100644
index 000000000..9c5c8eb36
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.rs
@@ -0,0 +1,4 @@
+fn foo() {
+    { #[A] bar!()? }
+    #[B] &()
+}
diff --git a/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.txt b/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.txt
new file mode 100644
index 000000000..4af64559c
--- /dev/null
+++ b/crates/ra_syntax/tests/data/parser/inline/ok/0127_attr_on_last_expr_in_block.txt
@@ -0,0 +1,54 @@
+SOURCE_FILE@[0; 47)
+  FN_DEF@[0; 46)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 46)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 15)
+      EXPR_STMT@[15; 31)
+        BLOCK_EXPR@[15; 31)
+          BLOCK@[15; 31)
+            L_CURLY@[15; 16)
+            WHITESPACE@[16; 17)
+            TRY_EXPR@[17; 29)
+              ATTR@[17; 21)
+                POUND@[17; 18)
+                TOKEN_TREE@[18; 21)
+                  L_BRACK@[18; 19)
+                  IDENT@[19; 20) "A"
+                  R_BRACK@[20; 21)
+              WHITESPACE@[21; 22)
+              MACRO_CALL@[22; 28)
+                PATH@[22; 25)
+                  PATH_SEGMENT@[22; 25)
+                    NAME_REF@[22; 25)
+                      IDENT@[22; 25) "bar"
+                EXCL@[25; 26)
+                TOKEN_TREE@[26; 28)
+                  L_PAREN@[26; 27)
+                  R_PAREN@[27; 28)
+              QUESTION@[28; 29)
+            WHITESPACE@[29; 30)
+            R_CURLY@[30; 31)
+      WHITESPACE@[31; 36)
+      REF_EXPR@[36; 44)
+        ATTR@[36; 40)
+          POUND@[36; 37)
+          TOKEN_TREE@[37; 40)
+            L_BRACK@[37; 38)
+            IDENT@[38; 39) "B"
+            R_BRACK@[39; 40)
+        WHITESPACE@[40; 41)
+        AMP@[41; 42)
+        TUPLE_EXPR@[42; 44)
+          L_PAREN@[42; 43)
+          R_PAREN@[43; 44)
+      WHITESPACE@[44; 45)
+      R_CURLY@[45; 46)
+  WHITESPACE@[46; 47)