aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--grammar.ron3
-rw-r--r--src/lexer/mod.rs2
-rw-r--r--src/parser/event_parser/grammar.rs26
-rw-r--r--src/parser/event_parser/parser.rs8
-rw-r--r--src/syntax_kinds.rs8
-rw-r--r--tests/data/lexer/0005_symbols.txt2
-rw-r--r--tests/data/parser/ok/0006_inner_attributes.rs10
-rw-r--r--tests/data/parser/ok/0006_inner_attributes.txt135
-rw-r--r--validation.md5
9 files changed, 188 insertions, 11 deletions
diff --git a/grammar.ron b/grammar.ron
index 9beb7f7aa..39cb0a543 100644
--- a/grammar.ron
+++ b/grammar.ron
@@ -39,7 +39,7 @@ Grammar(
39 "EQEQ", 39 "EQEQ",
40 "FAT_ARROW", 40 "FAT_ARROW",
41 "NEQ", 41 "NEQ",
42 "NOT", 42 "EXCL",
43 "LIFETIME", 43 "LIFETIME",
44 "CHAR", 44 "CHAR",
45 "BYTE", 45 "BYTE",
@@ -65,5 +65,6 @@ Grammar(
65 "STRUCT_ITEM", 65 "STRUCT_ITEM",
66 "STRUCT_FIELD", 66 "STRUCT_FIELD",
67 "FN_ITEM", 67 "FN_ITEM",
68 "ATTR",
68 ] 69 ]
69) \ No newline at end of file 70) \ No newline at end of file
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index bc5344b5f..842059a42 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -129,7 +129,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
129 ptr.bump(); 129 ptr.bump();
130 NEQ 130 NEQ
131 } 131 }
132 _ => NOT, 132 _ => EXCL,
133 }, 133 },
134 '-' => return if ptr.next_is('>') { 134 '-' => return if ptr.next_is('>') {
135 ptr.bump(); 135 ptr.bump();
diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs
index 09b2f02b2..d657ee1cd 100644
--- a/src/parser/event_parser/grammar.rs
+++ b/src/parser/event_parser/grammar.rs
@@ -17,10 +17,6 @@ pub fn file(p: &mut Parser) {
17 }) 17 })
18} 18}
19 19
20fn inner_attributes(_: &mut Parser) {
21 //TODO
22}
23
24fn item_first(p: &Parser) -> bool { 20fn item_first(p: &Parser) -> bool {
25 let current = match p.current() { 21 let current = match p.current() {
26 Some(c) => c, 22 Some(c) => c,
@@ -58,6 +54,20 @@ fn fn_item(p: &mut Parser) {
58 54
59// Paths, types, attributes, and stuff // 55// Paths, types, attributes, and stuff //
60 56
57fn inner_attributes(p: &mut Parser) {
58 many(p, inner_attribute)
59}
60
61fn inner_attribute(p: &mut Parser) -> bool {
62 if !(p.lookahead(&[EXCL, POUND])) {
63 return false;
64 }
65 node(p, ATTR, |p| {
66 p.bump_n(2);
67 });
68 true
69}
70
61fn outer_attributes(_: &mut Parser) { 71fn outer_attributes(_: &mut Parser) {
62} 72}
63 73
@@ -143,9 +153,15 @@ impl<'p> Parser<'p> {
143 } 153 }
144 } 154 }
145 155
146 pub(crate) fn optional(&mut self, kind: SyntaxKind) { 156 fn optional(&mut self, kind: SyntaxKind) {
147 if self.current_is(kind) { 157 if self.current_is(kind) {
148 self.bump(); 158 self.bump();
149 } 159 }
150 } 160 }
161
162 fn bump_n(&mut self, n: u8) {
163 for _ in 0..n {
164 self.bump();
165 }
166 }
151} \ No newline at end of file 167} \ No newline at end of file
diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs
index eafa03521..f8330af4e 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/event_parser/parser.rs
@@ -86,6 +86,14 @@ impl<'t> Parser<'t> {
86 Some(kind) 86 Some(kind)
87 } 87 }
88 88
89 pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool {
90 if self.non_ws_tokens[self.pos..].len() < kinds.len() {
91 return false
92 }
93 kinds.iter().zip(self.non_ws_tokens[self.pos..].iter())
94 .all(|(&k1, &(idx, _))| k1 == self.raw_tokens[idx].kind)
95 }
96
89 pub(crate) fn curly_block<F: FnOnce(&mut Parser)>(&mut self, f: F) -> bool { 97 pub(crate) fn curly_block<F: FnOnce(&mut Parser)>(&mut self, f: F) -> bool {
90 let old_level = self.curly_level; 98 let old_level = self.curly_level;
91 let old_limit = self.curly_limit; 99 let old_limit = self.curly_limit;
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index 6099cd6e0..67c840a3e 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -38,7 +38,7 @@ pub const EQ: SyntaxKind = SyntaxKind(33);
38pub const EQEQ: SyntaxKind = SyntaxKind(34); 38pub const EQEQ: SyntaxKind = SyntaxKind(34);
39pub const FAT_ARROW: SyntaxKind = SyntaxKind(35); 39pub const FAT_ARROW: SyntaxKind = SyntaxKind(35);
40pub const NEQ: SyntaxKind = SyntaxKind(36); 40pub const NEQ: SyntaxKind = SyntaxKind(36);
41pub const NOT: SyntaxKind = SyntaxKind(37); 41pub const EXCL: SyntaxKind = SyntaxKind(37);
42pub const LIFETIME: SyntaxKind = SyntaxKind(38); 42pub const LIFETIME: SyntaxKind = SyntaxKind(38);
43pub const CHAR: SyntaxKind = SyntaxKind(39); 43pub const CHAR: SyntaxKind = SyntaxKind(39);
44pub const BYTE: SyntaxKind = SyntaxKind(40); 44pub const BYTE: SyntaxKind = SyntaxKind(40);
@@ -62,8 +62,9 @@ pub const FILE: SyntaxKind = SyntaxKind(57);
62pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58); 62pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58);
63pub const STRUCT_FIELD: SyntaxKind = SyntaxKind(59); 63pub const STRUCT_FIELD: SyntaxKind = SyntaxKind(59);
64pub const FN_ITEM: SyntaxKind = SyntaxKind(60); 64pub const FN_ITEM: SyntaxKind = SyntaxKind(60);
65pub const ATTR: SyntaxKind = SyntaxKind(61);
65 66
66static INFOS: [SyntaxInfo; 61] = [ 67static INFOS: [SyntaxInfo; 62] = [
67 SyntaxInfo { name: "USE_KW" }, 68 SyntaxInfo { name: "USE_KW" },
68 SyntaxInfo { name: "FN_KW" }, 69 SyntaxInfo { name: "FN_KW" },
69 SyntaxInfo { name: "STRUCT_KW" }, 70 SyntaxInfo { name: "STRUCT_KW" },
@@ -101,7 +102,7 @@ static INFOS: [SyntaxInfo; 61] = [
101 SyntaxInfo { name: "EQEQ" }, 102 SyntaxInfo { name: "EQEQ" },
102 SyntaxInfo { name: "FAT_ARROW" }, 103 SyntaxInfo { name: "FAT_ARROW" },
103 SyntaxInfo { name: "NEQ" }, 104 SyntaxInfo { name: "NEQ" },
104 SyntaxInfo { name: "NOT" }, 105 SyntaxInfo { name: "EXCL" },
105 SyntaxInfo { name: "LIFETIME" }, 106 SyntaxInfo { name: "LIFETIME" },
106 SyntaxInfo { name: "CHAR" }, 107 SyntaxInfo { name: "CHAR" },
107 SyntaxInfo { name: "BYTE" }, 108 SyntaxInfo { name: "BYTE" },
@@ -125,6 +126,7 @@ static INFOS: [SyntaxInfo; 61] = [
125 SyntaxInfo { name: "STRUCT_ITEM" }, 126 SyntaxInfo { name: "STRUCT_ITEM" },
126 SyntaxInfo { name: "STRUCT_FIELD" }, 127 SyntaxInfo { name: "STRUCT_FIELD" },
127 SyntaxInfo { name: "FN_ITEM" }, 128 SyntaxInfo { name: "FN_ITEM" },
129 SyntaxInfo { name: "ATTR" },
128]; 130];
129 131
130pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { 132pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
diff --git a/tests/data/lexer/0005_symbols.txt b/tests/data/lexer/0005_symbols.txt
index ebb41accd..0f99c24cd 100644
--- a/tests/data/lexer/0005_symbols.txt
+++ b/tests/data/lexer/0005_symbols.txt
@@ -58,7 +58,7 @@ EQ 1 "="
58WHITESPACE 1 " " 58WHITESPACE 1 " "
59FAT_ARROW 2 "=>" 59FAT_ARROW 2 "=>"
60WHITESPACE 1 "\n" 60WHITESPACE 1 "\n"
61NOT 1 "!" 61EXCL 1 "!"
62WHITESPACE 1 " " 62WHITESPACE 1 " "
63NEQ 2 "!=" 63NEQ 2 "!="
64WHITESPACE 1 "\n" 64WHITESPACE 1 "\n"
diff --git a/tests/data/parser/ok/0006_inner_attributes.rs b/tests/data/parser/ok/0006_inner_attributes.rs
new file mode 100644
index 000000000..e81f8b1e8
--- /dev/null
+++ b/tests/data/parser/ok/0006_inner_attributes.rs
@@ -0,0 +1,10 @@
1#![attr]
2#![attr(true)]
3#![attr(ident)]
4#![attr(ident, 100, true, "true", ident = 100, ident = "hello", ident(100))]
5#![attr(100)]
6#![attr(enabled = true)]
7#![enabled(true)]
8#![attr("hello")]
9#![repr(C, align = 4)]
10#![repr(C, align(4))] \ No newline at end of file
diff --git a/tests/data/parser/ok/0006_inner_attributes.txt b/tests/data/parser/ok/0006_inner_attributes.txt
new file mode 100644
index 000000000..c837979d1
--- /dev/null
+++ b/tests/data/parser/ok/0006_inner_attributes.txt
@@ -0,0 +1,135 @@
1FILE@[0; 236)
2 ERROR@[0; 236)
3 err: `expected item`
4 POUND@[0; 1)
5 EXCL@[1; 2)
6 L_BRACK@[2; 3)
7 IDENT@[3; 7)
8 R_BRACK@[7; 8)
9 WHITESPACE@[8; 9)
10 POUND@[9; 10)
11 EXCL@[10; 11)
12 L_BRACK@[11; 12)
13 IDENT@[12; 16)
14 L_PAREN@[16; 17)
15 IDENT@[17; 21)
16 R_PAREN@[21; 22)
17 R_BRACK@[22; 23)
18 WHITESPACE@[23; 24)
19 POUND@[24; 25)
20 EXCL@[25; 26)
21 L_BRACK@[26; 27)
22 IDENT@[27; 31)
23 L_PAREN@[31; 32)
24 IDENT@[32; 37)
25 R_PAREN@[37; 38)
26 R_BRACK@[38; 39)
27 WHITESPACE@[39; 40)
28 POUND@[40; 41)
29 EXCL@[41; 42)
30 L_BRACK@[42; 43)
31 IDENT@[43; 47)
32 L_PAREN@[47; 48)
33 IDENT@[48; 53)
34 COMMA@[53; 54)
35 WHITESPACE@[54; 55)
36 INT_NUMBER@[55; 58)
37 COMMA@[58; 59)
38 WHITESPACE@[59; 60)
39 IDENT@[60; 64)
40 COMMA@[64; 65)
41 WHITESPACE@[65; 66)
42 STRING@[66; 72)
43 COMMA@[72; 73)
44 WHITESPACE@[73; 74)
45 IDENT@[74; 79)
46 WHITESPACE@[79; 80)
47 EQ@[80; 81)
48 WHITESPACE@[81; 82)
49 INT_NUMBER@[82; 85)
50 COMMA@[85; 86)
51 WHITESPACE@[86; 87)
52 IDENT@[87; 92)
53 WHITESPACE@[92; 93)
54 EQ@[93; 94)
55 WHITESPACE@[94; 95)
56 STRING@[95; 102)
57 COMMA@[102; 103)
58 WHITESPACE@[103; 104)
59 IDENT@[104; 109)
60 L_PAREN@[109; 110)
61 INT_NUMBER@[110; 113)
62 R_PAREN@[113; 114)
63 R_PAREN@[114; 115)
64 R_BRACK@[115; 116)
65 WHITESPACE@[116; 117)
66 POUND@[117; 118)
67 EXCL@[118; 119)
68 L_BRACK@[119; 120)
69 IDENT@[120; 124)
70 L_PAREN@[124; 125)
71 INT_NUMBER@[125; 128)
72 R_PAREN@[128; 129)
73 R_BRACK@[129; 130)
74 WHITESPACE@[130; 131)
75 POUND@[131; 132)
76 EXCL@[132; 133)
77 L_BRACK@[133; 134)
78 IDENT@[134; 138)
79 L_PAREN@[138; 139)
80 IDENT@[139; 146)
81 WHITESPACE@[146; 147)
82 EQ@[147; 148)
83 WHITESPACE@[148; 149)
84 IDENT@[149; 153)
85 R_PAREN@[153; 154)
86 R_BRACK@[154; 155)
87 WHITESPACE@[155; 156)
88 POUND@[156; 157)
89 EXCL@[157; 158)
90 L_BRACK@[158; 159)
91 IDENT@[159; 166)
92 L_PAREN@[166; 167)
93 IDENT@[167; 171)
94 R_PAREN@[171; 172)
95 R_BRACK@[172; 173)
96 WHITESPACE@[173; 174)
97 POUND@[174; 175)
98 EXCL@[175; 176)
99 L_BRACK@[176; 177)
100 IDENT@[177; 181)
101 L_PAREN@[181; 182)
102 STRING@[182; 189)
103 R_PAREN@[189; 190)
104 R_BRACK@[190; 191)
105 WHITESPACE@[191; 192)
106 POUND@[192; 193)
107 EXCL@[193; 194)
108 L_BRACK@[194; 195)
109 IDENT@[195; 199)
110 L_PAREN@[199; 200)
111 IDENT@[200; 201)
112 COMMA@[201; 202)
113 WHITESPACE@[202; 203)
114 IDENT@[203; 208)
115 WHITESPACE@[208; 209)
116 EQ@[209; 210)
117 WHITESPACE@[210; 211)
118 INT_NUMBER@[211; 212)
119 R_PAREN@[212; 213)
120 R_BRACK@[213; 214)
121 WHITESPACE@[214; 215)
122 POUND@[215; 216)
123 EXCL@[216; 217)
124 L_BRACK@[217; 218)
125 IDENT@[218; 222)
126 L_PAREN@[222; 223)
127 IDENT@[223; 224)
128 COMMA@[224; 225)
129 WHITESPACE@[225; 226)
130 IDENT@[226; 231)
131 L_PAREN@[231; 232)
132 INT_NUMBER@[232; 233)
133 R_PAREN@[233; 234)
134 R_PAREN@[234; 235)
135 R_BRACK@[235; 236)
diff --git a/validation.md b/validation.md
index e72de1ea2..2739bfcdd 100644
--- a/validation.md
+++ b/validation.md
@@ -1,5 +1,6 @@
1Fixmes: 1Fixmes:
2 2
3Lexer:
3* Fix `is_whitespace`, add more tests 4* Fix `is_whitespace`, add more tests
4* Add more thorough tests for idents for XID_Start & XID_Continue 5* Add more thorough tests for idents for XID_Start & XID_Continue
5* Validate that float and integer literals use digits only of the appropriate 6* Validate that float and integer literals use digits only of the appropriate
@@ -7,3 +8,7 @@ Fixmes:
7* Validation for unclosed char literal 8* Validation for unclosed char literal
8* Strings are completely wrong: more tests and comparison with libsyntax. 9* Strings are completely wrong: more tests and comparison with libsyntax.
9* Comment lexing is completely wrong 10* Comment lexing is completely wrong
11
12Parser:
13* Figure out what is the expected state of attribute grammar.
14 Token trees or something more structured? Token trees would be unfortunate: no extend selection =/ \ No newline at end of file