aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lexer.rs10
-rw-r--r--src/lib.rs15
-rw-r--r--src/syntax_kinds.rs16
-rw-r--r--src/text.rs32
-rw-r--r--src/tree.rs31
-rw-r--r--tests/lexer.rs24
6 files changed, 117 insertions, 11 deletions
diff --git a/src/lexer.rs b/src/lexer.rs
new file mode 100644
index 000000000..cda9fe2b2
--- /dev/null
+++ b/src/lexer.rs
@@ -0,0 +1,10 @@
1use {Token, TextUnit};
2use syntax_kinds::*;
3
4pub fn next_token(text: &str) -> Token {
5 let c = text.chars().next().unwrap();
6 Token {
7 kind: IDENT,
8 len: TextUnit::len_of_char(c),
9 }
10} \ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index 31e1bb209..4385c0325 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,7 +1,8 @@
1#[cfg(test)] 1mod text;
2mod tests { 2mod tree;
3 #[test] 3mod lexer;
4 fn it_works() { 4
5 assert_eq!(2 + 2, 4); 5pub mod syntax_kinds;
6 } 6pub use text::TextUnit;
7} 7pub use tree::{SyntaxKind, Token};
8pub use lexer::next_token;
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
new file mode 100644
index 000000000..18574b7f5
--- /dev/null
+++ b/src/syntax_kinds.rs
@@ -0,0 +1,16 @@
1use tree::{SyntaxKind, SyntaxInfo};
2
3pub const IDENT: SyntaxKind = SyntaxKind(1);
4pub const WHITESPACE: SyntaxKind = SyntaxKind(2);
5
6
7static IDENT_INFO: SyntaxInfo = SyntaxInfo {
8 name: "IDENT",
9};
10
11pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
12 match kind {
13 IDENT => &IDENT_INFO,
14 _ => unreachable!(),
15 }
16} \ No newline at end of file
diff --git a/src/text.rs b/src/text.rs
new file mode 100644
index 000000000..5297275ed
--- /dev/null
+++ b/src/text.rs
@@ -0,0 +1,32 @@
1use std::fmt;
2
3#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
4pub struct TextUnit(u32);
5
6impl TextUnit {
7 pub fn len_of_char(c: char) -> TextUnit {
8 TextUnit(c.len_utf8() as u32)
9 }
10
11 pub fn new(val: u32) -> TextUnit {
12 TextUnit(val)
13 }
14}
15
16impl fmt::Debug for TextUnit {
17 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
18 <Self as fmt::Display>::fmt(self, f)
19 }
20}
21
22impl fmt::Display for TextUnit {
23 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
24 self.0.fmt(f)
25 }
26}
27
28impl From<TextUnit> for u32 {
29 fn from(tu: TextUnit) -> u32 {
30 tu.0
31 }
32}
diff --git a/src/tree.rs b/src/tree.rs
new file mode 100644
index 000000000..0924f38d0
--- /dev/null
+++ b/src/tree.rs
@@ -0,0 +1,31 @@
1use text::{TextUnit};
2use syntax_kinds::syntax_info;
3
4use std::fmt;
5
6#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
7pub struct SyntaxKind(pub(crate) u32);
8
9impl SyntaxKind {
10 fn info(self) -> &'static SyntaxInfo {
11 syntax_info(self)
12 }
13}
14
15impl fmt::Debug for SyntaxKind {
16 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
17 let name = self.info().name;
18 f.write_str(name)
19 }
20}
21
22
23pub(crate) struct SyntaxInfo {
24 pub name: &'static str,
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
28pub struct Token {
29 pub kind: SyntaxKind,
30 pub len: TextUnit,
31} \ No newline at end of file
diff --git a/tests/lexer.rs b/tests/lexer.rs
index de76f0a15..a27e7c395 100644
--- a/tests/lexer.rs
+++ b/tests/lexer.rs
@@ -1,9 +1,13 @@
1extern crate file; 1extern crate file;
2#[macro_use(assert_diff)] 2#[macro_use(assert_diff)]
3extern crate difference; 3extern crate difference;
4extern crate libsyntax2;
4 5
5use std::path::{PathBuf, Path}; 6use std::path::{PathBuf, Path};
6use std::fs::read_dir; 7use std::fs::read_dir;
8use std::fmt::Write;
9
10use libsyntax2::{Token, next_token};
7 11
8#[test] 12#[test]
9fn lexer_tests() { 13fn lexer_tests() {
@@ -46,10 +50,22 @@ fn lexer_test_case(path: &Path) {
46 ) 50 )
47} 51}
48 52
49fn tokenize(text: &str) -> Vec<()> { 53fn tokenize(text: &str) -> Vec<Token> {
50 Vec::new() 54 let mut text = text;
55 let mut acc = Vec::new();
56 while !text.is_empty() {
57 let token = next_token(text);
58 acc.push(token);
59 let len: u32 = token.len.into();
60 text = &text[len as usize..];
61 }
62 acc
51} 63}
52 64
53fn dump_tokens(tokens: &[()]) -> String { 65fn dump_tokens(tokens: &[Token]) -> String {
54 "IDENT 5\nKEYWORD 1\nIDENT 5\n".to_string() 66 let mut acc = String::new();
67 for token in tokens {
68 write!(acc, "{:?} {}\n", token.kind, token.len).unwrap()
69 }
70 acc
55} \ No newline at end of file 71} \ No newline at end of file