aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2018-08-14 13:03:27 +0100
committerAleksey Kladov <[email protected]>2018-08-14 13:03:27 +0100
commit109658332a75ca91d6dc2bf573e0ab77fa5619ca (patch)
tree9227430826c75e184577623c25e54489b1033865 /crates
parent199e3b73c712a74e36bbb75eebf9e9418f1b1341 (diff)
Support raw strings in lexer
Diffstat (limited to 'crates')
-rw-r--r--crates/libanalysis/src/lib.rs69
-rw-r--r--crates/libsyntax2/src/lexer/strings.rs14
-rw-r--r--crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs1
-rw-r--r--crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt2
4 files changed, 52 insertions, 34 deletions
diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs
index ffd81bef8..fc0b9ee9a 100644
--- a/crates/libanalysis/src/lib.rs
+++ b/crates/libanalysis/src/lib.rs
@@ -20,7 +20,6 @@ use std::{
20 atomic::{AtomicUsize, Ordering::SeqCst}, 20 atomic::{AtomicUsize, Ordering::SeqCst},
21 }, 21 },
22 collections::hash_map::HashMap, 22 collections::hash_map::HashMap,
23 path::{PathBuf, Path},
24 time::Instant, 23 time::Instant,
25}; 24};
26 25
@@ -38,6 +37,7 @@ pub type Result<T> = ::std::result::Result<T, ::failure::Error>;
38const INDEXING_THRESHOLD: usize = 128; 37const INDEXING_THRESHOLD: usize = 128;
39 38
40pub struct WorldState { 39pub struct WorldState {
40 next_file_id: u32,
41 data: Arc<WorldData> 41 data: Arc<WorldData>
42} 42}
43 43
@@ -46,9 +46,13 @@ pub struct World {
46 data: Arc<WorldData>, 46 data: Arc<WorldData>,
47} 47}
48 48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
50pub struct FileId(u32);
51
49impl WorldState { 52impl WorldState {
50 pub fn new() -> WorldState { 53 pub fn new() -> WorldState {
51 WorldState { 54 WorldState {
55 next_file_id: 0,
52 data: Arc::new(WorldData::default()) 56 data: Arc::new(WorldData::default())
53 } 57 }
54 } 58 }
@@ -57,21 +61,27 @@ impl WorldState {
57 World { data: self.data.clone() } 61 World { data: self.data.clone() }
58 } 62 }
59 63
60 pub fn change_file(&mut self, path: PathBuf, text: Option<String>) { 64 pub fn new_file_id(&mut self) -> FileId {
61 self.change_files(::std::iter::once((path, text))); 65 let id = FileId(self.next_file_id);
66 self.next_file_id += 1;
67 id
68 }
69
70 pub fn change_file(&mut self, file_id: FileId, text: Option<String>) {
71 self.change_files(::std::iter::once((file_id, text)));
62 } 72 }
63 73
64 pub fn change_files(&mut self, changes: impl Iterator<Item=(PathBuf, Option<String>)>) { 74 pub fn change_files(&mut self, changes: impl Iterator<Item=(FileId, Option<String>)>) {
65 let data = self.data_mut(); 75 let data = self.data_mut();
66 let mut cnt = 0; 76 let mut cnt = 0;
67 for (path, text) in changes { 77 for (id, text) in changes {
68 cnt += 1; 78 cnt += 1;
69 data.file_map.remove(&path); 79 data.file_map.remove(&id);
70 if let Some(text) = text { 80 if let Some(text) = text {
71 let file_data = FileData::new(text); 81 let file_data = FileData::new(text);
72 data.file_map.insert(path, Arc::new(file_data)); 82 data.file_map.insert(id, Arc::new(file_data));
73 } else { 83 } else {
74 data.file_map.remove(&path); 84 data.file_map.remove(&id);
75 } 85 }
76 } 86 }
77 *data.unindexed.get_mut() += cnt; 87 *data.unindexed.get_mut() += cnt;
@@ -92,37 +102,33 @@ impl WorldState {
92 102
93 103
94impl World { 104impl World {
95 pub fn file_syntax(&self, path: &Path) -> Result<ast::File> { 105 pub fn file_syntax(&self, file_id: FileId) -> Result<ast::File> {
96 let data = self.file_data(path)?; 106 let data = self.file_data(file_id)?;
97 Ok(data.syntax(path).clone()) 107 Ok(data.syntax().clone())
98 } 108 }
99 109
100 pub fn file_line_index(&self, path: &Path) -> Result<LineIndex> { 110 pub fn file_line_index(&self, id: FileId) -> Result<LineIndex> {
101 let data = self.file_data(path)?; 111 let data = self.file_data(id)?;
102 let index = data.lines 112 let index = data.lines
103 .get_or_init(|| { 113 .get_or_init(|| LineIndex::new(&data.text));
104 trace!("calc line index: {}", path.display());
105 LineIndex::new(&data.text)
106 });
107 Ok(index.clone()) 114 Ok(index.clone())
108 } 115 }
109 116
110 pub fn world_symbols<'a>(&'a self, mut query: Query) -> impl Iterator<Item=(&'a Path, &'a FileSymbol)> + 'a { 117 pub fn world_symbols<'a>(&'a self, mut query: Query) -> impl Iterator<Item=(FileId, &'a FileSymbol)> + 'a {
111 self.reindex(); 118 self.reindex();
112 self.data.file_map.iter() 119 self.data.file_map.iter()
113 .flat_map(move |(path, data)| { 120 .flat_map(move |(id, data)| {
114 let symbols = data.symbols(); 121 let symbols = data.symbols();
115 let path: &'a Path = path.as_path(); 122 query.process(symbols).into_iter().map(move |s| (*id, s))
116 query.process(symbols).into_iter().map(move |s| (path, s))
117 }) 123 })
118 } 124 }
119 125
120 pub fn approximately_resolve_symbol<'a>( 126 pub fn approximately_resolve_symbol<'a>(
121 &'a self, 127 &'a self,
122 path: &Path, 128 id: FileId,
123 offset: TextUnit, 129 offset: TextUnit,
124 ) -> Result<Vec<(&'a Path, &'a FileSymbol)>> { 130 ) -> Result<Vec<(FileId, &'a FileSymbol)>> {
125 let file = self.file_syntax(path)?; 131 let file = self.file_syntax(id)?;
126 let syntax = file.syntax(); 132 let syntax = file.syntax();
127 let syntax = syntax.as_ref(); 133 let syntax = syntax.as_ref();
128 let name_ref = 134 let name_ref =
@@ -160,10 +166,10 @@ impl World {
160 } 166 }
161 } 167 }
162 168
163 fn file_data(&self, path: &Path) -> Result<Arc<FileData>> { 169 fn file_data(&self, file_id: FileId) -> Result<Arc<FileData>> {
164 match self.data.file_map.get(path) { 170 match self.data.file_map.get(&file_id) {
165 Some(data) => Ok(data.clone()), 171 Some(data) => Ok(data.clone()),
166 None => bail!("unknown file: {}", path.display()), 172 None => bail!("unknown file: {:?}", file_id),
167 } 173 }
168 } 174 }
169} 175}
@@ -182,7 +188,7 @@ pub const BREAK: SearchResult = Err(Break);
182#[derive(Default, Debug)] 188#[derive(Default, Debug)]
183struct WorldData { 189struct WorldData {
184 unindexed: AtomicUsize, 190 unindexed: AtomicUsize,
185 file_map: HashMap<PathBuf, Arc<FileData>>, 191 file_map: HashMap<FileId, Arc<FileData>>,
186} 192}
187 193
188#[derive(Debug)] 194#[derive(Debug)]
@@ -203,12 +209,9 @@ impl FileData {
203 } 209 }
204 } 210 }
205 211
206 fn syntax(&self, path: &Path) -> &ast::File { 212 fn syntax(&self) -> &ast::File {
207 self.syntax 213 self.syntax
208 .get_or_init(|| { 214 .get_or_init(|| ast::File::parse(&self.text))
209 trace!("parsing: {}", path.display());
210 ast::File::parse(&self.text)
211 })
212 } 215 }
213 216
214 fn syntax_transient(&self) -> ast::File { 217 fn syntax_transient(&self) -> ast::File {
diff --git a/crates/libsyntax2/src/lexer/strings.rs b/crates/libsyntax2/src/lexer/strings.rs
index e3704fbb3..fbae767e5 100644
--- a/crates/libsyntax2/src/lexer/strings.rs
+++ b/crates/libsyntax2/src/lexer/strings.rs
@@ -55,6 +55,11 @@ pub(crate) fn scan_string(ptr: &mut Ptr) {
55} 55}
56 56
57pub(crate) fn scan_raw_string(ptr: &mut Ptr) { 57pub(crate) fn scan_raw_string(ptr: &mut Ptr) {
58 let mut hashes = 0;
59 while ptr.next_is('#') {
60 hashes += 1;
61 ptr.bump();
62 }
58 if !ptr.next_is('"') { 63 if !ptr.next_is('"') {
59 return; 64 return;
60 } 65 }
@@ -62,7 +67,14 @@ pub(crate) fn scan_raw_string(ptr: &mut Ptr) {
62 67
63 while let Some(c) = ptr.bump() { 68 while let Some(c) = ptr.bump() {
64 if c == '"' { 69 if c == '"' {
65 return; 70 let mut hashes_left = hashes;
71 while ptr.next_is('#') && hashes_left > 0{
72 hashes_left -= 1;
73 ptr.bump();
74 }
75 if hashes_left == 0 {
76 return;
77 }
66 } 78 }
67 } 79 }
68} 80}
diff --git a/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs
new file mode 100644
index 000000000..e5ed0b693
--- /dev/null
+++ b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs
@@ -0,0 +1 @@
r###"this is a r##"raw"## string"###
diff --git a/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt
new file mode 100644
index 000000000..9cf0957d1
--- /dev/null
+++ b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt
@@ -0,0 +1,2 @@
1RAW_STRING 36 "r###\"this is a r##\"raw\"## string\"###"
2WHITESPACE 1 "\n"