From 109658332a75ca91d6dc2bf573e0ab77fa5619ca Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 14 Aug 2018 15:03:27 +0300 Subject: Support raw strings in lexer --- crates/libanalysis/src/lib.rs | 69 +++++++++++----------- crates/libsyntax2/src/lexer/strings.rs | 14 ++++- .../tests/data/lexer/0013_raw_strings.rs | 1 + .../tests/data/lexer/0013_raw_strings.txt | 2 + 4 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs create mode 100644 crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt (limited to 'crates') diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index ffd81bef8..fc0b9ee9a 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -20,7 +20,6 @@ use std::{ atomic::{AtomicUsize, Ordering::SeqCst}, }, collections::hash_map::HashMap, - path::{PathBuf, Path}, time::Instant, }; @@ -38,6 +37,7 @@ pub type Result = ::std::result::Result; const INDEXING_THRESHOLD: usize = 128; pub struct WorldState { + next_file_id: u32, data: Arc } @@ -46,9 +46,13 @@ pub struct World { data: Arc, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FileId(u32); + impl WorldState { pub fn new() -> WorldState { WorldState { + next_file_id: 0, data: Arc::new(WorldData::default()) } } @@ -57,21 +61,27 @@ impl WorldState { World { data: self.data.clone() } } - pub fn change_file(&mut self, path: PathBuf, text: Option) { - self.change_files(::std::iter::once((path, text))); + pub fn new_file_id(&mut self) -> FileId { + let id = FileId(self.next_file_id); + self.next_file_id += 1; + id + } + + pub fn change_file(&mut self, file_id: FileId, text: Option) { + self.change_files(::std::iter::once((file_id, text))); } - pub fn change_files(&mut self, changes: impl Iterator)>) { + pub fn change_files(&mut self, changes: impl Iterator)>) { let data = self.data_mut(); let mut cnt = 0; - for (path, text) in changes { + for (id, text) in changes { cnt += 1; - data.file_map.remove(&path); + data.file_map.remove(&id); if let Some(text) = text { let file_data = FileData::new(text); - data.file_map.insert(path, Arc::new(file_data)); + data.file_map.insert(id, Arc::new(file_data)); } else { - data.file_map.remove(&path); + data.file_map.remove(&id); } } *data.unindexed.get_mut() += cnt; @@ -92,37 +102,33 @@ impl WorldState { impl World { - pub fn file_syntax(&self, path: &Path) -> Result { - let data = self.file_data(path)?; - Ok(data.syntax(path).clone()) + pub fn file_syntax(&self, file_id: FileId) -> Result { + let data = self.file_data(file_id)?; + Ok(data.syntax().clone()) } - pub fn file_line_index(&self, path: &Path) -> Result { - let data = self.file_data(path)?; + pub fn file_line_index(&self, id: FileId) -> Result { + let data = self.file_data(id)?; let index = data.lines - .get_or_init(|| { - trace!("calc line index: {}", path.display()); - LineIndex::new(&data.text) - }); + .get_or_init(|| LineIndex::new(&data.text)); Ok(index.clone()) } - pub fn world_symbols<'a>(&'a self, mut query: Query) -> impl Iterator + 'a { + pub fn world_symbols<'a>(&'a self, mut query: Query) -> impl Iterator + 'a { self.reindex(); self.data.file_map.iter() - .flat_map(move |(path, data)| { + .flat_map(move |(id, data)| { let symbols = data.symbols(); - let path: &'a Path = path.as_path(); - query.process(symbols).into_iter().map(move |s| (path, s)) + query.process(symbols).into_iter().map(move |s| (*id, s)) }) } pub fn approximately_resolve_symbol<'a>( &'a self, - path: &Path, + id: FileId, offset: TextUnit, - ) -> Result> { - let file = self.file_syntax(path)?; + ) -> Result> { + let file = self.file_syntax(id)?; let syntax = file.syntax(); let syntax = syntax.as_ref(); let name_ref = @@ -160,10 +166,10 @@ impl World { } } - fn file_data(&self, path: &Path) -> Result> { - match self.data.file_map.get(path) { + fn file_data(&self, file_id: FileId) -> Result> { + match self.data.file_map.get(&file_id) { Some(data) => Ok(data.clone()), - None => bail!("unknown file: {}", path.display()), + None => bail!("unknown file: {:?}", file_id), } } } @@ -182,7 +188,7 @@ pub const BREAK: SearchResult = Err(Break); #[derive(Default, Debug)] struct WorldData { unindexed: AtomicUsize, - file_map: HashMap>, + file_map: HashMap>, } #[derive(Debug)] @@ -203,12 +209,9 @@ impl FileData { } } - fn syntax(&self, path: &Path) -> &ast::File { + fn syntax(&self) -> &ast::File { self.syntax - .get_or_init(|| { - trace!("parsing: {}", path.display()); - ast::File::parse(&self.text) - }) + .get_or_init(|| ast::File::parse(&self.text)) } fn syntax_transient(&self) -> ast::File { diff --git a/crates/libsyntax2/src/lexer/strings.rs b/crates/libsyntax2/src/lexer/strings.rs index e3704fbb3..fbae767e5 100644 --- a/crates/libsyntax2/src/lexer/strings.rs +++ b/crates/libsyntax2/src/lexer/strings.rs @@ -55,6 +55,11 @@ pub(crate) fn scan_string(ptr: &mut Ptr) { } pub(crate) fn scan_raw_string(ptr: &mut Ptr) { + let mut hashes = 0; + while ptr.next_is('#') { + hashes += 1; + ptr.bump(); + } if !ptr.next_is('"') { return; } @@ -62,7 +67,14 @@ pub(crate) fn scan_raw_string(ptr: &mut Ptr) { while let Some(c) = ptr.bump() { if c == '"' { - return; + let mut hashes_left = hashes; + while ptr.next_is('#') && hashes_left > 0{ + hashes_left -= 1; + ptr.bump(); + } + if hashes_left == 0 { + return; + } } } } diff --git a/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs new file mode 100644 index 000000000..e5ed0b693 --- /dev/null +++ b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.rs @@ -0,0 +1 @@ +r###"this is a r##"raw"## string"### diff --git a/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt new file mode 100644 index 000000000..9cf0957d1 --- /dev/null +++ b/crates/libsyntax2/tests/data/lexer/0013_raw_strings.txt @@ -0,0 +1,2 @@ +RAW_STRING 36 "r###\"this is a r##\"raw\"## string\"###" +WHITESPACE 1 "\n" -- cgit v1.2.3