From bb381a7ff7a21cad98d80005a81f2586684f80a0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 3 Feb 2018 12:51:06 +0300 Subject: Move tools to a separate package --- .cargo/config | 3 ++ Cargo.toml | 8 ++- docs/TOOLS.md | 30 ++++++++++++ src/bin/gen.rs | 112 ------------------------------------------ src/bin/parse-rust.rs | 20 -------- tests/testutils/src/lib.rs | 34 ++++++------- tools/Cargo.toml | 12 +++++ tools/src/bin/gen.rs | 118 +++++++++++++++++++++++++++++++++++++++++++++ tools/src/bin/parse.rs | 20 ++++++++ 9 files changed, 202 insertions(+), 155 deletions(-) create mode 100644 .cargo/config create mode 100644 docs/TOOLS.md delete mode 100644 src/bin/gen.rs delete mode 100644 src/bin/parse-rust.rs create mode 100644 tools/Cargo.toml create mode 100644 tools/src/bin/gen.rs create mode 100644 tools/src/bin/parse.rs diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 000000000..1ebc0f748 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,3 @@ +[alias] +parse = "run --package tools --bin parse" +gen = "run --package tools --bin gen" diff --git a/Cargo.toml b/Cargo.toml index e5caa5d12..622abcc42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,11 @@ version = "0.1.0" authors = ["Aleksey Kladov "] license = "MIT OR Apache-2.0" +[workspace] +members = [ "tools" ] + [dependencies] unicode-xid = "0.1.0" -serde = "1.0.26" -serde_derive = "1.0.26" -file = "1.1.1" -ron = "0.1.5" - [dev-dependencies] testutils = { path = "./tests/testutils" } diff --git a/docs/TOOLS.md b/docs/TOOLS.md new file mode 100644 index 000000000..1fcfa2dec --- /dev/null +++ b/docs/TOOLS.md @@ -0,0 +1,30 @@ +# Tools used to implement libsyntax + +libsyntax uses several tools to help with development. + +Each tool is a binary in the [tools/](../tools) package. +You can run them via `cargo run` command. + +``` +cargo run --package tools --bin tool +``` + +There are also aliases in [./cargo/config](../.cargo/config), +so the following also works: + +``` +cargo tool +``` + + +# Tool: `gen` + +This tool reads a "grammar" from [grammar.ron](../grammar.ron) and +generates the `syntax_kinds.rs` file. You should run this tool if you +add new keywords or syntax elements. + + +# Tool: 'parse' + +This tool reads rust source code from the standard input, parses it, +and prints the result to stdout. diff --git a/src/bin/gen.rs b/src/bin/gen.rs deleted file mode 100644 index e32f5044e..000000000 --- a/src/bin/gen.rs +++ /dev/null @@ -1,112 +0,0 @@ -extern crate serde; -#[macro_use] -extern crate serde_derive; - -extern crate file; -extern crate ron; - -use std::path::PathBuf; -use std::fmt::Write; - -fn main() { - let grammar = Grammar::read(); - let text = grammar.to_syntax_kinds(); - file::put_text(&generated_file(), &text).unwrap(); -} - -#[derive(Deserialize)] -struct Grammar { - keywords: Vec, - tokens: Vec, - nodes: Vec, -} - -impl Grammar { - fn read() -> Grammar { - let text = file::get_text(&grammar_file()).unwrap(); - ron::de::from_str(&text).unwrap() - } - - fn to_syntax_kinds(&self) -> String { - let mut acc = String::new(); - acc.push_str("#![allow(bad_style, missing_docs, unreachable_pub)]\n"); - acc.push_str("#![cfg_attr(rustfmt, rustfmt_skip)]\n"); - acc.push_str("//! Generated from grammar.ron\n"); - acc.push_str("use tree::SyntaxInfo;\n"); - acc.push_str("\n"); - - let syntax_kinds: Vec = self.keywords - .iter() - .map(|kw| kw_token(kw)) - .chain(self.tokens.iter().cloned()) - .chain(self.nodes.iter().cloned()) - .collect(); - - // enum SyntaxKind - acc.push_str("/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.\n"); - acc.push_str("#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]\n"); - acc.push_str("pub enum SyntaxKind {\n"); - for kind in syntax_kinds.iter() { - write!(acc, " {},\n", scream(kind)).unwrap(); - } - acc.push_str("\n"); - acc.push_str(" // Technical SyntaxKinds: they appear temporally during parsing,\n"); - acc.push_str(" // but never end up in the final tree\n"); - acc.push_str(" #[doc(hidden)]\n"); - acc.push_str(" TOMBSTONE,\n"); - acc.push_str(" #[doc(hidden)]\n"); - acc.push_str(" EOF,\n"); - acc.push_str("}\n"); - acc.push_str("pub(crate) use self::SyntaxKind::*;\n"); - acc.push_str("\n"); - - // fn info - acc.push_str("impl SyntaxKind {\n"); - acc.push_str(" pub(crate) fn info(self) -> &'static SyntaxInfo {\n"); - acc.push_str(" match self {\n"); - for kind in syntax_kinds.iter() { - let sname = scream(kind); - write!( - acc, - " {sname} => &SyntaxInfo {{ name: \"{sname}\" }},\n", - sname = sname - ).unwrap(); - } - acc.push_str("\n"); - acc.push_str(" TOMBSTONE => &SyntaxInfo { name: \"TOMBSTONE\" },\n"); - acc.push_str(" EOF => &SyntaxInfo { name: \"EOF\" },\n"); - acc.push_str(" }\n"); - acc.push_str(" }\n"); - acc.push_str("}\n"); - acc.push_str("\n"); - - // fn ident_to_keyword - acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option {\n"); - acc.push_str(" match ident {\n"); - for kw in self.keywords.iter() { - write!(acc, " {:?} => Some({}),\n", kw, kw_token(kw)).unwrap(); - } - acc.push_str(" _ => None,\n"); - acc.push_str(" }\n"); - acc.push_str("}\n"); - acc - } -} - -fn grammar_file() -> PathBuf { - let dir = env!("CARGO_MANIFEST_DIR"); - PathBuf::from(dir).join("grammar.ron") -} - -fn generated_file() -> PathBuf { - let dir = env!("CARGO_MANIFEST_DIR"); - PathBuf::from(dir).join("src/syntax_kinds.rs") -} - -fn scream(word: &str) -> String { - word.chars().map(|c| c.to_ascii_uppercase()).collect() -} - -fn kw_token(keyword: &str) -> String { - format!("{}_KW", scream(keyword)) -} diff --git a/src/bin/parse-rust.rs b/src/bin/parse-rust.rs deleted file mode 100644 index af1325bfc..000000000 --- a/src/bin/parse-rust.rs +++ /dev/null @@ -1,20 +0,0 @@ -extern crate libsyntax2; - -use std::io::Read; - -use libsyntax2::{parse, tokenize}; -use libsyntax2::utils::dump_tree; - -fn main() { - let text = read_input(); - let tokens = tokenize(&text); - let file = parse(text, &tokens); - let tree = dump_tree(&file); - println!("{}", tree); -} - -fn read_input() -> String { - let mut buff = String::new(); - ::std::io::stdin().read_to_string(&mut buff).unwrap(); - buff -} diff --git a/tests/testutils/src/lib.rs b/tests/testutils/src/lib.rs index 80a94fb66..f829b243b 100644 --- a/tests/testutils/src/lib.rs +++ b/tests/testutils/src/lib.rs @@ -1,7 +1,7 @@ extern crate difference; extern crate file; -use std::path::{PathBuf, Path}; +use std::path::{Path, PathBuf}; use std::fs::read_dir; use difference::Changeset; @@ -21,12 +21,9 @@ fn read_text(path: &Path) -> String { file::get_text(path).unwrap().replace("\r\n", "\n") } -pub fn dir_tests( - paths: &[&str], - f: F -) +pub fn dir_tests(paths: &[&str], f: F) where - F: Fn(&str) -> String + F: Fn(&str) -> String, { for path in collect_tests(paths) { let actual = { @@ -47,21 +44,20 @@ where } } -fn assert_equal_text( - expected: &str, - actual: &str, - path: &Path -) { +fn assert_equal_text(expected: &str, actual: &str, path: &Path) { if expected != actual { print_difference(expected, actual, path) } } fn collect_tests(paths: &[&str]) -> Vec { - paths.iter().flat_map(|path| { - let path = test_data_dir().join(path); - test_from_dir(&path).into_iter() - }).collect() + paths + .iter() + .flat_map(|path| { + let path = test_data_dir().join(path); + test_from_dir(&path).into_iter() + }) + .collect() } fn test_from_dir(dir: &Path) -> Vec { @@ -95,11 +91,13 @@ fn print_difference(expected: &str, actual: &str, path: &Path) { fn project_dir() -> PathBuf { let dir = env!("CARGO_MANIFEST_DIR"); PathBuf::from(dir) - .parent().unwrap() - .parent().unwrap() + .parent() + .unwrap() + .parent() + .unwrap() .to_owned() } fn test_data_dir() -> PathBuf { project_dir().join("tests/data") -} \ No newline at end of file +} diff --git a/tools/Cargo.toml b/tools/Cargo.toml new file mode 100644 index 000000000..e46874929 --- /dev/null +++ b/tools/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "tools" +version = "0.1.0" +authors = ["Aleksey Kladov "] +publish = false + +[dependencies] +serde = "1.0.26" +serde_derive = "1.0.26" +file = "1.1.1" +ron = "0.1.5" +libsyntax2 = { path = "../" } diff --git a/tools/src/bin/gen.rs b/tools/src/bin/gen.rs new file mode 100644 index 000000000..17cdea7a1 --- /dev/null +++ b/tools/src/bin/gen.rs @@ -0,0 +1,118 @@ +extern crate serde; +#[macro_use] +extern crate serde_derive; + +extern crate file; +extern crate ron; + +use std::path::PathBuf; +use std::fmt::Write; + +fn main() { + let grammar = Grammar::read(); + let text = grammar.to_syntax_kinds(); + let target = generated_file(); + if text != file::get_text(&target).unwrap_or_default() { + file::put_text(&target, &text).unwrap(); + } +} + +#[derive(Deserialize)] +struct Grammar { + keywords: Vec, + tokens: Vec, + nodes: Vec, +} + +impl Grammar { + fn read() -> Grammar { + let text = file::get_text(&grammar_file()).unwrap(); + ron::de::from_str(&text).unwrap() + } + + fn to_syntax_kinds(&self) -> String { + let mut acc = String::new(); + acc.push_str("#![allow(bad_style, missing_docs, unreachable_pub)]\n"); + acc.push_str("#![cfg_attr(rustfmt, rustfmt_skip)]\n"); + acc.push_str("//! Generated from grammar.ron\n"); + acc.push_str("use tree::SyntaxInfo;\n"); + acc.push_str("\n"); + + let syntax_kinds: Vec = self.keywords + .iter() + .map(|kw| kw_token(kw)) + .chain(self.tokens.iter().cloned()) + .chain(self.nodes.iter().cloned()) + .collect(); + + // enum SyntaxKind + acc.push_str("/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.\n"); + acc.push_str("#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]\n"); + acc.push_str("pub enum SyntaxKind {\n"); + for kind in syntax_kinds.iter() { + write!(acc, " {},\n", scream(kind)).unwrap(); + } + acc.push_str("\n"); + acc.push_str(" // Technical SyntaxKinds: they appear temporally during parsing,\n"); + acc.push_str(" // but never end up in the final tree\n"); + acc.push_str(" #[doc(hidden)]\n"); + acc.push_str(" TOMBSTONE,\n"); + acc.push_str(" #[doc(hidden)]\n"); + acc.push_str(" EOF,\n"); + acc.push_str("}\n"); + acc.push_str("pub(crate) use self::SyntaxKind::*;\n"); + acc.push_str("\n"); + + // fn info + acc.push_str("impl SyntaxKind {\n"); + acc.push_str(" pub(crate) fn info(self) -> &'static SyntaxInfo {\n"); + acc.push_str(" match self {\n"); + for kind in syntax_kinds.iter() { + let sname = scream(kind); + write!( + acc, + " {sname} => &SyntaxInfo {{ name: \"{sname}\" }},\n", + sname = sname + ).unwrap(); + } + acc.push_str("\n"); + acc.push_str(" TOMBSTONE => &SyntaxInfo { name: \"TOMBSTONE\" },\n"); + acc.push_str(" EOF => &SyntaxInfo { name: \"EOF\" },\n"); + acc.push_str(" }\n"); + acc.push_str(" }\n"); + acc.push_str("}\n"); + acc.push_str("\n"); + + // fn ident_to_keyword + acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option {\n"); + acc.push_str(" match ident {\n"); + for kw in self.keywords.iter() { + write!(acc, " {:?} => Some({}),\n", kw, kw_token(kw)).unwrap(); + } + acc.push_str(" _ => None,\n"); + acc.push_str(" }\n"); + acc.push_str("}\n"); + acc + } +} + +fn grammar_file() -> PathBuf { + base_dir().join("grammar.ron") +} + +fn generated_file() -> PathBuf { + base_dir().join("src/syntax_kinds.rs") +} + +fn scream(word: &str) -> String { + word.chars().map(|c| c.to_ascii_uppercase()).collect() +} + +fn kw_token(keyword: &str) -> String { + format!("{}_KW", scream(keyword)) +} + +fn base_dir() -> PathBuf { + let dir = env!("CARGO_MANIFEST_DIR"); + PathBuf::from(dir).parent().unwrap().to_owned() +} diff --git a/tools/src/bin/parse.rs b/tools/src/bin/parse.rs new file mode 100644 index 000000000..af1325bfc --- /dev/null +++ b/tools/src/bin/parse.rs @@ -0,0 +1,20 @@ +extern crate libsyntax2; + +use std::io::Read; + +use libsyntax2::{parse, tokenize}; +use libsyntax2::utils::dump_tree; + +fn main() { + let text = read_input(); + let tokens = tokenize(&text); + let file = parse(text, &tokens); + let tree = dump_tree(&file); + println!("{}", tree); +} + +fn read_input() -> String { + let mut buff = String::new(); + ::std::io::stdin().read_to_string(&mut buff).unwrap(); + buff +} -- cgit v1.2.3 From 9435ea4b8e990521ee7a6206b6106bb3ce392746 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 3 Feb 2018 12:54:13 +0300 Subject: Drop unused extern crate --- tests/lexer.rs | 1 - tests/parser.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/lexer.rs b/tests/lexer.rs index 397ebafdd..46ac9fedd 100644 --- a/tests/lexer.rs +++ b/tests/lexer.rs @@ -1,4 +1,3 @@ -extern crate file; extern crate libsyntax2; extern crate testutils; diff --git a/tests/parser.rs b/tests/parser.rs index 37c9021ef..f681c066f 100644 --- a/tests/parser.rs +++ b/tests/parser.rs @@ -1,4 +1,3 @@ -extern crate file; extern crate libsyntax2; extern crate testutils; -- cgit v1.2.3