9 files changed, 75 insertions, 30 deletions

diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 000000000..1ebc0f748
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,3 @@
+[alias]
+parse = "run --package tools --bin parse"
+gen = "run --package tools --bin gen"
diff --git a/Cargo.toml b/Cargo.toml
index e5caa5d12..622abcc42 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,13 +4,11 @@ version = "0.1.0"
 authors = ["Aleksey Kladov <[email protected]>"]
 license = "MIT OR Apache-2.0"
 
+[workspace]
+members = [ "tools" ]
+
 [dependencies]
 unicode-xid = "0.1.0"
 
-serde = "1.0.26"
-serde_derive = "1.0.26"
-file = "1.1.1"
-ron = "0.1.5"
-
 [dev-dependencies]
 testutils = { path = "./tests/testutils" }
diff --git a/docs/TOOLS.md b/docs/TOOLS.md
new file mode 100644
index 000000000..1fcfa2dec
--- /dev/null
+++ b/docs/TOOLS.md
@@ -0,0 +1,30 @@
+# Tools used to implement libsyntax
+
+libsyntax uses several tools to help with development. 
+
+Each tool is a binary in the [tools/](../tools) package. 
+You can run them via `cargo run` command. 
+
+```
+cargo run --package tools --bin tool
+```
+
+There are also aliases in [./cargo/config](../.cargo/config), 
+so the following also works:
+
+```
+cargo tool
+```
+
+
+# Tool: `gen`
+
+This tool reads a "grammar" from [grammar.ron](../grammar.ron) and
+generates the `syntax_kinds.rs` file. You should run this tool if you 
+add new keywords or syntax elements.
+
+
+# Tool: 'parse'
+
+This tool reads rust source code from the standard input, parses it,
+and prints the result to stdout.
diff --git a/tests/lexer.rs b/tests/lexer.rs
index 397ebafdd..46ac9fedd 100644
--- a/tests/lexer.rs
+++ b/tests/lexer.rs
@@ -1,4 +1,3 @@
-extern crate file;
 extern crate libsyntax2;
 extern crate testutils;
 
diff --git a/tests/parser.rs b/tests/parser.rs
index 37c9021ef..f681c066f 100644
--- a/tests/parser.rs
+++ b/tests/parser.rs
@@ -1,4 +1,3 @@
-extern crate file;
 extern crate libsyntax2;
 extern crate testutils;
 
diff --git a/tests/testutils/src/lib.rs b/tests/testutils/src/lib.rs
index 80a94fb66..f829b243b 100644
--- a/tests/testutils/src/lib.rs
+++ b/tests/testutils/src/lib.rs
@@ -1,7 +1,7 @@
 extern crate difference;
 extern crate file;
 
-use std::path::{PathBuf, Path};
+use std::path::{Path, PathBuf};
 use std::fs::read_dir;
 
 use difference::Changeset;
@@ -21,12 +21,9 @@ fn read_text(path: &Path) -> String {
     file::get_text(path).unwrap().replace("\r\n", "\n")
 }
 
-pub fn dir_tests<F>(
-    paths: &[&str],
-    f: F
-)
+pub fn dir_tests<F>(paths: &[&str], f: F)
 where
-    F: Fn(&str) -> String
+    F: Fn(&str) -> String,
 {
     for path in collect_tests(paths) {
         let actual = {
@@ -47,21 +44,20 @@ where
     }
 }
 
-fn assert_equal_text(
-    expected: &str,
-    actual: &str,
-    path: &Path
-) {
+fn assert_equal_text(expected: &str, actual: &str, path: &Path) {
     if expected != actual {
         print_difference(expected, actual, path)
     }
 }
 
 fn collect_tests(paths: &[&str]) -> Vec<PathBuf> {
-    paths.iter().flat_map(|path|  {
-        let path = test_data_dir().join(path);
-        test_from_dir(&path).into_iter()
-    }).collect()
+    paths
+        .iter()
+        .flat_map(|path| {
+            let path = test_data_dir().join(path);
+            test_from_dir(&path).into_iter()
+        })
+        .collect()
 }
 
 fn test_from_dir(dir: &Path) -> Vec<PathBuf> {
@@ -95,11 +91,13 @@ fn print_difference(expected: &str, actual: &str, path: &Path) {
 fn project_dir() -> PathBuf {
     let dir = env!("CARGO_MANIFEST_DIR");
     PathBuf::from(dir)
-        .parent().unwrap()
-        .parent().unwrap()
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
         .to_owned()
 }
 
 fn test_data_dir() -> PathBuf {
     project_dir().join("tests/data")
-}
\ No newline at end of file
+}
diff --git a/tools/Cargo.toml b/tools/Cargo.toml
new file mode 100644
index 000000000..e46874929
--- /dev/null
+++ b/tools/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "tools"
+version = "0.1.0"
+authors = ["Aleksey Kladov <[email protected]>"]
+publish = false
+
+[dependencies]
+serde = "1.0.26"
+serde_derive = "1.0.26"
+file = "1.1.1"
+ron = "0.1.5"
+libsyntax2 = { path = "../" }
diff --git a/src/bin/gen.rs b/tools/src/bin/gen.rs
index e32f5044e..17cdea7a1 100644
--- a/src/bin/gen.rs
+++ b/tools/src/bin/gen.rs
@@ -11,7 +11,10 @@ use std::fmt::Write;
 fn main() {
     let grammar = Grammar::read();
     let text = grammar.to_syntax_kinds();
-    file::put_text(&generated_file(), &text).unwrap();
+    let target = generated_file();
+    if text != file::get_text(&target).unwrap_or_default() {
+        file::put_text(&target, &text).unwrap();
+    }
 }
 
 #[derive(Deserialize)]
@@ -94,13 +97,11 @@ impl Grammar {
 }
 
 fn grammar_file() -> PathBuf {
-    let dir = env!("CARGO_MANIFEST_DIR");
-    PathBuf::from(dir).join("grammar.ron")
+    base_dir().join("grammar.ron")
 }
 
 fn generated_file() -> PathBuf {
-    let dir = env!("CARGO_MANIFEST_DIR");
-    PathBuf::from(dir).join("src/syntax_kinds.rs")
+    base_dir().join("src/syntax_kinds.rs")
 }
 
 fn scream(word: &str) -> String {
@@ -110,3 +111,8 @@ fn scream(word: &str) -> String {
 fn kw_token(keyword: &str) -> String {
     format!("{}_KW", scream(keyword))
 }
+
+fn base_dir() -> PathBuf {
+    let dir = env!("CARGO_MANIFEST_DIR");
+    PathBuf::from(dir).parent().unwrap().to_owned()
+}
diff --git a/src/bin/parse-rust.rs b/tools/src/bin/parse.rs
index af1325bfc..af1325bfc 100644
--- a/src/bin/parse-rust.rs
+++ b/tools/src/bin/parse.rs