9 files changed, 112 insertions, 47 deletions

diff --git a/.cargo/config b/.cargo/config
index 767045bdb..ac7470c7e 100644
--- a/.cargo/config
+++ b/.cargo/config
@@ -1,4 +1,5 @@
 [alias]
+# Automatically generates the ast and syntax kinds files
 gen-kinds =    "run --package tools -- gen-kinds"
 gen-tests =    "run --package tools -- gen-tests"
 install-code = "run --package tools -- install-code"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 41954c02e..c952078cf 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,38 +1,58 @@
-The project is in its early stages: contributions are welcome and
-would be **very** helpful, but the project is not *yet* optimized for
-contribution. Moreover, it is doubly experimental, so there's no
-guarantee that any work here would reach production. That said, here
-are some areas where contributions would be **especially** welcome:
-
-
-* Designing internal data structures: RFC only outlines the
-  constraints, it's an open question how to satisfy them in the
-  optimal way. See `ARCHITECTURE.md` for current design questions.
-
-* Porting libsyntax parser to rust-analyzer: currently rust-analyzer parses
-  only a tiny subset of Rust. This should be fixed by porting parsing
-  functions from libsyntax one by one. Take a look at the
-  [libsyntax parser](https://github.com/rust-lang/rust/blob/6b99adeb11313197f409b4f7c4083c2ceca8a4fe/src/libsyntax/parse/parser.rs)
-  for "what to port" and at the
-  [Kotlin parser](https://github.com/JetBrains/kotlin/blob/4d951de616b20feca92f3e9cc9679b2de9e65195/compiler/frontend/src/org/jetbrains/kotlin/parsing/KotlinParsing.java)
-  for "how to port".
-
-* Writing validators: by design, rust-analyzer is very lax about the
-  input. For example, the lexer happily accepts unclosed strings. The
-  idea is that there should be a higher level visitor, which walks the
-  syntax tree after parsing and produces all the warnings. Alas,
-  there's no such visitor yet :( Would you like to write one? :)
-
-* Creating tests: it would be tremendously helpful to read each of
-  libsyntax and rust-analyzer parser functions and crate a small separate
-  test cases to cover each and every edge case.
-
-* Building stuff with rust-analyzer: it would be really cool to compile
-  rust-analyzer to WASM and add *client side* syntax validation to rust
+The project is in its early stages: contributions are welcome and would be
+**very** helpful, but the project is not _yet_ optimized for contribution.
+Moreover, it is doubly experimental, so there's no guarantee that any work here
+would reach production. That said, here are some areas where contributions would
+be **especially** welcome:
+
+- Designing internal data structures: RFC only outlines the constraints, it's an
+  open question how to satisfy them in the optimal way. See `ARCHITECTURE.md`
+  for current design questions.
+
+- Porting libsyntax parser to rust-analyzer: currently rust-analyzer parses only
+  a tiny subset of Rust. This should be fixed by porting parsing functions from
+  libsyntax one by one. Take a look at the [libsyntax parser] for "what to port"
+  and at the [Kotlin parser] for "how to port".
+
+- Writing validators: by design, rust-analyzer is very lax about the input. For
+  example, the lexer happily accepts unclosed strings. The idea is that there
+  should be a higher level visitor, which walks the syntax tree after parsing
+  and produces all the warnings. Alas, there's no such visitor yet :( Would you
+  like to write one? :)
+
+- Creating tests: it would be tremendously helpful to read each of libsyntax and
+  rust-analyzer parser functions and crate a small separate test cases to cover
+  each and every edge case.
+
+- Building stuff with rust-analyzer: it would be really cool to compile
+  rust-analyzer to WASM and add _client side_ syntax validation to rust
   playground!
 
-
 Do take a look at the issue tracker.
 
-If you don't know where to start, or have *any* questions or suggestions,
-don't hesitate to chat at [Gitter](https://gitter.im/libsyntax2/Lobby)!
+If you don't know where to start, or have _any_ questions or suggestions, don't
+hesitate to chat at [Gitter]!
+
+# Code generation
+
+Some of the components of this repository are generated through automatic
+processes. These are outlined below:
+
+- `gen-kinds`: The kinds of tokens are reused in several places, so a generator
+  is used. This process uses [tera] to generate, using data in [grammar.ron],
+  the files:
+  - [ast/generated.rs][ast generated] in `ra_syntax` based on
+    [ast/generated.tera.rs][ast source]
+  - [syntax_kinds/generated.rs][syntax_kinds generated] in `ra_syntax` based on
+    [syntax_kinds/generated.tera.rs][syntax_kinds source]
+
+[libsyntax parser]:
+  https://github.com/rust-lang/rust/blob/6b99adeb11313197f409b4f7c4083c2ceca8a4fe/src/libsyntax/parse/parser.rs
+[kotlin parser]:
+  https://github.com/JetBrains/kotlin/blob/4d951de616b20feca92f3e9cc9679b2de9e65195/compiler/frontend/src/org/jetbrains/kotlin/parsing/KotlinParsing.java
+[gitter]: https://gitter.im/libsyntax2/Lobby
+[tera]: https://tera.netlify.com/
+[grammar.ron]: ./crates/ra_syntax/src/grammar.ron
+[ast generated]: ./crates/ra_syntax/src/ast/generated.rs
+[ast source]: ./crates/ra_syntax/src/ast/generated.tera.rs
+[syntax_kinds generated]: ./crates/ra_syntax/src/syntax_kinds/generated.rs
+[syntax_kinds source]: ./crates/ra_syntax/src/syntax_kinds/generated.tera.rs
diff --git a/crates/ra_syntax/src/ast/generated.rs b/crates/ra_syntax/src/ast/generated.rs
index c945c094a..2db6dff1b 100644
--- a/crates/ra_syntax/src/ast/generated.rs
+++ b/crates/ra_syntax/src/ast/generated.rs
@@ -1,3 +1,6 @@
+// This file is automatically generated based on the file `./generated.rs.tera` when `cargo gen-kinds` is run
+// Do not edit manually
+
 use {
     ast,
     SyntaxNodeRef, AstNode,
diff --git a/crates/ra_syntax/src/ast/generated.rs.tera b/crates/ra_syntax/src/ast/generated.rs.tera
index a72e9b732..ceb2cd7e2 100644
--- a/crates/ra_syntax/src/ast/generated.rs.tera
+++ b/crates/ra_syntax/src/ast/generated.rs.tera
@@ -1,3 +1,8 @@
+{# THIS File is not automatically generated:
+the below applies to the result of this template
+#}// This file is automatically generated based on the file `./generated.rs.tera` when `cargo gen-kinds` is run
+// Do not edit manually
+
 use {
     ast,
     SyntaxNodeRef, AstNode,
diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron
index 77ae4c7db..4b990fd8d 100644
--- a/crates/ra_syntax/src/grammar.ron
+++ b/crates/ra_syntax/src/grammar.ron
@@ -1,3 +1,5 @@
+// Stores definitions which must be used in multiple places
+// See `cargo gen-kinds` (defined in crates/tools/src/main.rs)
 Grammar(
     single_byte_tokens: [
         [";", "SEMI"],
@@ -23,8 +25,9 @@ Grammar(
         ["^", "CARET"],
         ["%", "PERCENT"],
     ],
+    // TODO: Confirm surmision: the tokens which cannot be recorded in a single UTF-8 byte
     multi_byte_tokens: [
-        [".", "DOT"],
+        [".", "DOT"], // Note: DOT is here because <TODO: REASON>
         ["..", "DOTDOT"],
         ["...", "DOTDOTDOT"],
         ["..=", "DOTDOTEQ"],
diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs b/crates/ra_syntax/src/syntax_kinds/generated.rs
index 7882bded9..11e9150bb 100644
--- a/crates/ra_syntax/src/syntax_kinds/generated.rs
+++ b/crates/ra_syntax/src/syntax_kinds/generated.rs
@@ -1,3 +1,6 @@
+// This file is automatically generated based on the file `./generated.rs.tera` when `cargo gen-kinds` is run
+// Do not edit manually
+
 #![allow(bad_style, missing_docs, unreachable_pub)]
 #![cfg_attr(rustfmt, rustfmt_skip)]
 use super::SyntaxInfo;
diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
index 90618721a..018564b8a 100644
--- a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
+++ b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
@@ -1,3 +1,8 @@
+{# THIS File is not automatically generated:
+the below applies to the result of this template
+#}// This file is automatically generated based on the file `./generated.rs.tera` when `cargo gen-kinds` is run
+// Do not edit manually
+
 #![allow(bad_style, missing_docs, unreachable_pub)]
 #![cfg_attr(rustfmt, rustfmt_skip)]
 use super::SyntaxInfo;
diff --git a/editors/code/package.json b/editors/code/package.json
index 305d6eaf1..9f05fe91a 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -111,7 +111,7 @@
         ],
         "configuration": {
             "type": "object",
-            "title": "Rust Analyzer configuration",
+            "title": "Rust Analyzer",
             "properties": {
                 "ra-lsp.highlightingOn": {
                     "type": "boolean",
diff --git a/editors/code/src/extension.ts b/editors/code/src/extension.ts
index dc1792e94..fde6a480d 100644
--- a/editors/code/src/extension.ts
+++ b/editors/code/src/extension.ts
@@ -11,10 +11,22 @@ let uris = {
 let highlightingOn = true;
 
 export function activate(context: vscode.ExtensionContext) {
-    let config = vscode.workspace.getConfiguration('ra-lsp');
-    if (config.has('highlightingOn')) {
-        highlightingOn = config.get('highlightingOn') as boolean;
-    }
+    let applyHighlightingOn = () => {
+        let config = vscode.workspace.getConfiguration('ra-lsp');
+        if (config.has('highlightingOn')) {
+            highlightingOn = config.get('highlightingOn') as boolean;
+        };
+
+        if (!highlightingOn) {
+            removeHighlights();
+        }
+    };
+
+    // Apply the highlightingOn config now and whenever the config changes
+    applyHighlightingOn();
+    vscode.workspace.onDidChangeConfiguration(_ => {
+        applyHighlightingOn();
+    });
 
     let textDocumentContentProvider = new TextDocumentContentProvider()
     let dispose = (disposable: vscode.Disposable) => {
@@ -130,7 +142,7 @@ export function activate(context: vscode.ExtensionContext) {
         })
     }, null, context.subscriptions)
     vscode.window.onDidChangeActiveTextEditor(async (editor) => {
-        if (!editor || editor.document.languageId != 'rust') return
+        if (!highlightingOn || !editor || editor.document.languageId != 'rust') return
         let params: lc.TextDocumentIdentifier = {
             uri: editor.document.uri.toString()
         }
@@ -179,7 +191,7 @@ function startServer() {
                 let editor = vscode.window.visibleTextEditors.find(
                     (editor) => editor.document.uri.toString() == params.uri
                 )
-                if (editor == null) return;
+                if (!highlightingOn || !editor) return;
                 setHighlights(
                     editor,
                     params.decorations,
@@ -213,10 +225,11 @@ class TextDocumentContentProvider implements vscode.TextDocumentContentProvider
     }
 }
 
+let decorations: { [index: string]: vscode.TextEditorDecorationType } = {};
 
-const decorations: { [index: string]: vscode.TextEditorDecorationType } = (() => {
+function initDecorations() {
     const decor = (obj: any) => vscode.window.createTextEditorDecorationType({ color: obj })
-    return {
+    decorations = {
         background: decor("#3F3F3F"),
         error: vscode.window.createTextEditorDecorationType({
             borderColor: "red",
@@ -232,14 +245,26 @@ const decorations: { [index: string]: vscode.TextEditorDecorationType } = (() =>
         attribute: decor("#BFEBBF"),
         literal: decor("#DFAF8F"),
     }
-})()
+}
+
+function removeHighlights() {
+    for (let tag in decorations) {
+        decorations[tag].dispose();
+    }
+
+    decorations = {};
+}
 
 function setHighlights(
     editor: vscode.TextEditor,
     highlights: Array<Decoration>
 ) {
-    if (!highlightingOn) {
-        return;
+    // Initialize decorations if necessary
+    //
+    // Note: decoration objects need to be kept around so we can dispose them
+    // if the user disables syntax highlighting
+    if (Object.keys(decorations).length === 0) {
+        initDecorations();
     }
 
     let byTag: Map<string, vscode.Range[]> = new Map()