Merge #2061

2061: Theme loading and "editor.tokenColorCustomizations" support. r=matklad a=seivan

Fixes: [Issue#1294](https://github.com/rust-analyzer/rust-analyzer/issues/1294#issuecomment-497450325)

TODO: 
- [x] Load themes
- [x] Load existing `ralsp`-prefixed overrides from `"workbench.colorCustomizations"`.
- [x] Load overrides from `"editor.tokenColorCustomizations.textMateRules"`.
- [x] Use RA tags to load `vscode.DecorationRenderOptions` (colors) from theme & overrides.
- [x] Map RA tags to common TextMate scopes before loading colors.
- [x] Add default scope mappings in extension.
- [x] Cache mappings between settings updates. 
- [x] Add scope mapping configuration manifest in `package.json`
- [x] Load configurable scope mappings from settings.
- [x] Load JSON Scheme for text mate scope rules in settings.
- [x] Update [Readme](https://github.com/seivan/rust-analyzer/blob/feature/themes/docs/user/README.md#settings).

Borrowed the theme loading (`scopes.ts`) from `Tree Sitter` with some modifications to reading `"editor.tokenColorCustomizations"` for merging with loaded themes and had to remove the async portions to be able to load it from settings updates. 

~Just a PoC and an idea I toyed around with a lot of room for improvement.~
For starters, certain keywords aren't part of the standard TextMate grammar, so it still reads colors from the `ralsp` prefixed values in `"workbench.colorCustomizations"`. 

But I think there's more value making the extension work with existing themes by maping some of the decoration tags to existing key or keys. 

<img width="453" alt="Screenshot 2019-11-09 at 17 43 18" src="https://user-images.githubusercontent.com/55424/68531968-71b4e380-0318-11ea-924e-cdbb8d5eae06.png">
<img width="780" alt="Screenshot 2019-11-09 at 17 41 45" src="https://user-images.githubusercontent.com/55424/68531950-4b8f4380-0318-11ea-8f85-24a84efaf23b.png">
<img width="468" alt="Screenshot 2019-11-09 at 17 40 29" src="https://user-images.githubusercontent.com/55424/68531952-51852480-0318-11ea-800a-6ae9215f5368.png">


These will merge with the default ones coming with the extension, so you don't have to implement all of them and works well with overrides defined in settings. 

```jsonc
    "editor.tokenColorCustomizations": {
        "textMateRules": [
            {
                "scope": "keyword",
                "settings": {
                    "fontStyle": "bold",
                }
            },
        ]
    },
```


Edit: The idea is to work with 90% of the themes out there by working within existing scopes available that are generally styled. It's not to say I want to erase the custom Rust scopes - those should still remain and eventually worked into a custom grammar bundle for Rust specific themes that target those, I just want to make it work with generic themes offered on the market place for now. 

A custom grammar bundle and themes for Rust specific scopes is out of... scope for this PR. 
We'll make another round to tackle those issues. 


Current fallbacks implemented

```typescript
    [
        'comment',
        [
            'comment',
            'comment.block',
            'comment.line',
            'comment.block.documentation'
        ]
    ],
    ['string', ['string']],
    ['keyword', ['keyword']],
    ['keyword.control', ['keyword.control', 'keyword', 'keyword.other']],
    [
        'keyword.unsafe',
        ['storage.modifier', 'keyword.other', 'keyword.control', 'keyword']
    ],
    ['function', ['entity.name.function']],
    ['parameter', ['variable.parameter']],
    ['constant', ['constant', 'variable']],
    ['type', ['entity.name.type']],
    ['builtin', ['variable.language', 'support.type', 'support.type']],
    ['text', ['string', 'string.quoted', 'string.regexp']],
    ['attribute', ['keyword']],
    ['literal', ['string', 'string.quoted', 'string.regexp']],
    ['macro', ['support.other']],
    ['variable', ['variable']],
    ['variable.mut', ['variable', 'storage.modifier']],
    [
        'field',
        [
            'variable.object.property',
            'meta.field.declaration',
            'meta.definition.property',
            'variable.other'
        ]
    ],
    ['module', ['entity.name.section', 'entity.other']]
```


Co-authored-by: Seivan Heidari <[email protected]>

1 files changed, 146 insertions, 0 deletions

diff --git a/editors/code/src/scopes.ts b/editors/code/src/scopes.ts
new file mode 100644
index 000000000..cb250b853
--- /dev/null
+++ b/editors/code/src/scopes.ts
@@ -0,0 +1,146 @@
+import * as fs from 'fs';
+import * as jsonc from 'jsonc-parser';
+import * as path from 'path';
+import * as vscode from 'vscode';
+
+export interface TextMateRule {
+    scope: string | string[];
+    settings: TextMateRuleSettings;
+}
+
+export interface TextMateRuleSettings {
+    foreground: string | undefined;
+    background: string | undefined;
+    fontStyle: string | undefined;
+}
+
+// Current theme colors
+const rules = new Map<string, TextMateRuleSettings>();
+
+export function find(scope: string): TextMateRuleSettings | undefined {
+    return rules.get(scope);
+}
+
+// Load all textmate scopes in the currently active theme
+export function load() {
+    // Remove any previous theme
+    rules.clear();
+    // Find out current color theme
+    const themeName = vscode.workspace
+        .getConfiguration('workbench')
+        .get('colorTheme');
+
+    if (typeof themeName !== 'string') {
+        // console.warn('workbench.colorTheme is', themeName)
+        return;
+    }
+    // Try to load colors from that theme
+    try {
+        loadThemeNamed(themeName);
+    } catch (e) {
+        // console.warn('failed to load theme', themeName, e)
+    }
+}
+
+function filterThemeExtensions(extension: vscode.Extension<any>): boolean {
+    return (
+        extension.extensionKind === vscode.ExtensionKind.UI &&
+        extension.packageJSON.contributes &&
+        extension.packageJSON.contributes.themes
+    );
+}
+
+// Find current theme on disk
+function loadThemeNamed(themeName: string) {
+    const themePaths = vscode.extensions.all
+        .filter(filterThemeExtensions)
+        .reduce((list, extension) => {
+            return extension.packageJSON.contributes.themes
+                .filter(
+                    (element: any) =>
+                        (element.id || element.label) === themeName,
+                )
+                .map((element: any) =>
+                    path.join(extension.extensionPath, element.path),
+                )
+                .concat(list);
+        }, Array<string>());
+
+    themePaths.forEach(loadThemeFile);
+
+    const tokenColorCustomizations: [any] = [
+        vscode.workspace
+            .getConfiguration('editor')
+            .get('tokenColorCustomizations'),
+    ];
+
+    tokenColorCustomizations
+        .filter(custom => custom && custom.textMateRules)
+        .map(custom => custom.textMateRules)
+        .forEach(loadColors);
+}
+
+function loadThemeFile(themePath: string) {
+    const themeContent = [themePath]
+        .filter(isFile)
+        .map(readFileText)
+        .map(parseJSON)
+        .filter(theme => theme);
+
+    themeContent
+        .filter(theme => theme.tokenColors)
+        .map(theme => theme.tokenColors)
+        .forEach(loadColors);
+
+    themeContent
+        .filter(theme => theme.include)
+        .map(theme => path.join(path.dirname(themePath), theme.include))
+        .forEach(loadThemeFile);
+}
+
+function mergeRuleSettings(
+    defaultSetting: TextMateRuleSettings | undefined,
+    override: TextMateRuleSettings,
+): TextMateRuleSettings {
+    if (defaultSetting === undefined) {
+        return override;
+    }
+    const mergedRule = defaultSetting;
+
+    mergedRule.background = override.background || defaultSetting.background;
+    mergedRule.foreground = override.foreground || defaultSetting.foreground;
+    mergedRule.fontStyle = override.fontStyle || defaultSetting.foreground;
+
+    return mergedRule;
+}
+
+function updateRules(
+    scope: string,
+    updatedSettings: TextMateRuleSettings,
+): void {
+    [rules.get(scope)]
+        .map(settings => mergeRuleSettings(settings, updatedSettings))
+        .forEach(settings => rules.set(scope, settings));
+}
+
+function loadColors(textMateRules: TextMateRule[]): void {
+    textMateRules.forEach(rule => {
+        if (typeof rule.scope === 'string') {
+            updateRules(rule.scope, rule.settings);
+        } else if (rule.scope instanceof Array) {
+            rule.scope.forEach(scope => updateRules(scope, rule.settings));
+        }
+    });
+}
+
+function isFile(filePath: string): boolean {
+    return [filePath].map(fs.statSync).every(stat => stat.isFile());
+}
+
+function readFileText(filePath: string): string {
+    return fs.readFileSync(filePath, 'utf8');
+}
+
+function parseJSON(content: string): any {
+    return jsonc.parse(content);
+}