5 files changed, 182 insertions, 221 deletions

diff --git a/editors/code/src/config.ts b/editors/code/src/config.ts
index e323110a4..f63d1ddce 100644
--- a/editors/code/src/config.ts
+++ b/editors/code/src/config.ts
@@ -1,5 +1,4 @@
 import * as vscode from 'vscode';
-import * as scopes from './scopes';
 import * as scopesMapper from './scopes_mapper';
 
 const RA_LSP_DEBUG = process.env.__RA_LSP_SERVER_DEBUG;
@@ -60,7 +59,6 @@ export class Config {
         if (config.has('highlightingOn')) {
             this.highlightingOn = config.get('highlightingOn') as boolean;
             if (this.highlightingOn) {
-                scopes.load();
                 scopesMapper.load();
             }
         }
diff --git a/editors/code/src/ctx.ts b/editors/code/src/ctx.ts
index 39eddfcbd..30dd9811c 100644
--- a/editors/code/src/ctx.ts
+++ b/editors/code/src/ctx.ts
@@ -65,7 +65,7 @@ export class Ctx {
     async sendRequestWithRetry<R>(
         method: string,
         param: any,
-        token: vscode.CancellationToken,
+        token?: vscode.CancellationToken,
     ): Promise<R> {
         await this.client.onReady();
         for (const delay of [2, 4, 6, 8, 10, null]) {
diff --git a/editors/code/src/highlighting.ts b/editors/code/src/highlighting.ts
index 247673b5c..5e9cbe0de 100644
--- a/editors/code/src/highlighting.ts
+++ b/editors/code/src/highlighting.ts
@@ -3,7 +3,7 @@ import * as lc from 'vscode-languageclient';
 import * as seedrandom_ from 'seedrandom';
 const seedrandom = seedrandom_; // https://github.com/jvandemo/generator-angular2-library/issues/221#issuecomment-355945207
 
-import * as scopes from './scopes';
+import { loadThemeColors, TextMateRuleSettings } from './scopes';
 import * as scopesMapper from './scopes_mapper';
 
 import { Ctx } from './ctx';
@@ -47,7 +47,7 @@ export function activateHighlighting(ctx: Ctx) {
             const params: lc.TextDocumentIdentifier = {
                 uri: editor.document.uri.toString(),
             };
-            const decorations = await ctx.client.sendRequest<Decoration[]>(
+            const decorations = await ctx.sendRequestWithRetry<Decoration[]>(
                 'rust-analyzer/decorationsRequest',
                 params,
             );
@@ -62,7 +62,7 @@ interface PublishDecorationsParams {
     decorations: Decoration[];
 }
 
-export interface Decoration {
+interface Decoration {
     range: lc.Range;
     tag: string;
     bindingHash?: string;
@@ -81,116 +81,17 @@ function fancify(seed: string, shade: 'light' | 'dark') {
     return `hsl(${h},${s}%,${l}%)`;
 }
 
-function createDecorationFromTextmate(
-    themeStyle: scopes.TextMateRuleSettings,
-): vscode.TextEditorDecorationType {
-    const decorationOptions: vscode.DecorationRenderOptions = {};
-    decorationOptions.rangeBehavior = vscode.DecorationRangeBehavior.OpenOpen;
-
-    if (themeStyle.foreground) {
-        decorationOptions.color = themeStyle.foreground;
-    }
-
-    if (themeStyle.background) {
-        decorationOptions.backgroundColor = themeStyle.background;
-    }
-
-    if (themeStyle.fontStyle) {
-        const parts: string[] = themeStyle.fontStyle.split(' ');
-        parts.forEach(part => {
-            switch (part) {
-                case 'italic':
-                    decorationOptions.fontStyle = 'italic';
-                    break;
-                case 'bold':
-                    decorationOptions.fontWeight = 'bold';
-                    break;
-                case 'underline':
-                    decorationOptions.textDecoration = 'underline';
-                    break;
-                default:
-                    break;
-            }
-        });
-    }
-    return vscode.window.createTextEditorDecorationType(decorationOptions);
-}
-
 class Highlighter {
     private ctx: Ctx;
-
-    constructor(ctx: Ctx) {
-        this.ctx = ctx;
-    }
-
-    private static initDecorations(): Map<
-        string,
-        vscode.TextEditorDecorationType
-    > {
-        const decoration = (
-            tag: string,
-            textDecoration?: string,
-        ): [string, vscode.TextEditorDecorationType] => {
-            const rule = scopesMapper.toRule(tag, scopes.find);
-
-            if (rule) {
-                const decor = createDecorationFromTextmate(rule);
-                return [tag, decor];
-            } else {
-                const fallBackTag = 'ralsp.' + tag;
-                // console.log(' ');
-                // console.log('Missing theme for: <"' + tag + '"> for following mapped scopes:');
-                // console.log(scopesMapper.find(tag));
-                // console.log('Falling back to values defined in: ' + fallBackTag);
-                // console.log(' ');
-                const color = new vscode.ThemeColor(fallBackTag);
-                const decor = vscode.window.createTextEditorDecorationType({
-                    color,
-                    textDecoration,
-                });
-                return [tag, decor];
-            }
-        };
-
-        const decorations: Iterable<[
-            string,
-            vscode.TextEditorDecorationType,
-        ]> = [
-                decoration('comment'),
-                decoration('string'),
-                decoration('keyword'),
-                decoration('keyword.control'),
-                decoration('keyword.unsafe'),
-                decoration('function'),
-                decoration('parameter'),
-                decoration('constant'),
-                decoration('type.builtin'),
-                decoration('type.generic'),
-                decoration('type.lifetime'),
-                decoration('type.param'),
-                decoration('type.self'),
-                decoration('type'),
-                decoration('text'),
-                decoration('attribute'),
-                decoration('literal'),
-                decoration('literal.numeric'),
-                decoration('literal.char'),
-                decoration('literal.byte'),
-                decoration('macro'),
-                decoration('variable'),
-                decoration('variable.mut', 'underline'),
-                decoration('field'),
-                decoration('module'),
-            ];
-
-        return new Map<string, vscode.TextEditorDecorationType>(decorations);
-    }
-
     private decorations: Map<
         string,
         vscode.TextEditorDecorationType
     > | null = null;
 
+    constructor(ctx: Ctx) {
+        this.ctx = ctx;
+    }
+
     public removeHighlights() {
         if (this.decorations == null) {
             return;
@@ -210,7 +111,7 @@ class Highlighter {
         // Note: decoration objects need to be kept around so we can dispose them
         // if the user disables syntax highlighting
         if (this.decorations == null) {
-            this.decorations = Highlighter.initDecorations();
+            this.decorations = initDecorations();
         }
 
         const byTag: Map<string, vscode.Range[]> = new Map();
@@ -266,3 +167,103 @@ class Highlighter {
         }
     }
 }
+
+function initDecorations(): Map<
+    string,
+    vscode.TextEditorDecorationType
+> {
+    const themeColors = loadThemeColors();
+
+    const decoration = (
+        tag: string,
+        textDecoration?: string,
+    ): [string, vscode.TextEditorDecorationType] => {
+        const rule = scopesMapper.toRule(tag, it => themeColors.get(it));
+
+        if (rule) {
+            const decor = createDecorationFromTextmate(rule);
+            return [tag, decor];
+        } else {
+            const fallBackTag = 'ralsp.' + tag;
+            // console.log(' ');
+            // console.log('Missing theme for: <"' + tag + '"> for following mapped scopes:');
+            // console.log(scopesMapper.find(tag));
+            // console.log('Falling back to values defined in: ' + fallBackTag);
+            // console.log(' ');
+            const color = new vscode.ThemeColor(fallBackTag);
+            const decor = vscode.window.createTextEditorDecorationType({
+                color,
+                textDecoration,
+            });
+            return [tag, decor];
+        }
+    };
+
+    const decorations: Iterable<[
+        string,
+        vscode.TextEditorDecorationType,
+    ]> = [
+            decoration('comment'),
+            decoration('string'),
+            decoration('keyword'),
+            decoration('keyword.control'),
+            decoration('keyword.unsafe'),
+            decoration('function'),
+            decoration('parameter'),
+            decoration('constant'),
+            decoration('type.builtin'),
+            decoration('type.generic'),
+            decoration('type.lifetime'),
+            decoration('type.param'),
+            decoration('type.self'),
+            decoration('type'),
+            decoration('text'),
+            decoration('attribute'),
+            decoration('literal'),
+            decoration('literal.numeric'),
+            decoration('literal.char'),
+            decoration('literal.byte'),
+            decoration('macro'),
+            decoration('variable'),
+            decoration('variable.mut', 'underline'),
+            decoration('field'),
+            decoration('module'),
+        ];
+
+    return new Map<string, vscode.TextEditorDecorationType>(decorations);
+}
+
+function createDecorationFromTextmate(
+    themeStyle: TextMateRuleSettings,
+): vscode.TextEditorDecorationType {
+    const decorationOptions: vscode.DecorationRenderOptions = {};
+    decorationOptions.rangeBehavior = vscode.DecorationRangeBehavior.OpenOpen;
+
+    if (themeStyle.foreground) {
+        decorationOptions.color = themeStyle.foreground;
+    }
+
+    if (themeStyle.background) {
+        decorationOptions.backgroundColor = themeStyle.background;
+    }
+
+    if (themeStyle.fontStyle) {
+        const parts: string[] = themeStyle.fontStyle.split(' ');
+        parts.forEach(part => {
+            switch (part) {
+                case 'italic':
+                    decorationOptions.fontStyle = 'italic';
+                    break;
+                case 'bold':
+                    decorationOptions.fontWeight = 'bold';
+                    break;
+                case 'underline':
+                    decorationOptions.textDecoration = 'underline';
+                    break;
+                default:
+                    break;
+            }
+        });
+    }
+    return vscode.window.createTextEditorDecorationType(decorationOptions);
+}
diff --git a/editors/code/src/scopes.ts b/editors/code/src/scopes.ts
index cb250b853..73fabbf54 100644
--- a/editors/code/src/scopes.ts
+++ b/editors/code/src/scopes.ts
@@ -3,28 +3,14 @@ import * as jsonc from 'jsonc-parser';
 import * as path from 'path';
 import * as vscode from 'vscode';
 
-export interface TextMateRule {
-    scope: string | string[];
-    settings: TextMateRuleSettings;
-}
-
 export interface TextMateRuleSettings {
-    foreground: string | undefined;
-    background: string | undefined;
-    fontStyle: string | undefined;
-}
-
-// Current theme colors
-const rules = new Map<string, TextMateRuleSettings>();
-
-export function find(scope: string): TextMateRuleSettings | undefined {
-    return rules.get(scope);
+    foreground?: string;
+    background?: string;
+    fontStyle?: string;
 }
 
 // Load all textmate scopes in the currently active theme
-export function load() {
-    // Remove any previous theme
-    rules.clear();
+export function loadThemeColors(): Map<string, TextMateRuleSettings> {
     // Find out current color theme
     const themeName = vscode.workspace
         .getConfiguration('workbench')
@@ -32,115 +18,91 @@ export function load() {
 
     if (typeof themeName !== 'string') {
         // console.warn('workbench.colorTheme is', themeName)
-        return;
-    }
-    // Try to load colors from that theme
-    try {
-        loadThemeNamed(themeName);
-    } catch (e) {
-        // console.warn('failed to load theme', themeName, e)
+        return new Map();
     }
+    return loadThemeNamed(themeName);
 }
 
-function filterThemeExtensions(extension: vscode.Extension<any>): boolean {
-    return (
-        extension.extensionKind === vscode.ExtensionKind.UI &&
-        extension.packageJSON.contributes &&
-        extension.packageJSON.contributes.themes
-    );
-}
-
-// Find current theme on disk
-function loadThemeNamed(themeName: string) {
-    const themePaths = vscode.extensions.all
-        .filter(filterThemeExtensions)
-        .reduce((list, extension) => {
-            return extension.packageJSON.contributes.themes
-                .filter(
-                    (element: any) =>
-                        (element.id || element.label) === themeName,
-                )
-                .map((element: any) =>
-                    path.join(extension.extensionPath, element.path),
-                )
-                .concat(list);
-        }, Array<string>());
-
-    themePaths.forEach(loadThemeFile);
-
-    const tokenColorCustomizations: [any] = [
-        vscode.workspace
-            .getConfiguration('editor')
-            .get('tokenColorCustomizations'),
-    ];
-
-    tokenColorCustomizations
-        .filter(custom => custom && custom.textMateRules)
-        .map(custom => custom.textMateRules)
-        .forEach(loadColors);
-}
+function loadThemeNamed(themeName: string): Map<string, TextMateRuleSettings> {
+    function isTheme(extension: vscode.Extension<any>): boolean {
+        return (
+            extension.extensionKind === vscode.ExtensionKind.UI &&
+            extension.packageJSON.contributes &&
+            extension.packageJSON.contributes.themes
+        );
+    }
 
-function loadThemeFile(themePath: string) {
-    const themeContent = [themePath]
-        .filter(isFile)
-        .map(readFileText)
-        .map(parseJSON)
-        .filter(theme => theme);
+    let themePaths = vscode.extensions.all
+        .filter(isTheme)
+        .flatMap(ext => {
+            return ext.packageJSON.contributes.themes
+                .filter((it: any) => (it.id || it.label) === themeName)
+                .map((it: any) => path.join(ext.extensionPath, it.path));
+        })
+
+    const res = new Map();
+    for (const themePath of themePaths) {
+        mergeInto(res, loadThemeFile(themePath))
+    }
 
-    themeContent
-        .filter(theme => theme.tokenColors)
-        .map(theme => theme.tokenColors)
-        .forEach(loadColors);
+    const customizations: any = vscode.workspace.getConfiguration('editor').get('tokenColorCustomizations');
+    mergeInto(res, loadColors(customizations?.textMateRules ?? []))
 
-    themeContent
-        .filter(theme => theme.include)
-        .map(theme => path.join(path.dirname(themePath), theme.include))
-        .forEach(loadThemeFile);
+    return res;
 }
 
-function mergeRuleSettings(
-    defaultSetting: TextMateRuleSettings | undefined,
-    override: TextMateRuleSettings,
-): TextMateRuleSettings {
-    if (defaultSetting === undefined) {
-        return override;
+function loadThemeFile(themePath: string): Map<string, TextMateRuleSettings> {
+    let text;
+    try {
+        text = fs.readFileSync(themePath, 'utf8')
+    } catch {
+        return new Map();
+    }
+    const obj = jsonc.parse(text);
+    const tokenColors = obj?.tokenColors ?? [];
+    const res = loadColors(tokenColors);
+
+    for (const include in obj?.include ?? []) {
+        const includePath = path.join(path.dirname(themePath), include);
+        const tmp = loadThemeFile(includePath);
+        mergeInto(res, tmp);
     }
-    const mergedRule = defaultSetting;
-
-    mergedRule.background = override.background || defaultSetting.background;
-    mergedRule.foreground = override.foreground || defaultSetting.foreground;
-    mergedRule.fontStyle = override.fontStyle || defaultSetting.foreground;
 
-    return mergedRule;
+    return res;
 }
 
-function updateRules(
-    scope: string,
-    updatedSettings: TextMateRuleSettings,
-): void {
-    [rules.get(scope)]
-        .map(settings => mergeRuleSettings(settings, updatedSettings))
-        .forEach(settings => rules.set(scope, settings));
+interface TextMateRule {
+    scope: string | string[];
+    settings: TextMateRuleSettings;
 }
 
-function loadColors(textMateRules: TextMateRule[]): void {
-    textMateRules.forEach(rule => {
-        if (typeof rule.scope === 'string') {
-            updateRules(rule.scope, rule.settings);
-        } else if (rule.scope instanceof Array) {
-            rule.scope.forEach(scope => updateRules(scope, rule.settings));
+function loadColors(textMateRules: TextMateRule[]): Map<string, TextMateRuleSettings> {
+    const res = new Map();
+    for (const rule of textMateRules) {
+        const scopes = typeof rule.scope === 'string'
+            ? [rule.scope]
+            : rule.scope;
+        for (const scope of scopes) {
+            res.set(scope, rule.settings)
         }
-    });
-}
-
-function isFile(filePath: string): boolean {
-    return [filePath].map(fs.statSync).every(stat => stat.isFile());
+    }
+    return res
 }
 
-function readFileText(filePath: string): string {
-    return fs.readFileSync(filePath, 'utf8');
+function mergeRuleSettings(
+    defaultSetting: TextMateRuleSettings | undefined,
+    override: TextMateRuleSettings,
+): TextMateRuleSettings {
+    return {
+        foreground: defaultSetting?.foreground ?? override.foreground,
+        background: defaultSetting?.background ?? override.background,
+        fontStyle: defaultSetting?.fontStyle ?? override.fontStyle,
+    }
 }
 
-function parseJSON(content: string): any {
-    return jsonc.parse(content);
+function mergeInto(dst: Map<string, TextMateRuleSettings>, addition: Map<string, TextMateRuleSettings>) {
+    addition.forEach((value, key) => {
+        const merged = mergeRuleSettings(dst.get(key), value)
+        dst.set(key, merged)
+    })
 }
diff --git a/editors/code/tsconfig.json b/editors/code/tsconfig.json
index fe3b40f34..e60eb8e5e 100644
--- a/editors/code/tsconfig.json
+++ b/editors/code/tsconfig.json
@@ -4,7 +4,7 @@
         "target": "es2018",
         "outDir": "out",
         "lib": [
-            "es2018"
+            "es2019"
         ],
         "sourceMap": true,
         "rootDir": "src",