Improved TextMate integration

- Update to TextMateSharp 1.0.9. Use the NuGet package, instead a submodule. - Use latest changes in TMModel available in TextMateSharp 1.0.9. - Allow to dispose the TMModel in order to stop the tokenizer thread. - Do not process invalidate ranges in the TextMateColoringTransformer. - Tokenize the viewport when the scroll offset changes. This helps to highlight visible lines for big documents before the tokenizer thread finishes.
2021-11-09 13:37:56 +01:00 · 2021-11-09 13:37:56 +01:00 · 430ee97e1c
--- a/AvaloniaEdit.sln
+++ b/AvaloniaEdit.sln
@ -16,9 +16,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Directory.Build.props = Directory.Build.props
 	EndProjectSection
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TextMateSharp", "TextMateSharp\src\TextMateSharp\TextMateSharp.csproj", "{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvaloniaEdit.TextMate", "src\AvaloniaEdit.TextMate\AvaloniaEdit.TextMate.csproj", "{63826C17-C08C-4E5F-AABB-443EBA565E92}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AvaloniaEdit.TextMate", "src\AvaloniaEdit.TextMate\AvaloniaEdit.TextMate.csproj", "{63826C17-C08C-4E5F-AABB-443EBA565E92}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -52,14 +50,6 @@ Global
 		{9E5D4372-D362-44A2-984D-578288870AB8}.Release|Any CPU.Build.0 = Release|Any CPU
 		{9E5D4372-D362-44A2-984D-578288870AB8}.Release|x64.ActiveCfg = Release|Any CPU
 		{9E5D4372-D362-44A2-984D-578288870AB8}.Release|x64.Build.0 = Release|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Debug|x64.Build.0 = Debug|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Release|x64.ActiveCfg = Release|Any CPU
-		{EC15E8B3-8426-404A-A4B7-E082AC4F11E7}.Release|x64.Build.0 = Release|Any CPU
 		{63826C17-C08C-4E5F-AABB-443EBA565E92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{63826C17-C08C-4E5F-AABB-443EBA565E92}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{63826C17-C08C-4E5F-AABB-443EBA565E92}.Debug|x64.ActiveCfg = Debug|Any CPU
--- a/src/AvaloniaEdit.TextMate/AvaloniaEdit.TextMate.csproj
+++ b/src/AvaloniaEdit.TextMate/AvaloniaEdit.TextMate.csproj
@ -19,7 +19,7 @@

  <ItemGroup>
    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="TextMateSharp" Version="1.0.6" />
+    <PackageReference Include="TextMateSharp" Version="1.0.9" />
  </ItemGroup>

 </Project>
--- a/src/AvaloniaEdit.TextMate/TextEditorModel.cs
+++ b/src/AvaloniaEdit.TextMate/TextEditorModel.cs
@ -5,7 +5,7 @@ using TextMateSharp.Model;

 namespace AvaloniaEdit.TextMate
 {
-    class TextEditorModel : AbstractLineList, IModelTokensChangedListener
+    class TextEditorModel : AbstractLineList
    {
        private object _lock = new object();
        private readonly TextDocument _document;
@ -16,19 +16,39 @@ namespace AvaloniaEdit.TextMate
        {
            _editor = editor;
            _document = document;
-            
+
            _lineCount = _document.LineCount;
-            
-            _document.Changing +=  DocumentOnChanging;
+
+            for (int i = 0; i < _document.LineCount; i++)
+                AddLine(i);
+
+            _document.Changing += DocumentOnChanging;
            _document.Changed += DocumentOnChanged;
            _document.LineCountChanged += DocumentOnLineCountChanged;
-            
-            for (int i = 0; i < _document.LineCount; i++)
-            {
-                AddLine(i);
-            }   
+            _editor.TextArea.TextView.ScrollOffsetChanged += TextView_ScrollOffsetChanged;
        }
-        
+
+        public override void Dispose()
+        {
+            _document.Changing -= DocumentOnChanging;
+            _document.Changed -= DocumentOnChanged;
+            _document.LineCountChanged -= DocumentOnLineCountChanged;
+            _editor.TextArea.TextView.ScrollOffsetChanged -= TextView_ScrollOffsetChanged;
+        }
+
+        private void TextView_ScrollOffsetChanged(object sender, EventArgs e)
+        {
+            Dispatcher.UIThread.InvokeAsync(() =>
+            {
+                if (!_editor.TextArea.TextView.VisualLinesValid)
+                    return;
+
+                ForceTokenization(
+                    _editor.TextArea.TextView.VisualLines[0].FirstDocumentLine.LineNumber - 1,
+                    _editor.TextArea.TextView.VisualLines[_editor.TextArea.TextView.VisualLines.Count - 1].LastDocumentLine.LineNumber - 1);
+            }, DispatcherPriority.Layout - 1);
+        }
+
        private void DocumentOnLineCountChanged(object? sender, EventArgs e)
        {
            lock (_lock)
@ -73,7 +93,7 @@ namespace AvaloniaEdit.TextMate
            {
                UpdateLine(startLine);
            }
-            
+
            InvalidateLine(startLine);
        }

@ -107,31 +127,5 @@ namespace AvaloniaEdit.TextMate
                return _document.Lines[lineIndex].Length;
            }).GetAwaiter().GetResult();
        }
-
-        public override void Dispose()
-        {
-            // todo implement dispose.
-        }
-
-        public void ModelTokensChanged(ModelTokensChangedEvent e)
-        {
-            Dispatcher.UIThread.Post(() =>
-            {
-                try
-                {
-                    foreach (var range in e.ranges)
-                    {
-                        var startLine = _document.GetLineByNumber(range.fromLineNumber);
-                        var endLine = _document.GetLineByNumber(range.toLineNumber);
-
-                        _editor.TextArea.TextView.Redraw(startLine.Offset, endLine.EndOffset - startLine.Offset);
-                    }
-                }
-                catch (Exception e)
-                {
-                    _editor.TextArea.TextView.Redraw();
-                }
-            });
-        }
    }
 }
--- a/src/AvaloniaEdit.TextMate/TextMate.cs
+++ b/src/AvaloniaEdit.TextMate/TextMate.cs
@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using TextMateSharp.Grammars;
 using TextMateSharp.Model;
@ -10,55 +11,122 @@ namespace AvaloniaEdit.TextMate
    {
        public static void InstallTextMate(this TextEditor editor, Theme theme, IGrammar grammar)
        {
-            editor.InstallTheme(theme);
-            editor.InstallGrammar(grammar);
-
-
-            void OnEditorOnDocumentChanged(object sender, EventArgs args)
+            lock(_lock)
            {
-                var editorModel = new TextEditorModel(editor, editor.Document);
-                var model = new TMModel(editorModel);
-                
-                editor.GetOrCreateTransformer().SetModel(editor.Document, model);
-                model.AddModelTokensChangedListener(editor.GetOrCreateTransformer());
-                model.AddModelTokensChangedListener(editorModel);
+                _installations.Add(editor, new TextMateInstallation(editor, theme, grammar));
            }
-            
-            OnEditorOnDocumentChanged(editor, EventArgs.Empty);
-
-            editor.DocumentChanged += OnEditorOnDocumentChanged;
        }
-        
+
+        public static void DisposeTextMate(this TextEditor editor)
+        {
+            lock (_lock)
+            {
+                if (!_installations.ContainsKey(editor))
+                    return;
+
+                _installations[editor].Dispose();
+                _installations.Remove(editor);
+            }
+        }
+
        public static void InstallGrammar(this TextEditor editor, IGrammar grammar)
        {
-            var transformer = editor.GetOrCreateTransformer();
-            
-            transformer.SetGrammar(grammar);
+            lock (_lock)
+            {
+                if (!_installations.ContainsKey(editor))
+                    return;

-            editor.TextArea.TextView.Redraw();
+                _installations[editor].SetGrammar(grammar);
+            }
        }

        public static void InstallTheme(this TextEditor editor, Theme theme)
        {
-            var transformer = editor.GetOrCreateTransformer();
-            
-            transformer.SetTheme(theme);
+            lock (_lock)
+            {
+                if (!_installations.ContainsKey(editor))
+                    return;

-            editor.TextArea.TextView.Redraw();
+                _installations[editor].SetTheme(theme);
+            }
        }

-        private static TextMateColoringTransformer GetOrCreateTransformer(this TextEditor editor)
-        {
-            var transformer = editor.TextArea.TextView.LineTransformers.OfType<TextMateColoringTransformer>().FirstOrDefault();
+        static object _lock = new object();
+        static Dictionary<TextEditor, TextMateInstallation> _installations = new Dictionary<TextEditor, TextMateInstallation>();

-            if (transformer is null)
+        class TextMateInstallation
+        {
+            internal TextMateInstallation(TextEditor editor, Theme theme, IGrammar grammar)
            {
-                transformer = new TextMateColoringTransformer();
-                
-                editor.TextArea.TextView.LineTransformers.Add(transformer);
+                _editor = editor;
+
+                SetTheme(theme);
+                SetGrammar(grammar);
+
+                editor.DocumentChanged += OnEditorOnDocumentChanged;
+
+                OnEditorOnDocumentChanged(editor, EventArgs.Empty);
            }

-            return transformer;
+            internal void SetGrammar(IGrammar grammar)
+            {
+                _grammar = grammar;
+
+                GetOrCreateTransformer().SetGrammar(grammar);
+
+                _editor.TextArea.TextView.Redraw();
+            }
+
+            internal void SetTheme(Theme theme)
+            {
+                GetOrCreateTransformer().SetTheme(theme);
+
+                _editor.TextArea.TextView.Redraw();
+            }
+
+            internal void Dispose()
+            {
+                _editor.DocumentChanged -= OnEditorOnDocumentChanged;
+
+                DisposeTMModel(_tmModel);
+            }
+
+            void OnEditorOnDocumentChanged(object sender, EventArgs args)
+            {
+                DisposeTMModel(_tmModel);
+
+                var editorModel = new TextEditorModel(_editor, _editor.Document);
+                _tmModel = new TMModel(editorModel);
+                _tmModel.SetGrammar(_grammar);
+                GetOrCreateTransformer().SetModel(_editor.Document, _editor.TextArea.TextView, _tmModel);
+                _tmModel.AddModelTokensChangedListener(GetOrCreateTransformer());
+            }
+
+            TextMateColoringTransformer GetOrCreateTransformer()
+            {
+                var transformer = _editor.TextArea.TextView.LineTransformers.OfType<TextMateColoringTransformer>().FirstOrDefault();
+
+                if (transformer is null)
+                {
+                    transformer = new TextMateColoringTransformer();
+
+                    _editor.TextArea.TextView.LineTransformers.Add(transformer);
+                }
+
+                return transformer;
+            }
+
+            static void DisposeTMModel(TMModel tmModel)
+            {
+                if (tmModel == null)
+                    return;
+
+                tmModel.Dispose();
+            }
+
+            TextEditor _editor;
+            IGrammar _grammar;
+            TMModel _tmModel;
        }
    }
 }
--- a/src/AvaloniaEdit.TextMate/TextMateColoringTransformer.cs
+++ b/src/AvaloniaEdit.TextMate/TextMateColoringTransformer.cs
@ -16,6 +16,7 @@ namespace AvaloniaEdit.TextMate
        private IGrammar _grammar;
        private TMModel _model;
        private TextDocument _document;
+        private TextView _textView;

        private readonly Dictionary<int, IBrush> _brushes;
        private TextSegmentCollection<TextTransformation> _transformations;
@ -25,10 +26,11 @@ namespace AvaloniaEdit.TextMate
            _brushes = new Dictionary<int, IBrush>();
        }

-        public void SetModel(TextDocument document, TMModel model)
+        public void SetModel(TextDocument document, TextView textView, TMModel model)
        {
            _document = document;
            _model = model;
+            _textView = textView;

            _transformations = new TextSegmentCollection<TextTransformation>(_document);

@ -120,17 +122,15 @@ namespace AvaloniaEdit.TextMate

        private void ProcessRange(Range range)
        {
-            
-            
            for (int i = range.fromLineNumber; i <= range.toLineNumber; i++)
            {
                var tokens = _model.GetLineTokens(i - 1);

-                if (tokens is { })
-                {
-                    RemoveLineTransformations(i);
-                    ProcessTokens(i, tokens);
-                }
+                if (tokens == null)
+                    continue;
+
+                RemoveLineTransformations(i);
+                ProcessTokens(i, tokens);
            }
        }

@ -140,11 +140,33 @@ namespace AvaloniaEdit.TextMate

            Dispatcher.UIThread.Post(() =>
            {
+                if (_model.IsStopped)
+                    return;
+
                foreach (var range in ranges)
                {
+                    if (!IsValidRange(range, _document.LineCount))
+                        continue;
+
                    ProcessRange(range);
+
+                    var startLine = _document.GetLineByNumber(range.fromLineNumber);
+                    var endLine = _document.GetLineByNumber(range.toLineNumber);
+
+                    _textView.Redraw(startLine.Offset, endLine.EndOffset - startLine.Offset);
                }
            });
        }
+
+        static bool IsValidRange(Range range, int lineCount)
+        {
+            if (range.fromLineNumber < 0 || range.fromLineNumber > lineCount)
+                return false;
+
+            if (range.toLineNumber < 0 || range.toLineNumber > lineCount)
+                return false;
+
+            return true;
+        }
    }
 }