Merge branch 'master' of ssh://github.com/Microsoft/GazeInteractionApps

Apps/MinAAC/MainPage.xaml

@@ -37,6 +37,7 @@
 
         <Grid.RowDefinitions>
             <RowDefinition Height="200"/>
+            <RowDefinition Height="100"/>
             <RowDefinition Height="*"/>
         </Grid.RowDefinitions>
         <Grid.ColumnDefinitions>
@@ -61,8 +62,14 @@
             <Button     x:Name="EnterButton" Grid.Row="0" Grid.Column="8" Content="&#xE768;" Style="{StaticResource Symbol}" Click="OnSpeak"/>
             <Button     x:Name="CloseButton" Grid.Row="0" Grid.Column="9" Content="&#xE711;" Style="{StaticResource Symbol}" Click="OnExit"/>
         </Grid>
-
-        <gk:GazeKeyboard x:Name="GazeKeyboard" Grid.Row="1"  Grid.Column="0" />
+        <StackPanel Orientation="Horizontal" Grid.Row="1">
+            <Button x:Name="Prediction0" Height="100" MinWidth="300" Style="{StaticResource Alpha}" />
+            <Button x:Name="Prediction1" Height="100" MinWidth="300" Style="{StaticResource Alpha}" />
+            <Button x:Name="Prediction2" Height="100" MinWidth="300" Style="{StaticResource Alpha}" />
+            <Button x:Name="Prediction3" Height="100" MinWidth="300" Style="{StaticResource Alpha}" />
+            <Button x:Name="Prediction4" Height="100" MinWidth="300" Style="{StaticResource Alpha}" />
+        </StackPanel>
+        <gk:GazeKeyboard x:Name="GazeKeyboard" Grid.Row="2"  Grid.Column="0" />
     </Grid>
 
 </Page>

Apps/MinAAC/MainPage.xaml.cs

@@ -12,12 +12,20 @@ namespace MinAAC
 {
     public sealed partial class MainPage : Page
     {
-        MediaElement        _mediaElement;
+        const int NUM_PREDICTIONS = 5;
+        MediaElement _mediaElement;
         SpeechSynthesizer   _speechSynthesizer;
+        Button[]         _predictions;
 
         public MainPage()
         {
             InitializeComponent();
+            _predictions = new Button[NUM_PREDICTIONS];
+            _predictions[0] = Prediction0;
+            _predictions[1] = Prediction1;
+            _predictions[2] = Prediction2;
+            _predictions[3] = Prediction3;
+            _predictions[4] = Prediction4;
 
             ApplicationView.PreferredLaunchWindowingMode = ApplicationViewWindowingMode.FullScreen;
             _mediaElement = new MediaElement();
@@ -29,6 +37,7 @@ namespace MinAAC
         {
             GazeKeyboard.Target = TextControl;
             await GazeKeyboard.LoadLayout("MinAAC.xaml");
+            GazeKeyboard.PredictionTargets = _predictions;
         }
 
         private async void OnSpeak(object sender, RoutedEventArgs e)

EyeGazeUserControls/EyeGazeUserControls.csproj

@@ -11,7 +11,7 @@
     <AssemblyName>EyeGazeUserControls</AssemblyName>
     <DefaultLanguage>en-US</DefaultLanguage>
     <TargetPlatformIdentifier>UAP</TargetPlatformIdentifier>
-    <TargetPlatformVersion Condition=" '$(TargetPlatformVersion)' == '' ">10.0.17134.0</TargetPlatformVersion>
+    <TargetPlatformVersion Condition=" '$(TargetPlatformVersion)' == '' ">10.0.18362.0</TargetPlatformVersion>
     <TargetPlatformMinVersion>10.0.17134.0</TargetPlatformMinVersion>
     <MinimumVisualStudioVersion>14</MinimumVisualStudioVersion>
     <FileAlignment>512</FileAlignment>

EyeGazeUserControls/GazeKeyboard.xaml.cs

@@ -9,6 +9,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 using Windows.ApplicationModel.DataTransfer;
+using Windows.Data.Text;
 using Windows.Storage;
 using Windows.System;
 using Windows.UI.Input.Preview.Injection;
@@ -45,14 +46,52 @@ namespace EyeGazeUserControls
         InputInjector _injector;
         List<ButtonBase> _keyboardButtons;
         KeyboardPage _rootPage;
-
-        public Control Target;
+        TextPredictionGenerator _textPredictionGenerator;
+        WordsSegmenter _wordsSegmenter;
+        string _predictionLanguage;
+        Button[] _predictionTargets;
 
         public bool GazePlusClickMode;
 
+        public TextBox Target;
+
+        public string PredictionLanguage
+        {
+            get { return _predictionLanguage; }
+            set
+            {
+                _predictionLanguage = value;
+                _textPredictionGenerator = new TextPredictionGenerator(value);
+                _textPredictionGenerator.InputScope = Windows.UI.Text.Core.CoreTextInputScope.Text;
+                _wordsSegmenter = new WordsSegmenter(value);
+            }
+        }
+
+        public Button[] PredictionTargets
+        {
+            get { return _predictionTargets; }
+            set
+            {
+                if (_predictionTargets != null)
+                {
+                    foreach (var target in _predictionTargets)
+                    {
+                        target.Click -= OnPredictionSelected;
+                    }
+                }
+                _predictionTargets = value;
+                foreach (var target in _predictionTargets)
+                {
+                    target.Click += OnPredictionSelected;
+                }
+
+            }
+        }
+
         public GazeKeyboard()
         {
             InitializeComponent();
+            PredictionLanguage = "en-US";
             _injector = InputInjector.TryCreate();
         }
 
@@ -130,6 +169,8 @@ namespace EyeGazeUserControls
             var key = new InjectedInputKeyboardInfo();
             key.VirtualKey = (ushort)vk;
             _injector.InjectKeyboardInput(new[] { key });
+
+            UpdatePredictions();
             return true;
         }
 
@@ -153,6 +194,8 @@ namespace EyeGazeUserControls
                 keys.Add(key);
             }
             _injector.InjectKeyboardInput(keys);
+
+            UpdatePredictions();
             return true;
         }
 
@@ -282,6 +325,7 @@ namespace EyeGazeUserControls
                 key.ScanCode = button.Content.ToString()[0];
                 key.KeyOptions = InjectedInputKeyOptions.Unicode;
                 _injector.InjectKeyboardInput(new[] { key });
+                UpdatePredictions();
                 injected = true;
             }
 
@@ -290,5 +334,127 @@ namespace EyeGazeUserControls
                 RevertTempPage(button);
             }
         }
+
+        private void InjectString(string str, bool addSpace)
+        {
+            if (addSpace)
+            {
+                str += " ";
+            }
+
+            var keys = new List<InjectedInputKeyboardInfo>();
+            foreach (var ch in str)
+            {
+                var key = new InjectedInputKeyboardInfo()
+                {
+                    ScanCode = ch,
+                    KeyOptions = InjectedInputKeyOptions.Unicode
+                };
+                keys.Add(key);
+
+                key = new InjectedInputKeyboardInfo()
+                {
+                    ScanCode = ch,
+                    KeyOptions = InjectedInputKeyOptions.Unicode | InjectedInputKeyOptions.KeyUp
+                };
+                keys.Add(key);
+            }
+            _injector.InjectKeyboardInput(keys);
+        }
+
+        private void OnPredictionSelected(object sender, RoutedEventArgs e)
+        {
+            WordSegment replaceSegment = null;
+            if (Target.SelectionStart < Target.Text.Length)
+            {
+                replaceSegment = _wordsSegmenter.GetTokenAt(Target.Text, (uint)Target.SelectionStart);
+            }
+            else if (Target.Text[Target.Text.Length - 1] != ' ')
+            {
+                var tokens = _wordsSegmenter.GetTokens(Target.Text);
+                if (tokens == null || tokens.Count == 0)
+                {
+                    return;
+                }
+
+                replaceSegment = tokens[tokens.Count - 1];
+            }
+
+            if (replaceSegment != null)
+            {
+                Target.Select((int)replaceSegment.SourceTextSegment.StartPosition, (int)replaceSegment.SourceTextSegment.Length);
+            }
+
+            string prediction = (sender as Button).Content.ToString();
+            InjectString(prediction, true);
+            UpdatePredictions();
+        }
+
+        private List<string> GetPrevWords()
+        {
+            var segments = _wordsSegmenter.GetTokens(Target.Text);
+            if ((segments == null) || (segments.Count == 0))
+            {
+                return null;
+            }
+
+            int i = 0;
+            var words = new List<string>(segments.Count);
+            foreach (var segment in segments)
+            {
+                words.Add(segment.Text);
+                i++;
+            }
+            words.Reverse();
+            return words;
+        }
+
+        private async void UpdateNextWordPredictions()
+        {
+            var prevWords = GetPrevWords();
+            var predictions = await _textPredictionGenerator.GetNextWordCandidatesAsync((uint)PredictionTargets.Length, prevWords);
+            DisplayPredictions(predictions);
+        }
+
+        private async void UpdatePredictions()
+        {
+            // IMPORTANT: Wait for the text box to be updated with the injected keys
+            await Task.Delay(1);
+
+            if ((PredictionTargets == null) || (PredictionTargets.Length <= 0))
+            {
+                return;
+            }
+
+            var prevWords = GetPrevWords();
+
+            if ((prevWords == null) || (prevWords.Count == 0))
+            {
+                return;
+            }
+
+            IReadOnlyList<string> predictions;
+            var prevWordsExceptLast = prevWords.GetRange(1, prevWords.Count - 1);
+            // It looks like we need to send in a larger number than necessary to get good quality predictions. 
+            uint maxCandidates = (uint)PredictionTargets.Length * 2; 
+            predictions = await _textPredictionGenerator.GetCandidatesAsync(prevWords[0], 
+                                                                            maxCandidates,
+                                                                            TextPredictionOptions.Corrections | TextPredictionOptions.Predictions,
+                                                                            prevWordsExceptLast);
+            DisplayPredictions(predictions);
+        }
+
+        private void DisplayPredictions(IReadOnlyList<string> predictions)
+        {
+            int i;
+            for (i = 0; (i < predictions.Count) && (i < PredictionTargets.Length); i++)
+            {
+                PredictionTargets[i].Content = predictions[i];
+            }
+            for (; i < PredictionTargets.Length; i++)
+            {
+                PredictionTargets[i].Content = "";
+            }
+        }
     }
 }