Fix wide character support

Fixes https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1501927

XtermSharp/BufferLine.cs

@@ -45,7 +45,7 @@ namespace XtermSharp {
 		   * from real empty cells.
 		   * */
 		   // TODO: not sue this is completely right
-		public bool HasContent (int index) => data [index].Code != 0 || data [index].Attribute != CharData.DefaultAttr;
+		public bool HasContent (int index) => !data [index].IsNullChar() || data [index].Attribute != CharData.DefaultAttr;
 
 		public bool HasAnyContent()
 		{
@@ -164,7 +164,7 @@ namespace XtermSharp {
 		public int GetTrimmedLength ()
 		{
 			for (int i = data.Length - 1; i >= 0; --i)
-				if (data [i].Code != 0) {
+				if (!data [i].IsNullChar()) {
 					int width = 0;
 					for (int j = 0; j <= i; j++)
 						width += data [i].Width;

XtermSharp/CharData.cs

@@ -27,7 +27,11 @@ namespace XtermSharp {
 		public static CharData RightParenthesis = new CharData (DefaultAttr, ')', 1, 41);
 		public static CharData Period = new CharData (DefaultAttr, '.', 1, 46);
 
-		public CharData (int attribute, Rune rune, int width, int code)
+		public CharData (int attribute, char rune, int width) : this(attribute, rune, width, rune)
+		{ 
+		}
+
+		public CharData (int attribute, char rune, int width, int code)
 		{
 			Attribute = attribute;
 			Rune = rune;

XtermSharp/CharWidth.cs

@@ -86,7 +86,7 @@ namespace XtermSharp {
 			if (rune >= 0x7f && rune <= 0xa0)
 				return 0;
 			/* binary search in table of non-spacing characters */
-			if (bisearch (rune, combining, combining.GetLength (0)) != 0)
+			if (bisearch (rune, combining, combining.GetLength (0) - 1) != 0)
 				return 0;
 			/* if we arrive here, ucs is not a combining or C0/C1 control character */
 			return 1 +

XtermSharp/InputHandlers/InputHandler.cs

@@ -1200,56 +1200,75 @@ namespace XtermSharp {
 			var curAttr = terminal.CurAttr;
 			var bufferRow = buffer.Lines [buffer.Y + buffer.YBase];
 
-
 			terminal.UpdateRange (buffer.Y);
 
-			while (readingBuffer.HasNext ()) {
-				int code;
-				byte bufferValue = readingBuffer.GetNext ();
-				var n = RuneExt.ExpectedSizeFromFirstByte (bufferValue);
-				if (n == -1) {
-					// Invalid UTF-8 sequence, client sent us some junk, happens if we run with the wrong locale set
-					// for example if LANG=en
-					code = (int)((uint)bufferValue);
-				} else if (n == 1) {
-					code = bufferValue;
-				} else {
-					var bytesRemaining = readingBuffer.BytesLeft ();
-					if (readingBuffer.BytesLeft () >= (n - 1)) {
-						var x = new byte [n];
-						x [0] = bufferValue;
-						for (int j = 1; j < n; j++)
-							x [j] = readingBuffer.GetNext ();
-
-						(var r, var size) = Rune.DecodeRune (x);
-						code = (int)(uint)r;
-					} else {
-						readingBuffer.Putback (bufferValue);
-						return;
+			while (readingBuffer.HasNext()) {
+				var ch = ' ';
+				var chWidth = 0;
+				int code = 32;
+				byte bufferValue = readingBuffer.GetNext();
+				var n = RuneExt.ExpectedSizeFromFirstByte(bufferValue);
+				if (n == -1 || n == 1) {
+					var chSet = false;
+					if (bufferValue < 127 && charset != null) {
+						var str = charset[bufferValue];
+						if (!string.IsNullOrEmpty(str)) {
+							ch = str[0];
+							// Every single mapping in the charset only takes one slot
+							chWidth = 1;
+							chSet = true;
+						}
 					}
+
+					if (!chSet) {
+						(var rune, var size) = Rune.DecodeRune(new byte[] { bufferValue });
+						chWidth = size;
+						ch = rune.ToString()[0];
+					}
+
 				}
+				// Invalid UTF-8 sequence, client sent us some junk, happens if we run with the wrong locale set
+				// for example if LANG=en
+				else if (readingBuffer.BytesLeft() >= (n - 1)) {
+					var bytes = new byte[n];
+					bytes[0] = bufferValue;
+					for (int j = 1; j < n; j++)
+						bytes[j] = readingBuffer.GetNext();
 
-				// MIGUEL-TODO: I suspect this needs to be a stirng in C# to cope with Grapheme clusters
-				var ch = code;
+					string s = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+					if (s.Length > 0) {
+						ch = s[0];
+					}
+					else {
+						ch = ' ';
+					}
 
-				// calculate print space
-				// expensive call, therefore we save width in line buffer
+					// Now the challenge is that we have a character, not a rune, and we want to compute
+					// the width of it.
+					if (s.Length == 1) {
+						chWidth = RuneHelper.ConsoleWidth(s[0]);
+					}
+					else {
+						chWidth = 0;
 
-				// TODO: This is wrong, we only have one byte at this point, we do not have a full rune.
-				// The correct fix includes the upper parser tracking the "pending" data across invocations
-				// until a valid UTF-8 string comes in, and *then* we can call this method
-				// var chWidth = Rune.ColumnWidth ((Rune)code);
+						foreach (var scalar in s) {
+							chWidth = Math.Max(chWidth, RuneHelper.ConsoleWidth(scalar));
+						}
+					}
 
-				// 1 until we get a fixed NStack
-				var chWidth = 1;
+				}
+				else {
+					readingBuffer.Putback(bufferValue);
+					return;
+				}
 
 				// get charset replacement character
 				// charset are only defined for ASCII, therefore we only
 				// search for an replacement char if code < 127
-				if (code < 127 && charset != null) {
+				if (bufferValue < 127 && charset != null) {
 
 					// MIGUEL-FIXME - this is broken for dutch cahrset that returns two letters "ij", need to figure out what to do
-					if (charset.TryGetValue ((byte)code, out var str)) {
+					if (charset.TryGetValue (bufferValue, out var str)) {
 						ch = str [0];
 						code = ch;
 					}
@@ -1336,7 +1355,7 @@ namespace XtermSharp {
 				}
 
 				// write current char to buffer and advance cursor
-				var charData = new CharData (curAttr, (uint)code, chWidth, ch);
+				var charData = new CharData (curAttr, ch, chWidth);
 				bufferRow [buffer.X++] = charData;
 
 				// fullwidth char - also set next cell to placeholder stub and advance cursor

XtermSharp/InputHandlers/TerminalBufferManipulationCommandExtensions.cs

@@ -171,7 +171,7 @@ namespace XtermSharp.CommandExtensions {
 						//for (scalar in ch.unicodeScalars) {
 						//	checksum += scalar.value;
 						//}
-						checksum += cd.Code == 0 ? 32 : cd.Code;
+						checksum += cd.IsNullChar() ? 32 : cd.Code;
 					}
 				}
 

XtermSharp/Terminal.cs

@@ -985,7 +985,7 @@ namespace XtermSharp {
 					}
 				}
 			} else {
-				if (buffer.X < left) {
+				if (buffer.X < left && buffer.X > 0) {
 					// This compensates for the scenario where backspace is supposed to move one step
 					// backwards if the "x" position is behind the left margin.
 					// Test BS_MovesLeftWhenLeftOfLeftMargin