Add support for Base64Vlq encode and source map serialize

Extract Base64VlqConstants from Base64VlqDecoder so that it can be
shared by Base64VlqEncoder
Implement Base64Converter.ToBase64 and Base64VlqEncoder
Implement SourceMapGenerator to support serializing SourceMap to json
string
Add unit tests for classes/methods added
This commit is contained in:
Nile Liao 2016-11-17 19:11:37 +08:00
Родитель 0ff79ec40a
Коммит 3b9a887cf7
11 изменённых файлов: 523 добавлений и 15 удалений

Просмотреть файл

@ -2,7 +2,7 @@
This is a C# library for working with JavaScript source maps and deminifying JavaScript callstacks.
## Source Map Parsing
The `SourcemapToolkit.SourcemapParser.dll` provides an API for parsing a souce map into an object that is easy to work with. The source map class has a method `GetMappingEntryForGeneratedSourcePosition`, which can be used to find a source map mapping entry that likely corresponds to a piece of generated code.
The source map class has a method `GetMappingEntryForGeneratedSourcePosition`, which can be used to find a source map mapping entry that likely corresponds to a piece of generated code.
## Call Stack Deminification
The `SourcemapToolkit.CallstackDeminifier.dll` allows for the deminification of JavaScript call stacks.

Просмотреть файл

@ -32,5 +32,18 @@ namespace SourcemapToolkit.SourcemapParser
return result;
}
}
/// <summary>
/// Converts a integer to base64 value
/// </summary>
internal static char ToBase64(int value)
{
if (value < 0 || value >= Base64Alphabet.Length)
{
throw new ArgumentOutOfRangeException(nameof(value));
}
return Base64Alphabet[value];
}
}
}

Просмотреть файл

@ -0,0 +1,36 @@
/* Based on the Base 64 VLQ implementation in Closure Compiler:
* https://github.com/google/closure-compiler/blob/master/src/com/google/debugging/sourcemap/Base64VLQ.java
*
* Copyright 2011 The Closure Compiler Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace SourcemapToolkit.SourcemapParser
{
/// <summary>
/// Constants used in Base64 VLQ encode/decode
/// </summary>
internal static class Base64VlqConstants
{
// A Base64 VLQ digit can represent 5 bits, so it is base-32.
public const int VlqBaseShift = 5;
public const int VlqBase = 1 << VlqBaseShift;
// A mask of bits for a VLQ digit (11111), 31 decimal.
public const int VlqBaseMask = VlqBase - 1;
// The continuation bit is the 6th bit.
public const int VlqContinuationBit = VlqBase;
}
}

Просмотреть файл

@ -26,16 +26,6 @@ namespace SourcemapToolkit.SourcemapParser
/// </summary>
internal static class Base64VlqDecoder
{
// A Base64 VLQ digit can represent 5 bits, so it is base-32.
private const int VlqBaseShift = 5;
private const int VlqBase = 1 << VlqBaseShift;
// A mask of bits for a VLQ digit (11111), 31 decimal.
private static readonly int VlqBaseMask = VlqBase - 1;
// The continuation bit is the 6th bit.
private static readonly int VlqContinuationBit = VlqBase;
/// <summary>
/// Converts to a two-complement value from a value where the sign bit is
/// is placed in the least significant bit.For example, as decimals:
@ -103,10 +93,10 @@ namespace SourcemapToolkit.SourcemapParser
{
char c = charProvider.GetNextCharacter();
int digit = Base64Converter.FromBase64(c);
continuation = (digit & VlqContinuationBit) != 0;
digit &= VlqBaseMask;
continuation = (digit & Base64VlqConstants.VlqContinuationBit) != 0;
digit &= Base64VlqConstants.VlqBaseMask;
result = result + (digit << shift);
shift = shift + VlqBaseShift;
shift = shift + Base64VlqConstants.VlqBaseShift;
} while (continuation);
return FromVlqSigned(result);

Просмотреть файл

@ -0,0 +1,49 @@
/* Based on the Base 64 VLQ implementation in Closure Compiler:
* https://github.com/google/closure-compiler/blob/master/src/com/google/debugging/sourcemap/Base64VLQ.java
*
* Copyright 2011 The Closure Compiler Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Collections.Generic;
namespace SourcemapToolkit.SourcemapParser
{
/// <summary>
/// This class provides a mechanism for converting an interger to Base64 Variable-length quantity (VLQ)
/// </summary>
internal static class Base64VlqEncoder
{
public static void Encode(ICollection<char> output, int value)
{
int vlq = ToVlqSigned(value);
do
{
int maskResult = vlq & Base64VlqConstants.VlqBaseMask;
vlq = vlq >> Base64VlqConstants.VlqBaseShift;
if (vlq > 0)
{
maskResult |= Base64VlqConstants.VlqContinuationBit;
}
output.Add(Base64Converter.ToBase64(maskResult));
} while (vlq > 0);
}
private static int ToVlqSigned(int value)
{
return value < 0 ? ((-value << 1) + 1) : (value << 1) + 0;
}
}
}

Просмотреть файл

@ -0,0 +1,185 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections.Generic;
using System.Runtime.Serialization;
namespace SourcemapToolkit.SourcemapParser
{
/// <summary>
/// Class to track the internal state during source map serialize
/// </summary>
internal class MappingGenerateState
{
/// <summary>
/// Last location of the code in the transformed code
/// </summary>
public readonly SourcePosition LastGeneratedPosition = new SourcePosition();
/// <summary>
/// Last location of the code in the source code
/// </summary>
public readonly SourcePosition LastOriginalPosition = new SourcePosition();
/// <summary>
/// List that contains the symbol names
/// </summary>
public readonly IList<string> Names;
/// <summary>
/// List that contains the file sources
/// </summary>
public readonly IList<string> Sources;
/// <summary>
/// Index of last file source
/// </summary>
public int LastSourceIndex { get; set; }
/// <summary>
/// Index of last symbol name
/// </summary>
public int LastNameIndex { get; set; }
/// <summary>
/// Whether this is the first segment in current line
/// </summary>
public bool IsFirstSegment { get; set; }
public MappingGenerateState(IList<string> names, IList<string> sources)
{
Names = names;
Sources = sources;
IsFirstSegment = true;
}
}
public class SourceMapGenerator
{
/// <summary>
/// Serialize SourceMap object to json string
/// </summary>
public string SerializeMapping(SourceMap sourceMap)
{
}
/// <summary>
/// Serialize SourceMap object to json string with given serialize settings
/// </summary>
public string SerializeMapping(SourceMap sourceMap, JsonSerializerSettings jsonSerializerSettings)
{
if (sourceMap == null)
{
return null;
}
SourceMap mapToSerialize = new SourceMap()
{
File = sourceMap.File,
Names = sourceMap.Names,
Sources = sourceMap.Sources,
Version = sourceMap.Version,
};
if (sourceMap.ParsedMappings != null && sourceMap.ParsedMappings.Count > 0)
{
MappingGenerateState state = new MappingGenerateState(sourceMap.Names, sourceMap.Sources);
List<char> output = new List<char>();
foreach (MappingEntry entry in sourceMap.ParsedMappings)
{
SerializeMappingEntry(entry, state, output);
}
output.Add(';');
mapToSerialize.Mappings = new string(output.ToArray());
}
return JsonConvert.SerializeObject(mapToSerialize,
jsonSerializerSettings ?? new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
NullValueHandling = NullValueHandling.Ignore,
});
}
/// <summary>
/// Convert each mapping entry to VLQ encoded segments
/// </summary>
internal void SerializeMappingEntry(MappingEntry entry, MappingGenerateState state, ICollection<char> output)
{
// Each line of generated code is separated using semicolons
while (entry.GeneratedSourcePosition.ZeroBasedLineNumber != state.LastGeneratedPosition.ZeroBasedLineNumber)
{
state.LastGeneratedPosition.ZeroBasedColumnNumber = 0;
state.LastGeneratedPosition.ZeroBasedLineNumber++;
state.IsFirstSegment = true;
output.Add(';');
}
// The V3 source map format calls for all Base64 VLQ segments to be seperated by commas.
if (!state.IsFirstSegment)
output.Add(',');
state.IsFirstSegment = false;
/*
* The following description was taken from the Sourcemap V3 spec https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/mobilebasic?pref=2&pli=1
* The Sourcemap V3 spec is under a Creative Commons Attribution-ShareAlike 3.0 Unported License. https://creativecommons.org/licenses/by-sa/3.0/
*
* Each VLQ segment has 1, 4, or 5 variable length fields.
* The fields in each segment are:
* 1. The zero-based starting column of the line in the generated code that the segment represents.
* If this is the first field of the first segment, or the first segment following a new generated line(;),
* then this field holds the whole base 64 VLQ.Otherwise, this field contains a base 64 VLQ that is relative to
* the previous occurrence of this field.Note that this is different than the fields below because the previous
* value is reset after every generated line.
* 2. If present, an zero - based index into the sources list.This field is a base 64 VLQ relative to the previous
* occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented.
* 3. If present, the zero-based starting line in the original source represented. This field is a base 64 VLQ relative to the
* previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is
* represented.Always present if there is a source field.
* 4. If present, the zero - based starting column of the line in the source represented.This field is a base 64 VLQ relative to
* the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is
* represented.Always present if there is a source field.
* 5. If present, the zero - based index into the names list associated with this segment.This field is a base 64 VLQ relative
* to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value
* is represented.
*/
Base64VlqEncoder.Encode(output, entry.GeneratedSourcePosition.ZeroBasedColumnNumber - state.LastGeneratedPosition.ZeroBasedColumnNumber);
state.LastGeneratedPosition.ZeroBasedColumnNumber = entry.GeneratedSourcePosition.ZeroBasedColumnNumber;
if (entry.OriginalFileName != null)
{
int sourceIndex = state.Sources.IndexOf(entry.OriginalFileName);
if (sourceIndex < 0)
{
throw new SerializationException("Source map contains original source that cannot be found in provided sources array");
}
Base64VlqEncoder.Encode(output, sourceIndex - state.LastSourceIndex);
state.LastSourceIndex = sourceIndex;
Base64VlqEncoder.Encode(output, entry.OriginalSourcePosition.ZeroBasedLineNumber - state.LastOriginalPosition.ZeroBasedLineNumber);
state.LastOriginalPosition.ZeroBasedLineNumber = entry.OriginalSourcePosition.ZeroBasedLineNumber;
Base64VlqEncoder.Encode(output, entry.OriginalSourcePosition.ZeroBasedColumnNumber - state.LastOriginalPosition.ZeroBasedColumnNumber);
state.LastOriginalPosition.ZeroBasedColumnNumber = entry.OriginalSourcePosition.ZeroBasedColumnNumber;
if (entry.OriginalName != null)
{
int nameIndex = state.Names.IndexOf(entry.OriginalName);
if (nameIndex < 0)
{
throw new SerializationException("Source map contains original name that cannot be found in provided names array");
}
Base64VlqEncoder.Encode(output, nameIndex - state.LastNameIndex);
state.LastNameIndex = nameIndex;
}
}
}
}
}

Просмотреть файл

@ -52,11 +52,14 @@
</ItemGroup>
<ItemGroup>
<Compile Include="Base64Converter.cs" />
<Compile Include="Base64VlqConstants.cs" />
<Compile Include="Base64VlqDecoder.cs" />
<Compile Include="Base64VlqEncoder.cs" />
<Compile Include="MappingEntry.cs" />
<Compile Include="MappingListParser.cs" />
<Compile Include="SourceMap.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="SourceMapGenerator.cs" />
<Compile Include="SourceMapParser.cs" />
<Compile Include="SourcePosition.cs" />
</ItemGroup>

Просмотреть файл

@ -33,5 +33,31 @@ namespace SourcemapToolkit.SourcemapParser.UnitTests
// Act
Base64Converter.FromBase64('@');
}
[TestMethod]
public void ToBase64_ValidIntegerInput61_CorrectBase64Output9()
{
// Act
char value = Base64Converter.ToBase64(61);
// Assert
Assert.AreEqual('9', value);
}
[TestMethod]
[ExpectedException(typeof(ArgumentOutOfRangeException))]
public void ToBase64_NegativeIntegerInput_ThrowsException()
{
// Act
Base64Converter.ToBase64(-1);
}
[TestMethod]
[ExpectedException(typeof(ArgumentOutOfRangeException))]
public void ToBase64_InvalidIntegerInput_ThrowsException()
{
// Act
Base64Converter.ToBase64(64);
}
}
}

Просмотреть файл

@ -0,0 +1,43 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
namespace SourcemapToolkit.SourcemapParser.UnitTests
{
[TestClass]
public class Base64VlqEncoderUnitTests
{
[TestMethod]
public void Base64VlqEncoder_SmallValue_ListWithOnlyOneValue()
{
// Act
List<char> result = new List<char>();
Base64VlqEncoder.Encode( result, 15 );
// Assert
Assert.AreEqual( "e", new string( result.ToArray() ) );
}
[TestMethod]
public void Base64VlqEncoder_LargeValue_ListWithOnlyMultipleValues()
{
// Act
List<char> result = new List<char>();
Base64VlqEncoder.Encode( result, 701 );
// Assert
Assert.AreEqual( "6rB", new string( result.ToArray() ) );
}
[TestMethod]
public void Base64VlqEncoder_NegativeValue_ListWithCorrectValue()
{
// Act
List<char> result = new List<char>();
Base64VlqEncoder.Encode( result, -15 );
// Assert
Assert.AreEqual( "f", new string( result.ToArray() ) );
}
}
}

Просмотреть файл

@ -0,0 +1,161 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
namespace SourcemapToolkit.SourcemapParser.UnitTests
{
[TestClass]
public class SourceMapGeneratorUnitTests
{
[TestMethod]
public void SerializeMappingEntry_DifferentLineNumber_SemicolonAdded()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
MappingGenerateState state = new MappingGenerateState( new List<string>() { "Name" }, new List<string>() { "Source" } );
state.LastGeneratedPosition.ZeroBasedColumnNumber = 1;
MappingEntry entry = new MappingEntry()
{
GeneratedSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 0 },
OriginalFileName = state.Sources[ 0 ],
OriginalName = state.Names[ 0 ],
OriginalSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 0 },
};
// Act
List<char> output = new List<char>();
sourceMapGenerator.SerializeMappingEntry( entry, state, output );
// Assert
Assert.IsTrue( output.IndexOf( ';' ) >= 0 );
}
[TestMethod]
public void SerializeMappingEntry_NoOriginalFileName_OneSegment()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
MappingGenerateState state = new MappingGenerateState( new List<string>() { "Name" }, new List<string>() { "Source" } );
MappingEntry entry = new MappingEntry()
{
GeneratedSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 0, ZeroBasedColumnNumber = 10 },
OriginalSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 0, ZeroBasedColumnNumber = 1 },
};
// Act
List<char> output = new List<char>();
sourceMapGenerator.SerializeMappingEntry( entry, state, output );
// Assert
Assert.AreEqual( "U", new string( output.ToArray() ) );
}
[TestMethod]
public void SerializeMappingEntry_WithOriginalFileNameNoOriginalName_FourSegment()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
MappingGenerateState state = new MappingGenerateState( new List<string>() { "Name" }, new List<string>() { "Source" } );
state.IsFirstSegment = false;
MappingEntry entry = new MappingEntry()
{
GeneratedSourcePosition = new SourcePosition() { ZeroBasedColumnNumber = 10 },
OriginalFileName = state.Sources[0],
OriginalSourcePosition = new SourcePosition() { ZeroBasedColumnNumber = 5 },
};
// Act
List<char> output = new List<char>();
sourceMapGenerator.SerializeMappingEntry( entry, state, output );
// Assert
Assert.AreEqual( ",UAAK", new string( output.ToArray() ) );
}
[TestMethod]
public void SerializeMappingEntry_WithOriginalFileNameAndOriginalName_FiveSegment()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
MappingGenerateState state = new MappingGenerateState( new List<string>() { "Name" }, new List<string>() { "Source" } );
state.LastGeneratedPosition.ZeroBasedLineNumber = 1;
MappingEntry entry = new MappingEntry()
{
GeneratedSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 5 },
OriginalSourcePosition = new SourcePosition() { ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 6 },
OriginalFileName = state.Sources[ 0 ],
OriginalName = state.Names[ 0 ],
};
// Act
List<char> output = new List<char>();
sourceMapGenerator.SerializeMappingEntry( entry, state, output );
// Assert
Assert.AreEqual( "KACMA", new string( output.ToArray() ) );
}
[TestMethod]
public void SerializeMapping_NullInput_ReturnsNull()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
SourceMap input = null;
// Act
string output = sourceMapGenerator.SerializeMapping( input );
// Assert
Assert.IsNull( output );
}
[TestMethod]
public void SerializeMapping_SimpleSourceMap_CorrectlySerialized()
{
// Arrange
SourceMapGenerator sourceMapGenerator = new SourceMapGenerator();
SourceMap input = new SourceMap()
{
File = "CommonIntl",
Names = new List<string>() { "CommonStrings", "afrikaans" },
Sources = new List<string>() { "input/CommonIntl.js" },
Version = 3,
};
input.ParsedMappings = new List<MappingEntry>()
{
new MappingEntry
{
GeneratedSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 0, ZeroBasedColumnNumber = 0 },
OriginalFileName = input.Sources[0],
OriginalName = input.Names[0],
OriginalSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 0 },
},
new MappingEntry
{
GeneratedSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 0, ZeroBasedColumnNumber = 13 },
OriginalFileName = input.Sources[0],
OriginalSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 0 },
},
new MappingEntry
{
GeneratedSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 0, ZeroBasedColumnNumber = 14 },
OriginalFileName = input.Sources[0],
OriginalSourcePosition = new SourcePosition() {ZeroBasedLineNumber = 1, ZeroBasedColumnNumber = 14 },
},
};
// Act
string output = sourceMapGenerator.SerializeMapping( input );
// Assert
Assert.AreEqual( "{\"version\":3,\"file\":\"CommonIntl\",\"mappings\":\"AACAA,aAAA,CAAc;\",\"sources\":[\"input/CommonIntl.js\"],\"names\":[\"CommonStrings\",\"afrikaans\"]}", output );
}
}
}

Просмотреть файл

@ -60,9 +60,11 @@
<ItemGroup>
<Compile Include="Base64ConverterUnitTests.cs" />
<Compile Include="Base64VlqDecoderUnitTests.cs" />
<Compile Include="Base64VlqEncoderUnitTests.cs" />
<Compile Include="NumericMappingEntryUnitTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="MappingsListParserUnitTests.cs" />
<Compile Include="SourceMapGeneratorUnitTests.cs" />
<Compile Include="SourceMapParserUnitTests.cs" />
<Compile Include="SourceMapUnitTests.cs" />
<Compile Include="SourcePositionUnitTests.cs" />