Initial support for Power Fx Modules. (#2661)

Only exposed in REPL, and must be explicitly activated.

Only new concept in core is adding **FileLocation**. We need to report
errors within a file (line,col), not just character offset in an
expression.

Also implement the Assert() function in the repl - this helps with
wrigin modules that act as unit tests.

Module support is defined via Repl. Module format is deserialized via
YamlDotNet.
Modules only support:
- a single Formulas section that behaves like app.formulas. 
- an "imports" section for importing other modules. 
The tests demonstrate the various combinations for public/private,
conflict, shadowing, etc.

Add management functions to repl:
Import(path)
DeleteModule(path) - remove from list 
ListModules() - show which moduels are loaded
This commit is contained in:
Mike Stall 2024-10-10 13:51:34 -07:00 коммит произвёл GitHub
Родитель ac3d51d049
Коммит 13fa198864
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
43 изменённых файлов: 1790 добавлений и 13 удалений

Просмотреть файл

@ -44,6 +44,7 @@ using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Connectors, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Connectors, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.LanguageServerProtocol, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.LanguageServerProtocol, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Json, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Json, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Repl, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Core.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Core.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Interpreter.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Interpreter.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Json.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Json.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]

Просмотреть файл

@ -39,10 +39,15 @@ namespace Microsoft.PowerFx
// If this is set directly, it will skip localization. // If this is set directly, it will skip localization.
set => _message = value; set => _message = value;
} }
/// <summary> /// <summary>
/// Source location for this error. /// Optional - provide file context for where this expression is from.
/// </summary>
public FileLocation FragmentLocation { get; set; }
/// <summary>
/// Source location for this error within a single expression.
/// </summary> /// </summary>
public Span Span { get; set; } public Span Span { get; set; }
@ -209,6 +214,24 @@ namespace Microsoft.PowerFx
return errors.Select(x => ExpressionError.New(x, locale)).ToArray(); return errors.Select(x => ExpressionError.New(x, locale)).ToArray();
} }
} }
// Translate Span in original text (start,end) to something more useful for a file.
internal static IEnumerable<ExpressionError> NewFragment(IEnumerable<IDocumentError> errors, string originalText, FileLocation fragmentLocation)
{
if (errors == null)
{
return Array.Empty<ExpressionError>();
}
else
{
return errors.Select(x =>
{
var error = ExpressionError.New(x, null);
error.FragmentLocation = fragmentLocation.Apply(originalText, error.Span);
return error;
});
}
}
} }
/// <summary> /// <summary>

Просмотреть файл

@ -0,0 +1,66 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Localization;
using Microsoft.PowerFx.Types;
namespace Microsoft.PowerFx.Syntax
{
/// <summary>
/// File-aware, Multi-line source span with Line and Column.
/// Wheras <see cref="Span"/> is a character span within a single expression.
/// </summary>
public class FileLocation
{
public string Filename { get; init; }
// 1-based index
public int LineStart { get; init; }
// 1-based index.
public int ColStart { get; init; }
/// <summary>
/// Apply a span (which is a character offset within a single expression)
/// to this file offset.
/// </summary>
/// <param name="originalText"></param>
/// <param name="s2"></param>
/// <returns></returns>
public FileLocation Apply(string originalText, Span s2)
{
// Convert index span to line span.
int iLine = this.LineStart;
int iCol = ColStart;
for (int i = 0; i < s2.Min; i++)
{
if (originalText[i] == '\r')
{
// ignore
}
else if (originalText[i] == '\n')
{
iLine++;
iCol = ColStart; // reset
}
else
{
iCol++;
}
}
return new FileLocation
{
Filename = Filename,
ColStart = iCol,
LineStart = iLine
};
}
}
}

Просмотреть файл

@ -1,4 +1,8 @@
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: InternalsVisibleTo("Microsoft.PowerFx.Repl.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")]

Просмотреть файл

@ -0,0 +1,49 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.PowerFx.Types;
#pragma warning disable CS0618 // Type or member is obsolete
namespace Microsoft.PowerFx.Repl.Functions
{
/// <summary>
/// Delete a modules.
/// </summary>
internal class DeleteModuleFunction : ReflectionFunction
{
private readonly PowerFxREPL _repl;
public DeleteModuleFunction(PowerFxREPL repl)
: base("DeleteModule", FormulaType.Void, new[] { FormulaType.String })
{
ConfigType = typeof(IReplOutput);
_repl = repl;
}
public async Task<VoidValue> Execute(IReplOutput output, StringValue value, CancellationToken cancel)
{
if (!_repl.TryResolveModule(value.Value, out var module))
{
await output.WriteLineAsync($"Can't resolve module '{value.Value}'. Try ListModules() to see loaded modules.", OutputKind.Error, cancel)
.ConfigureAwait(false);
}
else
{
_repl.DeleteModule(module);
await output.WriteLineAsync("Removed module: " + module.FullPath, OutputKind.Notify, cancel)
.ConfigureAwait(false);
}
return FormulaValue.NewVoid();
}
}
}

Просмотреть файл

@ -0,0 +1,87 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.PowerFx.Types;
#pragma warning disable CS0618 // Type or member is obsolete
namespace Microsoft.PowerFx.Repl.Functions
{
/// <summary>
/// Import a module.
/// </summary>
internal class ImportFunction : ReflectionFunction
{
private readonly PowerFxREPL _repl;
public ImportFunction(PowerFxREPL repl)
: base("Import", FormulaType.Void, new[] { FormulaType.String })
{
ConfigType = typeof(IReplOutput);
_repl = repl;
}
public async Task<VoidValue> Execute(IReplOutput output, StringValue name, CancellationToken cancel)
{
var errors = new List<ExpressionError>();
var ctx = new ModuleLoadContext(_repl.Engine.GetCombinedEngineSymbols());
string filename = name.Value;
var module = await ctx.LoadFromFileAsync(filename, errors).ConfigureAwait(false);
foreach (var error in errors)
{
// Adjust to file...?
var loc = error.FragmentLocation;
var shortName = Path.GetFileName(loc.Filename);
var prefix = error.IsWarning ? "Warning" : "Error";
var kind = error.IsWarning ? OutputKind.Warning : OutputKind.Error;
var msg = $"{prefix}: {shortName} ({loc.LineStart},{loc.ColStart}): {error.Message}";
await output.WriteLineAsync(msg, kind, cancel)
.ConfigureAwait(false);
}
var hasErrors = errors.Where(error => !error.IsWarning).Any();
if (!hasErrors)
{
// Apply these functions to engine.
string header = "Defined functions:";
await output.WriteLineAsync(header, OutputKind.Notify, cancel)
.ConfigureAwait(false);
await PrintModuleAsync(module, output, cancel)
.ConfigureAwait(false);
_repl.AddModule(module);
}
return FormulaValue.NewVoid();
}
internal static async Task PrintModuleAsync(Module module, IReplOutput output, CancellationToken cancel)
{
foreach (var funcName in module.Symbols.FunctionNames)
{
await output.WriteLineAsync($" {funcName}", OutputKind.Notify, cancel)
.ConfigureAwait(false);
}
await output.WriteLineAsync(string.Empty, OutputKind.Notify, cancel)
.ConfigureAwait(false);
}
}
}

Просмотреть файл

@ -0,0 +1,50 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.PowerFx.Types;
#pragma warning disable CS0618 // Type or member is obsolete
namespace Microsoft.PowerFx.Repl.Functions
{
/// <summary>
/// List the modules that are loaded.
/// </summary>
internal class ListModulesFunction : ReflectionFunction
{
private readonly PowerFxREPL _repl;
public ListModulesFunction(PowerFxREPL repl)
: base("ListModules", FormulaType.Void)
{
ConfigType = typeof(IReplOutput);
_repl = repl;
}
public async Task<VoidValue> Execute(IReplOutput output, CancellationToken cancel)
{
var modules = _repl.Modules;
await output.WriteLineAsync("Modules loaded:", OutputKind.Notify, cancel)
.ConfigureAwait(false);
foreach (var module in modules)
{
await output.WriteLineAsync(module.FullPath, OutputKind.Notify, cancel)
.ConfigureAwait(false);
await ImportFunction.PrintModuleAsync(module, output, cancel)
.ConfigureAwait(false);
}
return FormulaValue.NewVoid();
}
}
}

Просмотреть файл

@ -24,6 +24,7 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Microsoft.PowerFx.Core\Microsoft.PowerFx.Core.csproj" /> <ProjectReference Include="..\Microsoft.PowerFx.Core\Microsoft.PowerFx.Core.csproj" />
<ProjectReference Include="..\Microsoft.PowerFx.Interpreter\Microsoft.PowerFx.Interpreter.csproj" /> <ProjectReference Include="..\Microsoft.PowerFx.Interpreter\Microsoft.PowerFx.Interpreter.csproj" />
<PackageReference Include="YamlDotNet" Version="13.4.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

Просмотреть файл

@ -0,0 +1,33 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
namespace Microsoft.PowerFx.Repl
{
// Ensure uniqueness
internal class ConflictTracker
{
// Map from symbol Name to the module it was defined in.
private readonly Dictionary<string, Module> _defined = new Dictionary<string, Module>();
/// <summary>
/// Verify that the symbols added by Module are unique.
/// </summary>
/// <param name="module"></param>
/// <exception cref="InvalidOperationException">If a symbol is already defined.</exception>
public void VerifyUnique(Module module)
{
foreach (var name in module.Symbols.FunctionNames)
{
if (_defined.TryGetValue(name, out var original))
{
throw new InvalidOperationException($"Symbol '{name}' is already defined in both '{original.FullPath}' and '{module.FullPath}'");
}
_defined[name] = module;
}
}
}
}

Просмотреть файл

@ -0,0 +1,57 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.PowerFx.Core.Binding;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Functions;
using Microsoft.PowerFx.Core.Glue;
using Microsoft.PowerFx.Core.Utils;
using Microsoft.PowerFx.Syntax;
using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NodeDeserializers;
namespace Microsoft.PowerFx.Repl
{
internal class FileLoader : IFileLoader
{
// Root directory that we load from.
private readonly string _root;
public FileLoader(string root)
{
if (!Path.IsPathRooted(root))
{
throw new ArgumentException($"Path must be rooted: {root}", nameof(root));
}
_root = root;
}
public async Task<(ModulePoco, IFileLoader)> LoadAsync(string name)
{
name = name.Trim();
var fullPath = Path.Combine(_root, name);
var txt = File.ReadAllText(fullPath);
// Deserialize.
var deserializer = new DeserializerBuilder()
.WithTypeConverter(new StringWithSourceConverter(fullPath, txt))
.Build();
var modulePoco = deserializer.Deserialize<ModulePoco>(txt);
modulePoco.Src_Filename = fullPath;
return (modulePoco, this);
}
}
}

Просмотреть файл

@ -0,0 +1,20 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System.Threading.Tasks;
namespace Microsoft.PowerFx.Repl
{
/// <summary>
/// A load can resolve a name to Module contents.
/// </summary>
internal interface IFileLoader
{
/// <summary>
/// Load a module poco from the given fulename.
/// </summary>
/// <param name="filename">a filename. Could be full path or relative. The loader will resolve.</param>
/// <returns>The loaded module contents and a new file loader for resolving any subsequent imports in this module.</returns>
Task<(ModulePoco, IFileLoader)> LoadAsync(string filename);
}
}

Просмотреть файл

@ -0,0 +1,32 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
namespace Microsoft.PowerFx.Repl
{
internal class Module
{
/// <summary>
/// Public symbols exported by this module.
/// </summary>
public ReadOnlySymbolTable Symbols { get; private set; }
/// <summary>
/// Identity of the module. We should never have two different modules with the same identity.
/// </summary>
public ModuleIdentity Identity { get; init; }
/// <summary>
/// Optional Full path that this module was loaded from. Null if not loaded from a file.
/// Primarily useful for helping makers debugging ("where did this module come from").
/// </summary>
public string FullPath { get; init; }
internal Module(ModuleIdentity identity, ReadOnlySymbolTable exports)
{
this.Identity = identity;
this.Symbols = exports ?? throw new ArgumentNullException(nameof(exports));
}
}
}

Просмотреть файл

@ -0,0 +1,70 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using Microsoft.CodeAnalysis.VisualBasic.Syntax;
namespace Microsoft.PowerFx.Repl
{
/// <summary>
/// A comparable handle representing module identity.
/// </summary>
[DebuggerDisplay("{_value}")]
internal struct ModuleIdentity
{
private readonly string _value;
private ModuleIdentity(string value)
{
_value = value;
}
/// <summary>
/// Get an identify for a file.
/// </summary>
/// <param name="fullPath">full path to file. File does not need to actually exist.</param>
/// <returns>An identity.</returns>
public static ModuleIdentity FromFile(string fullPath)
{
if (fullPath == null)
{
throw new ArgumentNullException(nameof(fullPath));
}
if (!Path.IsPathRooted(fullPath))
{
throw new ArgumentException("Path must be rooted", nameof(fullPath));
}
// GetFullPath will normalize ".." to a canonical representation.
// Path does not need to actually exist.
string canonicalPath = Path.GetFullPath(fullPath);
// Lower to avoid case sensitivity.
var value = canonicalPath.ToLowerInvariant();
return new ModuleIdentity(value);
}
public override bool Equals(object obj)
{
if (obj is ModuleIdentity other)
{
return _value == other._value;
}
return false;
}
public override int GetHashCode()
{
return _value == null ? 0 : _value.GetHashCode();
}
// Do not implement ToString - identity should be treated opaque.
}
}

Просмотреть файл

@ -0,0 +1,229 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Microsoft.PowerFx.Core.Binding;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Functions;
using Microsoft.PowerFx.Core.Glue;
using Microsoft.PowerFx.Core.Types;
using Microsoft.PowerFx.Syntax;
using YamlDotNet.Serialization;
namespace Microsoft.PowerFx.Repl
{
internal static class ModuleLoadContextExtentions
{
// Load a module from disk.
// This may throw on hard error (missing file), or return null with compile errors.
public static async Task<Module> LoadFromFileAsync(this ModuleLoadContext context, string path, List<ExpressionError> errors)
{
var dir = Path.GetDirectoryName(path);
var name = Path.GetFileName(path);
var loader = new FileLoader(dir);
return await context.LoadAsync(name, loader, errors).ConfigureAwait(false);
}
}
/// <summary>
/// Context for loading a single module.
/// This can ensure the module dependencies don't have cycles.
/// </summary>
internal class ModuleLoadContext
{
// Core symbols we resolve against.
// This is needed for all the builtins.
private readonly ReadOnlySymbolTable _commonIncomingSymbols;
// Set of modules we visited - used to detect cycles.
// Key String is the module's full path.
// Value is null if module is in-progress of being loaded (this detects cycles).
// We still allow diamond dependencies.
private readonly Dictionary<ModuleIdentity, Module> _alreadyLoaded = new Dictionary<ModuleIdentity, Module>();
public ModuleLoadContext(ReadOnlySymbolTable commonIncomingSymbols)
{
_commonIncomingSymbols = commonIncomingSymbols;
}
/// <summary>
/// Load and return a resolved module. May throw on errors.
/// </summary>
/// <param name="name">name to resolve via loader parameter.</param>
/// <param name="loader">file loader to resolve the name and any further imports.</param>
/// <param name="errors">error collection to populate if there are errors in parsing, binding, etc. </param>
/// <returns>A module. Or possible null if there are errors. Or may throw. </returns>
public async Task<Module> LoadAsync(string name, IFileLoader loader, List<ExpressionError> errors)
{
(var poco, var loader2) = await loader.LoadAsync(name).ConfigureAwait(false);
ModuleIdentity fullPathIdentity = poco.GetIdentity();
if (_alreadyLoaded.TryGetValue(fullPathIdentity, out var existing))
{
if (existing != null)
{
// If we fully loaded the module, then it's not circular.
return existing;
}
// Circular reference.
throw new InvalidOperationException($"Circular reference: {name}");
}
_alreadyLoaded.Add(fullPathIdentity, null); // In-progress
// Load all imports first.
var symbolList = new List<ReadOnlySymbolTable>
{
_commonIncomingSymbols
};
var unique = new ConflictTracker();
if (poco.Imports != null)
{
foreach (var import in poco.Imports)
{
string file = import.File;
if (file != null)
{
var m2 = await LoadAsync(file, loader2, errors).ConfigureAwait(false);
if (m2 == null)
{
return null;
}
unique.VerifyUnique(m2);
symbolList.Add(m2.Symbols);
}
}
}
var incomingSymbols = ReadOnlySymbolTable.Compose(symbolList.ToArray());
var moduleExports = new SymbolTable { DebugName = name };
bool ok = ResolveBody(poco, moduleExports, incomingSymbols, errors);
if (!ok)
{
// On errors
return null;
}
var module = new Module(fullPathIdentity, moduleExports)
{
FullPath = poco.Src_Filename
};
_alreadyLoaded[fullPathIdentity] = module; // done loading.
return module;
}
// poco - the yaml file contents.
// incomingSymbols - what body can bind to. Builtins, etc.
// any module dependencies should already be resolved and included here.
// moduleExports - symbols to add to.
// Return true on successful load, false on failure
private bool ResolveBody(ModulePoco poco, SymbolTable moduleExports, ReadOnlySymbolTable incomingSymbols, List<ExpressionError> errors)
{
var str = poco.Formulas.Value;
bool allowSideEffects = true;
var options = new ParserOptions
{
AllowParseAsTypeLiteral = true,
AllowsSideEffects = true
};
var parseResult = UserDefinitions.Parse(str, options);
var fragmentLocation = poco.Formulas.Location;
// Scan for errors.
errors.AddRange(ExpressionError.NewFragment(parseResult.Errors, str, fragmentLocation));
if (errors.Any(x => !x.IsWarning))
{
// Abort on errors.
return false;
}
{
var errors2 = parseResult.UDFs.Where(udf => !udf.IsParseValid).FirstOrDefault();
if (errors2 != null)
{
// Should have been caught above.
throw new InvalidOperationException($"Errors should have already been caught");
}
}
var definedTypes = parseResult.DefinedTypes;
if (definedTypes != null && definedTypes.Any())
{
var resolvedTypes = DefinedTypeResolver.ResolveTypes(definedTypes, incomingSymbols, out var errors4);
errors.AddRange(ExpressionError.NewFragment(errors4, str, fragmentLocation));
if (errors.Any(x => !x.IsWarning))
{
// Abort on errors.
return false;
}
moduleExports.AddTypes(resolvedTypes);
}
var s2 = ReadOnlySymbolTable.Compose(moduleExports, incomingSymbols);
// Convert parse --> TexlFunctions
// fail if duplicates detected within this batch.
// fail if any names are resverd
// Basic type checking
IEnumerable<UserDefinedFunction> udfs = UserDefinedFunction.CreateFunctions(parseResult.UDFs, s2, out var errors3);
errors.AddRange(ExpressionError.NewFragment(errors3, str, fragmentLocation));
if (errors.Any(x => !x.IsWarning))
{
return false;
}
// Body can refer to other functions defined in this batch. So we need 2 pass.
// First add all definitions.
foreach (var udf in udfs)
{
moduleExports.AddFunction(udf);
}
// Then bind all bodies.
foreach (var udf in udfs)
{
var config = new BindingConfig(allowsSideEffects: allowSideEffects, useThisRecordForRuleScope: false, numberIsFloat: false);
Features features = Features.PowerFxV1;
var binding = udf.BindBody(s2, new Glue2DocumentBinderGlue(), config, features);
List<TexlError> bindErrors = new List<TexlError>();
binding.ErrorContainer.GetErrors(ref bindErrors);
errors.AddRange(ExpressionError.NewFragment(bindErrors, str, fragmentLocation));
if (errors.Any(x => !x.IsWarning))
{
return false;
}
}
return true;
}
}
}

Просмотреть файл

@ -0,0 +1,46 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using Microsoft.PowerFx.Syntax;
namespace Microsoft.PowerFx.Repl
{
/// <summary>
/// The yaml representation of a module.
/// Properties with 'Src_' prefix are set by deserializer and not part of the yaml file contents.
/// </summary>
internal class ModulePoco
{
/// <summary>
/// Full path of file this was loaded from.
/// This is set by the deserializer.
/// </summary>
public string Src_Filename { get; set; }
public ModuleIdentity GetIdentity()
{
return ModuleIdentity.FromFile(Src_Filename);
}
/// <summary>
/// Contain Power Fx UDF declarations.
/// </summary>
public StringWithSource Formulas { get; set; }
/// <summary>
/// Set of modules that this depends on.
/// </summary>
public ImportPoco[] Imports { get; set; }
}
internal class ImportPoco
{
/// <summary>
/// File path to import from. "foo.fx.yml" or ".\path\foo.fx.yml".
/// </summary>
public string File { get; set; }
// identifier resolved against the host.
public string Host { get; set; }
}
}

Просмотреть файл

@ -0,0 +1,25 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using Microsoft.PowerFx.Syntax;
namespace Microsoft.PowerFx.Repl
{
/// <summary>
/// This is a string object, but also tagged with a solution location for where the string contents start.
/// This is restircted to Inline strings (and rejects other formats like fodling, quotes, etc).
/// Use a custom converter to populate the source location of the string. See <see cref="StringWithSourceConverter"/> .
/// </summary>
internal class StringWithSource
{
/// <summary>
/// The contents of the string from the yaml.
/// </summary>
public string Value { get; set; }
/// <summary>
/// The location in the file for where the contents start. Useful for error reporting.
/// </summary>
public FileLocation Location { get; set; }
}
}

Просмотреть файл

@ -0,0 +1,110 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.PowerFx.Core.Binding;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Functions;
using Microsoft.PowerFx.Core.Glue;
using Microsoft.PowerFx.Core.Utils;
using Microsoft.PowerFx.Syntax;
using YamlDotNet.Core;
using YamlDotNet.Core.Events;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NodeDeserializers;
namespace Microsoft.PowerFx.Repl
{
/// <summary>
/// Yaml converter to parse a <see cref="StringWithSource"/> and tag
/// it with the source location.
/// </summary>
internal class StringWithSourceConverter : IYamlTypeConverter
{
private readonly string _filename;
private readonly string[] _lines; // to infer indent level.
/// <summary>
/// Initializes a new instance of the <see cref="StringWithSourceConverter"/> class.
/// </summary>
/// <param name="filename">filename to tag with.</param>
/// <param name="contents">contents of the file - this is used to infer indenting depth that can't be gotten from th yaml parser.</param>
public StringWithSourceConverter(string filename, string contents)
{
_filename = filename;
_lines = contents.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
}
public bool Accepts(Type type)
{
return type == typeof(StringWithSource);
}
public object ReadYaml(IParser parser, Type type)
{
var val = parser.Consume<Scalar>();
// Empty case
if (val.Style == ScalarStyle.Plain &&
val.Value.Length == 0)
{
return new StringWithSource
{
Value = null, // Preserve same semantics as String deserialization.
Location = new FileLocation
{
Filename = _filename,
LineStart = val.Start.Line,
ColStart = val.Start.Column
}
};
}
// https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines
// Yaml has many multi-line encodings. Fortunately, we can determine this based on ScalarStyle
// We want to force | , which is Literal. This means the fx is copy & paste into yaml, with an indent level
// and no other mutation.
//
// This will prevent all other forms, including:
// > is "fold" which will remove newlines.
// "" is double quotes.
// plain where the string is single-line embedded.
if (val.Style != ScalarStyle.Literal)
{
// This is important if we want to preserve character index mapping.
throw new InvalidOperationException($"must be literal string encoding at {val.Start.Line}");
}
int line1PropertyName = val.Start.Line; // 1-based Line that the "Formulas" property is on. Content starts on next line.
// YamlParser does not tell us indent level. Must discern it.
var line = _lines[line1PropertyName];
var colStart0 = line.FindIndex(x => !char.IsWhiteSpace(x));
return new StringWithSource
{
Value = val.Value,
Location = new FileLocation
{
Filename = _filename,
LineStart = line1PropertyName + 1,
ColStart = colStart0 + 1
}
};
}
public void WriteYaml(IEmitter emitter, object value, Type type)
{
// Only reads.
throw new NotImplementedException();
}
} // end class
}

Просмотреть файл

@ -37,6 +37,13 @@ namespace Microsoft.PowerFx
// Allow repl to create new UserDefinedFunctions. // Allow repl to create new UserDefinedFunctions.
public bool AllowUserDefinedFunctions { get; set; } public bool AllowUserDefinedFunctions { get; set; }
/// <summary>
/// Enable the Import() function for importing modules.
/// Defaults to false.
/// </summary>
[Obsolete("preview")]
public bool AllowImport { get; set; }
// Do we print each command before evaluation? // Do we print each command before evaluation?
// Useful if we're running a file and are debugging, or if input UI is separated from output UI. // Useful if we're running a file and are debugging, or if input UI is separated from output UI.
public bool Echo { get; set; } = false; public bool Echo { get; set; } = false;
@ -76,6 +83,11 @@ namespace Microsoft.PowerFx
/// </summary> /// </summary>
public ReadOnlySymbolValues ExtraSymbolValues { get; set; } public ReadOnlySymbolValues ExtraSymbolValues { get; set; }
// Map from Module full path to Module.
private readonly Dictionary<string, Module> _loadedModules = new Dictionary<string, Module>(StringComparer.OrdinalIgnoreCase);
internal IEnumerable<Module> Modules => _loadedModules.Values;
/// <summary> /// <summary>
/// Get sorted names of all functions. This includes functions from the <see cref="Engine"/> as well as <see cref="MetaFunctions"/>. /// Get sorted names of all functions. This includes functions from the <see cref="Engine"/> as well as <see cref="MetaFunctions"/>.
/// </summary> /// </summary>
@ -94,6 +106,68 @@ namespace Microsoft.PowerFx
} }
} }
// Get combined symbol values (extra, modules, etc)
public ReadOnlySymbolValues GetCombined()
{
// Combine with symbols from modules.
var list = new List<ReadOnlySymbolTable>();
foreach (var module in _loadedModules.Values)
{
list.Add(module.Symbols);
}
if (this.ExtraSymbolValues != null)
{
list.Add(this.ExtraSymbolValues.SymbolTable);
}
var m1 = ReadOnlySymbolTable.Compose(list.ToArray());
var values = m1.CreateValues(this.ExtraSymbolValues);
return values;
}
internal bool TryResolveModule(string path, out Module module)
{
if (_loadedModules.TryGetValue(path, out module))
{
return true;
}
// can we resolve by short name?
List<Module> list = new List<Module>();
foreach (var m in _loadedModules.Values)
{
string shortName = System.IO.Path.GetFileName(m.FullPath);
if (string.Equals(shortName, path, StringComparison.OrdinalIgnoreCase))
{
list.Add(m);
}
}
if (list.Count == 1)
{
module = list[0];
return true;
}
return false;
}
internal void DeleteModule(Module module)
{
_loadedModules.Remove(module.FullPath);
}
internal void AddModule(Module module)
{
string id = module.FullPath;
_loadedModules[id] = module;
}
// Interpreter should normally not throw. // Interpreter should normally not throw.
// Exceptions should be caught and converted to ErrorResult. // Exceptions should be caught and converted to ErrorResult.
// Not called for OperationCanceledException since those are expected. // Not called for OperationCanceledException since those are expected.
@ -123,6 +197,8 @@ namespace Microsoft.PowerFx
this.MetaFunctions.AddFunction(new Help1Function(this)); this.MetaFunctions.AddFunction(new Help1Function(this));
} }
private bool _finishInit = false;
private bool _userEnabled = false; private bool _userEnabled = false;
public void EnableSampleUserObject() public void EnableSampleUserObject()
@ -217,6 +293,22 @@ namespace Microsoft.PowerFx
} }
} }
// Property ctor is run before Init properties are set.
// So apply final initialization after property initializers but before we execute commands.
private void FinishInit()
{
if (!_finishInit)
{
_finishInit = true;
if (this.AllowImport)
{
this.MetaFunctions.AddFunction(new ImportFunction(this));
this.MetaFunctions.AddFunction(new ListModulesFunction(this));
this.MetaFunctions.AddFunction(new DeleteModuleFunction(this));
}
}
}
/// <summary> /// <summary>
/// Directly invoke a command. This skips multiline handling. /// Directly invoke a command. This skips multiline handling.
/// </summary> /// </summary>
@ -239,6 +331,8 @@ namespace Microsoft.PowerFx
return new ReplResult(); return new ReplResult();
} }
FinishInit();
if (this.Echo) if (this.Echo)
{ {
await this.WritePromptAsync(cancel); await this.WritePromptAsync(cancel);
@ -247,9 +341,10 @@ namespace Microsoft.PowerFx
await this.Output.WriteLineAsync(expression.TrimEnd(), OutputKind.Repl, cancel); await this.Output.WriteLineAsync(expression.TrimEnd(), OutputKind.Repl, cancel);
} }
var extraSymbolTable = this.ExtraSymbolValues?.SymbolTable; var extraValues = this.GetCombined();
var extraSymbolTable = extraValues.SymbolTable;
var runtimeConfig = new RuntimeConfig(this.ExtraSymbolValues) var runtimeConfig = new RuntimeConfig(extraValues)
{ {
ServiceProvider = new BasicServiceProvider(this.InnerServices) ServiceProvider = new BasicServiceProvider(this.InnerServices)
}; };
@ -363,7 +458,7 @@ namespace Microsoft.PowerFx
// Get the type. // Get the type.
var rhsExpr = declare._rhs.GetCompleteSpan().GetFragment(expression); var rhsExpr = declare._rhs.GetCompleteSpan().GetFragment(expression);
var setCheck = this.Engine.Check(rhsExpr, ParserOptions, this.ExtraSymbolValues?.SymbolTable); var setCheck = this.Engine.Check(rhsExpr, ParserOptions, this.GetCombined().SymbolTable);
if (!setCheck.IsSuccess) if (!setCheck.IsSuccess)
{ {
await this.Output.WriteLineAsync($"Error: Failed to initialize '{name}'.", OutputKind.Error, cancel) await this.Output.WriteLineAsync($"Error: Failed to initialize '{name}'.", OutputKind.Error, cancel)

Просмотреть файл

@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Localization;
using Microsoft.PowerFx.Syntax;
using Xunit;
namespace Microsoft.PowerFx.Core.Tests
{
public class FileLocationTests
{
[Theory]
[InlineData(0, 200, 300)]
[InlineData(1, 200, 301)]
[InlineData(4, 201, 300)]
[InlineData(9, 202, 301)]
public void Apply(int min, int expectedLine, int expectedCol)
{
string file =
// 012 345 6 789
"ABC\nDE\r\nFG";
var loc = new FileLocation
{
Filename = "test",
LineStart = 200,
ColStart = 300
};
var span = new Span(min, min + 1);
var loc2 = loc.Apply(file, span);
Assert.Same(loc.Filename, loc2.Filename);
Assert.Equal(expectedLine, loc2.LineStart);
Assert.Equal(expectedCol, loc2.ColStart);
}
}
}

Просмотреть файл

@ -68,6 +68,7 @@ namespace Microsoft.PowerFx.Core.Tests
"Microsoft.PowerFx.Syntax.IdentToken", "Microsoft.PowerFx.Syntax.IdentToken",
"Microsoft.PowerFx.Syntax.NumLitToken", "Microsoft.PowerFx.Syntax.NumLitToken",
"Microsoft.PowerFx.Syntax.Span", "Microsoft.PowerFx.Syntax.Span",
"Microsoft.PowerFx.Syntax.FileLocation",
"Microsoft.PowerFx.Syntax.StrLitToken", "Microsoft.PowerFx.Syntax.StrLitToken",
"Microsoft.PowerFx.Syntax.Token", "Microsoft.PowerFx.Syntax.Token",
"Microsoft.PowerFx.Syntax.TokKind", "Microsoft.PowerFx.Syntax.TokKind",

Просмотреть файл

@ -10,8 +10,20 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Compile Include="$(MSBuildThisFileDirectory)AssemblyProperties2.cs" /> <Compile Include="$(MSBuildThisFileDirectory)AssemblyProperties2.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Mocks\TempFileHolder.cs" />
<Compile Include="$(MSBuildThisFileDirectory)ModuleIdentityTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)ModuleTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)MultilineProcessorTests.cs" /> <Compile Include="$(MSBuildThisFileDirectory)MultilineProcessorTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)ReplTests.cs" /> <Compile Include="$(MSBuildThisFileDirectory)ReplTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Mocks\TestReplOutput.cs" /> <Compile Include="$(MSBuildThisFileDirectory)Mocks\TestReplOutput.cs" />
<Compile Include="$(MSBuildThisFileDirectory)StringWithSourceTests.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="$(MSBuildThisFileDirectory)Modules\" />
</ItemGroup>
<ItemGroup>
<None Include="$(MSBuildThisFileDirectory)modules\*.fx.yml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup> </ItemGroup>
</Project> </Project>

Просмотреть файл

@ -0,0 +1,27 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.IO;
namespace Microsoft.PowerFx.Repl.Tests
{
// Helper for creating temp files and cleaning up.
internal class TempFileHolder : IDisposable
{
public string FullPath { get; } = Path.GetTempFileName();
public void Dispose()
{
// Cleanup the file.
try
{
File.Delete(this.FullPath);
}
catch
{
// Ignore
}
}
}
}

Просмотреть файл

@ -0,0 +1,81 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using Xunit;
namespace Microsoft.PowerFx.Repl.Tests
{
public class ModuleIdentityTests
{
[Fact]
public void Collections()
{
var id1a = ModuleIdentity.FromFile(@"z:\path1.txt");
var id1b = ModuleIdentity.FromFile(@"z:\Path1.txt");
var id2 = ModuleIdentity.FromFile(@"z:\path2.txt");
Assert.Equal(id1a, id1a); // identity
Assert.Equal(id1a.GetHashCode(), id1a.GetHashCode());
Assert.NotEqual(id1a, id2);
// paths should be normalized
Assert.Equal(id1a, id1b);
// Work with collections.
var set = new HashSet<ModuleIdentity>();
var added = set.Add(id1a);
Assert.True(added);
added = set.Add(id1a);
Assert.False(added);
added = set.Add(id1b);
Assert.False(added);
Assert.Contains(id1a, set);
Assert.Contains(id1b, set);
Assert.DoesNotContain(id2, set);
added = set.Add(id2);
Assert.True(added);
}
// Ensure we normalize file paths, particularly when using ".."
[Fact]
public void Canonical()
{
var id1a = ModuleIdentity.FromFile(@"z:\foo\path1.txt");
var id1b = ModuleIdentity.FromFile(@"z:\foo\bar\..\Path1.txt");
Assert.Equal(id1a, id1a); // identity
}
[Fact]
public void NoToString()
{
var id1 = ModuleIdentity.FromFile(@"z:\foo\path1.txt");
// *Don't implement ToString* - we want to treat identity as opauque.
var str = id1.ToString();
Assert.Equal("Microsoft.PowerFx.Repl.ModuleIdentity", str);
}
[Fact]
public void MustBeFullPath()
{
Assert.Throws<ArgumentNullException>(() => ModuleIdentity.FromFile(null));
// Must be a full path
Assert.Throws<ArgumentException>(() => ModuleIdentity.FromFile("path1.txt"));
Assert.Throws<ArgumentException>(() => ModuleIdentity.FromFile(@".\path1.txt"));
Assert.Throws<ArgumentException>(() => ModuleIdentity.FromFile(string.Empty));
}
}
}

Просмотреть файл

@ -0,0 +1,306 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.VisualBasic.Syntax;
using Microsoft.PowerFx.Types;
using Xunit;
#pragma warning disable CS0618 // Type or member is obsolete
#pragma warning disable SA1118 // Parameter should not span multiple lines
namespace Microsoft.PowerFx.Repl.Tests
{
// Thesde tests should target public Module Import() function
// (and not internal services)
public class ModuleTests
{
private PowerFxREPL _repl;
private readonly TestReplOutput _output = new TestReplOutput();
public ModuleTests()
{
var config = new PowerFxConfig();
config.SymbolTable.EnableMutationFunctions();
// config.EnableSetFunction();
var engine = new RecalcEngine(config);
_repl = new PowerFxREPL
{
Engine = engine,
Output = _output,
AllowSetDefinitions = true,
AllowUserDefinedFunctions = true,
AllowImport = true,
ParserOptions = new ParserOptions() { AllowsSideEffects = true }
};
}
// Run line, return the normal output.
private string HandleLine(string fx, bool expectErrors = false)
{
Debugger.Log(0, string.Empty, ">> " + fx);
_repl.HandleLine(fx);
if (!expectErrors)
{
AssertNoErrors();
}
var log = _output.Get(OutputKind.Repl);
Debugger.Log(0, string.Empty, log);
return log;
}
private void AssertNoErrors()
{
var errors = _output.Get(OutputKind.Error);
Assert.Empty(errors);
}
private string GetModuleFullPath(string path)
{
string fullpath = Path.Combine(
Environment.CurrentDirectory,
"Modules",
path);
return fullpath;
}
private void Import(string path, bool expectErrors = false)
{
string fullpath = GetModuleFullPath(path);
var expr = $"Import(\"{fullpath}\")";
HandleLine(expr, expectErrors);
if (expectErrors)
{
var modules = _repl.Modules.ToArray();
Assert.Empty(modules); // nothing actually loaded.
}
}
// Ensure that we only can access modules if they're enabled.
[Fact]
public void MustEnable()
{
var engine = new RecalcEngine();
_repl = new PowerFxREPL
{
Engine = engine,
Output = _output
};
// Defaults to false
Assert.False(_repl.AllowUserDefinedFunctions);
Import("basic1.fx.yml", expectErrors: true);
// Failed to import.
var msg = _output.Get(OutputKind.Error);
Assert.Contains("'Import' is an unknown or unsupported function.", msg);
}
[Fact]
public void Test1()
{
Import("basic1.fx.yml");
// Call method defined in moduel.
var log = HandleLine("DoubleIt(3)");
Assert.Equal("6", log);
}
// Load multiple modules
[Fact]
public void Test2()
{
Import("basic1.fx.yml");
Import("basic2.fx.yml");
// Call method defined in moduel.
var log = HandleLine("DoubleIt(Add1(3))");
Assert.Equal("8", log);
}
// Load with imports
[Fact]
public void TestImport()
{
Import("Depend2.fx.yml");
// Call method defined in moduel.
var log = HandleLine("Func2(3)");
Assert.Equal("8", log);
}
// Load with imports
[Fact]
public void TestImport2()
{
Import("Depend2.fx.yml");
// Call method defined in moduel.
var log = HandleLine("Func2(3)");
Assert.Equal("8", log);
// Inner dependencies are not exported
log = HandleLine("Add1(3)", expectErrors: true);
var msg = _output.Get(OutputKind.Error);
Assert.Contains("Add1' is an unknown or unsupported function.", msg);
// But we can import the inner module and call.
Import("basic2.fx.yml");
log = HandleLine("Add1(3)");
Assert.Equal("4", log);
}
// Load with imports
[Fact]
public void Recursion()
{
Import("recursion.fx.yml");
// 0,1, 1, 2, 3, 5, 8,13
var log = HandleLine("Fib(7)");
Assert.Equal("13", log);
}
// Load with imports
[Fact]
public void Shadow()
{
Import("shadow.fx.yml");
var log = HandleLine("Foo()");
Assert.Equal("-2", log);
log = HandleLine("Abs(-7)"); // calls new one from module
}
// Diamond inheritence.
// 1 --> {2a, 2b}. 2a-->3. 2b-->3.
// This is interesting since module 3 gets used multiple times, but it's not a cycle.
[Fact]
public void Diamond()
{
Import("diamond_1.fx.yml");
var log = HandleLine("Func1(5)");
Assert.Equal("\"2A(3(5)),2B(3(5))\"", log);
}
// Conflict if we import 2 modules that define the same symbols.
[Fact]
public void DuplicateSymbolsConflict()
{
Import("conflict1.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
// message contains useful information.
Assert.Contains("DoubleIt", errorMsg); // conflicting symbol
Assert.Contains("basic1.fx.yml", errorMsg); // module 1
Assert.Contains("basic1_dup.fx.yml", errorMsg); // module 2
}
// Import() is a meta function and can only be called
// at top-levle repl and not within a module itself.
[Fact]
public void ImportIsAMetafunction()
{
Import("ErrorImport.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
Assert.Contains("'Import' is an unknown or unsupported function", errorMsg);
}
// When we import a file twice, we get the updated contents.
[Fact]
public void GetsLatest()
{
using var temp = new TempFileHolder();
string fullpath = temp.FullPath;
// First version
File.WriteAllText(fullpath, @"
Formulas: |
Func(x: Number) : Number = x * 10;
");
HandleLine($"Import(\"{fullpath}\")");
// Call method defined in moduel.
var before = HandleLine("Func(3)");
Assert.Equal("30", before);
// Update the file, 2nd version
File.WriteAllText(fullpath, @"
Formulas: |
Func(x: Number) : Number = x * 20;
");
HandleLine($"Import(\"{fullpath}\")");
var after = HandleLine("Func(3)");
Assert.Equal("60", after);
}
// Error gets spans.
[Fact]
public void ErrorShowsFileRange()
{
Import("Error1.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
// Key elements here are that:
// - the message has the filename
// - the location (5,9) is relative into the file, not just the expression.
Assert.Equal("Error: Error1.fx.yml (5,9): Name isn't valid. 'missing' isn't recognized.", errorMsg);
}
// Circular references are detected
[Fact]
public void Cycles()
{
Import("cycle1.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
Assert.Contains("Circular reference", errorMsg);
}
// Loading a missing file
[Fact]
public void Missing()
{
Import("missing_file.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
Assert.Contains("missing_file.fx.yml", errorMsg);
}
// Loading a file with yaml parse errors.
[Fact]
public void YamlParseErrors()
{
Import("yaml_parse_errors.fx.yml", expectErrors: true);
var errorMsg = _output.Get(OutputKind.Error);
}
}
}

Просмотреть файл

@ -0,0 +1,7 @@
Imports:
- File: Basic1.fx.yml
- File: Basic2.fx.yml
# Depends on Basic1
Formulas: |
Func2(x: Number) : Number = DoubleIt(Add1(x));

Просмотреть файл

@ -0,0 +1,6 @@
# Has an error in expression.
# Test that error reporting gets right line.
Formulas: |
DoubleIt(x: Number) : Number =
missing // <-- error is on line 5
* 2;

Просмотреть файл

@ -0,0 +1,7 @@
# Error - we can't call Import() within a function.
# must be a top-level yaml construct.
Formulas: |
Func2(x: Number) : Void = {
Import("Basic1.fx.yml")
};

Просмотреть файл

@ -0,0 +1,3 @@
# Most basic Power Fx module.
Formulas: |
DoubleIt(x: Number) : Number = x * 2;

Просмотреть файл

@ -0,0 +1,4 @@
# test symbol conflicts.
# Same symbols as base1.fx.yml
Formulas: |
DoubleIt(x: Number) : Number = x * 200;

Просмотреть файл

@ -0,0 +1,3 @@
# Most basic Power Fx module.
Formulas: |
Add1(x: Number) : Number = x +1;

Просмотреть файл

@ -0,0 +1,6 @@
Imports:
- File: basic1.fx.yml
- File: basic1_dup.fx.yml
Formulas: |
Run() : Number = DoubleIt(5);

Просмотреть файл

@ -0,0 +1,6 @@
# Test circular dependency
Imports:
- File: cycle2.fx.yml
Formulas: |
Func1(x: Number) : Number = 1;

Просмотреть файл

@ -0,0 +1,6 @@
# Test circular dependency
Imports:
- File: cycle1.fx.yml
Formulas: |
Func2(x: Number) : Number = 1;

Просмотреть файл

@ -0,0 +1,7 @@
Imports:
- File: diamond_2a.fx.yml
- File: diamond_2b.fx.yml
# Depends on 2a,2b. Both depend on 3.
Formulas: |
Func1(x: Text) : Text = Func2a(x) & "," & Func2b(x);

Просмотреть файл

@ -0,0 +1,8 @@
Imports:
- File: diamond_3.fx.yml
# Depends on Basic1
Formulas: |
Func2a(x: Text) : Text = $"2A({Func3(x)})";

Просмотреть файл

@ -0,0 +1,8 @@
Imports:
- File: diamond_3.fx.yml
# Depends on Basic1
Formulas: |
Func2b(x: Text) : Text = $"2B({Func3(x)})";

Просмотреть файл

@ -0,0 +1,3 @@
Formulas: |
Func3(x: Text) : Text = $"3({x})";

Просмотреть файл

@ -0,0 +1,4 @@
# Define a basic recursive function.
Formulas: |
Fib(n: Number) : Number =
If(n <= 1, n, Fib(n - 1) + Fib(n - 2));

Просмотреть файл

@ -0,0 +1,8 @@
Formulas: |
// Redefine a builtin.
// Give a different (wrong) implementation so we know which is called.
Abs(x: Number) : Number = x *2;
// Binds to the one in this scope.
Foo() : Number = Abs(-1);

Просмотреть файл

@ -0,0 +1,3 @@
# not a valid yaml file
Formulas::

Просмотреть файл

@ -0,0 +1,176 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.CodeFixes;
using Microsoft.CodeAnalysis.VisualBasic.Syntax;
using Microsoft.PowerFx.Types;
using Xunit;
using YamlDotNet.Core;
using YamlDotNet.Serialization;
namespace Microsoft.PowerFx.Repl.Tests
{
#pragma warning disable SA1117 // Parameters should be on same line or separate lines
public class StringWithSourceTests
{
private class Poco1
{
public StringWithSource Prop { get; set; }
}
private class Poco1Direct
{
public string Prop { get; set; }
}
// common fake filename used in locations.
private const string Filename = "myfile1.txt";
private T Parse<T>(string contents)
{
// Deserialize.
var deserializer = new DeserializerBuilder()
.WithTypeConverter(new StringWithSourceConverter(Filename, contents))
.Build();
var poco = deserializer.Deserialize<T>(contents);
return poco;
}
private T ParseExpectError<T>(string contents, string expectedError)
{
try
{
// Deserialize.
var deserializer = new DeserializerBuilder()
.WithTypeConverter(new StringWithSourceConverter(Filename, contents))
.Build();
var poco = deserializer.Deserialize<T>(contents);
Assert.Fail($"Expected error: {expectedError}");
return poco;
}
catch (YamlException e)
{
// Exceptions from parser will be wrapped in YamlException.
var msg = e?.InnerException?.Message ?? e.Message;
Assert.Contains(expectedError, msg);
}
return default;
}
// In all cases, parse as 'string' should succeed.
// But Parse with SourceLocaiton has more limitations.
// expectedError specifies restrictions.
private StringWithSource Test(string contents, string expectedError = null)
{
// Should always work with 'string'
var p1 = Parse<Poco1Direct>(contents);
if (expectedError != null)
{
ParseExpectError<Poco1>(contents, expectedError);
return null;
}
else
{
var p2 = Parse<Poco1>(contents);
Assert.Equal(p1.Prop, p2.Prop.Value);
Assert.Same(Filename, p2.Prop.Location.Filename);
if (p2.Prop.Value != null)
{
p2.Prop.Value = p2.Prop.Value.Replace("\r", string.Empty);
}
return p2.Prop;
}
}
[Theory]
[InlineData(
@"Prop: |
123
", 2, 3, "123\n")]
// Leading space in front of property.
[InlineData(
@" Prop: |
123
", 2, 3, "123\n")]
[InlineData(
@"# another line
Prop: |
123
456", 3, 6, "123\n456")]
[InlineData(
@"Prop: |-
123
", 2, 3, "123")]
[InlineData(
@"Prop: |-
true
", 2, 3, "true")] // YDN will still return as string.
// Other encodings (not | ) are not supported.
[InlineData(
@"Prop: >
123
456
", 0, 0, "123456\n", "literal")] // Folding
[InlineData(
@"Prop: 123456
", 0, 0, "123456", "literal")] // plain
[InlineData(
@"Prop: ""123456""
", 0, 0, "123456", "literal")] // quotes
[InlineData(
@"Prop: ", 1, 6, null)] // empty
public void Test1(string contents, int lineStart, int colStart, string value, string errorMessage = null)
{
var p1 = Test(contents, errorMessage);
if (errorMessage == null)
{
Assert.Equal(value, p1.Value);
Assert.Equal(colStart, p1.Location.ColStart);
Assert.Equal(lineStart, p1.Location.LineStart);
}
else
{
Assert.Null(p1);
}
}
// Error conditions.
[Theory]
[InlineData(
"Prop: { }")] // object , expecting string.
public void TestError(string contents)
{
ParseExpectError<Poco1Direct>(contents, "Failed");
ParseExpectError<Poco1>(contents, "Expected 'Scalar'");
}
}
}

Просмотреть файл

@ -0,0 +1,45 @@
// <copyright file="AssertFunction.cs" company="PlaceholderCompany">
// Copyright (c) PlaceholderCompany. All rights reserved.
// </copyright>
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.PowerFx.Types;
#pragma warning disable CS0618 // Type or member is obsolete
namespace Microsoft.PowerFx.Repl.Functions
{
/// <summary>
/// Assert Function.
/// </summary>
internal class AssertFunction : ReflectionFunction
{
public AssertFunction()
: base("Assert", FormulaType.Void, new[] { FormulaType.Boolean, FormulaType.String })
{
ConfigType = typeof(IReplOutput);
}
public async Task<VoidValue> Execute(IReplOutput output, BooleanValue test, StringValue message, CancellationToken cancel)
{
if (test.Value)
{
await output.WriteLineAsync($"PASSED: {message.Value}", OutputKind.Notify, cancel)
.ConfigureAwait(false);
}
else
{
await output.WriteLineAsync($"FAILED: {message.Value}", OutputKind.Error, cancel)
.ConfigureAwait(false);
}
return FormulaValue.NewVoid();
}
}
}

Просмотреть файл

@ -18,6 +18,8 @@ using Microsoft.PowerFx.Types;
namespace Microsoft.PowerFx namespace Microsoft.PowerFx
{ {
#pragma warning disable CS0618 // Type or member is obsolete
public static class ConsoleRepl public static class ConsoleRepl
{ {
private const string OptionFormatTable = "FormatTable"; private const string OptionFormatTable = "FormatTable";
@ -84,9 +86,9 @@ namespace Microsoft.PowerFx
config.EnableSetFunction(); config.EnableSetFunction();
config.EnableJsonFunctions(); config.EnableJsonFunctions();
#pragma warning disable CS0618 // Type or member is obsolete
config.EnableOptionSetInfo(); config.EnableOptionSetInfo();
#pragma warning restore CS0618 // Type or member is obsolete
config.AddFunction(new AssertFunction());
config.AddFunction(new ResetFunction()); config.AddFunction(new ResetFunction());
config.AddFunction(new Option0Function()); config.AddFunction(new Option0Function());
@ -97,9 +99,7 @@ namespace Microsoft.PowerFx
var optionsSet = new OptionSet("Options", DisplayNameUtility.MakeUnique(options)); var optionsSet = new OptionSet("Options", DisplayNameUtility.MakeUnique(options));
#pragma warning disable CS0618 // Type or member is obsolete
config.EnableRegExFunctions(new TimeSpan(0, 0, 5)); config.EnableRegExFunctions(new TimeSpan(0, 0, 5));
#pragma warning restore CS0618 // Type or member is obsolete
config.AddOptionSet(optionsSet); config.AddOptionSet(optionsSet);
@ -124,9 +124,7 @@ namespace Microsoft.PowerFx
} }
// Hook repl engine with customizations. // Hook repl engine with customizations.
#pragma warning disable CS0618 // Type or member is obsolete
private class MyRepl : PowerFxREPL private class MyRepl : PowerFxREPL
#pragma warning restore CS0618 // Type or member is obsolete
{ {
public MyRepl() public MyRepl()
{ {
@ -138,6 +136,8 @@ namespace Microsoft.PowerFx
this.AllowSetDefinitions = true; this.AllowSetDefinitions = true;
this.AllowUserDefinedFunctions = _enableUDFs; this.AllowUserDefinedFunctions = _enableUDFs;
this.AllowImport = true;
this.EnableSampleUserObject(); this.EnableSampleUserObject();
this.AddPseudoFunction(new IRPseudoFunction()); this.AddPseudoFunction(new IRPseudoFunction());
this.AddPseudoFunction(new SuggestionsPseudoFunction()); this.AddPseudoFunction(new SuggestionsPseudoFunction());
@ -427,9 +427,7 @@ namespace Microsoft.PowerFx
private class MyHelpProvider : HelpProvider private class MyHelpProvider : HelpProvider
{ {
#pragma warning disable CS0618 // Type or member is obsolete
public override async Task Execute(PowerFxREPL repl, CancellationToken cancel, string context = null) public override async Task Execute(PowerFxREPL repl, CancellationToken cancel, string context = null)
#pragma warning restore CS0618 // Type or member is obsolete
{ {
if (context?.ToLowerInvariant() == "options" || context?.ToLowerInvariant() == "option") if (context?.ToLowerInvariant() == "options" || context?.ToLowerInvariant() == "option")
{ {