diff --git a/src/libraries/Microsoft.PowerFx.Core/AssemblyProperties.cs b/src/libraries/Microsoft.PowerFx.Core/AssemblyProperties.cs index 5b2f40023..ad4afdc24 100644 --- a/src/libraries/Microsoft.PowerFx.Core/AssemblyProperties.cs +++ b/src/libraries/Microsoft.PowerFx.Core/AssemblyProperties.cs @@ -44,6 +44,7 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.PowerFx.Connectors, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.LanguageServerProtocol, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Json, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] +[assembly: InternalsVisibleTo("Microsoft.PowerFx.Repl, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Core.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Interpreter.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] [assembly: InternalsVisibleTo("Microsoft.PowerFx.Json.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/ExpressionError.cs b/src/libraries/Microsoft.PowerFx.Core/Public/ExpressionError.cs index 50b43d131..92f8e7756 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Public/ExpressionError.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Public/ExpressionError.cs @@ -39,10 +39,15 @@ namespace Microsoft.PowerFx // If this is set directly, it will skip localization. set => _message = value; - } + } /// - /// Source location for this error. + /// Optional - provide file context for where this expression is from. + /// + public FileLocation FragmentLocation { get; set; } + + /// + /// Source location for this error within a single expression. /// public Span Span { get; set; } @@ -209,6 +214,24 @@ namespace Microsoft.PowerFx return errors.Select(x => ExpressionError.New(x, locale)).ToArray(); } } + + // Translate Span in original text (start,end) to something more useful for a file. + internal static IEnumerable NewFragment(IEnumerable errors, string originalText, FileLocation fragmentLocation) + { + if (errors == null) + { + return Array.Empty(); + } + else + { + return errors.Select(x => + { + var error = ExpressionError.New(x, null); + error.FragmentLocation = fragmentLocation.Apply(originalText, error.Span); + return error; + }); + } + } } /// diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/FileLocation.cs b/src/libraries/Microsoft.PowerFx.Core/Public/FileLocation.cs new file mode 100644 index 000000000..7032b7af1 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Public/FileLocation.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Syntax +{ + /// + /// File-aware, Multi-line source span with Line and Column. + /// Wheras is a character span within a single expression. + /// + public class FileLocation + { + public string Filename { get; init; } + + // 1-based index + public int LineStart { get; init; } + + // 1-based index. + public int ColStart { get; init; } + + /// + /// Apply a span (which is a character offset within a single expression) + /// to this file offset. + /// + /// + /// + /// + public FileLocation Apply(string originalText, Span s2) + { + // Convert index span to line span. + int iLine = this.LineStart; + int iCol = ColStart; + + for (int i = 0; i < s2.Min; i++) + { + if (originalText[i] == '\r') + { + // ignore + } + else if (originalText[i] == '\n') + { + iLine++; + iCol = ColStart; // reset + } + else + { + iCol++; + } + } + + return new FileLocation + { + Filename = Filename, + ColStart = iCol, + LineStart = iLine + }; + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/AssemblyProperties.cs b/src/libraries/Microsoft.PowerFx.Repl/AssemblyProperties.cs index bd083c2c6..4f70d6c1e 100644 --- a/src/libraries/Microsoft.PowerFx.Repl/AssemblyProperties.cs +++ b/src/libraries/Microsoft.PowerFx.Repl/AssemblyProperties.cs @@ -1,4 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: InternalsVisibleTo("Microsoft.PowerFx.Repl.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] diff --git a/src/libraries/Microsoft.PowerFx.Repl/Functions/DeleteModule.cs b/src/libraries/Microsoft.PowerFx.Repl/Functions/DeleteModule.cs new file mode 100644 index 000000000..7c5910668 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Functions/DeleteModule.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Types; + +#pragma warning disable CS0618 // Type or member is obsolete + +namespace Microsoft.PowerFx.Repl.Functions +{ + /// + /// Delete a modules. + /// + internal class DeleteModuleFunction : ReflectionFunction + { + private readonly PowerFxREPL _repl; + + public DeleteModuleFunction(PowerFxREPL repl) + : base("DeleteModule", FormulaType.Void, new[] { FormulaType.String }) + { + ConfigType = typeof(IReplOutput); + _repl = repl; + } + + public async Task Execute(IReplOutput output, StringValue value, CancellationToken cancel) + { + if (!_repl.TryResolveModule(value.Value, out var module)) + { + await output.WriteLineAsync($"Can't resolve module '{value.Value}'. Try ListModules() to see loaded modules.", OutputKind.Error, cancel) + .ConfigureAwait(false); + } + else + { + _repl.DeleteModule(module); + + await output.WriteLineAsync("Removed module: " + module.FullPath, OutputKind.Notify, cancel) + .ConfigureAwait(false); + } + + return FormulaValue.NewVoid(); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Functions/ImportFunction.cs b/src/libraries/Microsoft.PowerFx.Repl/Functions/ImportFunction.cs new file mode 100644 index 000000000..58a4ff8b4 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Functions/ImportFunction.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Types; + +#pragma warning disable CS0618 // Type or member is obsolete + +namespace Microsoft.PowerFx.Repl.Functions +{ + /// + /// Import a module. + /// + internal class ImportFunction : ReflectionFunction + { + private readonly PowerFxREPL _repl; + + public ImportFunction(PowerFxREPL repl) + : base("Import", FormulaType.Void, new[] { FormulaType.String }) + { + ConfigType = typeof(IReplOutput); + _repl = repl; + } + + public async Task Execute(IReplOutput output, StringValue name, CancellationToken cancel) + { + var errors = new List(); + + var ctx = new ModuleLoadContext(_repl.Engine.GetCombinedEngineSymbols()); + + string filename = name.Value; + var module = await ctx.LoadFromFileAsync(filename, errors).ConfigureAwait(false); + + foreach (var error in errors) + { + // Adjust to file...? + var loc = error.FragmentLocation; + + var shortName = Path.GetFileName(loc.Filename); + + var prefix = error.IsWarning ? "Warning" : "Error"; + var kind = error.IsWarning ? OutputKind.Warning : OutputKind.Error; + + var msg = $"{prefix}: {shortName} ({loc.LineStart},{loc.ColStart}): {error.Message}"; + + await output.WriteLineAsync(msg, kind, cancel) + .ConfigureAwait(false); + } + + var hasErrors = errors.Where(error => !error.IsWarning).Any(); + + if (!hasErrors) + { + // Apply these functions to engine. + + string header = "Defined functions:"; + await output.WriteLineAsync(header, OutputKind.Notify, cancel) + .ConfigureAwait(false); + + await PrintModuleAsync(module, output, cancel) + .ConfigureAwait(false); + + _repl.AddModule(module); + } + + return FormulaValue.NewVoid(); + } + + internal static async Task PrintModuleAsync(Module module, IReplOutput output, CancellationToken cancel) + { + foreach (var funcName in module.Symbols.FunctionNames) + { + await output.WriteLineAsync($" {funcName}", OutputKind.Notify, cancel) + .ConfigureAwait(false); + } + + await output.WriteLineAsync(string.Empty, OutputKind.Notify, cancel) + .ConfigureAwait(false); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Functions/ListModules.cs b/src/libraries/Microsoft.PowerFx.Repl/Functions/ListModules.cs new file mode 100644 index 000000000..50827818d --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Functions/ListModules.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Types; + +#pragma warning disable CS0618 // Type or member is obsolete + +namespace Microsoft.PowerFx.Repl.Functions +{ + /// + /// List the modules that are loaded. + /// + internal class ListModulesFunction : ReflectionFunction + { + private readonly PowerFxREPL _repl; + + public ListModulesFunction(PowerFxREPL repl) + : base("ListModules", FormulaType.Void) + { + ConfigType = typeof(IReplOutput); + _repl = repl; + } + + public async Task Execute(IReplOutput output, CancellationToken cancel) + { + var modules = _repl.Modules; + + await output.WriteLineAsync("Modules loaded:", OutputKind.Notify, cancel) + .ConfigureAwait(false); + + foreach (var module in modules) + { + await output.WriteLineAsync(module.FullPath, OutputKind.Notify, cancel) + .ConfigureAwait(false); + + await ImportFunction.PrintModuleAsync(module, output, cancel) + .ConfigureAwait(false); + } + + return FormulaValue.NewVoid(); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Microsoft.PowerFx.Repl.csproj b/src/libraries/Microsoft.PowerFx.Repl/Microsoft.PowerFx.Repl.csproj index ecae66fc9..fa43d1026 100644 --- a/src/libraries/Microsoft.PowerFx.Repl/Microsoft.PowerFx.Repl.csproj +++ b/src/libraries/Microsoft.PowerFx.Repl/Microsoft.PowerFx.Repl.csproj @@ -24,6 +24,7 @@ + diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/ConflictTracker.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/ConflictTracker.cs new file mode 100644 index 000000000..74c6f3e6f --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/ConflictTracker.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; + +namespace Microsoft.PowerFx.Repl +{ + // Ensure uniqueness + internal class ConflictTracker + { + // Map from symbol Name to the module it was defined in. + private readonly Dictionary _defined = new Dictionary(); + + /// + /// Verify that the symbols added by Module are unique. + /// + /// + /// If a symbol is already defined. + public void VerifyUnique(Module module) + { + foreach (var name in module.Symbols.FunctionNames) + { + if (_defined.TryGetValue(name, out var original)) + { + throw new InvalidOperationException($"Symbol '{name}' is already defined in both '{original.FullPath}' and '{module.FullPath}'"); + } + + _defined[name] = module; + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/FileLoader.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/FileLoader.cs new file mode 100644 index 000000000..6293ed073 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/FileLoader.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Glue; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Syntax; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NodeDeserializers; + +namespace Microsoft.PowerFx.Repl +{ + internal class FileLoader : IFileLoader + { + // Root directory that we load from. + private readonly string _root; + + public FileLoader(string root) + { + if (!Path.IsPathRooted(root)) + { + throw new ArgumentException($"Path must be rooted: {root}", nameof(root)); + } + + _root = root; + } + + public async Task<(ModulePoco, IFileLoader)> LoadAsync(string name) + { + name = name.Trim(); + var fullPath = Path.Combine(_root, name); + + var txt = File.ReadAllText(fullPath); + + // Deserialize. + var deserializer = new DeserializerBuilder() + .WithTypeConverter(new StringWithSourceConverter(fullPath, txt)) + .Build(); + var modulePoco = deserializer.Deserialize(txt); + + modulePoco.Src_Filename = fullPath; + + return (modulePoco, this); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/IFileLoader.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/IFileLoader.cs new file mode 100644 index 000000000..e20cce10c --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/IFileLoader.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Threading.Tasks; + +namespace Microsoft.PowerFx.Repl +{ + /// + /// A load can resolve a name to Module contents. + /// + internal interface IFileLoader + { + /// + /// Load a module poco from the given fulename. + /// + /// a filename. Could be full path or relative. The loader will resolve. + /// The loaded module contents and a new file loader for resolving any subsequent imports in this module. + Task<(ModulePoco, IFileLoader)> LoadAsync(string filename); + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/Module.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/Module.cs new file mode 100644 index 000000000..f80f568c8 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/Module.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Microsoft.PowerFx.Repl +{ + internal class Module + { + /// + /// Public symbols exported by this module. + /// + public ReadOnlySymbolTable Symbols { get; private set; } + + /// + /// Identity of the module. We should never have two different modules with the same identity. + /// + public ModuleIdentity Identity { get; init; } + + /// + /// Optional Full path that this module was loaded from. Null if not loaded from a file. + /// Primarily useful for helping makers debugging ("where did this module come from"). + /// + public string FullPath { get; init; } + + internal Module(ModuleIdentity identity, ReadOnlySymbolTable exports) + { + this.Identity = identity; + this.Symbols = exports ?? throw new ArgumentNullException(nameof(exports)); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleIdentity.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleIdentity.cs new file mode 100644 index 000000000..5ac263ad6 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleIdentity.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using Microsoft.CodeAnalysis.VisualBasic.Syntax; + +namespace Microsoft.PowerFx.Repl +{ + /// + /// A comparable handle representing module identity. + /// + [DebuggerDisplay("{_value}")] + internal struct ModuleIdentity + { + private readonly string _value; + + private ModuleIdentity(string value) + { + _value = value; + } + + /// + /// Get an identify for a file. + /// + /// full path to file. File does not need to actually exist. + /// An identity. + public static ModuleIdentity FromFile(string fullPath) + { + if (fullPath == null) + { + throw new ArgumentNullException(nameof(fullPath)); + } + + if (!Path.IsPathRooted(fullPath)) + { + throw new ArgumentException("Path must be rooted", nameof(fullPath)); + } + + // GetFullPath will normalize ".." to a canonical representation. + // Path does not need to actually exist. + string canonicalPath = Path.GetFullPath(fullPath); + + // Lower to avoid case sensitivity. + var value = canonicalPath.ToLowerInvariant(); + + return new ModuleIdentity(value); + } + + public override bool Equals(object obj) + { + if (obj is ModuleIdentity other) + { + return _value == other._value; + } + + return false; + } + + public override int GetHashCode() + { + return _value == null ? 0 : _value.GetHashCode(); + } + + // Do not implement ToString - identity should be treated opaque. + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleLoadContext.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleLoadContext.cs new file mode 100644 index 000000000..d788cad5a --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleLoadContext.cs @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Glue; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Syntax; +using YamlDotNet.Serialization; + +namespace Microsoft.PowerFx.Repl +{ + internal static class ModuleLoadContextExtentions + { + // Load a module from disk. + // This may throw on hard error (missing file), or return null with compile errors. + public static async Task LoadFromFileAsync(this ModuleLoadContext context, string path, List errors) + { + var dir = Path.GetDirectoryName(path); + var name = Path.GetFileName(path); + + var loader = new FileLoader(dir); + + return await context.LoadAsync(name, loader, errors).ConfigureAwait(false); + } + } + + /// + /// Context for loading a single module. + /// This can ensure the module dependencies don't have cycles. + /// + internal class ModuleLoadContext + { + // Core symbols we resolve against. + // This is needed for all the builtins. + private readonly ReadOnlySymbolTable _commonIncomingSymbols; + + // Set of modules we visited - used to detect cycles. + // Key String is the module's full path. + // Value is null if module is in-progress of being loaded (this detects cycles). + // We still allow diamond dependencies. + private readonly Dictionary _alreadyLoaded = new Dictionary(); + + public ModuleLoadContext(ReadOnlySymbolTable commonIncomingSymbols) + { + _commonIncomingSymbols = commonIncomingSymbols; + } + + /// + /// Load and return a resolved module. May throw on errors. + /// + /// name to resolve via loader parameter. + /// file loader to resolve the name and any further imports. + /// error collection to populate if there are errors in parsing, binding, etc. + /// A module. Or possible null if there are errors. Or may throw. + public async Task LoadAsync(string name, IFileLoader loader, List errors) + { + (var poco, var loader2) = await loader.LoadAsync(name).ConfigureAwait(false); + + ModuleIdentity fullPathIdentity = poco.GetIdentity(); + if (_alreadyLoaded.TryGetValue(fullPathIdentity, out var existing)) + { + if (existing != null) + { + // If we fully loaded the module, then it's not circular. + return existing; + } + + // Circular reference. + throw new InvalidOperationException($"Circular reference: {name}"); + } + + _alreadyLoaded.Add(fullPathIdentity, null); // In-progress + + // Load all imports first. + + var symbolList = new List + { + _commonIncomingSymbols + }; + var unique = new ConflictTracker(); + + if (poco.Imports != null) + { + foreach (var import in poco.Imports) + { + string file = import.File; + if (file != null) + { + var m2 = await LoadAsync(file, loader2, errors).ConfigureAwait(false); + if (m2 == null) + { + return null; + } + + unique.VerifyUnique(m2); + symbolList.Add(m2.Symbols); + } + } + } + + var incomingSymbols = ReadOnlySymbolTable.Compose(symbolList.ToArray()); + + var moduleExports = new SymbolTable { DebugName = name }; + + bool ok = ResolveBody(poco, moduleExports, incomingSymbols, errors); + + if (!ok) + { + // On errors + return null; + } + + var module = new Module(fullPathIdentity, moduleExports) + { + FullPath = poco.Src_Filename + }; + + _alreadyLoaded[fullPathIdentity] = module; // done loading. + + return module; + } + + // poco - the yaml file contents. + // incomingSymbols - what body can bind to. Builtins, etc. + // any module dependencies should already be resolved and included here. + // moduleExports - symbols to add to. + // Return true on successful load, false on failure + private bool ResolveBody(ModulePoco poco, SymbolTable moduleExports, ReadOnlySymbolTable incomingSymbols, List errors) + { + var str = poco.Formulas.Value; + + bool allowSideEffects = true; + var options = new ParserOptions + { + AllowParseAsTypeLiteral = true, + AllowsSideEffects = true + }; + + var parseResult = UserDefinitions.Parse(str, options); + + var fragmentLocation = poco.Formulas.Location; + + // Scan for errors. + errors.AddRange(ExpressionError.NewFragment(parseResult.Errors, str, fragmentLocation)); + + if (errors.Any(x => !x.IsWarning)) + { + // Abort on errors. + return false; + } + + { + var errors2 = parseResult.UDFs.Where(udf => !udf.IsParseValid).FirstOrDefault(); + if (errors2 != null) + { + // Should have been caught above. + throw new InvalidOperationException($"Errors should have already been caught"); + } + } + + var definedTypes = parseResult.DefinedTypes; + if (definedTypes != null && definedTypes.Any()) + { + var resolvedTypes = DefinedTypeResolver.ResolveTypes(definedTypes, incomingSymbols, out var errors4); + errors.AddRange(ExpressionError.NewFragment(errors4, str, fragmentLocation)); + if (errors.Any(x => !x.IsWarning)) + { + // Abort on errors. + return false; + } + + moduleExports.AddTypes(resolvedTypes); + } + + var s2 = ReadOnlySymbolTable.Compose(moduleExports, incomingSymbols); + + // Convert parse --> TexlFunctions + // fail if duplicates detected within this batch. + // fail if any names are resverd + // Basic type checking + IEnumerable udfs = UserDefinedFunction.CreateFunctions(parseResult.UDFs, s2, out var errors3); + + errors.AddRange(ExpressionError.NewFragment(errors3, str, fragmentLocation)); + if (errors.Any(x => !x.IsWarning)) + { + return false; + } + + // Body can refer to other functions defined in this batch. So we need 2 pass. + // First add all definitions. + foreach (var udf in udfs) + { + moduleExports.AddFunction(udf); + } + + // Then bind all bodies. + foreach (var udf in udfs) + { + var config = new BindingConfig(allowsSideEffects: allowSideEffects, useThisRecordForRuleScope: false, numberIsFloat: false); + + Features features = Features.PowerFxV1; + + var binding = udf.BindBody(s2, new Glue2DocumentBinderGlue(), config, features); + + List bindErrors = new List(); + + binding.ErrorContainer.GetErrors(ref bindErrors); + errors.AddRange(ExpressionError.NewFragment(bindErrors, str, fragmentLocation)); + + if (errors.Any(x => !x.IsWarning)) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/ModulePoco.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModulePoco.cs new file mode 100644 index 000000000..1943727b8 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModulePoco.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Microsoft.PowerFx.Syntax; + +namespace Microsoft.PowerFx.Repl +{ + /// + /// The yaml representation of a module. + /// Properties with 'Src_' prefix are set by deserializer and not part of the yaml file contents. + /// + internal class ModulePoco + { + /// + /// Full path of file this was loaded from. + /// This is set by the deserializer. + /// + public string Src_Filename { get; set; } + + public ModuleIdentity GetIdentity() + { + return ModuleIdentity.FromFile(Src_Filename); + } + + /// + /// Contain Power Fx UDF declarations. + /// + public StringWithSource Formulas { get; set; } + + /// + /// Set of modules that this depends on. + /// + public ImportPoco[] Imports { get; set; } + } + + internal class ImportPoco + { + /// + /// File path to import from. "foo.fx.yml" or ".\path\foo.fx.yml". + /// + public string File { get; set; } + + // identifier resolved against the host. + public string Host { get; set; } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSource.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSource.cs new file mode 100644 index 000000000..69e72ffc1 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSource.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Microsoft.PowerFx.Syntax; + +namespace Microsoft.PowerFx.Repl +{ + /// + /// This is a string object, but also tagged with a solution location for where the string contents start. + /// This is restircted to Inline strings (and rejects other formats like fodling, quotes, etc). + /// Use a custom converter to populate the source location of the string. See . + /// + internal class StringWithSource + { + /// + /// The contents of the string from the yaml. + /// + public string Value { get; set; } + + /// + /// The location in the file for where the contents start. Useful for error reporting. + /// + public FileLocation Location { get; set; } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSourceConverter.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSourceConverter.cs new file mode 100644 index 000000000..ef5963210 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Repl/Modules/StringWithSourceConverter.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Glue; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Syntax; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NodeDeserializers; + +namespace Microsoft.PowerFx.Repl +{ + /// + /// Yaml converter to parse a and tag + /// it with the source location. + /// + internal class StringWithSourceConverter : IYamlTypeConverter + { + private readonly string _filename; + private readonly string[] _lines; // to infer indent level. + + /// + /// Initializes a new instance of the class. + /// + /// filename to tag with. + /// contents of the file - this is used to infer indenting depth that can't be gotten from th yaml parser. + public StringWithSourceConverter(string filename, string contents) + { + _filename = filename; + + _lines = contents.Split(new[] { Environment.NewLine }, StringSplitOptions.None); + } + + public bool Accepts(Type type) + { + return type == typeof(StringWithSource); + } + + public object ReadYaml(IParser parser, Type type) + { + var val = parser.Consume(); + + // Empty case + if (val.Style == ScalarStyle.Plain && + val.Value.Length == 0) + { + return new StringWithSource + { + Value = null, // Preserve same semantics as String deserialization. + Location = new FileLocation + { + Filename = _filename, + LineStart = val.Start.Line, + ColStart = val.Start.Column + } + }; + } + + // https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines + // Yaml has many multi-line encodings. Fortunately, we can determine this based on ScalarStyle + // We want to force | , which is Literal. This means the fx is copy & paste into yaml, with an indent level + // and no other mutation. + // + // This will prevent all other forms, including: + // > is "fold" which will remove newlines. + // "" is double quotes. + // plain where the string is single-line embedded. + + if (val.Style != ScalarStyle.Literal) + { + // This is important if we want to preserve character index mapping. + throw new InvalidOperationException($"must be literal string encoding at {val.Start.Line}"); + } + + int line1PropertyName = val.Start.Line; // 1-based Line that the "Formulas" property is on. Content starts on next line. + + // YamlParser does not tell us indent level. Must discern it. + var line = _lines[line1PropertyName]; + var colStart0 = line.FindIndex(x => !char.IsWhiteSpace(x)); + + return new StringWithSource + { + Value = val.Value, + Location = new FileLocation + { + Filename = _filename, + LineStart = line1PropertyName + 1, + ColStart = colStart0 + 1 + } + }; + } + + public void WriteYaml(IEmitter emitter, object value, Type type) + { + // Only reads. + throw new NotImplementedException(); + } + } // end class +} diff --git a/src/libraries/Microsoft.PowerFx.Repl/Repl.cs b/src/libraries/Microsoft.PowerFx.Repl/Repl.cs index 1d0657ff1..ef4d41d9e 100644 --- a/src/libraries/Microsoft.PowerFx.Repl/Repl.cs +++ b/src/libraries/Microsoft.PowerFx.Repl/Repl.cs @@ -37,6 +37,13 @@ namespace Microsoft.PowerFx // Allow repl to create new UserDefinedFunctions. public bool AllowUserDefinedFunctions { get; set; } + /// + /// Enable the Import() function for importing modules. + /// Defaults to false. + /// + [Obsolete("preview")] + public bool AllowImport { get; set; } + // Do we print each command before evaluation? // Useful if we're running a file and are debugging, or if input UI is separated from output UI. public bool Echo { get; set; } = false; @@ -76,6 +83,11 @@ namespace Microsoft.PowerFx /// public ReadOnlySymbolValues ExtraSymbolValues { get; set; } + // Map from Module full path to Module. + private readonly Dictionary _loadedModules = new Dictionary(StringComparer.OrdinalIgnoreCase); + + internal IEnumerable Modules => _loadedModules.Values; + /// /// Get sorted names of all functions. This includes functions from the as well as . /// @@ -94,6 +106,68 @@ namespace Microsoft.PowerFx } } + // Get combined symbol values (extra, modules, etc) + public ReadOnlySymbolValues GetCombined() + { + // Combine with symbols from modules. + var list = new List(); + + foreach (var module in _loadedModules.Values) + { + list.Add(module.Symbols); + } + + if (this.ExtraSymbolValues != null) + { + list.Add(this.ExtraSymbolValues.SymbolTable); + } + + var m1 = ReadOnlySymbolTable.Compose(list.ToArray()); + + var values = m1.CreateValues(this.ExtraSymbolValues); + + return values; + } + + internal bool TryResolveModule(string path, out Module module) + { + if (_loadedModules.TryGetValue(path, out module)) + { + return true; + } + + // can we resolve by short name? + List list = new List(); + + foreach (var m in _loadedModules.Values) + { + string shortName = System.IO.Path.GetFileName(m.FullPath); + if (string.Equals(shortName, path, StringComparison.OrdinalIgnoreCase)) + { + list.Add(m); + } + } + + if (list.Count == 1) + { + module = list[0]; + return true; + } + + return false; + } + + internal void DeleteModule(Module module) + { + _loadedModules.Remove(module.FullPath); + } + + internal void AddModule(Module module) + { + string id = module.FullPath; + _loadedModules[id] = module; + } + // Interpreter should normally not throw. // Exceptions should be caught and converted to ErrorResult. // Not called for OperationCanceledException since those are expected. @@ -123,6 +197,8 @@ namespace Microsoft.PowerFx this.MetaFunctions.AddFunction(new Help1Function(this)); } + private bool _finishInit = false; + private bool _userEnabled = false; public void EnableSampleUserObject() @@ -217,6 +293,22 @@ namespace Microsoft.PowerFx } } + // Property ctor is run before Init properties are set. + // So apply final initialization after property initializers but before we execute commands. + private void FinishInit() + { + if (!_finishInit) + { + _finishInit = true; + if (this.AllowImport) + { + this.MetaFunctions.AddFunction(new ImportFunction(this)); + this.MetaFunctions.AddFunction(new ListModulesFunction(this)); + this.MetaFunctions.AddFunction(new DeleteModuleFunction(this)); + } + } + } + /// /// Directly invoke a command. This skips multiline handling. /// @@ -239,6 +331,8 @@ namespace Microsoft.PowerFx return new ReplResult(); } + FinishInit(); + if (this.Echo) { await this.WritePromptAsync(cancel); @@ -247,9 +341,10 @@ namespace Microsoft.PowerFx await this.Output.WriteLineAsync(expression.TrimEnd(), OutputKind.Repl, cancel); } - var extraSymbolTable = this.ExtraSymbolValues?.SymbolTable; + var extraValues = this.GetCombined(); + var extraSymbolTable = extraValues.SymbolTable; - var runtimeConfig = new RuntimeConfig(this.ExtraSymbolValues) + var runtimeConfig = new RuntimeConfig(extraValues) { ServiceProvider = new BasicServiceProvider(this.InnerServices) }; @@ -363,7 +458,7 @@ namespace Microsoft.PowerFx // Get the type. var rhsExpr = declare._rhs.GetCompleteSpan().GetFragment(expression); - var setCheck = this.Engine.Check(rhsExpr, ParserOptions, this.ExtraSymbolValues?.SymbolTable); + var setCheck = this.Engine.Check(rhsExpr, ParserOptions, this.GetCombined().SymbolTable); if (!setCheck.IsSuccess) { await this.Output.WriteLineAsync($"Error: Failed to initialize '{name}'.", OutputKind.Error, cancel) diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/FileLocationTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/FileLocationTests.cs new file mode 100644 index 000000000..12d814dc4 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/FileLocationTests.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Syntax; +using Xunit; + +namespace Microsoft.PowerFx.Core.Tests +{ + public class FileLocationTests + { + [Theory] + [InlineData(0, 200, 300)] + [InlineData(1, 200, 301)] + [InlineData(4, 201, 300)] + [InlineData(9, 202, 301)] + public void Apply(int min, int expectedLine, int expectedCol) + { + string file = + + // 012 345 6 789 + "ABC\nDE\r\nFG"; + + var loc = new FileLocation + { + Filename = "test", + LineStart = 200, + ColStart = 300 + }; + + var span = new Span(min, min + 1); + var loc2 = loc.Apply(file, span); + + Assert.Same(loc.Filename, loc2.Filename); + Assert.Equal(expectedLine, loc2.LineStart); + Assert.Equal(expectedCol, loc2.ColStart); + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs index f771f39e1..873916a79 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/PublicSurfaceTests.cs @@ -68,6 +68,7 @@ namespace Microsoft.PowerFx.Core.Tests "Microsoft.PowerFx.Syntax.IdentToken", "Microsoft.PowerFx.Syntax.NumLitToken", "Microsoft.PowerFx.Syntax.Span", + "Microsoft.PowerFx.Syntax.FileLocation", "Microsoft.PowerFx.Syntax.StrLitToken", "Microsoft.PowerFx.Syntax.Token", "Microsoft.PowerFx.Syntax.TokKind", diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Microsoft.PowerFx.Repl.Tests.Shared.projitems b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Microsoft.PowerFx.Repl.Tests.Shared.projitems index d40b69ace..71b973926 100644 --- a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Microsoft.PowerFx.Repl.Tests.Shared.projitems +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Microsoft.PowerFx.Repl.Tests.Shared.projitems @@ -10,8 +10,20 @@ + + + + + + + + + + + Always + \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Mocks/TempFileHolder.cs b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Mocks/TempFileHolder.cs new file mode 100644 index 000000000..e84793a92 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Mocks/TempFileHolder.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.PowerFx.Repl.Tests +{ + // Helper for creating temp files and cleaning up. + internal class TempFileHolder : IDisposable + { + public string FullPath { get; } = Path.GetTempFileName(); + + public void Dispose() + { + // Cleanup the file. + try + { + File.Delete(this.FullPath); + } + catch + { + // Ignore + } + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleIdentityTests.cs b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleIdentityTests.cs new file mode 100644 index 000000000..0c3c9be01 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleIdentityTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.PowerFx.Repl.Tests +{ + public class ModuleIdentityTests + { + [Fact] + public void Collections() + { + var id1a = ModuleIdentity.FromFile(@"z:\path1.txt"); + var id1b = ModuleIdentity.FromFile(@"z:\Path1.txt"); + + var id2 = ModuleIdentity.FromFile(@"z:\path2.txt"); + + Assert.Equal(id1a, id1a); // identity + Assert.Equal(id1a.GetHashCode(), id1a.GetHashCode()); + + Assert.NotEqual(id1a, id2); + + // paths should be normalized + Assert.Equal(id1a, id1b); + + // Work with collections. + var set = new HashSet(); + + var added = set.Add(id1a); + Assert.True(added); + + added = set.Add(id1a); + Assert.False(added); + + added = set.Add(id1b); + Assert.False(added); + + Assert.Contains(id1a, set); + Assert.Contains(id1b, set); + + Assert.DoesNotContain(id2, set); + + added = set.Add(id2); + Assert.True(added); + } + + // Ensure we normalize file paths, particularly when using ".." + [Fact] + public void Canonical() + { + var id1a = ModuleIdentity.FromFile(@"z:\foo\path1.txt"); + var id1b = ModuleIdentity.FromFile(@"z:\foo\bar\..\Path1.txt"); + + Assert.Equal(id1a, id1a); // identity + } + + [Fact] + public void NoToString() + { + var id1 = ModuleIdentity.FromFile(@"z:\foo\path1.txt"); + + // *Don't implement ToString* - we want to treat identity as opauque. + var str = id1.ToString(); + + Assert.Equal("Microsoft.PowerFx.Repl.ModuleIdentity", str); + } + + [Fact] + public void MustBeFullPath() + { + Assert.Throws(() => ModuleIdentity.FromFile(null)); + + // Must be a full path + Assert.Throws(() => ModuleIdentity.FromFile("path1.txt")); + Assert.Throws(() => ModuleIdentity.FromFile(@".\path1.txt")); + Assert.Throws(() => ModuleIdentity.FromFile(string.Empty)); + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleTests.cs b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleTests.cs new file mode 100644 index 000000000..7d7c438cf --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/ModuleTests.cs @@ -0,0 +1,306 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.VisualBasic.Syntax; +using Microsoft.PowerFx.Types; +using Xunit; + +#pragma warning disable CS0618 // Type or member is obsolete +#pragma warning disable SA1118 // Parameter should not span multiple lines + +namespace Microsoft.PowerFx.Repl.Tests +{ + // Thesde tests should target public Module Import() function + // (and not internal services) + public class ModuleTests + { + private PowerFxREPL _repl; + private readonly TestReplOutput _output = new TestReplOutput(); + + public ModuleTests() + { + var config = new PowerFxConfig(); + config.SymbolTable.EnableMutationFunctions(); + + // config.EnableSetFunction(); + var engine = new RecalcEngine(config); + + _repl = new PowerFxREPL + { + Engine = engine, + Output = _output, + AllowSetDefinitions = true, + AllowUserDefinedFunctions = true, + AllowImport = true, + ParserOptions = new ParserOptions() { AllowsSideEffects = true } + }; + } + + // Run line, return the normal output. + private string HandleLine(string fx, bool expectErrors = false) + { + Debugger.Log(0, string.Empty, ">> " + fx); + _repl.HandleLine(fx); + + if (!expectErrors) + { + AssertNoErrors(); + } + + var log = _output.Get(OutputKind.Repl); + + Debugger.Log(0, string.Empty, log); + return log; + } + + private void AssertNoErrors() + { + var errors = _output.Get(OutputKind.Error); + Assert.Empty(errors); + } + + private string GetModuleFullPath(string path) + { + string fullpath = Path.Combine( + Environment.CurrentDirectory, + "Modules", + path); + + return fullpath; + } + + private void Import(string path, bool expectErrors = false) + { + string fullpath = GetModuleFullPath(path); + + var expr = $"Import(\"{fullpath}\")"; + HandleLine(expr, expectErrors); + + if (expectErrors) + { + var modules = _repl.Modules.ToArray(); + Assert.Empty(modules); // nothing actually loaded. + } + } + + // Ensure that we only can access modules if they're enabled. + [Fact] + public void MustEnable() + { + var engine = new RecalcEngine(); + + _repl = new PowerFxREPL + { + Engine = engine, + Output = _output + }; + + // Defaults to false + Assert.False(_repl.AllowUserDefinedFunctions); + + Import("basic1.fx.yml", expectErrors: true); + + // Failed to import. + var msg = _output.Get(OutputKind.Error); + Assert.Contains("'Import' is an unknown or unsupported function.", msg); + } + + [Fact] + public void Test1() + { + Import("basic1.fx.yml"); + + // Call method defined in moduel. + var log = HandleLine("DoubleIt(3)"); + + Assert.Equal("6", log); + } + + // Load multiple modules + [Fact] + public void Test2() + { + Import("basic1.fx.yml"); + Import("basic2.fx.yml"); + + // Call method defined in moduel. + var log = HandleLine("DoubleIt(Add1(3))"); + + Assert.Equal("8", log); + } + + // Load with imports + [Fact] + public void TestImport() + { + Import("Depend2.fx.yml"); + + // Call method defined in moduel. + var log = HandleLine("Func2(3)"); + + Assert.Equal("8", log); + } + + // Load with imports + [Fact] + public void TestImport2() + { + Import("Depend2.fx.yml"); + + // Call method defined in moduel. + var log = HandleLine("Func2(3)"); + Assert.Equal("8", log); + + // Inner dependencies are not exported + log = HandleLine("Add1(3)", expectErrors: true); + var msg = _output.Get(OutputKind.Error); + Assert.Contains("Add1' is an unknown or unsupported function.", msg); + + // But we can import the inner module and call. + Import("basic2.fx.yml"); + log = HandleLine("Add1(3)"); + Assert.Equal("4", log); + } + + // Load with imports + [Fact] + public void Recursion() + { + Import("recursion.fx.yml"); + + // 0,1, 1, 2, 3, 5, 8,13 + var log = HandleLine("Fib(7)"); + Assert.Equal("13", log); + } + + // Load with imports + [Fact] + public void Shadow() + { + Import("shadow.fx.yml"); + + var log = HandleLine("Foo()"); + Assert.Equal("-2", log); + + log = HandleLine("Abs(-7)"); // calls new one from module + } + + // Diamond inheritence. + // 1 --> {2a, 2b}. 2a-->3. 2b-->3. + // This is interesting since module 3 gets used multiple times, but it's not a cycle. + [Fact] + public void Diamond() + { + Import("diamond_1.fx.yml"); + + var log = HandleLine("Func1(5)"); + Assert.Equal("\"2A(3(5)),2B(3(5))\"", log); + } + + // Conflict if we import 2 modules that define the same symbols. + [Fact] + public void DuplicateSymbolsConflict() + { + Import("conflict1.fx.yml", expectErrors: true); + var errorMsg = _output.Get(OutputKind.Error); + + // message contains useful information. + Assert.Contains("DoubleIt", errorMsg); // conflicting symbol + Assert.Contains("basic1.fx.yml", errorMsg); // module 1 + Assert.Contains("basic1_dup.fx.yml", errorMsg); // module 2 + } + + // Import() is a meta function and can only be called + // at top-levle repl and not within a module itself. + [Fact] + public void ImportIsAMetafunction() + { + Import("ErrorImport.fx.yml", expectErrors: true); + var errorMsg = _output.Get(OutputKind.Error); + + Assert.Contains("'Import' is an unknown or unsupported function", errorMsg); + } + + // When we import a file twice, we get the updated contents. + [Fact] + public void GetsLatest() + { + using var temp = new TempFileHolder(); + string fullpath = temp.FullPath; + + // First version + File.WriteAllText(fullpath, @" +Formulas: | + Func(x: Number) : Number = x * 10; +"); + HandleLine($"Import(\"{fullpath}\")"); + + // Call method defined in moduel. + var before = HandleLine("Func(3)"); + Assert.Equal("30", before); + + // Update the file, 2nd version + File.WriteAllText(fullpath, @" +Formulas: | + Func(x: Number) : Number = x * 20; +"); + + HandleLine($"Import(\"{fullpath}\")"); + + var after = HandleLine("Func(3)"); + Assert.Equal("60", after); + } + + // Error gets spans. + [Fact] + public void ErrorShowsFileRange() + { + Import("Error1.fx.yml", expectErrors: true); + + var errorMsg = _output.Get(OutputKind.Error); + + // Key elements here are that: + // - the message has the filename + // - the location (5,9) is relative into the file, not just the expression. + Assert.Equal("Error: Error1.fx.yml (5,9): Name isn't valid. 'missing' isn't recognized.", errorMsg); + } + + // Circular references are detected + [Fact] + public void Cycles() + { + Import("cycle1.fx.yml", expectErrors: true); + + var errorMsg = _output.Get(OutputKind.Error); + Assert.Contains("Circular reference", errorMsg); + } + + // Loading a missing file + [Fact] + public void Missing() + { + Import("missing_file.fx.yml", expectErrors: true); + + var errorMsg = _output.Get(OutputKind.Error); + Assert.Contains("missing_file.fx.yml", errorMsg); + } + + // Loading a file with yaml parse errors. + [Fact] + public void YamlParseErrors() + { + Import("yaml_parse_errors.fx.yml", expectErrors: true); + + var errorMsg = _output.Get(OutputKind.Error); + } + } +} diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Depend2.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Depend2.fx.yml new file mode 100644 index 000000000..bb1769d6b --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Depend2.fx.yml @@ -0,0 +1,7 @@ +Imports: +- File: Basic1.fx.yml +- File: Basic2.fx.yml + +# Depends on Basic1 +Formulas: | + Func2(x: Number) : Number = DoubleIt(Add1(x)); diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Error1.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Error1.fx.yml new file mode 100644 index 000000000..14d82011b --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/Error1.fx.yml @@ -0,0 +1,6 @@ +# Has an error in expression. +# Test that error reporting gets right line. +Formulas: | + DoubleIt(x: Number) : Number = + missing // <-- error is on line 5 + * 2; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/ErrorImport.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/ErrorImport.fx.yml new file mode 100644 index 000000000..ad78e7094 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/ErrorImport.fx.yml @@ -0,0 +1,7 @@ +# Error - we can't call Import() within a function. +# must be a top-level yaml construct. + +Formulas: | + Func2(x: Number) : Void = { + Import("Basic1.fx.yml") + }; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1.fx.yml new file mode 100644 index 000000000..e7913abd1 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1.fx.yml @@ -0,0 +1,3 @@ +# Most basic Power Fx module. +Formulas: | + DoubleIt(x: Number) : Number = x * 2; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1_dup.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1_dup.fx.yml new file mode 100644 index 000000000..73e2c514b --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic1_dup.fx.yml @@ -0,0 +1,4 @@ +# test symbol conflicts. +# Same symbols as base1.fx.yml +Formulas: | + DoubleIt(x: Number) : Number = x * 200; \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic2.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic2.fx.yml new file mode 100644 index 000000000..78aeb00d3 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/basic2.fx.yml @@ -0,0 +1,3 @@ +# Most basic Power Fx module. +Formulas: | + Add1(x: Number) : Number = x +1; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/conflict1.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/conflict1.fx.yml new file mode 100644 index 000000000..bedea06a7 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/conflict1.fx.yml @@ -0,0 +1,6 @@ +Imports: +- File: basic1.fx.yml +- File: basic1_dup.fx.yml + +Formulas: | + Run() : Number = DoubleIt(5); \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle1.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle1.fx.yml new file mode 100644 index 000000000..7395675eb --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle1.fx.yml @@ -0,0 +1,6 @@ +# Test circular dependency +Imports: +- File: cycle2.fx.yml + +Formulas: | + Func1(x: Number) : Number = 1; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle2.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle2.fx.yml new file mode 100644 index 000000000..bb48463c9 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/cycle2.fx.yml @@ -0,0 +1,6 @@ +# Test circular dependency +Imports: +- File: cycle1.fx.yml + +Formulas: | + Func2(x: Number) : Number = 1; diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_1.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_1.fx.yml new file mode 100644 index 000000000..a5d90a235 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_1.fx.yml @@ -0,0 +1,7 @@ +Imports: +- File: diamond_2a.fx.yml +- File: diamond_2b.fx.yml + +# Depends on 2a,2b. Both depend on 3. +Formulas: | + Func1(x: Text) : Text = Func2a(x) & "," & Func2b(x); diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2a.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2a.fx.yml new file mode 100644 index 000000000..813a91085 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2a.fx.yml @@ -0,0 +1,8 @@ +Imports: +- File: diamond_3.fx.yml + + +# Depends on Basic1 +Formulas: | + Func2a(x: Text) : Text = $"2A({Func3(x)})"; + diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2b.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2b.fx.yml new file mode 100644 index 000000000..aa65f242b --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_2b.fx.yml @@ -0,0 +1,8 @@ +Imports: +- File: diamond_3.fx.yml + + +# Depends on Basic1 +Formulas: | + Func2b(x: Text) : Text = $"2B({Func3(x)})"; + diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_3.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_3.fx.yml new file mode 100644 index 000000000..454ce6759 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/diamond_3.fx.yml @@ -0,0 +1,3 @@ + +Formulas: | + Func3(x: Text) : Text = $"3({x})"; \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/recursion.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/recursion.fx.yml new file mode 100644 index 000000000..5b0d6c0a9 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/recursion.fx.yml @@ -0,0 +1,4 @@ +# Define a basic recursive function. +Formulas: | + Fib(n: Number) : Number = + If(n <= 1, n, Fib(n - 1) + Fib(n - 2)); diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/shadow.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/shadow.fx.yml new file mode 100644 index 000000000..3782bf116 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/shadow.fx.yml @@ -0,0 +1,8 @@ + +Formulas: | + // Redefine a builtin. + // Give a different (wrong) implementation so we know which is called. + Abs(x: Number) : Number = x *2; + + // Binds to the one in this scope. + Foo() : Number = Abs(-1); \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/yaml_parse_errors.fx.yml b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/yaml_parse_errors.fx.yml new file mode 100644 index 000000000..9aa6d975f --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/Modules/yaml_parse_errors.fx.yml @@ -0,0 +1,3 @@ +# not a valid yaml file +Formulas:: + \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/StringWithSourceTests.cs b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/StringWithSourceTests.cs new file mode 100644 index 000000000..604de09bc --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Repl.Tests.Shared/StringWithSourceTests.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.VisualBasic.Syntax; +using Microsoft.PowerFx.Types; +using Xunit; +using YamlDotNet.Core; +using YamlDotNet.Serialization; + +namespace Microsoft.PowerFx.Repl.Tests +{ +#pragma warning disable SA1117 // Parameters should be on same line or separate lines + + public class StringWithSourceTests + { + private class Poco1 + { + public StringWithSource Prop { get; set; } + } + + private class Poco1Direct + { + public string Prop { get; set; } + } + + // common fake filename used in locations. + private const string Filename = "myfile1.txt"; + + private T Parse(string contents) + { + // Deserialize. + var deserializer = new DeserializerBuilder() + .WithTypeConverter(new StringWithSourceConverter(Filename, contents)) + .Build(); + + var poco = deserializer.Deserialize(contents); + + return poco; + } + + private T ParseExpectError(string contents, string expectedError) + { + try + { + // Deserialize. + var deserializer = new DeserializerBuilder() + .WithTypeConverter(new StringWithSourceConverter(Filename, contents)) + .Build(); + + var poco = deserializer.Deserialize(contents); + + Assert.Fail($"Expected error: {expectedError}"); + return poco; + } + catch (YamlException e) + { + // Exceptions from parser will be wrapped in YamlException. + var msg = e?.InnerException?.Message ?? e.Message; + Assert.Contains(expectedError, msg); + } + + return default; + } + + // In all cases, parse as 'string' should succeed. + // But Parse with SourceLocaiton has more limitations. + // expectedError specifies restrictions. + private StringWithSource Test(string contents, string expectedError = null) + { + // Should always work with 'string' + var p1 = Parse(contents); + + if (expectedError != null) + { + ParseExpectError(contents, expectedError); + return null; + } + else + { + var p2 = Parse(contents); + + Assert.Equal(p1.Prop, p2.Prop.Value); + Assert.Same(Filename, p2.Prop.Location.Filename); + + if (p2.Prop.Value != null) + { + p2.Prop.Value = p2.Prop.Value.Replace("\r", string.Empty); + } + + return p2.Prop; + } + } + + [Theory] + [InlineData( +@"Prop: | + 123 +", 2, 3, "123\n")] + + // Leading space in front of property. + [InlineData( +@" Prop: | + 123 +", 2, 3, "123\n")] + + [InlineData( +@"# another line +Prop: | + 123 + 456", 3, 6, "123\n456")] + + [InlineData( +@"Prop: |- + 123 +", 2, 3, "123")] + + [InlineData( +@"Prop: |- + true +", 2, 3, "true")] // YDN will still return as string. + + // Other encodings (not | ) are not supported. + [InlineData( +@"Prop: > + 123 + 456 +", 0, 0, "123456\n", "literal")] // Folding + + [InlineData( +@"Prop: 123456 +", 0, 0, "123456", "literal")] // plain + + [InlineData( +@"Prop: ""123456"" +", 0, 0, "123456", "literal")] // quotes + + [InlineData( +@"Prop: ", 1, 6, null)] // empty + + public void Test1(string contents, int lineStart, int colStart, string value, string errorMessage = null) + { + var p1 = Test(contents, errorMessage); + + if (errorMessage == null) + { + Assert.Equal(value, p1.Value); + Assert.Equal(colStart, p1.Location.ColStart); + Assert.Equal(lineStart, p1.Location.LineStart); + } + else + { + Assert.Null(p1); + } + } + + // Error conditions. + [Theory] + [InlineData( +"Prop: { }")] // object , expecting string. + public void TestError(string contents) + { + ParseExpectError(contents, "Failed"); + ParseExpectError(contents, "Expected 'Scalar'"); + } + } +} diff --git a/src/tools/Repl/AssertFunction.cs b/src/tools/Repl/AssertFunction.cs new file mode 100644 index 000000000..c03b1ca04 --- /dev/null +++ b/src/tools/Repl/AssertFunction.cs @@ -0,0 +1,45 @@ +// +// Copyright (c) PlaceholderCompany. All rights reserved. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Types; + +#pragma warning disable CS0618 // Type or member is obsolete + +namespace Microsoft.PowerFx.Repl.Functions +{ + /// + /// Assert Function. + /// + internal class AssertFunction : ReflectionFunction + { + public AssertFunction() + : base("Assert", FormulaType.Void, new[] { FormulaType.Boolean, FormulaType.String }) + { + ConfigType = typeof(IReplOutput); + } + + public async Task Execute(IReplOutput output, BooleanValue test, StringValue message, CancellationToken cancel) + { + if (test.Value) + { + await output.WriteLineAsync($"PASSED: {message.Value}", OutputKind.Notify, cancel) + .ConfigureAwait(false); + } + else + { + await output.WriteLineAsync($"FAILED: {message.Value}", OutputKind.Error, cancel) + .ConfigureAwait(false); + } + + return FormulaValue.NewVoid(); + } + } +} diff --git a/src/tools/Repl/Program.cs b/src/tools/Repl/Program.cs index f23481dc6..045c7919e 100644 --- a/src/tools/Repl/Program.cs +++ b/src/tools/Repl/Program.cs @@ -18,6 +18,8 @@ using Microsoft.PowerFx.Types; namespace Microsoft.PowerFx { +#pragma warning disable CS0618 // Type or member is obsolete + public static class ConsoleRepl { private const string OptionFormatTable = "FormatTable"; @@ -84,9 +86,9 @@ namespace Microsoft.PowerFx config.EnableSetFunction(); config.EnableJsonFunctions(); -#pragma warning disable CS0618 // Type or member is obsolete config.EnableOptionSetInfo(); -#pragma warning restore CS0618 // Type or member is obsolete + + config.AddFunction(new AssertFunction()); config.AddFunction(new ResetFunction()); config.AddFunction(new Option0Function()); @@ -97,9 +99,7 @@ namespace Microsoft.PowerFx var optionsSet = new OptionSet("Options", DisplayNameUtility.MakeUnique(options)); -#pragma warning disable CS0618 // Type or member is obsolete config.EnableRegExFunctions(new TimeSpan(0, 0, 5)); -#pragma warning restore CS0618 // Type or member is obsolete config.AddOptionSet(optionsSet); @@ -124,9 +124,7 @@ namespace Microsoft.PowerFx } // Hook repl engine with customizations. -#pragma warning disable CS0618 // Type or member is obsolete private class MyRepl : PowerFxREPL -#pragma warning restore CS0618 // Type or member is obsolete { public MyRepl() { @@ -138,6 +136,8 @@ namespace Microsoft.PowerFx this.AllowSetDefinitions = true; this.AllowUserDefinedFunctions = _enableUDFs; + this.AllowImport = true; + this.EnableSampleUserObject(); this.AddPseudoFunction(new IRPseudoFunction()); this.AddPseudoFunction(new SuggestionsPseudoFunction()); @@ -427,9 +427,7 @@ namespace Microsoft.PowerFx private class MyHelpProvider : HelpProvider { -#pragma warning disable CS0618 // Type or member is obsolete public override async Task Execute(PowerFxREPL repl, CancellationToken cancel, string context = null) -#pragma warning restore CS0618 // Type or member is obsolete { if (context?.ToLowerInvariant() == "options" || context?.ToLowerInvariant() == "option") {