Preliminary support for user-defined functions in RecalcEngine (#485)

* main implementation of UDF

* fixed nit changes.

* Defining a function twice now throws an exception

* nit changes

src/libraries/Microsoft.PowerFx.Core/Glue/SimpleResolver.cs

@@ -99,7 +99,7 @@ namespace Microsoft.PowerFx.Core.Glue
             return false;
         }
 
-        public IEnumerable<TexlFunction> LookupFunctions(DPath theNamespace, string name, bool localeInvariant = false)
+        public virtual IEnumerable<TexlFunction> LookupFunctions(DPath theNamespace, string name, bool localeInvariant = false)
         {
             Contracts.Check(theNamespace.IsValid, "The namespace is invalid.");
             Contracts.CheckNonEmpty(name, "name");

src/libraries/Microsoft.PowerFx.Interpreter/EvalVisitor.cs

@@ -170,9 +170,14 @@ namespace Microsoft.PowerFx
                 var result = await asyncFunc.InvokeAsync(args, _cancel);
                 return result;
             }
-            else if (func is CustomTexlFunction customFunc)
+            else if (func is UserDefinedTexlFunction udtf)
             {
-                var result = customFunc.Invoke(args);
+                var result = await udtf.InvokeAsync(args, _cancel, context.StackDepthCounter.Increment());
+                return result;
+            }
+            else if (func is CustomTexlFunction customTexlFunc)
+            {
+                var result = customTexlFunc.Invoke(args);
                 return result;
             }
             else
@@ -182,7 +187,6 @@ namespace Microsoft.PowerFx
                     var result = await ptr(this, context.IncrementStackDepthCounter(childContext), node.IRContext, args);
 
                     Contract.Assert(result.IRContext.ResultType == node.IRContext.ResultType || result is ErrorValue || result.IRContext.ResultType is BlankType);
-
                     return result;
                 }
 

src/libraries/Microsoft.PowerFx.Interpreter/ParsedExpression.cs

@@ -39,5 +39,14 @@ namespace Microsoft.PowerFx
                 return maxCallDepthException.ToErrorValue(_irnode.IRContext);
             }
         }
+
+        internal async Task<FormulaValue> EvalAsyncInternal(RecordValue parameters, CancellationToken cancel, StackDepthCounter stackMarker)
+        {
+            // We don't catch the max call depth exception here becuase someone could swallow the error with an "IfError" check.
+            // Instead we only catch at the top of parsed expression, which is the above function.
+            var ev2 = new EvalVisitor(_cultureInfo, cancel);
+            var newValue = await _irnode.Accept(ev2, new EvalVisitorContext(SymbolContext.NewTopScope(_topScopeSymbol, parameters), stackMarker));
+            return newValue;
+        }
     }
 }

src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs

@@ -3,16 +3,21 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.PowerFx.Core.Binding;
+using Microsoft.PowerFx.Core.Functions;
 using Microsoft.PowerFx.Core.Glue;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Core.Texl;
+using Microsoft.PowerFx.Core.Types;
 using Microsoft.PowerFx.Core.Utils;
 using Microsoft.PowerFx.Functions;
 using Microsoft.PowerFx.Interpreter;
+using Microsoft.PowerFx.Interpreter.UDF;
 using Microsoft.PowerFx.Types;
+using static Microsoft.PowerFx.Interpreter.UDFHelper;
 
 namespace Microsoft.PowerFx
 {
@@ -23,6 +28,8 @@ namespace Microsoft.PowerFx
     {
         internal Dictionary<string, RecalcFormulaInfo> Formulas { get; } = new Dictionary<string, RecalcFormulaInfo>();
 
+        internal Dictionary<string, TexlFunction> _customFuncs = new Dictionary<string, TexlFunction>();
+
         /// <summary>
         /// Initializes a new instance of the <see cref="RecalcEngine"/> class.
         /// Create a new power fx engine. 
@@ -169,6 +176,77 @@ namespace Microsoft.PowerFx
             return await check.Expression.EvalAsync(parameters, cancel);
         }
 
+        /// <summary>
+        /// For private use because we don't want anyone defining a function without binding it.
+        /// </summary>
+        /// <returns></returns>
+        private UDFLazyBinder DefineFunction(UDFDefinition definition)
+        {
+            // $$$ Would be a good helper function 
+            var record = RecordType.Empty();
+            foreach (var p in definition.Parameters)
+            {
+                record = record.Add(p);
+            }
+
+            var check = new CheckWrapper(this, definition.Body, record);
+
+            var func = new UserDefinedTexlFunction(definition.Name, definition.ReturnType, definition.Parameters, check);
+            if (_customFuncs.ContainsKey(definition.Name))
+            {
+                throw new InvalidOperationException($"Function {definition.Name} is already defined");
+            }
+
+            _customFuncs[definition.Name] = func;
+            return new UDFLazyBinder(func, definition.Name);
+        }
+
+        private void RemoveFunction(string name)
+        {
+            _customFuncs.Remove(name);
+        }
+
+        /// <summary>
+        /// Tries to define and bind all the functions here. If any function names conflict returns an expression error. 
+        /// Also returns any errors from binding failing. All functions defined here are removed if any of them contain errors.
+        /// </summary>
+        /// <param name="udfDefinitions"></param>
+        /// <returns></returns>
+        internal IEnumerable<ExpressionError> DefineFunctions(IEnumerable<UDFDefinition> udfDefinitions)
+        {
+            var expressionErrors = new List<ExpressionError>();
+
+            var binders = new List<UDFLazyBinder>();
+            foreach (UDFDefinition definition in udfDefinitions)
+            {
+                binders.Add(DefineFunction(definition));
+            }
+            
+            foreach (UDFLazyBinder lazyBinder in binders)
+            {
+                var possibleErrors = lazyBinder.Bind();
+                if (possibleErrors.Any())
+                {
+                    expressionErrors.AddRange(possibleErrors);
+                }
+            }
+
+            if (expressionErrors.Any())
+            {
+                foreach (UDFLazyBinder lazyBinder in binders)
+                {
+                    RemoveFunction(lazyBinder.Name);
+                }
+            }
+
+            return expressionErrors;
+        }
+
+        internal IEnumerable<ExpressionError> DefineFunctions(params UDFDefinition[] udfDefinitions)
+        {
+            return DefineFunctions(udfDefinitions.AsEnumerable());
+        }
+
         // Invoke onUpdate() each time this formula is changed, passing in the new value. 
         public void SetFormula(string name, string expr, Action<string, FormulaValue> onUpdate)
         {

src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngineResolver.cs

@@ -6,6 +6,7 @@ using System.Collections.ObjectModel;
 using System.Linq;
 using Microsoft.PowerFx.Core.Binding;
 using Microsoft.PowerFx.Core.Binding.BindInfo;
+using Microsoft.PowerFx.Core.Functions;
 using Microsoft.PowerFx.Core.Glue;
 using Microsoft.PowerFx.Core.Utils;
 
@@ -33,6 +34,19 @@ namespace Microsoft.PowerFx
             _powerFxConfig = powerFxConfig;
         }
 
+        public override IEnumerable<TexlFunction> LookupFunctions(DPath theNamespace, string name, bool localeInvariant = false)
+        {
+            if (theNamespace.IsRoot)
+            {
+                if (_parent._customFuncs.TryGetValue(name, out var func))
+                {
+                    return new TexlFunction[] { func };
+                }
+            }
+
+            return base.LookupFunctions(theNamespace, name, localeInvariant);
+        }
+
         public override bool Lookup(DName name, out NameLookupInfo nameInfo, NameLookupPreferences preferences = NameLookupPreferences.None)
         {
             // Kinds of globals:

src/libraries/Microsoft.PowerFx.Interpreter/ReflectionFunction.cs

@@ -4,10 +4,8 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.PowerFx.Core;
 using Microsoft.PowerFx.Core.Functions;
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Types;
@@ -33,7 +31,7 @@ namespace Microsoft.PowerFx
 
         public CustomTexlFunction(string name, DType returnType, params DType[] paramTypes)
             : base(DPath.Root, name, name, SG("Custom func " + name), FunctionCategories.MathAndStat, returnType, 0, paramTypes.Length, paramTypes.Length, paramTypes)
-        {            
+        {
         }
 
         public override bool IsSelfContained => true;

src/libraries/Microsoft.PowerFx.Interpreter/RuntimeMaxCallDepthException.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    [Serializable]
+    internal class RuntimeMaxCallDepthException : Exception
+    {
+        public RuntimeMaxCallDepthException()
+            : base()
+        {
+        }
+
+        public RuntimeMaxCallDepthException(string message)
+            : base(message)
+        { 
+        }
+
+        public RuntimeMaxCallDepthException(string message, Exception inner)
+            : base(message, inner)
+        {
+        }
+
+        protected RuntimeMaxCallDepthException(
+            System.Runtime.Serialization.SerializationInfo info,
+            System.Runtime.Serialization.StreamingContext context)
+            : base(info, context) 
+        { 
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/CheckWrapper.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerFx.Interpreter.UDF
+{
+     /// <summary>
+     /// CheckWrapper delays the evaluation of the body to the Get call while taking in all the parameters needed to make the Check call.
+     /// </summary>
+    internal class CheckWrapper
+    {
+        private readonly string _expressionText;
+        private readonly RecordType _parameterType;
+        private readonly RecalcEngine _engine;
+
+        public CheckWrapper(RecalcEngine engine, string expressionText, RecordType parameterType = null)
+        {
+            _engine = engine;
+            _expressionText = expressionText;
+            _parameterType = parameterType;
+        }
+
+        public CheckResult Get() => _engine.Check(_expressionText, _parameterType);
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/UDFBindingMissingException.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    [Serializable]
+    internal class UDFBindingMissingException : Exception
+    {
+        public UDFBindingMissingException()
+           : base()
+        {
+        }
+
+        public UDFBindingMissingException(string message)
+            : base(message)
+        {
+        }
+
+        public UDFBindingMissingException(string message, Exception inner)
+            : base(message, inner)
+        {
+        }
+
+        protected UDFBindingMissingException(
+            System.Runtime.Serialization.SerializationInfo info,
+            System.Runtime.Serialization.StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/UDFDefinition.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    internal class UDFDefinition
+    {
+        internal readonly string Name;
+
+        internal readonly string Body;
+
+        internal readonly FormulaType ReturnType;
+
+        internal readonly IEnumerable<NamedFormulaType> Parameters;
+
+        public UDFDefinition(string name, string body, FormulaType returnType, IEnumerable<NamedFormulaType> parameters)
+        {
+            Name = name;
+            Body = body;
+            ReturnType = returnType;
+            Parameters = parameters;
+        }
+
+        public UDFDefinition(string name, string body, FormulaType returnType, params NamedFormulaType[] parameters)
+            : this(name, body, returnType, parameters.AsEnumerable())
+        {
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/UDFHelper.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    internal static class UDFHelper
+    {
+        internal static IEnumerable<NamedValue> Zip(NamedFormulaType[] parameters, FormulaValue[] args)
+        {
+            if (parameters.Length != args.Length)
+            {
+                throw new ArgumentException();
+            }
+
+            var result = new NamedValue[args.Length];
+            for (var i = 0; i < args.Length; i++)
+            {
+                result[i] = new NamedValue(parameters[i].Name, args[i]);
+            }
+
+            return result;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/UDFLazyBinder.cs

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    internal class UDFLazyBinder
+    {
+        private readonly UserDefinedTexlFunction _function;
+        
+        private readonly List<ExpressionError> _expressionError = new List<ExpressionError>();
+
+        public readonly string Name;
+
+        public UDFLazyBinder(UserDefinedTexlFunction texlFunction, string name)
+        { 
+            _function = texlFunction;
+            Name = name;
+        }
+
+        // Used when trying to define a function, there is an error before binding, such as the function name already being defined.
+        public UDFLazyBinder(ExpressionError expressionError, string name)
+        {
+            _expressionError.Add(expressionError);
+            Name = name;
+        }
+
+        public IEnumerable<ExpressionError> Bind()
+        {
+            if (_expressionError.Any())
+            {
+                return _expressionError;
+            }
+
+            return _function.Bind();
+        } 
+    }
+}

src/libraries/Microsoft.PowerFx.Interpreter/UDF/UserDefinedTexlFunction.cs

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Interpreter.UDF;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerFx.Interpreter
+{
+    internal class UserDefinedTexlFunction : CustomTexlFunction
+    {
+        private readonly IEnumerable<NamedFormulaType> _parameterNames;
+        private ParsedExpression _expr;
+        private readonly CheckWrapper _check;
+
+        public override bool SupportsParamCoercion => false;
+
+        public UserDefinedTexlFunction(string name, FormulaType returnType, IEnumerable<NamedFormulaType> parameterNames, CheckWrapper lazyCheck)
+            : base(name, returnType, parameterNames.Select(x => x.Type).ToArray())
+        {
+            _parameterNames = parameterNames;
+            _check = lazyCheck;
+        }
+
+        public async Task<FormulaValue> InvokeAsync(FormulaValue[] args, CancellationToken cancel, StackDepthCounter stackMarker)
+        {
+            // $$$ There's a lot of unnecessary string packing overhead here 
+            // because Eval wants a Record rather than a resolved arg array.                 
+            var parameters = FormulaValue.NewRecordFromFields(UDFHelper.Zip(_parameterNames.ToArray(), args));
+
+            var result = await GetExpression().EvalAsyncInternal(parameters, cancel, stackMarker);
+
+            return result;
+        }
+
+        public IEnumerable<ExpressionError> Bind()
+        {
+            var check = _check.Get();
+            if (!check.IsSuccess)
+            {
+                return check.Errors;
+            }
+
+            if (check.Expression is ParsedExpression parsed)
+            {
+                _expr = parsed;
+            } 
+            else
+            {
+                var errorList = new List<ExpressionError>
+                {
+                    new ExpressionError()
+                };
+                return errorList;
+            }
+
+            return new List<ExpressionError>();
+        }
+
+        public ParsedExpression GetExpression()
+        {
+            if (_expr == null)
+            {
+                throw new UDFBindingMissingException();
+            }
+
+            return _expr;
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/RecalcEngineTests.cs

@@ -9,6 +9,8 @@ using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.PowerFx.Core;
 using Microsoft.PowerFx.Core.Functions;
+using Microsoft.PowerFx.Core.Localization;
+using Microsoft.PowerFx.Core.Parser;
 using Microsoft.PowerFx.Core.Tests;
 using Microsoft.PowerFx.Core.Texl;
 using Microsoft.PowerFx.Core.Types.Enums;
@@ -17,6 +19,7 @@ using Microsoft.PowerFx.Interpreter;
 using Microsoft.PowerFx.Types;
 using Xunit;
 using Xunit.Sdk;
+using static Microsoft.PowerFx.Interpreter.UDFHelper;
 
 namespace Microsoft.PowerFx.Tests
 {
@@ -239,6 +242,130 @@ namespace Microsoft.PowerFx.Tests
                engine.SetFormula("A", "3", OnUpdate));
         }
 
+        [Fact]
+        public void DefFunc()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+
+            IEnumerable<ExpressionError> enumerable = recalcEngine.DefineFunctions(
+            new UDFDefinition(
+                "foo",
+                "x * y", 
+                FormulaType.Number, 
+                new NamedFormulaType("x", FormulaType.Number),
+                new NamedFormulaType("y", FormulaType.Number)));
+            Assert.False(enumerable.Any());
+            Assert.Equal(17.0, recalcEngine.Eval("foo(3,4) + 5").ToObject());
+        }
+
+        [Fact]
+        public void DefRecursiveFunc()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+            var body = @"If(x=0,foo(1),If(x=1,foo(2),If(x=2,2)))";
+            IEnumerable<ExpressionError> enumerable = recalcEngine.DefineFunctions(
+                new UDFDefinition(
+                    "foo",
+                    body,
+                    FormulaType.Number,
+                    new NamedFormulaType("x", FormulaType.Number)));
+            var result = recalcEngine.Eval("foo(0)");
+            Assert.Equal(2.0, result.ToObject());
+            Assert.False(enumerable.Any());
+        }
+
+        [Fact]
+        public void DefSimpleRecursiveFunc()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+            Assert.False(recalcEngine.DefineFunctions(
+                new UDFDefinition(
+                    "foo",
+                    "foo()",
+                    FormulaType.Blank)).Any());
+            var result = recalcEngine.Eval("foo()");
+            Assert.IsType<ErrorValue>(result);
+        }
+
+        [Fact]
+        public void DefHailstoneSequence()
+        {
+            var config = new PowerFxConfig(null)
+            {
+                MaxCallDepth = 100
+            };
+            var recalcEngine = new RecalcEngine(config);
+            var body = @"If(Not(x = 1), If(Mod(x, 2)=0, hailstone(x/2), hailstone(3*x+1)), x)";
+            var funcName = "hailstone";
+            var returnType = FormulaType.Number;
+            var variable = new NamedFormulaType("x", FormulaType.Number);
+
+            Assert.False(recalcEngine.DefineFunctions(
+                new UDFDefinition(funcName, body, returnType, variable)).Any());
+            Assert.Equal(1.0, recalcEngine.Eval("hailstone(192)").ToObject());
+        }
+
+        [Fact]
+        public void DefMutualRecursionFunc()
+        {
+            var config = new PowerFxConfig(null)
+            {
+                MaxCallDepth = 100
+            };
+            var recalcEngine = new RecalcEngine(config);
+            var bodyEven = @"If(number = 0, true, odd(Abs(number)-1))";
+            var bodyOdd = @"If(number = 0, false, even(Abs(number)-1))";
+
+            var udfOdd = new UDFDefinition(
+                "odd",
+                bodyOdd,
+                FormulaType.Boolean,
+                new NamedFormulaType("number", FormulaType.Number));
+            var udfEven = new UDFDefinition(
+                "even",
+                bodyEven,
+                FormulaType.Boolean,
+                new NamedFormulaType("number", FormulaType.Number));
+
+            Assert.False(recalcEngine.DefineFunctions(udfOdd, udfEven).Any());
+
+            Assert.Equal(true, recalcEngine.Eval("odd(17)").ToObject());
+            Assert.Equal(false, recalcEngine.Eval("even(17)").ToObject());
+        }
+
+        [Fact]
+        public async void RedefinitionError()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+            Assert.Throws<InvalidOperationException>(() => recalcEngine.DefineFunctions(
+                new UDFDefinition("foo", "foo()", FormulaType.Blank),
+                new UDFDefinition("foo", "x+1", FormulaType.Number)));
+        }
+
+        [Fact]
+        public void UDFBodySyntaxErrorTest()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+            Assert.True(recalcEngine.DefineFunctions(new UDFDefinition("foo", "x[", FormulaType.Blank)).Any());
+        }
+
+        [Fact]
+        public async void UDFIncorrectParametersTest()
+        {
+            var config = new PowerFxConfig(null);
+            var recalcEngine = new RecalcEngine(config);
+            Assert.False(recalcEngine.DefineFunctions(new UDFDefinition("foo", "x+1", FormulaType.Number, new NamedFormulaType("x", FormulaType.Number))).Any());
+            Assert.False(recalcEngine.Check("foo(False)").IsSuccess);
+            Assert.False(recalcEngine.Check("foo(Table( { Value: \"Strawberry\" }, { Value: \"Vanilla\" } ))").IsSuccess);
+            Assert.True(recalcEngine.Check("foo(1)").IsSuccess);
+            await Assert.ThrowsAsync<InvalidOperationException>(async () => await recalcEngine.EvalAsync("foo(False)", CancellationToken.None));
+        }
+
         [Fact]
         public void PropagateNull()
         {