Catch exceptions (#54)

* Implimenting conditional catch with exception details

* Testing for catching result status codes in particular
* Catch can add output to working memory from result and error details
* Catch can stop errors conditionally by jmespath expression
* Improving the exception thrown by response status code
* Adds a request and response to the exception
* adds error.type.name and error.type.fullName properties

examples/202-throwing-errors/workflow.yaml

@@ -0,0 +1,27 @@
+
+values:
+  first: 
+    color: green
+  second:
+    color: blue
+
+operations:
+- message: An operation can throw an exception with a message and structured details
+
+- message: This is a good way to break out of a workflow if something is going wrong
+
+- message: Having a condition to the throw operation enables you determine if the throw should happen
+
+- condition: first.color != 'green'
+  message: Unexpected first color
+  throw: 
+    message: (['Color must be green, not ', first.color])
+    details: (first)
+
+- condition: second.color != 'green'
+  message: Unexpected second color
+  throw: 
+    message: (['Color must be green, not ', second.color])
+    details: (second)
+
+- message: This operation will never be reached

examples/203-catching-errors/fetch-github-file.yaml

@@ -0,0 +1,2 @@
+method: GET
+url: https://raw.githubusercontent.com/Microsoft/Atlas/master/{{ path }}

examples/203-catching-errors/workflow.yaml

@@ -0,0 +1,64 @@
+
+operations:
+- message: An operation can catch errors which it throws
+
+- message: Rendering a missing template
+  template: no-such-template.yaml
+  write: example.yaml
+  catch:
+    output: 
+      missing-file-error: (error.{ type:type.name, message:message, file:FileName })
+
+- message: An operation can catch errors thrown from any nested operations
+
+- message: Catching nested errors
+  operations:
+  - message: This operation is fine
+  - message: This operation throws an error
+    throw: 
+      message: Something bad
+      details:
+        one: two
+  - message: This operation won't be reached
+  catch:
+    output: 
+      nested-operation-error: (error.{ type:type.name, message:message, details:details })
+
+- message: Errors can be caught conditionally
+
+- message: It's a good idea to catch a specific, known condition if possible
+
+- message: otherwise your workflow will continue to execute when anything unexpected happens
+
+- message: ([ 'Fetching ', foreach[0] ])
+  values:
+    downloads: []
+    foreach: 
+    - examples/107-outputs/workflow.yaml
+    - No-Such-File.txt
+    - examples/108-looping/workflow.yaml
+    - README.md
+  operations:
+  - values:
+      path: (foreach[0])
+    request: fetch-github-file.yaml
+    output:
+      append: 
+        path: (path)
+        contentlength: ( to_number( result.headers."Content-Length"[0] ) )
+    catch:
+      condition: result.status == `404` || contains(['SyntaxErrorException', 'YamlException'], error.type.name)
+      output:
+        append: 
+          path: (path)
+          exception: (error.type.name)
+          message: (error.message)
+  output:
+    downloads: ([downloads, append] [])
+    foreach: (foreach[1:])
+  repeat:
+    condition: (foreach[0] != null)
+
+- message: (['Number of downloads - ', length(downloads[].contentlength) ])
+- message: (['Total content length - ', sum(downloads[].contentlength) ])
+- message: (['Number of errors - ', length(downloads[].message) ])

src/Microsoft.Atlas.CommandLine/CommandLineApplicationExtensions.cs

@@ -73,7 +73,7 @@ namespace Microsoft.Atlas.CommandLine
             return defaultValue;
         }
 
-        public static void OnExecute<TCommand>(this CommandLineApplication app, Func<TCommand, Task<int>> onExecute)
+        public static void OnExecute<TCommand>(this CommandLineApplication app, Func<TCommand, int> onExecute)
         {
             app.OnExecute(() =>
             {
@@ -110,11 +110,12 @@ namespace Microsoft.Atlas.CommandLine
                 if (helpOption.HasValue())
                 {
                     app.ShowHelp();
-                    return Task.FromResult(0);
+                    return 0;
                 }
                 else
                 {
-                    return (Task<int>)method.Invoke(cmd, new object[0]);
+                    var task = (Task<int>)method.Invoke(cmd, new object[0]);
+                    return task.GetAwaiter().GetResult();
                 }
             });
         }

src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs

@@ -295,9 +295,6 @@ namespace Microsoft.Atlas.CommandLine.Commands
 
             var patternOkay = context.PatternMatcher.IsMatch(context.Path);
 
-            var message = ConvertToString(ProcessValues(operation.message, context.Values));
-            var write = ConvertToString(ProcessValues(operation.write, context.Values));
-
             var conditionOkay = true;
             if (!string.IsNullOrEmpty(operation.condition))
             {
@@ -307,6 +304,9 @@ namespace Microsoft.Atlas.CommandLine.Commands
 
             for (var shouldExecute = patternOkay && conditionOkay; shouldExecute; shouldExecute = await EvaluateRepeat(context))
             {
+                var message = ConvertToString(ProcessValues(operation.message, context.Values));
+                var write = ConvertToString(ProcessValues(operation.write, context.Values));
+
                 if (!string.IsNullOrEmpty(message))
                 {
                     _console.WriteLine();
@@ -338,14 +338,18 @@ namespace Microsoft.Atlas.CommandLine.Commands
                         { "response", null },
                         { "cumulativeValues", context.Values },
                     };
+
                     try
                     {
-                        object result = null;
+                        // object result = null;
+                        object outputContext = context.Values;
 
+                        try
+                        {
                             // First special type of operation - executing a request
                             if (!string.IsNullOrWhiteSpace(operation.request))
                             {
-                            WorkflowModel.Request request = context.TemplateEngine.Render<WorkflowModel.Request>(
+                                var request = context.TemplateEngine.Render<WorkflowModel.Request>(
                                     operation.request,
                                     context.Values);
 
@@ -372,8 +376,6 @@ namespace Microsoft.Atlas.CommandLine.Commands
                                     _console.WriteLine($"Skipping {method.Method.ToString().Color(ConsoleColor.DarkYellow)} {request.url}");
                                 }
                                 else
-                            {
-                                try
                                 {
                                     var jsonRequest = new JsonRequest
                                     {
@@ -385,24 +387,26 @@ namespace Microsoft.Atlas.CommandLine.Commands
                                     };
 
                                     var jsonResponse = await client.SendAsync(jsonRequest);
-                                    if ((int)jsonResponse.status >= 400)
-                                    {
-                                        throw new ApplicationException($"Request failed with status code {jsonResponse.status}");
-                                    }
 
-                                    result = new WorkflowModel.Response
+                                    var response = new WorkflowModel.Response
                                     {
                                         status = (int)jsonResponse.status,
                                         headers = jsonResponse.headers,
                                         body = jsonResponse.body,
                                     };
 
-                                    logentry["response"] = result;
-                                }
-                                catch
+                                    logentry["response"] = response;
+
+                                    outputContext = MergeUtils.Merge(new Dictionary<object, object> { { "result", response } }, outputContext);
+
+                                    if (response.status >= 400)
                                     {
-                                    // TODO - retry logic here?
-                                    throw;
+                                        var error = new RequestException($"Request failed with status code {jsonResponse.status}")
+                                        {
+                                            Request = request,
+                                            Response = response,
+                                        };
+                                        throw error;
                                     }
                                 }
                             }
@@ -424,38 +428,36 @@ namespace Microsoft.Atlas.CommandLine.Commands
                                         }
                                     }
                                 }
-                            else
+
+                                if (operation.output != null)
                                 {
-                                result = context.TemplateEngine.Render<object>(operation.template, context.Values);
+                                    var templateResult = context.TemplateEngine.Render<object>(operation.template, context.Values);
+
+                                    outputContext = MergeUtils.Merge(new Dictionary<object, object> { { "result", templateResult } }, outputContext);
                                 }
                             }
 
                             // Third special type of operation - nested operations
                             if (operation.operations != null)
                             {
-                            result = await ExecuteOperations(context, operation.operations);
+                                var nestedResult = await ExecuteOperations(context, operation.operations);
+
+                                if (operation.output == null)
+                                {
+                                    // if output is unstated, and there are nested operations with output - those flows back as effective output
+                                    context.AddValuesOut(nestedResult);
+                                }
+                                else
+                                {
+                                    // if output is stated, nested operations with output are visible to output queries
+                                    outputContext = MergeUtils.Merge(nestedResult, context.Values) ?? context.Values;
+                                }
                             }
 
                             // If output is specifically stated - use it to query
                             if (operation.output != null)
                             {
-                            if (operation.operations != null)
-                            {
-                                // for nested operations, output expressions can pull in the current operation's cumulative values as well
-                                context.AddValuesOut(ProcessValues(operation.output, MergeUtils.Merge(result, context.Values) ?? context.Values));
-                            }
-                            else if (result != null)
-                            {
-                                // for request and template operations, the current operation result is a well-known property to avoid collisions
-                                var merged = MergeUtils.Merge(new Dictionary<object, object> { { "result", result } }, context.Values);
-
-                                context.AddValuesOut(ProcessValues(operation.output, merged));
-                            }
-                            else
-                            {
-                                // there are no values coming out of this operation - output queries are based only on cumulative values
-                                context.AddValuesOut(ProcessValues(operation.output, context.Values));
-                            }
+                                context.AddValuesOut(ProcessValues(operation.output, outputContext));
                             }
 
                             if (operation.@throw != null)
@@ -476,11 +478,15 @@ namespace Microsoft.Atlas.CommandLine.Commands
                                     Details = throwDetails
                                 };
                             }
-
-                        // otherwise if output is unstated, and there are nested operations with output - those flows back as effective output
-                        if (operation.output == null && operation.operations != null && result != null)
+                        }
+                        catch (Exception ex) when (CatchCondition(ex, operation.@catch, outputContext))
                         {
-                            context.AddValuesOut(result);
+                            if (operation.@catch.output != null)
+                            {
+                                var mergedContext = MergeError(ex, outputContext);
+                                var catchDetails = ProcessValues(operation.@catch.output, mergedContext);
+                                context.AddValuesOut(catchDetails);
+                            }
                         }
                     }
                     catch (Exception ex)
@@ -505,6 +511,45 @@ namespace Microsoft.Atlas.CommandLine.Commands
             }
         }
 
+        private bool CatchCondition(Exception ex, WorkflowModel.Catch @catch, object outputContext)
+        {
+            try
+            {
+                if (@catch == null)
+                {
+                    return false;
+                }
+
+                if (string.IsNullOrEmpty(@catch.condition))
+                {
+                    return true;
+                }
+
+                var mergedContext = MergeError(ex, outputContext);
+                var conditionResult = _jmesPathQuery.Search(@catch.condition, mergedContext);
+                var conditionIsTrue = ConditionBoolean(conditionResult);
+                return conditionIsTrue;
+            }
+            catch (Exception ex2)
+            {
+                Console.Error.WriteLine($"{"Fatal".Color(ConsoleColor.Red)}: exception processing catch condition: {ex2.Message.Color(ConsoleColor.DarkRed)}");
+                throw;
+            }
+        }
+
+        private object MergeError(Exception exception, object context)
+        {
+            var yaml = _serializers.YamlSerializer.Serialize(new { error = exception });
+            var error = _serializers.YamlDeserializer.Deserialize<object>($@"
+{yaml}
+  type: 
+    name: {exception.GetType().Name}
+    fullName: {exception.GetType().FullName}
+");
+            var mergedContext = MergeUtils.Merge(error, context);
+            return mergedContext;
+        }
+
         private async Task<bool> EvaluateRepeat(ExecutionContext context)
         {
             var repeat = context.Operation.repeat;

src/Microsoft.Atlas.CommandLine/Execution/OperationException.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using YamlDotNet.Serialization;
 
 namespace Microsoft.Atlas.CommandLine.Execution
 {
@@ -43,6 +44,7 @@ namespace Microsoft.Atlas.CommandLine.Execution
         {
         }
 
+        [YamlMember(Alias = "details")]
         public object Details { get; internal set; }
     }
 }

src/Microsoft.Atlas.CommandLine/Execution/RequestException.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.Runtime.Serialization;
+using YamlDotNet.Serialization;
+
+namespace Microsoft.Atlas.CommandLine.Execution
+{
+    public class RequestException : Exception
+    {
+        public RequestException()
+        {
+        }
+
+        public RequestException(string message)
+            : base(message)
+        {
+        }
+
+        public RequestException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        protected RequestException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+
+        [YamlMember(Alias = "request")]
+        public Models.Workflow.WorkflowModel.Request Request { get; internal set; }
+
+        [YamlMember(Alias = "response")]
+        public Models.Workflow.WorkflowModel.Response Response { get; internal set; }
+    }
+}

src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs

@@ -29,6 +29,9 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow
             [YamlMember(Alias = "throw")]
             public Throw @throw { get; set; }
 
+            [YamlMember(Alias = "catch")]
+            public Catch @catch { get; set; }
+
             public string request { get; set; }
             public string template { get; set; }
             public string write { get; set; }
@@ -71,5 +74,11 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow
             public string message { get; set; }
             public object details { get; set; }
         }
+
+        public class Catch
+        {
+            public string condition { get; set; }
+            public object output { get; set; }
+        }
     }
 }

src/Microsoft.Atlas.CommandLine/Serialization/YamlSerializers.cs

@@ -3,6 +3,7 @@
 
 using System;
 using Microsoft.Atlas.CommandLine.Commands;
+using Microsoft.Atlas.CommandLine.Execution;
 using Newtonsoft.Json.Linq;
 using YamlDotNet.Serialization;
 
@@ -20,6 +21,8 @@ namespace Microsoft.Atlas.CommandLine.Serialization
 
             YamlSerializer = new SerializerBuilder()
                 .DisableAliases()
+                .WithAttributeOverride<Exception>(e => e.TargetSite, new YamlIgnoreAttribute())
+                .WithAttributeOverride<Exception>(e => e.Message, new YamlMemberAttribute { Alias = "message" })
                 .WithEventEmitter(DoubleQuoteAmbiguousStringScalarEmitter.Factory)
                 .WithTypeConverter(new ByteArrayConverter())
                 .Build();

test/Microsoft.Atlas.CommandLine.Tests/CatchOperationTests.cs

@@ -0,0 +1,247 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.IO;
+using Microsoft.Atlas.CommandLine.Execution;
+using Microsoft.Atlas.CommandLine.Secrets;
+using Microsoft.Atlas.CommandLine.Tests.Stubs;
+using Microsoft.Atlas.CommandLine.Tests.Utilities;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.Atlas.CommandLine.Tests
+{
+    [TestClass]
+    public class CatchOperationTests : ServiceContextTestsBase<RequestOperationTests.ServiceContext>
+    {
+        [TestMethod]
+        public void ThrowOperationCanBeCaught()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching operation
+                          operations:
+                          - message: Throwing operation
+                            throw: { message: boom }
+                          catch: {}
+                        - message: Still Running
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder("Catching operation", "Throwing operation", "Still Running");
+        }
+
+        [TestMethod]
+        public void CatchesIfConditionIsTrue()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching operation
+                          operations:
+                          - message: Throwing operation
+                            throw: { message: boom }
+                          catch:
+                            condition: error.message == 'boom'
+                        - message: Still Running
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder("Catching operation", "Throwing operation", "Still Running");
+        }
+
+        [TestMethod]
+        public void NotCaughtIfConditionIsFalse()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching operation
+                          operations:
+                          - message: Throwing operation
+                            throw: { message: different }
+                          catch:
+                            condition: Message == 'boom'
+                        - message: Still Running
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var ex = Assert.ThrowsException<OperationException>(() => Services.App.Execute("deploy", "the-test"));
+
+            Assert.AreEqual("different", ex.Message);
+
+            Console.AssertContainsInOrder("Catching operation", "Throwing operation", "different");
+        }
+
+        [TestMethod]
+        public void CatchByStatusCode()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching 404
+                          request: notfound.yaml
+                          catch:
+                            condition: result.status == `404`
+                        - message: Not Catching 503
+                          request: servererror.yaml
+                          catch:
+                            condition: result.status == `404`
+                        - message: Still Running
+                      notfound.yaml: |
+                        method: GET
+                        url: https://localhost/notfound
+                      servererror.yaml: |
+                        method: GET
+                        url: https://localhost/servererror
+            ");
+
+            var stubHttpClients = Yaml<StubHttpClientHandlerFactory>(@"
+                Responses:
+                  https://localhost/notfound:
+                    GET:
+                      status: 404
+                      body: Page error
+                  https://localhost/servererror:
+                    GET:
+                      status: 503
+                      body: Page error
+            ");
+
+            InitializeServices(stubBlueprints, stubHttpClients);
+
+            var ex = Assert.ThrowsException<RequestException>(() => Services.App.Execute("deploy", "the-test"));
+
+            Assert.IsTrue(ex.Message.Contains("503"));
+
+            Assert.AreEqual(503, ex.Response.status);
+
+            Console.AssertContainsInOrder("Catching 404", "Not Catching 503");
+
+            Console.AssertNotContains("Still Running");
+        }
+
+        [TestMethod]
+        public void ExceptionAndResultDetailsCanBeSavedInCatch()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching 400
+                          request: badrequest.yaml
+                          catch:
+                            output:
+                              the-result: (result)
+                              the-error: (error)
+                        - message: Still Running
+                      badrequest.yaml: |
+                        method: GET
+                        url: https://localhost/badrequest
+            ");
+
+            var stubHttpClients = Yaml<StubHttpClientHandlerFactory>(@"
+                Responses:
+                  https://localhost/badrequest:
+                    GET:
+                      status: 400
+                      headers: { Content-Type: ['application/json'] }
+                      body: 
+                        oops:
+                          code: 1234
+                          summary: This is a bad request
+            ");
+
+            InitializeServices(stubBlueprints, stubHttpClients);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder(
+                "Catching 400",
+                "Still Running",
+                "the-result:",
+                "status: 400",
+                "body:",
+                "oops:",
+                "code:",
+                "1234",
+                "summary: This is a bad request",
+                "the-error:",
+                "message:",
+                "400");
+        }
+
+        [TestMethod]
+        public void ThrowDetailsCanBeSavedInCatch()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - message: Catching Exception
+                          operations:
+                          - message: Throwing Exception
+                            throw:
+                              message: one
+                              details:
+                                x: two
+                                y: three
+                          catch:
+                            output:
+                              the-details: (error.details)
+                              the-message: (error.message)
+                        - message: Still Running
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder(
+                "Catching Exception",
+                "Throwing Exception",
+                "Still Running",
+                "the-details:",
+                "x: two",
+                "y: three",
+                "the-message: one");
+        }
+
+        public class ServiceContext : ServiceContextBase
+        {
+            public CommandLineApplicationServices App { get; set; }
+        }
+    }
+}

test/Microsoft.Atlas.CommandLine.Tests/Stubs/StubConsole.cs

@@ -35,5 +35,12 @@ namespace Microsoft.Atlas.CommandLine.Tests.Stubs
                 startIndex = segmentIndex + segment.Length;
             }
         }
+
+        internal void AssertNotContains(string text)
+        {
+            var output = OutStringWriter.GetStringBuilder().ToString();
+
+            Assert.IsFalse(output.Contains(text), $"Output does not contain {text}");
+        }
     }
 }

test/Microsoft.Atlas.CommandLine.Tests/WorkflowCommandsTests.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.IO;
 using System.Threading.Tasks;
 using Microsoft.Atlas.CommandLine.Commands;
 using Microsoft.Atlas.CommandLine.Tests.Stubs;
@@ -62,7 +63,7 @@ Blueprints:
 
             InitializeServices(stubs);
 
-            var ex = Assert.ThrowsException<AggregateException>(() =>
+            var ex = Assert.ThrowsException<ApplicationException>(() =>
             {
                 Services.App.Execute("deploy", "bad-file-name");
             });
@@ -84,7 +85,7 @@ Blueprints:
 
             InitializeServices(stubs);
 
-            var ex = Assert.ThrowsException<AggregateException>(() =>
+            var ex = Assert.ThrowsException<FileNotFoundException>(() =>
             {
                 var result = Services.App.Execute("deploy", "-f", "missing-values.yaml", "the-test");
             });
@@ -289,7 +290,7 @@ Responses:
 
             InitializeServices(stubBlueprints, stubRequests);
 
-            var error = Assert.ThrowsException<AggregateException>(() =>
+            var error = Assert.ThrowsException<InvalidOperationException>(() =>
             {
                 Services.App.Execute("deploy", "the-test", "--non-interactive");
             });