Operation foreach (#98)

* Any operation may have a wrapping foreach loop
* foreach.values which are arrays cause the operation to
be executed multiple times
* foreach.output which are arrays are concatinated to produce
final results

Resolves #71

azure-rest-api-specs

@@ -0,0 +1 @@
+Subproject commit baa81416330fbb712ad9f39d2e997ba8afbbb13b

examples/108-foreach/values.yaml

@@ -0,0 +1,5 @@
+
+items:
+- url: https://en.wikipedia.org/wiki/Cheshire_Cat
+- url: https://en.wikipedia.org/wiki/Bill_the_Lizard
+- url: https://en.wikipedia.org/wiki/Hatter_(Alice%27s_Adventures_in_Wonderland)

examples/108-foreach/workflow.yaml

@@ -0,0 +1,22 @@
+
+operations:
+
+- message: This is an example of looping through items in an array 
+
+- message: Any jmespath expression that results in an array can be iterated over
+
+- foreach:
+    values:
+      item: ( items[] )
+    output:
+      items: ( [item] )
+ 
+  operations:
+  - message: Nested operations are a typical way of working with a loop
+ 
+  - message: ( ['Current item.url is ', item.url])
+ 
+  - message: Adding item.length property
+    output:
+      item:
+        length: ( length(item.url) )

examples/108-looping/workflow.yaml

@@ -1,43 +0,0 @@
-
-operations:
-
-- message: This is an example of counting from 0 to 9
-- message: and building an `append` object to add to a `data` array
-
-- message: (['Looping... Index is ', index])
-
-  values: # count from 0 to 9 by 1
-    index: 0
-    data: []
-
-  operations:
-  - message: Nested work would go here
-    output: 
-      append: # append[] is added to data[] below
-      - id: (index)
-        formula: ( bits(index) | [].to_string(@) | join(' + ', @) )
-
-  output: 
-    data: ( [data, append] [] ) # combines array-of-arrays, then uses [] operator to flatten
-    index: (sum( [index, `1`] )) # increment index
-
-  repeat: # repeat until index is 10
-    condition: (index < `10`)
-
-
-- message: This is an example of looping through items in an array 
-  
-- message: (['Looping... Current is ', to_string(current)])
-  values: 
-    foreach: (data) # keep track of remaining data to iterate
-    current: (data[0]) # and make the current item easy to access
-  condition: ( length(foreach) > `0` ) # don't run if there's no data
-  
-  operations:
-  - message: Nested work would go here
-
-  output:
-    current: (foreach[1]) # advance to the next data item
-    foreach: (foreach[1:]) # reduce the remaining data to iterate
-  repeat:
-    condition: ( length(foreach) > `0` ) # stop when there is no remaining data

examples/203-catching-errors/workflow.yaml

@@ -30,34 +30,29 @@ operations:
 
 - message: otherwise your workflow will continue to execute when anything unexpected happens
 
-- message: ([ 'Fetching ', foreach[0] ])
-  values:
-    downloads: []
-    foreach: 
-    - examples/107-outputs/workflow.yaml
-    - No-Such-File.txt
-    - examples/108-looping/workflow.yaml
-    - README.md
+- foreach:
+    values:
+      path:
+      - examples/107-outputs/workflow.yaml
+      - No-Such-File.txt
+      - examples/108-looping/workflow.yaml
+      - README.md
+    output:
+      downloads: ([download])
   operations:
-  - values:
-      path: (foreach[0])
+  - message: ([ 'Fetching ', path ])
     request: fetch-github-file.yaml
     output:
-      append: 
+      download: 
         path: (path)
         contentlength: ( to_number( result.headers."Content-Length"[0] ) )
     catch:
       condition: result.status == `404` || contains(['SyntaxErrorException', 'YamlException'], error.type.name)
       output:
-        append: 
+        download: 
           path: (path)
           exception: (error.type.name)
           message: (error.message)
-  output:
-    downloads: ([downloads, append] [])
-    foreach: (foreach[1:])
-  repeat:
-    condition: (foreach[0] != null)
 
 - message: (['Number of downloads - ', length(downloads[].contentlength) ])
 - message: (['Total content length - ', sum(downloads[].contentlength) ])

examples/302-gather-info/workflow.yaml

@@ -1,51 +1,42 @@
 
 
 operations:
-- message: REST APIs can be called
-
 - message: Listing Microsoft Azure tenants, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list
   request: apis/azure/SubscriptionClient/Tenants/List.yaml
   output:
-    tenantIds: ( result.body.value[].tenantId )
+    tenants: ( result.body.value[] )
 
-- condition: ( length(tenantIds) != `0` )  
-  values:
-    tenants: []
-
-  operations:
-  - message: (['Looking at tenant ', tenantIds[0]])
-    values: # These values apply to all three of the operations
+- message: ( ['Processing tenant ', tenant.tenantId] )
+  foreach:
+    values:
+      tenant: ( tenants[] )
       azure:
-        tenant: ( tenantIds[0] )
+        tenant: ( tenants[].tenantId )
       request:
         parameters:
-          tenantID: ( tenantIds[0] )
+          tenantID: ( tenants[].tenantId )
+    output:
+      tenants: ( [tenant] )
 
-    operations:
-    - message: Getting Microsoft Graph user info, see https://docs.microsoft.com/en-us/rest/api/graphrbac/objects/getcurrentuser
-      request: apis/graph/GraphRbacManagementClient/SignedInUser/Get.yaml
-      output:
-        current:
-          tenantId: ( tenantIds[0] )
-          user: ( result.body.{ "odata.type":"odata.type", objectId:objectId, displayName:displayName, userPrincipalName:userPrincipalName, otherMails:otherMails } )
+  operations:
+  - message: Getting Microsoft Graph user info, see https://docs.microsoft.com/en-us/rest/api/graphrbac/objects/getcurrentuser
+    request: apis/graph/GraphRbacManagementClient/SignedInUser/Get.yaml
+    output:
+      tenant:
+        user: ( result.body.{ "odata.type":"odata.type", objectId:objectId, displayName:displayName, userPrincipalName:userPrincipalName, otherMails:otherMails } )
 
-    - message: Listing Active Directory domains, see https://docs.microsoft.com/en-us/rest/api/graphrbac/domains/list
-      request: apis/graph/GraphRbacManagementClient/Domain/Domains_List.yaml
-      output:
-        current:
-          domains: ( result.body.value[].name )
+  - message: Listing Active Directory domains, see https://docs.microsoft.com/en-us/rest/api/graphrbac/domains/list
+    request: apis/graph/GraphRbacManagementClient/Domain/Domains_List.yaml
+    output:
+      tenant:
+        domains: ( result.body.value[].name )
 
-    - message: Listing Microsoft Azure subscriptions, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list
-      request: apis/azure/SubscriptionClient/Subscriptions/List.yaml
-      output:
-        current:
-          subscriptions: ( result.body.value[].{ subscriptionId:subscriptionId, displayName:displayName } )
+  - message: Listing Microsoft Azure subscriptions, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list
+    request: apis/azure/SubscriptionClient/Subscriptions/List.yaml
+    output:
+      tenant:
+        subscriptions: ( result.body.value[].{ subscriptionId:subscriptionId, displayName:displayName } )
 
-  output:
-    tenants: ( [tenants, current] [] )
-    tenantIds: ( tenantIds[1:] )
-  repeat:
-    condition: ( length(tenantIds) != `0` )
 
 - message: Arranging selection of azure.tenant and azure.subscription
   template: selection.yaml

examples/values.yaml

@@ -1,3 +0,0 @@
-azure:
-  tenant: 72f988bf-86f1-41af-91ab-2d7cd011db47
-  subscription: cd0fa82d-b6b6-4361-b002-050c32f71353

src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs

@@ -311,6 +311,43 @@ namespace Microsoft.Atlas.CommandLine.Commands
         {
             var operation = context.Operation;
 
+            if (context.Operation.@foreach == null)
+            {
+                await ExecuteOperationInner(context);
+            }
+            else
+            {
+                var foreachContexts = new List<ExecutionContext>();
+                var foreachValuesInList = ProcessValuesForeachIn(operation.@foreach.values, context.Values);
+
+                foreach (var foreachValuesIn in foreachValuesInList)
+                {
+                    var foreachContext = context.CreateChildContext(operation, MergeUtils.Merge(foreachValuesIn, context.Values));
+                    foreachContexts.Add(foreachContext);
+                    await ExecuteOperationInner(foreachContext);
+                }
+
+                var valuesOut = default(object);
+                if (operation.@foreach.output != null)
+                {
+                    valuesOut = ProcessValuesForeachOut(operation.@foreach.output, foreachContexts.Select(foreachContext => foreachContext.Values).ToList());
+                }
+                else
+                {
+                    foreach (var foreachValuesOut in foreachContexts.Select(foreachContext => foreachContext.ValuesOut))
+                    {
+                        valuesOut = MergeUtils.Merge(foreachValuesOut, valuesOut);
+                    }
+                }
+
+                context.AddValuesOut(valuesOut);
+            }
+        }
+
+        private async Task ExecuteOperationInner(ExecutionContext context)
+        {
+            var operation = context.Operation;
+
             if (operation.values != null)
             {
                 context.AddValuesIn(ProcessValues(operation.values, context.Values));
@@ -618,21 +655,66 @@ namespace Microsoft.Atlas.CommandLine.Commands
         }
 
         private object ProcessValues(object source, object context)
+        {
+            return ProcessValuesRecursive(source, new[] { context }, promoteArrays: false);
+        }
+
+        private IList<object> ProcessValuesForeachIn(object source, object context)
+        {
+            var result = ProcessValuesRecursive(source, new[] { context }, promoteArrays: true);
+            if (result is IList<object> resultList)
+            {
+                return resultList;
+            }
+
+            throw new ApplicationException("Foreach values contained no arrays");
+        }
+
+        private object ProcessValuesForeachOut(object source, IList<object> contexts)
+        {
+            return ProcessValuesRecursive(source, contexts, promoteArrays: false);
+        }
+
+        private object ProcessValuesRecursive(object source, IList<object> contexts, bool promoteArrays)
         {
             if (source is IDictionary<object, object> sourceDictionary)
             {
+                var arrayIsPromoting = false;
+                var arrayLength = 0;
+
                 var output = new Dictionary<object, object>();
                 foreach (var kv in sourceDictionary)
                 {
-                    var isQuery = IsQuery(kv);
-                    if (isQuery)
+                    var result = ProcessValuesRecursive(kv.Value, contexts, promoteArrays: promoteArrays);
+                    output[kv.Key] = result;
+
+                    if (promoteArrays && result is IList<object> resultArray)
                     {
-                        output[kv.Key.ToString().TrimEnd('?')] = ProcessQueries(kv.Value, context);
+                        if (!arrayIsPromoting)
+                        {
+                            arrayIsPromoting = true;
+                            arrayLength = resultArray.Count();
+                        }
+                        else
+                        {
+                            if (arrayLength != resultArray.Count())
+                            {
+                                throw new ApplicationException("Foreach arrays must all be same size");
+                            }
+                        }
                     }
-                    else
+                }
+
+                if (arrayIsPromoting)
+                {
+                    var arrayOutput = new List<object>();
+                    for (var index = 0; index < arrayLength; ++index)
                     {
-                        output[kv.Key] = ProcessValues(kv.Value, context);
+                        var arrayItem = output.ToDictionary(kv => kv.Key, kv => kv.Value is IList<object> valueArray ? valueArray[index] : kv.Value);
+                        arrayOutput.Add(arrayItem);
                     }
+
+                    return arrayOutput;
                 }
 
                 return output;
@@ -640,51 +722,35 @@ namespace Microsoft.Atlas.CommandLine.Commands
 
             if (source is IList<object> sourceList)
             {
-                return sourceList.Select(value => ProcessValues(value, context)).ToList();
+                return sourceList.Select(value => ProcessValuesRecursive(value, contexts, promoteArrays: promoteArrays)).ToList();
             }
 
             if (source is string sourceString)
             {
                 if (sourceString.StartsWith('(') && sourceString.EndsWith(')'))
                 {
-                    return ProcessQueries(sourceString.Substring(1, sourceString.Length - 2), context);
+                    var expression = sourceString.Substring(1, sourceString.Length - 2);
+                    var mergedResult = default(object);
+                    foreach (var context in contexts)
+                    {
+                        var result = _jmesPathQuery.Search(sourceString, context);
+                        if (result is IList<object> resultList && mergedResult is IList<object> mergedList)
+                        {
+                            mergedResult = mergedList.Concat(resultList).ToList();
+                        }
+                        else
+                        {
+                            mergedResult = MergeUtils.Merge(result, mergedResult);
+                        }
+                    }
+
+                    return mergedResult;
                 }
             }
 
             return source;
         }
 
-        private object ProcessQueries(object source, object context)
-        {
-            if (source is IDictionary<object, object> sourceDictionary)
-            {
-                var output = new Dictionary<object, object>();
-                foreach (var kv in sourceDictionary)
-                {
-                    output[kv.Key] = ProcessQueries(kv.Value, context);
-                }
-
-                return output;
-            }
-
-            if (source is IList<object> sourceList)
-            {
-                return sourceList.Select(value => ProcessQueries(value, context)).ToList();
-            }
-
-            if (source is string sourceString)
-            {
-                return _jmesPathQuery.Search(sourceString, context);
-            }
-
-            throw new ApplicationException($"Unexpected value type {source.GetType().FullName} {source}");
-        }
-
-        private bool IsQuery(KeyValuePair<object, object> kv)
-        {
-            return kv.Key?.ToString()?.EndsWith('?') ?? false;
-        }
-
         private string ConvertToString(object source)
         {
             if (source is IDictionary<object, object> sourceDictionary)

src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs

@@ -22,6 +22,9 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow
             public string message { get; set; }
             public string target { get; set; }
             public string condition { get; set; }
+
+            [YamlMember(Alias = "foreach")]
+            public Foreach @foreach { get; set; }
             public Repeat repeat { get; set; }
             public object values { get; set; }
             public object output { get; set; }
@@ -63,6 +66,12 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow
             public object body { get; set; }
         }
 
+        public class Foreach
+        {
+            public object values { get; set; }
+            public object output { get; set; }
+        }
+
         public class Repeat
         {
             public string condition { get; set; }

src/Microsoft.Atlas.CommandLine/Properties/launchSettings.json

@@ -23,6 +23,11 @@
       "commandLineArgs": "deploy sandbox k8s/lodejard-atlas-dev01",
       "workingDirectory": "..\\..\\..\\AtlasSandbox"
     },
+    "deploy 105-partials": {
+      "commandName": "Project",
+      "commandLineArgs": "deploy 105-partials",
+      "workingDirectory": "..\\..\\examples"
+    },
     "deploy 106-looping": {
       "commandName": "Project",
       "commandLineArgs": "deploy 106-looping",

test/Microsoft.Atlas.CommandLine.Tests/ForeachOperationTests.cs

@@ -0,0 +1,150 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.IO;
+using Microsoft.Atlas.CommandLine.Execution;
+using Microsoft.Atlas.CommandLine.Secrets;
+using Microsoft.Atlas.CommandLine.Tests.Stubs;
+using Microsoft.Atlas.CommandLine.Tests.Utilities;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.Atlas.CommandLine.Tests
+{
+    [TestClass]
+    public class ForeachOperationTests : ServiceContextTestsBase<RequestOperationTests.ServiceContext>
+    {
+        [TestMethod]
+        public void ForeachValuesAreIterated()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - foreach:
+                            values:
+                              x: ['a', 'b', 'c']
+                          message: (['x is ', x])
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder("x is a", "x is b", "x is c");
+        }
+
+        [TestMethod]
+        public void SeveralArraysCanBeIterated()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - foreach:
+                            values:
+                              x: ['a', 'b', 'c']
+                              y: 
+                                z: ['d', 'e', 'f']
+                          message: (['x is ', x, ' y.z is ', y.z])
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder(
+                "x is a",
+                "y.z is d",
+                "x is b",
+                "y.z is e",
+                "x is c",
+                "y.z is f");
+        }
+
+
+        [TestMethod]
+        public void ForeachHashObject()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      values.yaml: |
+                          hash:
+                            a: b
+                            c: d
+                            e: f
+                      workflow.yaml: |
+                        operations:
+                        - foreach:
+                            values: ( map(&{ key:[0], value:[1] }, items(hash)) )
+                          message: ""(['1: ', key, ' is ', value])""
+                        - foreach:
+                            values:
+                              key: ( keys(hash) )
+                              value: ( values(hash) )
+                          message: ""(['2: ', key, ' is ', value])""
+                        - foreach:
+                            values:
+                              kv: ( items(hash) )
+                          message: ""(['3: ', kv[0], ' is ', kv[1]])""
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder(
+                "1: a is b",
+                "1: c is d",
+                "1: e is f",
+                "2: a is b",
+                "2: c is d",
+                "2: e is f",
+                "3: a is b",
+                "3: c is d",
+                "3: e is f");
+        }
+
+        [TestMethod]
+        public void ForeachOutputIsConcatinated()
+        {
+            var stubBlueprints = Yaml<StubBlueprintManager>(@"
+                Blueprints:
+                  the-test:
+                    Files:
+                      workflow.yaml: |
+                        operations:
+                        - foreach:
+                            values:
+                              x: ['a', 'b', 'c']
+                            output:
+                              y: ([x])
+                        - message: (join('+', y))
+            ");
+
+            InitializeServices(stubBlueprints);
+
+            var result = Services.App.Execute("deploy", "the-test");
+
+            Assert.AreEqual(0, result);
+
+            Console.AssertContainsInOrder("a+b+c");
+        }
+        public class ServiceContext : ServiceContextBase
+        {
+            public CommandLineApplicationServices App { get; set; }
+        }
+    }
+}