From 40fabb5b2bf91cc6c596f8a0786a4d28370d193b Mon Sep 17 00:00:00 2001 From: Louis DeJardin Date: Thu, 8 Nov 2018 16:18:59 -0800 Subject: [PATCH] Operation foreach (#98) * Any operation may have a wrapping foreach loop * foreach.values which are arrays cause the operation to be executed multiple times * foreach.output which are arrays are concatinated to produce final results Resolves #71 --- azure-rest-api-specs | 1 + examples/108-foreach/values.yaml | 5 + examples/108-foreach/workflow.yaml | 22 +++ examples/108-looping/workflow.yaml | 43 ----- examples/203-catching-errors/workflow.yaml | 29 ++-- examples/302-gather-info/workflow.yaml | 61 +++---- examples/values.yaml | 3 - .../Commands/WorkflowCommands.cs | 142 ++++++++++++----- .../Models/Workflow/Model.cs | 9 ++ .../Properties/launchSettings.json | 5 + .../ForeachOperationTests.cs | 150 ++++++++++++++++++ 11 files changed, 334 insertions(+), 136 deletions(-) create mode 160000 azure-rest-api-specs create mode 100644 examples/108-foreach/values.yaml create mode 100644 examples/108-foreach/workflow.yaml delete mode 100644 examples/108-looping/workflow.yaml delete mode 100644 examples/values.yaml create mode 100644 test/Microsoft.Atlas.CommandLine.Tests/ForeachOperationTests.cs diff --git a/azure-rest-api-specs b/azure-rest-api-specs new file mode 160000 index 0000000..baa8141 --- /dev/null +++ b/azure-rest-api-specs @@ -0,0 +1 @@ +Subproject commit baa81416330fbb712ad9f39d2e997ba8afbbb13b diff --git a/examples/108-foreach/values.yaml b/examples/108-foreach/values.yaml new file mode 100644 index 0000000..1c06fb4 --- /dev/null +++ b/examples/108-foreach/values.yaml @@ -0,0 +1,5 @@ + +items: +- url: https://en.wikipedia.org/wiki/Cheshire_Cat +- url: https://en.wikipedia.org/wiki/Bill_the_Lizard +- url: https://en.wikipedia.org/wiki/Hatter_(Alice%27s_Adventures_in_Wonderland) diff --git a/examples/108-foreach/workflow.yaml b/examples/108-foreach/workflow.yaml new file mode 100644 index 0000000..ea91e62 --- /dev/null +++ b/examples/108-foreach/workflow.yaml @@ -0,0 +1,22 @@ + +operations: + +- message: This is an example of looping through items in an array + +- message: Any jmespath expression that results in an array can be iterated over + +- foreach: + values: + item: ( items[] ) + output: + items: ( [item] ) + + operations: + - message: Nested operations are a typical way of working with a loop + + - message: ( ['Current item.url is ', item.url]) + + - message: Adding item.length property + output: + item: + length: ( length(item.url) ) diff --git a/examples/108-looping/workflow.yaml b/examples/108-looping/workflow.yaml deleted file mode 100644 index 1f7d386..0000000 --- a/examples/108-looping/workflow.yaml +++ /dev/null @@ -1,43 +0,0 @@ - -operations: - -- message: This is an example of counting from 0 to 9 -- message: and building an `append` object to add to a `data` array - -- message: (['Looping... Index is ', index]) - - values: # count from 0 to 9 by 1 - index: 0 - data: [] - - operations: - - message: Nested work would go here - output: - append: # append[] is added to data[] below - - id: (index) - formula: ( bits(index) | [].to_string(@) | join(' + ', @) ) - - output: - data: ( [data, append] [] ) # combines array-of-arrays, then uses [] operator to flatten - index: (sum( [index, `1`] )) # increment index - - repeat: # repeat until index is 10 - condition: (index < `10`) - - -- message: This is an example of looping through items in an array - -- message: (['Looping... Current is ', to_string(current)]) - values: - foreach: (data) # keep track of remaining data to iterate - current: (data[0]) # and make the current item easy to access - condition: ( length(foreach) > `0` ) # don't run if there's no data - - operations: - - message: Nested work would go here - - output: - current: (foreach[1]) # advance to the next data item - foreach: (foreach[1:]) # reduce the remaining data to iterate - repeat: - condition: ( length(foreach) > `0` ) # stop when there is no remaining data diff --git a/examples/203-catching-errors/workflow.yaml b/examples/203-catching-errors/workflow.yaml index 64a5ed3..e0c0f8a 100644 --- a/examples/203-catching-errors/workflow.yaml +++ b/examples/203-catching-errors/workflow.yaml @@ -30,34 +30,29 @@ operations: - message: otherwise your workflow will continue to execute when anything unexpected happens -- message: ([ 'Fetching ', foreach[0] ]) - values: - downloads: [] - foreach: - - examples/107-outputs/workflow.yaml - - No-Such-File.txt - - examples/108-looping/workflow.yaml - - README.md +- foreach: + values: + path: + - examples/107-outputs/workflow.yaml + - No-Such-File.txt + - examples/108-looping/workflow.yaml + - README.md + output: + downloads: ([download]) operations: - - values: - path: (foreach[0]) + - message: ([ 'Fetching ', path ]) request: fetch-github-file.yaml output: - append: + download: path: (path) contentlength: ( to_number( result.headers."Content-Length"[0] ) ) catch: condition: result.status == `404` || contains(['SyntaxErrorException', 'YamlException'], error.type.name) output: - append: + download: path: (path) exception: (error.type.name) message: (error.message) - output: - downloads: ([downloads, append] []) - foreach: (foreach[1:]) - repeat: - condition: (foreach[0] != null) - message: (['Number of downloads - ', length(downloads[].contentlength) ]) - message: (['Total content length - ', sum(downloads[].contentlength) ]) diff --git a/examples/302-gather-info/workflow.yaml b/examples/302-gather-info/workflow.yaml index 2c0cb46..d89dd8d 100644 --- a/examples/302-gather-info/workflow.yaml +++ b/examples/302-gather-info/workflow.yaml @@ -1,52 +1,43 @@ operations: -- message: REST APIs can be called - - message: Listing Microsoft Azure tenants, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list request: apis/azure/SubscriptionClient/Tenants/List.yaml output: - tenantIds: ( result.body.value[].tenantId ) + tenants: ( result.body.value[] ) -- condition: ( length(tenantIds) != `0` ) - values: - tenants: [] - - operations: - - message: (['Looking at tenant ', tenantIds[0]]) - values: # These values apply to all three of the operations +- message: ( ['Processing tenant ', tenant.tenantId] ) + foreach: + values: + tenant: ( tenants[] ) azure: - tenant: ( tenantIds[0] ) + tenant: ( tenants[].tenantId ) request: parameters: - tenantID: ( tenantIds[0] ) + tenantID: ( tenants[].tenantId ) + output: + tenants: ( [tenant] ) - operations: - - message: Getting Microsoft Graph user info, see https://docs.microsoft.com/en-us/rest/api/graphrbac/objects/getcurrentuser - request: apis/graph/GraphRbacManagementClient/SignedInUser/Get.yaml - output: - current: - tenantId: ( tenantIds[0] ) - user: ( result.body.{ "odata.type":"odata.type", objectId:objectId, displayName:displayName, userPrincipalName:userPrincipalName, otherMails:otherMails } ) + operations: + - message: Getting Microsoft Graph user info, see https://docs.microsoft.com/en-us/rest/api/graphrbac/objects/getcurrentuser + request: apis/graph/GraphRbacManagementClient/SignedInUser/Get.yaml + output: + tenant: + user: ( result.body.{ "odata.type":"odata.type", objectId:objectId, displayName:displayName, userPrincipalName:userPrincipalName, otherMails:otherMails } ) - - message: Listing Active Directory domains, see https://docs.microsoft.com/en-us/rest/api/graphrbac/domains/list - request: apis/graph/GraphRbacManagementClient/Domain/Domains_List.yaml - output: - current: - domains: ( result.body.value[].name ) + - message: Listing Active Directory domains, see https://docs.microsoft.com/en-us/rest/api/graphrbac/domains/list + request: apis/graph/GraphRbacManagementClient/Domain/Domains_List.yaml + output: + tenant: + domains: ( result.body.value[].name ) + + - message: Listing Microsoft Azure subscriptions, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list + request: apis/azure/SubscriptionClient/Subscriptions/List.yaml + output: + tenant: + subscriptions: ( result.body.value[].{ subscriptionId:subscriptionId, displayName:displayName } ) - - message: Listing Microsoft Azure subscriptions, see https://docs.microsoft.com/en-us/rest/api/resources/subscriptions/list - request: apis/azure/SubscriptionClient/Subscriptions/List.yaml - output: - current: - subscriptions: ( result.body.value[].{ subscriptionId:subscriptionId, displayName:displayName } ) - output: - tenants: ( [tenants, current] [] ) - tenantIds: ( tenantIds[1:] ) - repeat: - condition: ( length(tenantIds) != `0` ) - - message: Arranging selection of azure.tenant and azure.subscription template: selection.yaml write: selection.yaml diff --git a/examples/values.yaml b/examples/values.yaml deleted file mode 100644 index 43db7b2..0000000 --- a/examples/values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -azure: - tenant: 72f988bf-86f1-41af-91ab-2d7cd011db47 - subscription: cd0fa82d-b6b6-4361-b002-050c32f71353 diff --git a/src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs b/src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs index ec2752c..d0487d9 100644 --- a/src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs +++ b/src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs @@ -311,6 +311,43 @@ namespace Microsoft.Atlas.CommandLine.Commands { var operation = context.Operation; + if (context.Operation.@foreach == null) + { + await ExecuteOperationInner(context); + } + else + { + var foreachContexts = new List(); + var foreachValuesInList = ProcessValuesForeachIn(operation.@foreach.values, context.Values); + + foreach (var foreachValuesIn in foreachValuesInList) + { + var foreachContext = context.CreateChildContext(operation, MergeUtils.Merge(foreachValuesIn, context.Values)); + foreachContexts.Add(foreachContext); + await ExecuteOperationInner(foreachContext); + } + + var valuesOut = default(object); + if (operation.@foreach.output != null) + { + valuesOut = ProcessValuesForeachOut(operation.@foreach.output, foreachContexts.Select(foreachContext => foreachContext.Values).ToList()); + } + else + { + foreach (var foreachValuesOut in foreachContexts.Select(foreachContext => foreachContext.ValuesOut)) + { + valuesOut = MergeUtils.Merge(foreachValuesOut, valuesOut); + } + } + + context.AddValuesOut(valuesOut); + } + } + + private async Task ExecuteOperationInner(ExecutionContext context) + { + var operation = context.Operation; + if (operation.values != null) { context.AddValuesIn(ProcessValues(operation.values, context.Values)); @@ -618,21 +655,66 @@ namespace Microsoft.Atlas.CommandLine.Commands } private object ProcessValues(object source, object context) + { + return ProcessValuesRecursive(source, new[] { context }, promoteArrays: false); + } + + private IList ProcessValuesForeachIn(object source, object context) + { + var result = ProcessValuesRecursive(source, new[] { context }, promoteArrays: true); + if (result is IList resultList) + { + return resultList; + } + + throw new ApplicationException("Foreach values contained no arrays"); + } + + private object ProcessValuesForeachOut(object source, IList contexts) + { + return ProcessValuesRecursive(source, contexts, promoteArrays: false); + } + + private object ProcessValuesRecursive(object source, IList contexts, bool promoteArrays) { if (source is IDictionary sourceDictionary) { + var arrayIsPromoting = false; + var arrayLength = 0; + var output = new Dictionary(); foreach (var kv in sourceDictionary) { - var isQuery = IsQuery(kv); - if (isQuery) + var result = ProcessValuesRecursive(kv.Value, contexts, promoteArrays: promoteArrays); + output[kv.Key] = result; + + if (promoteArrays && result is IList resultArray) { - output[kv.Key.ToString().TrimEnd('?')] = ProcessQueries(kv.Value, context); + if (!arrayIsPromoting) + { + arrayIsPromoting = true; + arrayLength = resultArray.Count(); + } + else + { + if (arrayLength != resultArray.Count()) + { + throw new ApplicationException("Foreach arrays must all be same size"); + } + } } - else + } + + if (arrayIsPromoting) + { + var arrayOutput = new List(); + for (var index = 0; index < arrayLength; ++index) { - output[kv.Key] = ProcessValues(kv.Value, context); + var arrayItem = output.ToDictionary(kv => kv.Key, kv => kv.Value is IList valueArray ? valueArray[index] : kv.Value); + arrayOutput.Add(arrayItem); } + + return arrayOutput; } return output; @@ -640,51 +722,35 @@ namespace Microsoft.Atlas.CommandLine.Commands if (source is IList sourceList) { - return sourceList.Select(value => ProcessValues(value, context)).ToList(); + return sourceList.Select(value => ProcessValuesRecursive(value, contexts, promoteArrays: promoteArrays)).ToList(); } if (source is string sourceString) { if (sourceString.StartsWith('(') && sourceString.EndsWith(')')) { - return ProcessQueries(sourceString.Substring(1, sourceString.Length - 2), context); + var expression = sourceString.Substring(1, sourceString.Length - 2); + var mergedResult = default(object); + foreach (var context in contexts) + { + var result = _jmesPathQuery.Search(sourceString, context); + if (result is IList resultList && mergedResult is IList mergedList) + { + mergedResult = mergedList.Concat(resultList).ToList(); + } + else + { + mergedResult = MergeUtils.Merge(result, mergedResult); + } + } + + return mergedResult; } } return source; } - private object ProcessQueries(object source, object context) - { - if (source is IDictionary sourceDictionary) - { - var output = new Dictionary(); - foreach (var kv in sourceDictionary) - { - output[kv.Key] = ProcessQueries(kv.Value, context); - } - - return output; - } - - if (source is IList sourceList) - { - return sourceList.Select(value => ProcessQueries(value, context)).ToList(); - } - - if (source is string sourceString) - { - return _jmesPathQuery.Search(sourceString, context); - } - - throw new ApplicationException($"Unexpected value type {source.GetType().FullName} {source}"); - } - - private bool IsQuery(KeyValuePair kv) - { - return kv.Key?.ToString()?.EndsWith('?') ?? false; - } - private string ConvertToString(object source) { if (source is IDictionary sourceDictionary) diff --git a/src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs b/src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs index 716a280..067ed88 100644 --- a/src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs +++ b/src/Microsoft.Atlas.CommandLine/Models/Workflow/Model.cs @@ -22,6 +22,9 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow public string message { get; set; } public string target { get; set; } public string condition { get; set; } + + [YamlMember(Alias = "foreach")] + public Foreach @foreach { get; set; } public Repeat repeat { get; set; } public object values { get; set; } public object output { get; set; } @@ -63,6 +66,12 @@ namespace Microsoft.Atlas.CommandLine.Models.Workflow public object body { get; set; } } + public class Foreach + { + public object values { get; set; } + public object output { get; set; } + } + public class Repeat { public string condition { get; set; } diff --git a/src/Microsoft.Atlas.CommandLine/Properties/launchSettings.json b/src/Microsoft.Atlas.CommandLine/Properties/launchSettings.json index ee12fbf..7da8f82 100644 --- a/src/Microsoft.Atlas.CommandLine/Properties/launchSettings.json +++ b/src/Microsoft.Atlas.CommandLine/Properties/launchSettings.json @@ -23,6 +23,11 @@ "commandLineArgs": "deploy sandbox k8s/lodejard-atlas-dev01", "workingDirectory": "..\\..\\..\\AtlasSandbox" }, + "deploy 105-partials": { + "commandName": "Project", + "commandLineArgs": "deploy 105-partials", + "workingDirectory": "..\\..\\examples" + }, "deploy 106-looping": { "commandName": "Project", "commandLineArgs": "deploy 106-looping", diff --git a/test/Microsoft.Atlas.CommandLine.Tests/ForeachOperationTests.cs b/test/Microsoft.Atlas.CommandLine.Tests/ForeachOperationTests.cs new file mode 100644 index 0000000..6178eb3 --- /dev/null +++ b/test/Microsoft.Atlas.CommandLine.Tests/ForeachOperationTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.IO; +using Microsoft.Atlas.CommandLine.Execution; +using Microsoft.Atlas.CommandLine.Secrets; +using Microsoft.Atlas.CommandLine.Tests.Stubs; +using Microsoft.Atlas.CommandLine.Tests.Utilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Atlas.CommandLine.Tests +{ + [TestClass] + public class ForeachOperationTests : ServiceContextTestsBase + { + [TestMethod] + public void ForeachValuesAreIterated() + { + var stubBlueprints = Yaml(@" + Blueprints: + the-test: + Files: + workflow.yaml: | + operations: + - foreach: + values: + x: ['a', 'b', 'c'] + message: (['x is ', x]) + "); + + InitializeServices(stubBlueprints); + + var result = Services.App.Execute("deploy", "the-test"); + + Assert.AreEqual(0, result); + + Console.AssertContainsInOrder("x is a", "x is b", "x is c"); + } + + [TestMethod] + public void SeveralArraysCanBeIterated() + { + var stubBlueprints = Yaml(@" + Blueprints: + the-test: + Files: + workflow.yaml: | + operations: + - foreach: + values: + x: ['a', 'b', 'c'] + y: + z: ['d', 'e', 'f'] + message: (['x is ', x, ' y.z is ', y.z]) + "); + + InitializeServices(stubBlueprints); + + var result = Services.App.Execute("deploy", "the-test"); + + Assert.AreEqual(0, result); + + Console.AssertContainsInOrder( + "x is a", + "y.z is d", + "x is b", + "y.z is e", + "x is c", + "y.z is f"); + } + + + [TestMethod] + public void ForeachHashObject() + { + var stubBlueprints = Yaml(@" + Blueprints: + the-test: + Files: + values.yaml: | + hash: + a: b + c: d + e: f + workflow.yaml: | + operations: + - foreach: + values: ( map(&{ key:[0], value:[1] }, items(hash)) ) + message: ""(['1: ', key, ' is ', value])"" + - foreach: + values: + key: ( keys(hash) ) + value: ( values(hash) ) + message: ""(['2: ', key, ' is ', value])"" + - foreach: + values: + kv: ( items(hash) ) + message: ""(['3: ', kv[0], ' is ', kv[1]])"" + "); + + InitializeServices(stubBlueprints); + + var result = Services.App.Execute("deploy", "the-test"); + + Assert.AreEqual(0, result); + + Console.AssertContainsInOrder( + "1: a is b", + "1: c is d", + "1: e is f", + "2: a is b", + "2: c is d", + "2: e is f", + "3: a is b", + "3: c is d", + "3: e is f"); + } + + [TestMethod] + public void ForeachOutputIsConcatinated() + { + var stubBlueprints = Yaml(@" + Blueprints: + the-test: + Files: + workflow.yaml: | + operations: + - foreach: + values: + x: ['a', 'b', 'c'] + output: + y: ([x]) + - message: (join('+', y)) + "); + + InitializeServices(stubBlueprints); + + var result = Services.App.Execute("deploy", "the-test"); + + Assert.AreEqual(0, result); + + Console.AssertContainsInOrder("a+b+c"); + } + public class ServiceContext : ServiceContextBase + { + public CommandLineApplicationServices App { get; set; } + } + } +}