Fix subworkflow values (#119)

* Fixing sub-workflow effective values calculation

* values.yaml contains defaults
* calling workflow operation values overlay those
* optional model yaml calculates extended structure based on result
* workflow root values finally add inputs based on that accumulation
* all values (except workflow root values) can by used by handlebar
  rendering of the workflow

* Adding test for interaction between values and handlebar in subworkflows

* Fixing stylecop warnings and moving global supporessions into ruleset file

build/analyzers.ruleset

@@ -69,6 +69,8 @@
     <Rule Id="CA2242" Action="Warning" />
   </Rules>
   <Rules AnalyzerId="StyleCop.Analyzers" RuleNamespace="StyleCop.Analyzers">
+    <Rule Id="SA1008" Action="None" />
+    <Rule Id="SA1009" Action="None" />
     <Rule Id="SA1309" Action="None" />
     <Rule Id="SX1309" Action="Error" />
     <Rule Id="SA1101" Action="None" />

src/Microsoft.Atlas.CommandLine/Commands/WorkflowCommands.cs

@@ -130,17 +130,7 @@ namespace Microsoft.Atlas.CommandLine.Commands
             }
 
             var eachValues = new List<object>();
-
-            if (blueprint.Exists("values.yaml"))
-            {
-                using (var reader = blueprint.OpenText("values.yaml"))
-                {
-                    eachValues.Add(_serializers.YamlDeserializer.Deserialize(reader));
-                }
-            }
-
             var defaultValuesFiles =
-                File.Exists("atlas-values.yaml") ? new[] { "atlas-values.yaml" } :
                 File.Exists("values.yaml") ? new[] { "values.yaml" } :
                 new string[0];
 
@@ -199,7 +189,7 @@ namespace Microsoft.Atlas.CommandLine.Commands
                 values = (IDictionary<object, object>)MergeUtils.Merge(addValues, values) ?? values;
             }
 
-            var(templateEngine, workflow, model) = _workflowLoader.Load(blueprint, values, GenerateOutput);
+            var(templateEngine, workflow, effectiveValues) = _workflowLoader.Load(blueprint, values, GenerateOutput);
 
             if (generateOnly == false)
             {
@@ -212,7 +202,7 @@ namespace Microsoft.Atlas.CommandLine.Commands
                     .SetOutputDirectory(OutputDirectory.Required())
                     .SetNonInteractive(NonInteractive?.HasValue() ?? false)
                     .SetDryRun(DryRun?.HasValue() ?? false)
-                    .SetValues(model)
+                    .SetValues(effectiveValues)
                     .Build();
 
                 context.AddValuesIn(_valuesEngine.ProcessValues(workflow.values, context.Values) ?? context.Values);

src/Microsoft.Atlas.CommandLine/Execution/IWorkflowLoader.cs

@@ -11,6 +11,6 @@ namespace Microsoft.Atlas.CommandLine.Execution
 {
     public interface IWorkflowLoader
     {
-        (ITemplateEngine templateEngine, WorkflowModel workflow, object model) Load(IBlueprintPackage blueprint, object values, Action<string, Action<TextWriter>> generateOutput);
+        (ITemplateEngine templateEngine, WorkflowModel workflow, object effectiveValues) Load(IBlueprintPackage blueprint, object values, Action<string, Action<TextWriter>> generateOutput);
     }
 }

src/Microsoft.Atlas.CommandLine/Execution/MergeUtils.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
 using System.Collections.Generic;
@@ -8,19 +8,19 @@ namespace Microsoft.Atlas.CommandLine.Execution
 {
     public static class MergeUtils
     {
-        public static object Merge(object values1, object values2)
+        public static object Merge(object overlayValues, object underlayValues)
         {
-            var values1Properties = values1 as IDictionary<object, object>;
-            var values2Properties = values2 as IDictionary<object, object>;
+            var overlayProperties = overlayValues as IDictionary<object, object>;
+            var underlayProperties = underlayValues as IDictionary<object, object>;
 
-            if (values1Properties != null && values2Properties != null)
+            if (overlayProperties != null && underlayProperties != null)
             {
                 var result = new Dictionary<object, object>();
-                foreach (var key in values2Properties.Keys.Concat(values1Properties.Keys.Except(values2Properties.Keys)))
+                foreach (var key in underlayProperties.Keys.Concat(overlayProperties.Keys.Except(underlayProperties.Keys)))
                 {
-                    if (values1Properties.TryGetValue(key, out var childValue1))
+                    if (overlayProperties.TryGetValue(key, out var childValue1))
                     {
-                        if (values2Properties.TryGetValue(key, out var childValue2))
+                        if (underlayProperties.TryGetValue(key, out var childValue2))
                         {
                             result.Add(key, Merge(childValue1, childValue2));
                         }
@@ -31,7 +31,7 @@ namespace Microsoft.Atlas.CommandLine.Execution
                     }
                     else
                     {
-                        if (values2Properties.TryGetValue(key, out var childValue2))
+                        if (underlayProperties.TryGetValue(key, out var childValue2))
                         {
                             result.Add(key, childValue2);
                         }
@@ -46,7 +46,7 @@ namespace Microsoft.Atlas.CommandLine.Execution
                 return result;
             }
 
-            return values1;
+            return overlayValues;
         }
     }
 }

src/Microsoft.Atlas.CommandLine/Execution/WorkflowEngine.cs

@@ -273,15 +273,20 @@ namespace Microsoft.Atlas.CommandLine.Execution
                                     throw new OperationException($"Unable to load sub-workflow {workflow}");
                                 }
 
-                                var(subTemplateEngine, subWorkflow, subModel) = _workflowLoader.Load(subBlueprint, context.ValuesIn, GenerateOutput);
+                                var(subTemplateEngine, subWorkflow, subValues) = _workflowLoader.Load(subBlueprint, context.ValuesIn, GenerateOutput);
 
                                 var subContext = new ExecutionContext.Builder()
                                     .CopyFrom(context)
                                     .UseBlueprintPackage(subBlueprint)
                                     .UseTemplateEngine(subTemplateEngine)
-                                    .SetValues(subModel)
+                                    .SetValues(subValues)
                                     .Build();
 
+                                if (subWorkflow.values != null)
+                                {
+                                    subContext.AddValuesIn(_valuesEngine.ProcessValues(subWorkflow.values, subContext.Values));
+                                }
+
                                 var nestedResult = await ExecuteOperations(subContext, subWorkflow.operations);
 
                                 if (subWorkflow.output != null)

src/Microsoft.Atlas.CommandLine/Execution/WorkflowLoader.cs

@@ -26,35 +26,44 @@ namespace Microsoft.Atlas.CommandLine.Execution
             _serializers = serializers;
         }
 
-        public (ITemplateEngine templateEngine, WorkflowModel workflow, object model) Load(IBlueprintPackage blueprint, object values, Action<string, Action<TextWriter>> generateOutput)
+        public (ITemplateEngine templateEngine, WorkflowModel workflow, object effectiveValues) Load(IBlueprintPackage blueprint, object values, Action<string, Action<TextWriter>> generateOutput)
         {
             var templateEngine = _templateEngineFactory.Create(new TemplateEngineOptions
             {
                 FileSystem = new BlueprintPackageFileSystem(blueprint)
             });
 
-            object model;
-
-            var modelTemplate = "model.yaml";
-            var modelExists = blueprint.Exists(modelTemplate);
-            if (modelExists)
+            var providedValues = values;
+            if (blueprint.Exists("values.yaml"))
             {
-                model = templateEngine.Render<object>(modelTemplate, values);
-                if (values != null)
+                using (var reader = blueprint.OpenText("values.yaml"))
                 {
-                    model = MergeUtils.Merge(model, values);
+                    var defaultValues = _serializers.YamlDeserializer.Deserialize(reader);
+                    if (values == null)
+                    {
+                        values = defaultValues;
+                    }
+                    else
+                    {
+                        values = MergeUtils.Merge(values, defaultValues);
+                    }
                 }
             }
-            else
+
+            var premodelValues = values;
+            if (blueprint.Exists("model.yaml"))
             {
-                model = values;
+                var model = templateEngine.Render<object>("model.yaml", values);
+                if (model != null)
+                {
+                    values = MergeUtils.Merge(model, values);
+                }
             }
 
-            var workflowTemplate = "workflow.yaml";
             var workflowContents = new StringBuilder();
             using (var workflowWriter = new StringWriter(workflowContents))
             {
-                templateEngine.Render(workflowTemplate, model, workflowWriter);
+                templateEngine.Render("workflow.yaml", values, workflowWriter);
             }
 
             // NOTE: the workflow is rendered BEFORE writing these output files because it may contain
@@ -63,12 +72,6 @@ namespace Microsoft.Atlas.CommandLine.Execution
             // write values to output folder
             generateOutput("values.yaml", writer => _serializers.YamlSerializer.Serialize(writer, values));
 
-            if (modelExists)
-            {
-                // write normalized values to output folder
-                generateOutput("model.yaml", writer => templateEngine.Render(modelTemplate, model, writer));
-            }
-
             // write workflow to output folder
             generateOutput("workflow.yaml", writer => writer.Write(workflowContents.ToString()));
 
@@ -82,7 +85,7 @@ namespace Microsoft.Atlas.CommandLine.Execution
                 }
             }
 
-            return (templateEngine, workflow, model);
+            return (templateEngine, workflow, values);
         }
     }
 }

src/Microsoft.Atlas.CommandLine/GlobalSuppressions.cs

@@ -1,5 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT license.
-
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1009:Closing parenthesis must be spaced correctly", Justification = "StyleCop is irritating", Scope = "module")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1008:Opening parenthesis must be spaced correctly", Justification = "StyleCop is irritating", Scope = "module")]

src/Microsoft.Atlas.CommandLine/Queries/JmesPathQuery.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license.
 
+using System;
 using System.IO;
 using DevLab.JmesPath;
 using DevLab.JmesPath.Functions;
@@ -32,19 +33,26 @@ namespace Microsoft.Atlas.CommandLine.Queries
 
         public object Search(string expression, object json)
         {
-            var jtokenEmitter = new JTokenEmitter();
-            _serializers.ValueSerialier.SerializeValue(jtokenEmitter, json, json?.GetType() ?? typeof(object));
-            var transformOutput = _jmespath.Transform(jtokenEmitter.Root, expression);
-
-            using (var stringWriter = new StringWriter())
+            try
             {
-                using (var jsonWriter = new JsonTextWriter(stringWriter) { CloseOutput = false })
-                {
-                    transformOutput.WriteTo(jsonWriter);
-                }
+                var jtokenEmitter = new JTokenEmitter();
+                _serializers.ValueSerialier.SerializeValue(jtokenEmitter, json, json?.GetType() ?? typeof(object));
+                var transformOutput = _jmespath.Transform(jtokenEmitter.Root, expression);
 
-                var jsonText = stringWriter.GetStringBuilder().ToString();
-                return _serializers.YamlDeserializer.Deserialize<object>(jsonText);
+                using (var stringWriter = new StringWriter())
+                {
+                    using (var jsonWriter = new JsonTextWriter(stringWriter) { CloseOutput = false })
+                    {
+                        transformOutput.WriteTo(jsonWriter);
+                    }
+
+                    var jsonText = stringWriter.GetStringBuilder().ToString();
+                    return _serializers.YamlDeserializer.Deserialize<object>(jsonText);
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new QueryException(ex.Message + Environment.NewLine + expression) { Expression = expression };
             }
         }
     }

src/Microsoft.Atlas.CommandLine/Queries/QueryException.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using System;
+using System.Runtime.Serialization;
+
+namespace Microsoft.Atlas.CommandLine.Queries
+{
+    [Serializable]
+    internal class QueryException : Exception
+    {
+        public QueryException()
+        {
+        }
+
+        public QueryException(string message)
+            : base(message)
+        {
+        }
+
+        public QueryException(string message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        protected QueryException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+
+        public string Expression { get; set; }
+    }
+}

test/Microsoft.Atlas.CommandLine.Tests/SubWorkflowTests.cs

@@ -225,5 +225,122 @@ Files:
 
             Console.AssertContainsInOrder(@"everything is {""xOut"":""alpha""}");
         }
+
+        [TestMethod]
+        public async Task WorkflowValuesYamlFileIsInEffect()
+        {
+            var stubs = Yaml<StubHttpClientHandlerFactory>(@"
+Files:
+  https://localhost/the-test/workflow.yaml: |
+    operations:
+    - workflow: step1
+      output: (result)
+    - condition: (x != 'one')
+      throw: 
+        message: (['Expected x == one, actual <', x||'null', '>'])
+  https://localhost/the-test/step1/values.yaml: |
+    xValue: one
+  https://localhost/the-test/step1/workflow.yaml: |
+    operations:
+    - output: {x: (xValue)}
+");
+
+            InitializeServices(stubs);
+
+            var result = Services.App.Execute("deploy", "https://localhost/the-test");
+
+            Assert.AreEqual(0, result);
+        }
+
+        [TestMethod]
+        public async Task WorkflowValuesPropertyIsInEffect()
+        {
+            var stubs = Yaml<StubHttpClientHandlerFactory>(@"
+Files:
+  https://localhost/the-test/workflow.yaml: |
+    operations:
+    - workflow: step1
+      output: (result)
+    - condition: (y != 'two')
+      throw: 
+        message: (['Expected y == two, actual <', y||'null', '>'])
+  https://localhost/the-test/step1/workflow.yaml: |
+    values:
+      yValue: two
+    operations:
+    - output: {y: (yValue)}
+");
+
+            InitializeServices(stubs);
+
+            var result = Services.App.Execute("deploy", "https://localhost/the-test");
+
+            Assert.AreEqual(0, result);
+        }
+
+        [TestMethod]
+        public async Task WorkflowModelsYamlIsInEffect()
+        {
+            var stubs = Yaml<StubHttpClientHandlerFactory>(@"
+Files:
+  https://localhost/the-test/workflow.yaml: |
+    operations:
+    - workflow: step1
+      values:
+        zValueInput: three
+      output: (result)
+    - condition: (z != 'three')
+      throw: 
+        message: (['Expected z == three, actual <', z||'null', '>'])
+  https://localhost/the-test/step1/model.yaml: |
+    zValue: {{ zValueInput }}
+  https://localhost/the-test/step1/workflow.yaml: |
+    operations:
+    - output: {z: (zValue)}
+");
+
+            InitializeServices(stubs);
+
+            var result = Services.App.Execute("deploy", "https://localhost/the-test");
+
+            Assert.AreEqual(0, result);
+        }
+
+        [TestMethod]
+        public async Task HandlebarRenderingInModelAndWorkflowCanUseValues()
+        {
+            var stubs = Yaml<StubHttpClientHandlerFactory>(@"
+Files:
+  https://localhost/the-test/workflow.yaml: |
+    operations:
+    - workflow: step1
+      values:
+        x1: one
+  https://localhost/the-test/step1/values.yaml: |
+    x2: two
+  https://localhost/the-test/step1/model.yaml: |
+    x3: three
+    x1x2: m<{{x1}}{{x2}}{{x3}}{{x4}}>
+  https://localhost/the-test/step1/workflow.yaml: |
+    values:
+      x4: four
+      x1x2x3: w<{{x1}}{{x2}}{{x3}}{{x4}}>
+    operations:
+    - foreach:
+        values: 
+          variable: (['x1', 'x2', 'x3', 'x4', 'x1x2', 'x1x2x3'])
+          actual: ([x1, x2, x3, x4, x1x2, x1x2x3])
+          expected: (['one', 'two', 'three', 'four', 'm<onetwo>', 'w<onetwothree>'])
+      condition: ( actual != expected )
+      throw:
+        message: (['expected ', variable, '==<', expected || 'null', '> but actual <', actual || 'null', '>'])
+");
+
+            InitializeServices(stubs);
+
+            var result = Services.App.Execute("deploy", "https://localhost/the-test");
+
+            Assert.AreEqual(0, result);
+        }
     }
 }