Docker compose environment variable syntax (#669)

* docker-compose like environment variable syntax * documentation
2020-10-02 14:06:03 -04:00 · 2020-10-02 14:06:03 -04:00 · 7a0b24c9c7
--- a/docs/reference/schema.md
+++ b/docs/reference/schema.md
@ -184,10 +184,14 @@ Including `external: true` marks the service as *external*:

 External services are useful to provide bindings without any run or deployment behavior.

-#### `env` (`EnvironmentVariable[]`)
+#### `env` (`EnvironmentVariable[] | string[]`)

 A list of environment variable mappings for the service. Does not apply when the service is external.

+#### `env_file` (`string[]`)
+
+A list of files from which environment variables are taken. Does not apply when the service is external.
+
 #### `args` (string)

 Command-line arguments to use when launching the service. Does not apply when the service is external.
@ -256,6 +260,55 @@ The name of the environment variable.

 The value of the environment variable.

+Environment variables can also be provided using a compact syntax (similar to that of [docker-compose](https://docs.docker.com/compose/environment-variables/)).
+
+### Environment Variable Compact Syntax Example
+
+```yaml
+name: myapplication
+services:
+- name: backend
+  project: backend/backend.csproj
+
+  # environment variables appear here
+  env:
+  - SOME_KEY=SOME_VALUE
+  - SOME_KEY2="SOME VALUE"
+  - SOME_KEY3
+```
+
+Using the compact syntax, you provide environment variable name and value via a single string, separated by a `=` sign.  
+
+In the absence of an `=` sign, the value of the environment variable will be taken from the operating system/shell.
+
+## Environment Variables Files
+
+`string` elements appear in a list inside the `env_file` property of a `Service`.  
+
+These strings reference [`.env` files](https://docs.docker.com/compose/env-file/) from which the environment variables will be injected.  
+
+### Environment Variables File Example
+
+```yaml
+name: myapplication
+services:
+- name: backend
+  project: backend/backend.csproj
+
+  # environment variables files appear here
+  env_file:
+    - ./envfile_a.env
+    - ./envfile_b.env
+```
+
+### .env File Example
+
+```
+SOME_KEY=SOME_VALUE
+# This line is ignored because it start with '#'
+SOME_KEY2="SOME VALUE"
+```
+
 ## Build Properties

 Configuration that can be specified when building a project. These will be passed in as MSBuild properties when building a project. It appears in the list `buildProperties` of a `Service`.
--- a/src/Microsoft.Tye.Core/CoreStrings.resx
+++ b/src/Microsoft.Tye.Core/CoreStrings.resx
@ -141,12 +141,12 @@
  <data name="MultipleBindingWithSamePort" xml:space="preserve">
    <value>Cannot have multiple {0} bindings with the same port.</value>
  </data>
-    <data name="ProberRequired" xml:space="preserve">
-        <value>A prober must be configured for the {0} probe.</value>
-    </data>
-    <data name="SuccessThresholdMustBeOne" xml:space="preserve">
-        <value>"successThreshold" for {0} probe must be set to "1".</value>
-    </data>
+  <data name="ProberRequired" xml:space="preserve">
+    <value>A prober must be configured for the {0} probe.</value>
+  </data>
+  <data name="SuccessThresholdMustBeOne" xml:space="preserve">
+    <value>"successThreshold" for {0} probe must be set to "1".</value>
+  </data>
  <data name="MustBeABoolean" xml:space="preserve">
    <value>"{value}" must be a boolean value (true/false).</value>
  </data>
@ -156,9 +156,9 @@
  <data name="MustBePositive" xml:space="preserve">
    <value>"{value}" value cannot be negative.</value>
  </data>
-    <data name="MustBeGreaterThanZero" xml:space="preserve">
-        <value>"{value}" value must be greater than zero.</value>
-    </data>
+  <data name="MustBeGreaterThanZero" xml:space="preserve">
+    <value>"{value}" value must be greater than zero.</value>
+  </data>
  <data name="ProjectImageExecutableExclusive" xml:space="preserve">
    <value>Cannot have both "{0}" and "{1}" set for a service. Only one of project, image, and executable can be set for a given service.</value>
  </data>
@ -171,4 +171,13 @@
  <data name="UnrecognizedKey" xml:space="preserve">
    <value>Unexpected key "{key}" in tye.yaml.</value>
  </data>
-</root>
+  <data name="UnexpectedTypes" xml:space="preserve">
+    <value>Unexpected node type in tye.yaml. Expected one of ({expected}) but got "{actual}".</value>
+  </data>
+  <data name="PathNotFound" xml:space="preserve">
+    <value>Path "{path}" was not found.</value>
+  </data>
+  <data name="ExpectedEnvironmentVariableValue" xml:space="preserve">
+    <value>Expected a value for environment variable "{key}".</value>
+  </data>
+</root>
--- a/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs
+++ b/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs
@ -35,7 +35,7 @@ namespace Tye.Serialization
                        break;
                    case "services":
                        YamlParser.ThrowIfNotYamlSequence(key, child.Value);
-                        ConfigServiceParser.HandleServiceMapping((child.Value as YamlSequenceNode)!, app.Services);
+                        ConfigServiceParser.HandleServiceMapping((child.Value as YamlSequenceNode)!, app.Services, app);
                        break;
                    case "extensions":
                        YamlParser.ThrowIfNotYamlSequence(key, child.Value);
--- a/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs
+++ b/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs
@ -2,7 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.

+using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using Microsoft.Tye.ConfigModel;
 using YamlDotNet.RepresentationModel;
@ -11,18 +13,18 @@ namespace Tye.Serialization
 {
    public static class ConfigServiceParser
    {
-        public static void HandleServiceMapping(YamlSequenceNode yamlSequenceNode, List<ConfigService> services)
+        public static void HandleServiceMapping(YamlSequenceNode yamlSequenceNode, List<ConfigService> services, ConfigApplication application)
        {
            foreach (var child in yamlSequenceNode.Children)
            {
                YamlParser.ThrowIfNotYamlMapping(child);
                var service = new ConfigService();
-                HandleServiceNameMapping((YamlMappingNode)child, service);
+                HandleServiceNameMapping((YamlMappingNode)child, service, application);
                services.Add(service);
            }
        }

-        private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, ConfigService service)
+        private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, ConfigService service, ConfigApplication application)
        {
            foreach (var child in yamlMappingNode!.Children)
            {
@ -130,6 +132,14 @@ namespace Tye.Serialization

                        HandleServiceConfiguration((child.Value as YamlSequenceNode)!, service.Configuration);
                        break;
+                    case "env_file":
+                        if (child.Value.NodeType != YamlNodeType.Sequence)
+                        {
+                            throw new TyeYamlException(child.Value.Start, CoreStrings.FormatExpectedYamlSequence(key));
+                        }
+
+                        HandleServiceEnvFiles((child.Value as YamlSequenceNode)!, service.Configuration, application);
+                        break;
                    case "liveness":
                        service.Liveness = new ConfigProbe();
                        HandleServiceProbe((YamlMappingNode)child.Value, service.Liveness!);
@ -430,9 +440,25 @@ namespace Tye.Serialization
        {
            foreach (var child in yamlSequenceNode.Children)
            {
-                YamlParser.ThrowIfNotYamlMapping(child);
                var config = new ConfigConfigurationSource();
-                HandleServiceConfigurationNameMapping((YamlMappingNode)child, config);
+                switch (child)
+                {
+                    case YamlMappingNode childMappingNode:
+                        HandleServiceConfigurationNameMapping(childMappingNode, config);
+                        break;
+                    case YamlScalarNode childScalarNode:
+                        HandleServiceConfigurationCompact(childScalarNode, config);
+                        break;
+                    default:
+                        throw new TyeYamlException(child.Start, CoreStrings.FormatUnexpectedTypes($"\"{YamlNodeType.Mapping.ToString()}\", \"{YamlNodeType.Scalar.ToString()}\"", child.NodeType.ToString()));
+                }
+
+                // if no value is given, we take the value from the system/shell environment variables
+                if (config.Value == null)
+                {
+                    config.Value = Environment.GetEnvironmentVariable(config.Name) ?? string.Empty;
+                }
+
                configuration.Add(config);
            }
        }
@ -457,6 +483,66 @@ namespace Tye.Serialization
            }
        }

+        private static void HandleServiceConfigurationCompact(YamlScalarNode yamlScalarNode, ConfigConfigurationSource config)
+        {
+            var nodeValue = YamlParser.GetScalarValue(yamlScalarNode);
+            var keyValueSeparator = nodeValue.IndexOf('=');
+
+            if (keyValueSeparator != -1)
+            {
+                var key = nodeValue.Substring(0, keyValueSeparator).Trim();
+                var value = nodeValue.Substring(keyValueSeparator + 1)?.Trim();
+
+                config.Name = key;
+                config.Value = value?.Trim(new[] { ' ', '"' }) ?? string.Empty;
+            }
+            else
+            {
+                config.Name = nodeValue.Trim();
+            }
+        }
+
+        private static void HandleServiceEnvFiles(YamlSequenceNode yamlSequenceNode, List<ConfigConfigurationSource> configuration, ConfigApplication application)
+        {
+            foreach (var child in yamlSequenceNode.Children)
+            {
+                switch (child)
+                {
+                    case YamlScalarNode childScalarNode:
+                        var envFile = new FileInfo(Path.Combine(application.Source?.DirectoryName ?? Directory.GetCurrentDirectory(), YamlParser.GetScalarValue(childScalarNode)));
+                        if (!envFile.Exists)
+                            throw new TyeYamlException(child.Start, CoreStrings.FormatPathNotFound(envFile.FullName));
+                        HandleServiceEnvFile(childScalarNode, File.ReadAllLines(envFile.FullName), configuration);
+                        break;
+                    default:
+                        throw new TyeYamlException(child.Start, CoreStrings.FormatUnexpectedType(YamlNodeType.Scalar.ToString(), child.NodeType.ToString()));
+                }
+            }
+        }
+
+        private static void HandleServiceEnvFile(YamlScalarNode yamlScalarNode, string[] envLines, List<ConfigConfigurationSource> configuration)
+        {
+            foreach (var line in envLines)
+            {
+                var lineTrim = line?.Trim();
+                if (string.IsNullOrEmpty(lineTrim) || lineTrim[0] == '#')
+                {
+                    continue;
+                }
+
+                var keyValueSeparator = lineTrim.IndexOf('=');
+
+                if (keyValueSeparator == -1)
+                    throw new TyeYamlException(yamlScalarNode.Start, CoreStrings.FormatExpectedEnvironmentVariableValue(lineTrim));
+
+                configuration.Add(new ConfigConfigurationSource
+                {
+                    Name = lineTrim.Substring(0, keyValueSeparator).Trim(),
+                    Value = lineTrim.Substring(keyValueSeparator + 1)?.Trim(new[] { ' ', '"' }) ?? string.Empty
+                });
+            }
+        }
+
        private static void HandleServiceBuildPropertyNameMapping(YamlMappingNode yamlMappingNode, BuildProperty buildProperty)
        {
            foreach (var child in yamlMappingNode!.Children)
--- a/src/Microsoft.Tye.Core/Serialization/YamlParser.cs
+++ b/src/Microsoft.Tye.Core/Serialization/YamlParser.cs
@ -17,21 +17,21 @@ namespace Tye.Serialization
        private FileInfo? _fileInfo;
        private TextReader _reader;

-        public YamlParser(string yamlContent)
-            : this(new StringReader(yamlContent))
+        public YamlParser(string yamlContent, FileInfo? fileInfo = null)
+            : this(new StringReader(yamlContent), fileInfo)
        {
        }

        public YamlParser(FileInfo fileInfo)
-            : this(fileInfo.OpenText())
+            : this(fileInfo.OpenText(), fileInfo)
        {
-            _fileInfo = fileInfo;
        }

-        internal YamlParser(TextReader reader)
+        internal YamlParser(TextReader reader, FileInfo? fileInfo = null)
        {
            _reader = reader;
            _yamlStream = new YamlStream();
+            _fileInfo = fileInfo;
        }

        public ConfigApplication ParseConfigApplication()
@ -51,9 +51,11 @@ namespace Tye.Serialization
            var document = _yamlStream.Documents[0];
            var node = document.RootNode;
            ThrowIfNotYamlMapping(node);
-            ConfigApplicationParser.HandleConfigApplication((YamlMappingNode)node, app);

            app.Source = _fileInfo!;
+
+            ConfigApplicationParser.HandleConfigApplication((YamlMappingNode)node, app);
+
            app.Name ??= NameInferer.InferApplicationName(_fileInfo!);

            // TODO confirm if these are ever null.
--- a/test/UnitTests/Microsoft.Tye.UnitTests.csproj
+++ b/test/UnitTests/Microsoft.Tye.UnitTests.csproj
@ -24,4 +24,13 @@
    <ProjectReference Include="..\..\src\tye\tye.csproj" />
    <ProjectReference Include="..\Test.Infrastructure\Test.Infrastructure.csproj" />
  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="testassets\envfile_a.env">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="testassets\envfile_b.env">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>
--- a/test/UnitTests/TyeDeserializationTests.cs
+++ b/test/UnitTests/TyeDeserializationTests.cs
@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using System.Linq;
 using Microsoft.Tye.ConfigModel;
 using Test.Infrastructure;
@ -228,6 +229,121 @@ services:
            var app = parser.ParseConfigApplication();
        }

+        [Theory]
+        [InlineData("env")]
+        [InlineData("configuration")]
+        public void EnvSimpleSyntaxTest(string rootKeyName)
+        {
+            using var parser = new YamlParser(
+@$"
+services:
+    - {rootKeyName}:
+        - name: env1
+          value: value1
+        - name: env2
+          value: value2
+        - name: env3
+          value: ""long string""
+        - name: env4
+          value:
+");
+
+            var app = parser.ParseConfigApplication();
+            var serviceConfig = app.Services.First().Configuration;
+
+            Assert.Equal(4, serviceConfig.Count);
+            Assert.Equal("value1", serviceConfig.Where(env => env.Name == "env1").First().Value);
+            Assert.Equal("value2", serviceConfig.Where(env => env.Name == "env2").First().Value);
+            Assert.Equal("long string", serviceConfig.Where(env => env.Name == "env3").First().Value);
+            Assert.Equal(string.Empty, serviceConfig.Where(env => env.Name == "env4").First().Value);
+        }
+
+        [Fact]
+        public void EnvCompactSyntaxTest()
+        {
+            using var parser = new YamlParser(
+@"
+services:
+    - env:
+        - env1=value1
+        - env2=value2
+        - env3 = value3
+        - env4 = ""long string""
+        - name: env5
+          value: value5
+        - env6 =
+");
+
+            var app = parser.ParseConfigApplication();
+            var serviceConfig = app.Services.First().Configuration;
+
+            Assert.Equal(6, serviceConfig.Count);
+            Assert.Equal("value1", serviceConfig.Where(env => env.Name == "env1").First().Value);
+            Assert.Equal("value2", serviceConfig.Where(env => env.Name == "env2").First().Value);
+            Assert.Equal("value3", serviceConfig.Where(env => env.Name == "env3").First().Value);
+            Assert.Equal("long string", serviceConfig.Where(env => env.Name == "env4").First().Value);
+            Assert.Equal("value5", serviceConfig.Where(env => env.Name == "env5").First().Value);
+            Assert.Equal(string.Empty, serviceConfig.Where(env => env.Name == "env6").First().Value);
+        }
+
+        [Fact]
+        public void EnvTakeValueFromEnvironmentTest()
+        {
+            using var parser = new YamlParser(
+@"
+services:
+    - env:
+        - env1
+        - name: env2
+        - env3
+");
+
+            Environment.SetEnvironmentVariable("env1", "value1");
+            Environment.SetEnvironmentVariable("env2", "value2");
+
+            var app = parser.ParseConfigApplication();
+            var serviceConfig = app.Services.First().Configuration;
+
+            Assert.Equal(3, serviceConfig.Count);
+            Assert.Equal("value1", serviceConfig.Where(env => env.Name == "env1").First().Value);
+            Assert.Equal("value2", serviceConfig.Where(env => env.Name == "env2").First().Value);
+            Assert.Equal(string.Empty, serviceConfig.Where(env => env.Name == "env3").First().Value);
+        }
+
+        [Theory]
+        [InlineData(false)]
+        [InlineData(true)]
+        public void EnvFromFile(bool setWorkingDirectory)
+        {
+            var subDirectory = setWorkingDirectory ? string.Empty : "testassets/";
+            using var parser = new YamlParser(
+@$"
+services:
+    - env_file:
+        - ./{subDirectory}envfile_a.env
+        - ./{subDirectory}envfile_b.env
+", setWorkingDirectory ? new FileInfo(Path.Join(Directory.GetCurrentDirectory(), "testassets", "tye.yaml")) : null);
+
+
+            var app = parser.ParseConfigApplication();
+            var serviceConfig = app.Services.First().Configuration;
+
+            Assert.Equal(3, serviceConfig.Count);
+        }
+
+        [Fact]
+        public void PathNotFound_ThrowException()
+        {
+            using var parser = new YamlParser(
+@"
+services:
+    - env_file:
+        - ./envfile_c.env
+", new FileInfo(Path.Join(Directory.GetCurrentDirectory(), "testassets", "tye.yaml")));
+
+            var exception = Assert.Throws<TyeYamlException>(() => parser.ParseConfigApplication());
+            Assert.Contains(CoreStrings.FormatPathNotFound(Path.Join(Directory.GetCurrentDirectory(), "testassets", "envfile_c.env")), exception.Message);
+        }

        [Fact]
        public void UnrecognizedConfigApplicationField_ThrowException()
--- a/test/UnitTests/testassets/envfile_a.env
+++ b/test/UnitTests/testassets/envfile_a.env
@ -0,0 +1,3 @@
+env1=value1
+# Ignore comment
+env2 = value2
--- a/test/UnitTests/testassets/envfile_b.env
+++ b/test/UnitTests/testassets/envfile_b.env
@ -0,0 +1 @@
+env3 = "long string"