Added health probes to all services to ensure dapr sidecar is up.

Updated VirtualCustomers to ensure dapr sidecar is up.

Updated Bootstrapper to ensure dapr running.
Removed unused variables.

Added startup probes

Updated branch deployment kustomization

Added middle-out compression (#27)

Added middle-out compression

Tres commas
This commit is contained in:
Lynn Orrell 2021-05-28 01:02:07 -05:00
Родитель bfe2e42b0f
Коммит fbdd8d2a60
22 изменённых файлов: 437 добавлений и 43 удалений

11
.gitignore поставляемый
Просмотреть файл

@ -644,3 +644,14 @@ manifests/local/corporate/**
!manifests/local/corporate/.keep
**/*.pfx
# KEDA deployment yml with secrets
RedDog.CorporateTransferService/func-deployment.yaml
RedDog.CorporateTransferService/local.settings.json
# Brian
brian-notes.md
contoso-health-notes.md
# Joey
components/local/

Просмотреть файл

@ -1,4 +1,5 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
@ -12,11 +13,14 @@ namespace RedDog.AccountingService.Controllers
public class ProbesController : ControllerBase
{
private static bool isReady;
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger)
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
@ -43,5 +47,15 @@ namespace RedDog.AccountingService.Controllers
return Ok();
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -27,7 +27,7 @@ namespace RedDog.AccountingService
public void ConfigureServices(IServiceCollection services)
{
services.AddHttpClient();
services.AddControllers().AddDapr();
services.AddSwaggerGen(c =>
{

Просмотреть файл

@ -13,6 +13,7 @@ namespace RedDog.Bootstrapper
{
private const string SecretStoreName = "reddog.secretstore";
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private HttpClient _httpClient = new HttpClient();
static async Task Main(string[] args)
{
@ -27,26 +28,35 @@ namespace RedDog.Bootstrapper
public AccountingContext CreateDbContext(string[] args)
{
var daprClient = new DaprClientBuilder().Build();
EnsureDaprOrTerminate().Wait();
string connectionString = GetDbConnectionString().Result;
ShutdownDapr();
Dictionary<string, string> connectionString = null;
do
DbContextOptionsBuilder<AccountingContext> optionsBuilder = new DbContextOptionsBuilder<AccountingContext>().UseSqlServer(connectionString, b =>
{
try
{
Console.WriteLine("Attempting to retrieve database connection string from Dapr...");
connectionString = daprClient.GetSecretAsync(SecretStoreName, "reddog-sql").GetAwaiter().GetResult();
Console.WriteLine("Successfully retrieved database connection string.");
}
catch(Exception e)
{
Console.WriteLine($"An exception occured while retrieving the secret from the Dapr sidecar. Retrying in 5 seconds...");
Console.WriteLine(e.StackTrace);
Task.Delay(5000).Wait();
}
} while(connectionString == null);
b.MigrationsAssembly("RedDog.Bootstrapper");
b.EnableRetryOnFailure();
});
return new AccountingContext(optionsBuilder.Options);
}
private async Task EnsureDaprOrTerminate()
{
try
{
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
response.EnsureSuccessStatusCode();
}
catch(Exception e)
{
Console.WriteLine("Error communicating with Dapr sidecar. Exiting...", e.InnerException?.Message ?? e.Message);
Environment.Exit(1);
}
}
private void ShutdownDapr()
{
Console.WriteLine("Attempting to shutdown Dapr sidecar...");
bool isDaprShutdownSuccessful = false;
HttpClient httpClient = new HttpClient();
@ -56,7 +66,7 @@ namespace RedDog.Bootstrapper
{
var response = httpClient.PostAsync($"http://localhost:{DaprHttpPort}/v1.0/shutdown", null).Result;
if(response.IsSuccessStatusCode)
if (response.IsSuccessStatusCode)
{
isDaprShutdownSuccessful = true;
Console.WriteLine("Successfully shutdown Dapr sidecar.");
@ -67,7 +77,7 @@ namespace RedDog.Bootstrapper
Console.WriteLine($"Dapr error message: {response.Content.ReadAsStringAsync().Result}");
}
}
catch(Exception e)
catch (Exception e)
{
Console.WriteLine($"An exception occured while attempting to shutdown the Dapr sidecar.");
Console.WriteLine(e.StackTrace);
@ -76,15 +86,30 @@ namespace RedDog.Bootstrapper
{
Task.Delay(5000).Wait();
}
} while(!isDaprShutdownSuccessful);
} while (!isDaprShutdownSuccessful);
}
DbContextOptionsBuilder<AccountingContext> optionsBuilder = new DbContextOptionsBuilder<AccountingContext>().UseSqlServer(connectionString["reddog-sql"], b =>
private async Task<string> GetDbConnectionString()
{
var daprClient = new DaprClientBuilder().Build();
Dictionary<string, string> connectionString = null;
do
{
b.MigrationsAssembly("RedDog.Bootstrapper");
b.EnableRetryOnFailure();
});
return new AccountingContext(optionsBuilder.Options);
try
{
Console.WriteLine("Attempting to retrieve database connection string from Dapr...");
connectionString = await daprClient.GetSecretAsync(SecretStoreName, "reddog-sql");
Console.WriteLine("Successfully retrieved database connection string.");
}
catch (Exception e)
{
Console.WriteLine($"An exception occured while retrieving the secret from the Dapr sidecar. Retrying in 5 seconds...");
Console.WriteLine(e.StackTrace);
Task.Delay(5000).Wait();
}
} while (connectionString == null);
return connectionString["reddog-sql"];
}
}
}

Просмотреть файл

@ -0,0 +1,39 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace RedDog.LoyaltyService.Controllers
{
[ApiController]
[Route("[controller]")]
public class ProbesController : ControllerBase
{
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
public async Task<IActionResult> IsReady()
{
return await Task.FromResult(Ok());
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -0,0 +1,39 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace RedDog.MakeLineService.Controllers
{
[ApiController]
[Route("[controller]")]
public class ProbesController : ControllerBase
{
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
public async Task<IActionResult> IsReady()
{
return await Task.FromResult(Ok());
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -0,0 +1,40 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace RedDog.OrderService.Controllers
{
[ApiController]
[Route("[controller]")]
public class ProbesController : ControllerBase
{
private static bool isReady;
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
public async Task<IActionResult> IsReady()
{
return await Task.FromResult(Ok());
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -0,0 +1,39 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace RedDog.ReceiptGenerationService.Controllers
{
[ApiController]
[Route("[controller]")]
public class ProbesController : ControllerBase
{
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
public async Task<IActionResult> IsReady()
{
return await Task.FromResult(Ok());
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -25,7 +25,7 @@ namespace RedDog.VirtualCustomers
try
{
using (IHost host = new HostBuilder()
using IHost host = new HostBuilder()
.ConfigureHostConfiguration(configHost =>
{
configHost.SetBasePath(Directory.GetCurrentDirectory());
@ -42,14 +42,14 @@ namespace RedDog.VirtualCustomers
})
.ConfigureServices((hostContext, services) =>
{
services.AddHttpClient();
services.AddDaprClient();
services.AddHostedService<VirtualCustomers>();
})
.UseSerilog()
.Build())
{
await host.RunAsync();
}
.Build();
await host.RunAsync();
}
catch (Exception e)
{

Просмотреть файл

@ -21,8 +21,11 @@ namespace RedDog.VirtualCustomers
public static readonly int maxSecondsBetweenOrders = int.Parse(Environment.GetEnvironmentVariable("MAX_SEC_BETWEEN_ORDERS") ?? "5");
public static readonly int numOrders = int.Parse(Environment.GetEnvironmentVariable("NUM_ORDERS") ?? "-1");
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private IHostApplicationLifetime _lifetime;
private readonly ILogger<VirtualCustomers> _logger;
private readonly DaprClient _daprClient;
private readonly HttpClient _httpClient;
private Task _ordersTask = Task.CompletedTask;
private List<Product> _products;
private Random _random;
@ -230,22 +233,35 @@ namespace RedDog.VirtualCustomers
(200,"Rachel","Alvarez")
};
public VirtualCustomers(ILogger<VirtualCustomers> logger, DaprClient daprClient)
public VirtualCustomers(IHostApplicationLifetime lifetime, ILogger<VirtualCustomers> logger, DaprClient daprClient, IHttpClientFactory httpClientFactory)
{
_lifetime = lifetime;
_logger = logger;
_daprClient = daprClient;
_httpClient = httpClientFactory.CreateClient();
_random = new Random();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz", stoppingToken);
response.EnsureSuccessStatusCode();
}
catch(Exception e)
{
_logger.LogError("Error communicating with Dapr sidecar. Exiting...", e.InnerException?.Message ?? e.Message);
_lifetime.StopApplication();
}
_logger.LogInformation($"The customers are ready to place their orders!");
stoppingToken.Register(() =>
{
_ordersTask.Wait();
_logger.LogInformation($"The remaining customers have cancelled their orders and are leaving the store.");
Environment.Exit(1);
_lifetime.StopApplication();
});
int ordersCreated = 0;
@ -253,18 +269,18 @@ namespace RedDog.VirtualCustomers
{
try
{
_products = await _daprClient.InvokeMethodAsync<List<Product>>(HttpMethod.Get, OrderServiceDaprId, "product");
_products = await _daprClient.InvokeMethodAsync<List<Product>>(HttpMethod.Get, OrderServiceDaprId, "product", stoppingToken);
}
catch(Exception e)
{
_logger.LogError("Error retrieving products. Retrying in 5 seconds. Message: {Message}", e.InnerException?.Message ?? e.Message);
await Task.Delay(5000);
await Task.Delay(5000, stoppingToken);
}
} while (!stoppingToken.IsCancellationRequested && _products == null);
do
{
await Task.Delay(_random.Next(minSecondsBetweenOrders, maxSecondsBetweenOrders + 1) * 1000);
await Task.Delay(_random.Next(minSecondsBetweenOrders, maxSecondsBetweenOrders + 1) * 1000, stoppingToken);
await CreateOrder(stoppingToken);
ordersCreated += 1;

Просмотреть файл

@ -0,0 +1,39 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace RedDog.VirtualWorker.Controllers
{
[ApiController]
[Route("[controller]")]
public class ProbesController : ControllerBase
{
private string DaprHttpPort = Environment.GetEnvironmentVariable("DAPR_HTTP_PORT") ?? "3500";
private ILogger<ProbesController> _logger;
private HttpClient _httpClient;
public ProbesController(ILogger<ProbesController> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient();
}
[HttpGet("ready")]
public async Task<IActionResult> IsReady()
{
return await Task.FromResult(Ok());
}
[HttpGet("healthz")]
public async Task<IActionResult> IsHealthy()
{
// Ensure dapr sidecar is running and healthy. If not, fail the health check and have the pod restarted.
// This should prevent the case where the application container is running before dapr is installed in
// the case of a gitops deploy.
var response = await _httpClient.GetAsync($"http://localhost:{DaprHttpPort}/v1.0/healthz");
return new StatusCodeResult((int)response.StatusCode);
}
}
}

Просмотреть файл

@ -34,3 +34,9 @@ spec:
successThreshold: 1
failureThreshold: 12
periodSeconds: 10
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -25,4 +25,10 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-loyalty-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -25,4 +25,10 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-make-line-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -25,4 +25,10 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-order-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -25,4 +25,10 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-receipt-generation-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -24,6 +24,12 @@ spec:
- name: virtual-worker
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-virtual-worker:latest"
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10
env:
- name: STORE_ID
value: Denver

Просмотреть файл

@ -8,6 +8,7 @@ resources:
- ./components/reddog.secretstore.yaml
- ./components/reddog.state.loyalty.yaml
- ./components/reddog.state.makeline.yaml
- ./deployments/rbac.yaml
- ./deployments/accounting-service.yaml
- ./deployments/bootstrapper.yaml
- ./deployments/loyalty-service.yaml

Просмотреть файл

@ -0,0 +1,22 @@
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: azure-disk
provisioner: kubernetes.io/azure-disk
parameters:
storageaccounttype: Standard_LRS
kind: Managed
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: mssql-data
annotations:
volume.beta.kubernetes.io/storage-class: azure-disk
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
storageClassName: default

Просмотреть файл

@ -0,0 +1,53 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: mssql-deployment
spec:
replicas: 1
selector:
matchLabels:
app: mssql
template:
metadata:
labels:
app: mssql
spec:
terminationGracePeriodSeconds: 30
hostname: mssqlinst
securityContext:
fsGroup: 10001
containers:
- name: mssql
image: mcr.microsoft.com/mssql/server:2019-latest
ports:
- containerPort: 1433
env:
- name: MSSQL_PID
value: "Developer"
- name: ACCEPT_EULA
value: "Y"
- name: SA_PASSWORD
valueFrom:
secretKeyRef:
name: mssql
key: SA_PASSWORD
volumeMounts:
- name: mssqldb
mountPath: /var/opt/mssql
volumes:
- name: mssqldb
persistentVolumeClaim:
claimName: mssql-data
---
apiVersion: v1
kind: Service
metadata:
name: mssql-deployment
spec:
selector:
app: mssql
ports:
- protocol: TCP
port: 1433
targetPort: 1433
type: LoadBalancer

Просмотреть файл

@ -25,4 +25,18 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-accounting-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
readinessProbe:
httpGet:
path: /probes/ready
port: 80
timeoutSeconds: 30
successThreshold: 1
failureThreshold: 12
periodSeconds: 10
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10

Просмотреть файл

@ -25,4 +25,10 @@ spec:
image: "ghcr.io/cloudnativegbb/paas-vnext/reddog-loyalty-service:latest"
ports:
- containerPort: 80
imagePullPolicy: Always
imagePullPolicy: Always
startupProbe:
httpGet:
path: /probes/healthz
port: 80
failureThreshold: 6
periodSeconds: 10