diff --git a/Dockerfile b/Dockerfile index c109f9a..acf4d22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,15 +14,17 @@ COPY main.go main.go COPY api/ api/ COPY pkg/ pkg/ COPY controllers/ controllers/ +COPY config.yaml config.yaml # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM alpine:3.7 +RUN apk add --no-cache bash +RUN mkdir -p /etc/orkestra/charts/pull/ + WORKDIR / COPY --from=builder /workspace/manager . -USER nonroot:nonroot +COPY --from=builder /workspace/config.yaml /etc/controller/config.yaml ENTRYPOINT ["/manager"] diff --git a/README.md b/README.md index 3acb9b1..ca4987e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To solve the complex application orchestration problem Orkestra builds a [Direct 2. For each `Application` in `ApplicationGroup` download Helm chart from “primary” Helm Registry 3. (*optional) For each dependency in the `Application` chart, if dependency chart is embedded in `charts/` directory, push to ”staging” Helm Registry (Chart-museum). 4. Generate and submit Argo Workflow DAG -5. (Executor nodes only) Submit and probe deployment state of `HelmRelease CR. +5. (Executor nodes only) Submit and probe deployment state of `HelmRelease` CR. 6. Fetch and deploy Helm charts referred to by each `HelmRelease` CR to the Kubernetes cluster. (*optional) Embedded subcharts are fetched from the “staging” registry instead of the “primary/remote” registry. diff --git a/api/v1alpha1/application_types.go b/api/v1alpha1/application_types.go index fd870d0..63d768d 100644 --- a/api/v1alpha1/application_types.go +++ b/api/v1alpha1/application_types.go @@ -27,9 +27,10 @@ type ApplicationSpec struct { // ChartStatus denotes the current status of the Application Reconciliation type ChartStatus struct { - Ready bool `json:"ready,omitempty"` - Error string `json:"error,omitempty"` - Staged bool `json:"staged,omitempty"` + Ready bool `json:"ready,omitempty"` + Error string `json:"error,omitempty"` + Staged bool `json:"staged,omitempty"` + Version string `json:"version,omitempty"` } // ApplicationStatus defines the observed state of Application diff --git a/chart/orkestra/templates/NOTES.txt b/chart/orkestra/templates/NOTES.txt new file mode 100644 index 0000000..cd4d865 --- /dev/null +++ b/chart/orkestra/templates/NOTES.txt @@ -0,0 +1 @@ +Happy Helming with Azure/Orkestra \ No newline at end of file diff --git a/chart/orkestra/templates/_helpers.tpl b/chart/orkestra/templates/_helpers.tpl new file mode 100644 index 0000000..c18e045 --- /dev/null +++ b/chart/orkestra/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "orkestra.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "orkestra.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "orkestra.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "orkestra.labels" -}} +helm.sh/chart: {{ include "orkestra.chart" . }} +{{ include "orkestra.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "orkestra.selectorLabels" -}} +app.kubernetes.io/name: {{ include "orkestra.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "orkestra.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "orkestra.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/orkestra/templates/deployment.yaml b/chart/orkestra/templates/deployment.yaml new file mode 100644 index 0000000..49c0331 --- /dev/null +++ b/chart/orkestra/templates/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "orkestra.fullname" . }} + labels: + {{- include "orkestra.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "orkestra.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "orkestra.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "orkestra.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --staging-repo-name + - staging + - --config + - /etc/controller/config.yaml + - --chart-store-path + - /etc/orkestra/charts/pull + env: + - name: WORKFLOW_NAMESPACE + value: orkestra + # ports: + # - name: http + # containerPort: 80 + # protocol: TCP + # livenessProbe: + # httpGet: + # path: / + # port: http + # readinessProbe: + # httpGet: + # path: / + # port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/orkestra/templates/rbac.yaml b/chart/orkestra/templates/rbac.yaml new file mode 100644 index 0000000..860b640 --- /dev/null +++ b/chart/orkestra/templates/rbac.yaml @@ -0,0 +1,35 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "orkestra.serviceAccountName" . }} +rules: +- apiGroups: [""] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["orkestra.azure.microsoft.com"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["helm.fluxcd.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["argoproj.io"] + resources: ["*"] + verbs: ["*"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "orkestra.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + namespace: orkestra + name: {{ include "orkestra.serviceAccountName" . }} +roleRef: + kind: ClusterRole + name: {{ include "orkestra.serviceAccountName" . }} + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/chart/orkestra/templates/serviceaccount.yaml b/chart/orkestra/templates/serviceaccount.yaml new file mode 100644 index 0000000..c78b336 --- /dev/null +++ b/chart/orkestra/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "orkestra.serviceAccountName" . }} + labels: + {{- include "orkestra.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/orkestra/values.yaml b/chart/orkestra/values.yaml index 4d29b2d..f033534 100644 --- a/chart/orkestra/values.yaml +++ b/chart/orkestra/values.yaml @@ -1,29 +1,57 @@ -namespace: &namespace default -serviceAccount: &serviceAccount releaser-paas -clusterRole : &clusterRole "permissive-full-cr" +namespace: &namespace orkestra +serviceAccount: &serviceAccount orkestra +replicaCount: 1 + +image: + repository: azureorkestra/orkestra + pullPolicy: Always + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: *serviceAccount + +podAnnotations: {} + +podSecurityContext: {} + +securityContext: {} + +resources: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Dependency overlay values chartmuseum: env: open: DISABLE_API: false helm-operator: - # Add demo repository configureRepositories: enable: true repositories: - name: bitnami url: https://charts.bitnami.com/bitnami - - name: chartmuseum - url: http://localhost:8080 - # url: http://releaser-chartmuseum.default:8080 + - name: chartmuseum + url: http://orkestra-chartmuseum.orkestra:8080 rbac: - create: true - pspEnabled: true + create: false + pspEnabled: false serviceAccount: - create: true + create: false annotations: {} - name: "" + name: *serviceAccount helm: versions: "v3" @@ -37,23 +65,18 @@ argo: workflow: namespace: *namespace serviceAccount: - create: false name: *serviceAccount rbac: - create: false + enabled: false controller: - serviceAccount: *serviceAccount + # serviceAccount: *serviceAccount name: workflow-controller workflowNamespaces: - *namespace - - "default" - # containerRuntimeExecutor: docker + containerRuntimeExecutor: docker # For KinD use - - containerRuntimeExecutor: k8sapi - - # executor controls how the init and wait container should be customized - executor: + # containerRuntimeExecutor: k8sapi server: enabled: true diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..13ffe0b --- /dev/null +++ b/config.yaml @@ -0,0 +1,7 @@ +registries: + bitnami: + url: "https://charts.bitnami.com/bitnami" + insecureSkipVerify: true + + staging: + url: "http://orkestra-chartmuseum.orkestra:8080" \ No newline at end of file diff --git a/config/crd/bases/orkestra.azure.microsoft.com_applicationgroups.yaml b/config/crd/bases/orkestra.azure.microsoft.com_applicationgroups.yaml index f5910d2..f81af0a 100644 --- a/config/crd/bases/orkestra.azure.microsoft.com_applicationgroups.yaml +++ b/config/crd/bases/orkestra.azure.microsoft.com_applicationgroups.yaml @@ -69,6 +69,8 @@ spec: type: boolean staged: type: boolean + version: + type: string type: object name: type: string @@ -83,6 +85,8 @@ spec: type: boolean staged: type: boolean + version: + type: string type: object type: object required: diff --git a/config/crd/bases/orkestra.azure.microsoft.com_applications.yaml b/config/crd/bases/orkestra.azure.microsoft.com_applications.yaml index 38a9aac..e6e5abd 100644 --- a/config/crd/bases/orkestra.azure.microsoft.com_applications.yaml +++ b/config/crd/bases/orkestra.azure.microsoft.com_applications.yaml @@ -321,6 +321,8 @@ spec: type: boolean staged: type: boolean + version: + type: string type: object name: type: string @@ -335,6 +337,8 @@ spec: type: boolean staged: type: boolean + version: + type: string type: object type: object required: diff --git a/controllers/application_reconciler.go b/controllers/application_reconciler.go index 7c6788d..2b40c28 100644 --- a/controllers/application_reconciler.go +++ b/controllers/application_reconciler.go @@ -59,7 +59,7 @@ func (r *ApplicationReconciler) reconcile(ctx context.Context, l logr.Logger, ap if err != nil { cs.Error = err.Error() ll.Error(err, "failed to save subchart package as tgz") - return false, fmt.Errorf("failed to save subchart package as tgz : %w", err) + return false, fmt.Errorf("failed to save subchart package as tgz at location %s : %w", path, err) } err = r.RegistryClient.PushChart(ll, stagingRepoName, path, sc) @@ -71,6 +71,8 @@ func (r *ApplicationReconciler) reconcile(ctx context.Context, l logr.Logger, ap } cs.Staged = true + // TODO (nitishm) : Rather than the path we can just store the version of the chart + cs.Version = sc.Metadata.Version cs.Ready = true cs.Error = "" diff --git a/examples/simple/dev-applicationgroup.yaml b/examples/simple/dev-applicationgroup.yaml new file mode 100644 index 0000000..b45d06b --- /dev/null +++ b/examples/simple/dev-applicationgroup.yaml @@ -0,0 +1,14 @@ +apiVersion: orkestra.azure.microsoft.com/v1alpha1 +kind: ApplicationGroup +metadata: + name: dev +spec: + applications: + - name: kafka-dev + dependencies: + - redis-dev + - name: redis-dev +# TODO (nitishm) - define these fields using Argo Workflow spec +# strategy: +# rollout: +# backout: \ No newline at end of file diff --git a/examples/simple/kafka-dev-application.yaml b/examples/simple/kafka-dev-application.yaml new file mode 100644 index 0000000..7b3663c --- /dev/null +++ b/examples/simple/kafka-dev-application.yaml @@ -0,0 +1,25 @@ +# Example: Kafka with Zookeeper as a dependency +apiVersion: orkestra.azure.microsoft.com/v1alpha1 +kind: Application +metadata: + name: kafka-dev +spec: + # Namespace to which the HelmRelease object is deployed + namespace: "orkestra" + repo: bitnami + groupID: "dev" + subcharts: + # subchart ordering + - name: zookeeper + dependencies: [] + # HelmRelease spec fields + # https://docs.fluxcd.io/projects/helm-operator/en/1.0.0-rc9/references/helmrelease-custom-resource.html#helmrelease-custom-resource + chart: + repository: "https://charts.bitnami.com/bitnami" + name: kafka + version: 12.4.1 + overlays: "{\"global\":{\"imagePullSecrets\":[]},\"zookeeper\":{\"enable\":true}}" + # targetNamespace: "kafka-dev-ns" + + + diff --git a/examples/simple/redis-dev-application.yaml b/examples/simple/redis-dev-application.yaml new file mode 100644 index 0000000..581ec76 --- /dev/null +++ b/examples/simple/redis-dev-application.yaml @@ -0,0 +1,21 @@ +# Example: Kafka with Zookeeper as a dependency +apiVersion: orkestra.azure.microsoft.com/v1alpha1 +kind: Application +metadata: + name: redis-dev +spec: + namespace: "orkestra" + repo: bitnami + groupID: "dev" + subcharts: [] + # HelmRelease spec fields + # https://docs.fluxcd.io/projects/helm-operator/en/1.0.0-rc9/references/helmrelease-custom-resource.html#helmrelease-custom-resource + chart: + repository: "https://charts.bitnami.com/bitnami" + name: redis + version: 12.2.3 + overlays: "{\"global\":{\"imageRegistry\":\"nil\",\"imagePullSecrets\":[]},\"master\":{\"persistence\":{\"size\":\"4Gi\"}}}" + # targetNamespace: "redis-dev-ns" + + + diff --git a/pkg/workflow/argo.go b/pkg/workflow/argo.go index 1b8c77e..bf59e51 100644 --- a/pkg/workflow/argo.go +++ b/pkg/workflow/argo.go @@ -303,7 +303,12 @@ func generateSubchartAndAppDAGTasks(app *v1alpha1.Application, repo, targetNS st tasks := make([]v1alpha12.DAGTask, 0, len(app.Spec.Subcharts)+1) for _, sc := range app.Spec.Subcharts { - hr := generateSubchartHelmRelease(app.Spec.HelmReleaseSpec, sc.Name, repo, targetNS) + s, ok := app.Status.Subcharts[sc.Name] + if !ok { + return nil, fmt.Errorf("failed to find subchart info in applications status field") + } + + hr := generateSubchartHelmRelease(app.Spec.HelmReleaseSpec, sc.Name, s.Version, repo, targetNS) task := v1alpha12.DAGTask{ Name: sc.Name, Template: helmReleaseExecutor, @@ -370,7 +375,7 @@ func defaultExecutor() v1alpha12.Template { Name: helmReleaseExecutor, // FIXME (nitishm) : Hack // Replace with the actual service account in use - ServiceAccountName: "releaser-helm-operator", + ServiceAccountName: "orkestra", Inputs: v1alpha12.Inputs{ Parameters: []v1alpha12.Parameter{ { @@ -380,9 +385,10 @@ func defaultExecutor() v1alpha12.Template { }, Outputs: v1alpha12.Outputs{}, Resource: &v1alpha12.ResourceTemplate{ - Action: "create", - Manifest: "{{inputs.parameters.helmrelease}}", - SuccessCondition: "status.phase == Succeeded", + SetOwnerReference: true, + Action: "create", + Manifest: "{{inputs.parameters.helmrelease}}", + SuccessCondition: "status.phase == Succeeded", }, } } @@ -400,7 +406,7 @@ func hrToYAML(hr helmopv1.HelmRelease) string { return string(b) } -func generateSubchartHelmRelease(a helmopv1.HelmReleaseSpec, sc, repo, targetNS string) helmopv1.HelmRelease { +func generateSubchartHelmRelease(a helmopv1.HelmReleaseSpec, sc, version, repo, targetNS string) helmopv1.HelmRelease { hr := helmopv1.HelmRelease{ TypeMeta: v1.TypeMeta{ Kind: "HelmRelease", @@ -423,7 +429,7 @@ func generateSubchartHelmRelease(a helmopv1.HelmReleaseSpec, sc, repo, targetNS hr.Spec.ChartSource.RepoChartSource = a.DeepCopy().RepoChartSource hr.Spec.ChartSource.RepoChartSource.Name = sc hr.Spec.ChartSource.RepoChartSource.RepoURL = repo - hr.Spec.ChartSource.RepoChartSource.Version = a.Version + hr.Spec.ChartSource.RepoChartSource.Version = version hr.Spec.Values = subchartValues(sc, a.Values) return hr diff --git a/pkg/workflow/argo_test.go b/pkg/workflow/argo_test.go index c6cc936..f524a83 100644 --- a/pkg/workflow/argo_test.go +++ b/pkg/workflow/argo_test.go @@ -149,6 +149,19 @@ func Test_generateSubchartAndAppDAGTasks(t *testing.T) { }, }, }, + Status: v1alpha1.ApplicationStatus{ + Subcharts: map[string]v1alpha1.ChartStatus{ + "subchart-1": { + Version: "1.0.0", + }, + "subchart-2": { + Version: "1.0.0", + }, + "subchart-3": { + Version: "1.0.0", + }, + }, + }, }, }, want: []v1alpha12.DAGTask{ @@ -362,6 +375,19 @@ func Test_generateSubchartAndAppDAGTasks(t *testing.T) { }, }, }, + Status: v1alpha1.ApplicationStatus{ + Subcharts: map[string]v1alpha1.ChartStatus{ + "subchart-1": { + Version: "1.0.0", + }, + "subchart-2": { + Version: "1.0.0", + }, + "subchart-3": { + Version: "1.0.0", + }, + }, + }, }, }, want: []v1alpha12.DAGTask{ @@ -593,6 +619,19 @@ func Test_generateAppDAGTemplates(t *testing.T) { }, }, }, + Status: v1alpha1.ApplicationStatus{ + Subcharts: map[string]v1alpha1.ChartStatus{ + "subchart-1": { + Version: "1.0.0", + }, + "subchart-2": { + Version: "1.0.0", + }, + "subchart-3": { + Version: "1.0.0", + }, + }, + }, }, }, repo: "http://stagingrepo", @@ -779,6 +818,19 @@ func Test_generateAppDAGTemplates(t *testing.T) { }, }, }, + Status: v1alpha1.ApplicationStatus{ + Subcharts: map[string]v1alpha1.ChartStatus{ + "subchart-1": { + Version: "1.0.0", + }, + "subchart-2": { + Version: "1.0.0", + }, + "subchart-3": { + Version: "1.0.0", + }, + }, + }, }, }, repo: "http://stagingrepo", diff --git a/testwith/config.yaml b/testwith/config.yaml new file mode 100644 index 0000000..b73eb4a --- /dev/null +++ b/testwith/config.yaml @@ -0,0 +1,7 @@ +registries: + bitnami: + url: "https://charts.bitnami.com/bitnami" + insecureSkipVerify: true + + staging: + url: "http://localhost:8080" \ No newline at end of file