KFP Upgrade (#69)

.azdo/iac.yaml

@@ -44,7 +44,7 @@ stages:
              curl -L https://istio.io/downloadIstio | ISTIO_VERSION=$ISTIO_VERSION sh -
              cd istio-$ISTIO_VERSION
              export PATH=$PWD/bin:$PATH
-             istioctl manifest apply  --set profile=default
+             istioctl install  --set profile=default
       - task: Bash@3
         name: "Install_KFP"
         displayName: "Install Kubeflow Pipelines"

docs/Kubeflow-install.md

@@ -12,36 +12,7 @@
 
 ## Install Kubeflow core + Kubeflow pipelines
 
-* Use [kubeflow-install.sh](../setup/kfp/kubeflow-install.sh) script or follow the manual steps below
-* Download the kfctl v1.0.2 release from the [Kubeflow releases page](https://github.com/kubeflow/kfctl/releases/tag/v1.0.2).
-* Unpack the tar ball
-```
-tar -xvf kfctl_v1.0.2_<platform>.tar.gz
-```
-* Install the package
-```
-# The following command is optional, to make kfctl binary easier to use.
-export PATH=$PATH:<path to where kfctl was unpacked>
-
-# Set KF_NAME to the name of your Kubeflow deployment. This also becomes the
-# name of the directory containing your configuration.
-# For example, your deployment name can be 'my-kubeflow' or 'kf-test'.
-export KF_NAME=<your choice of name for the Kubeflow deployment>
-
-# Set the path to the base directory where you want to store one or more 
-# Kubeflow deployments. For example, /opt/.
-# Then set the Kubeflow application directory for this deployment.
-export BASE_DIR=<path to a base directory>
-export KF_DIR=${BASE_DIR}/${KF_NAME}
-
-# Set the configuration file to use, such as the file specified below:
-export CONFIG_URI="setup/kfp/kfctl_k8s_light.yaml"
-
-# Generate and deploy Kubeflow:
-mkdir -p ${KF_DIR}
-cd ${KF_DIR}
-kfctl apply -V -f ${CONFIG_URI}
-```
+* Use [kubeflow-install.sh](../setup/kfp/kubeflow-install.sh) script
 
 ## Verify the installation
 
@@ -50,10 +21,3 @@ Wait until pods in the *kubeflow* namespaces are up and running
 kubectl get pods -n kubeflow
 ```
 
-## Delete the installation
-
-```
-cd ${KF_DIR}
-kfctl delete -V -f ${CONFIG_URI}
-```
-

setup/iac/aks-create.sh

@@ -22,5 +22,7 @@ az aks create -n $AKS_NAME -g $RESOURCE_GROUP -l $LOCATION \
   --enable-managed-identity \
   --network-plugin azure \
   --generate-ssh-keys \
-  --node-resource-group kubeflowyo \
+  --service-principal $AKS_SERVICE_PRICIPAL \
+  --client-secret $AKS_SERVICE_PRINCIPAL_SECRET \
+  --node-resource-group $RESOURCE_GROUP-nodepool \
   --aks-custom-headers usegen2vm=true,ContainerRuntime=containerd

setup/kfp/crb.yaml

@@ -0,0 +1,42 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: csr-approver
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/*
+  verbs:
+  - approve
+  
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: csr-approver-binding
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: kubeflow
+roleRef:
+ kind: ClusterRole
+ name: csr-approver
+ apiGroup: ""

setup/kfp/istio-resources.yaml

@@ -0,0 +1,62 @@
+apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: istio-ingressgateway
+  namespace: istio-system
+  labels:
+      operator.istio.io/component: IngressGateways
+      operator.istio.io/managed: Reconcile
+      release: istio
+spec:
+  selector:
+    app: istio-ingressgateway
+    istio: ingressgateway
+  servers:
+  - hosts:
+    - '*'
+    port:
+      name: http
+      number: 80
+      protocol: HTTP
+---
+apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: kubeflow-gateway
+  namespace: kubeflow
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+  - hosts:
+    - '*'
+    port:
+      name: http
+      number: 80
+      protocol: HTTP
+---  
+apiVersion: networking.istio.io/v1beta1
+kind: VirtualService
+metadata:
+  labels:
+    app.kubernetes.io/component: pipelines-ui
+    app.kubernetes.io/name: pipelines-ui
+  name: ml-pipeline-ui
+  namespace: kubeflow
+spec:
+  gateways:
+  - kubeflow-gateway
+  hosts:
+  - '*'
+  http:
+  - match:
+    - uri:
+        prefix: /
+    rewrite:
+      uri: /
+    route:
+    - destination:
+        host: ml-pipeline-ui.kubeflow.svc.cluster.local
+        port:
+          number: 80
+    timeout: 300s

setup/kfp/kfctl_k8s_light.yaml

@@ -1,203 +0,0 @@
-apiVersion: kfdef.apps.kubeflow.org/v1
-kind: KfDef
-metadata:
-  namespace: kubeflow
-spec:
-  applications:
-  - kustomizeConfig:
-      parameters:
-      - name: namespace
-        value: istio-system
-      repoRef:
-        name: manifests
-        path: istio/cluster-local-gateway
-    name: cluster-local-gateway
-  - kustomizeConfig:
-      parameters:
-      - name: clusterRbacConfig
-        value: "OFF"
-      repoRef:
-        name: manifests
-        path: istio/istio
-    name: istio
-  - kustomizeConfig:
-      parameters:
-      - name: namespace
-        value: istio-system
-      repoRef:
-        name: manifests
-        path: istio/add-anonymous-user-filter
-    name: add-anonymous-user-filter
-  - kustomizeConfig:
-      repoRef:
-        name: manifests
-        path: application/application-crds
-    name: application-crds
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: application/application
-    name: application
-  - kustomizeConfig:
-      parameters:
-      - name: namespace
-        value: cert-manager
-      repoRef:
-        name: manifests
-        path: cert-manager/cert-manager-crds
-    name: cert-manager-crds
-  - kustomizeConfig:
-      parameters:
-      - name: namespace
-        value: kube-system
-      repoRef:
-        name: manifests
-        path: cert-manager/cert-manager-kube-system-resources
-    name: cert-manager-kube-system-resources
-  - kustomizeConfig:
-      overlays:
-      - self-signed
-      - application
-      parameters:
-      - name: namespace
-        value: cert-manager
-      repoRef:
-        name: manifests
-        path: cert-manager/cert-manager
-    name: cert-manager
-  - kustomizeConfig:
-      repoRef:
-        name: manifests
-        path: metacontroller
-    name: metacontroller
-  - kustomizeConfig:
-      overlays:
-      - istio
-      - application
-      repoRef:
-        name: manifests
-        path: argo
-    name: argo
-  - kustomizeConfig:
-      repoRef:
-        name: manifests
-        path: kubeflow-roles
-    name: kubeflow-roles
-  - kustomizeConfig:
-      overlays:
-      - istio
-      - application
-      repoRef:
-        name: manifests
-        path: common/centraldashboard
-    name: centraldashboard
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: admission-webhook/bootstrap
-    name: bootstrap
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: admission-webhook/webhook
-    name: webhook
-  - kustomizeConfig:
-      overlays:
-      - istio
-      - application
-      - db
-      repoRef:
-        name: manifests
-        path: metadata
-    name: metadata
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/api-service
-    name: api-service
-  - kustomizeConfig:
-      overlays:
-      - application
-      parameters:
-      - name: minioPvcName
-        value: minio-pv-claim
-      repoRef:
-        name: manifests
-        path: pipeline/minio
-    name: minio
-  - kustomizeConfig:
-      overlays:
-      - application
-      parameters:
-      - name: mysqlPvcName
-        value: mysql-pv-claim
-      repoRef:
-        name: manifests
-        path: pipeline/mysql
-    name: mysql
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/persistent-agent
-    name: persistent-agent
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/pipelines-runner
-    name: pipelines-runner
-  - kustomizeConfig:
-      overlays:
-      - istio
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/pipelines-ui
-    name: pipelines-ui
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/pipelines-viewer
-    name: pipelines-viewer
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/scheduledworkflow
-    name: scheduledworkflow
-  - kustomizeConfig:
-      overlays:
-      - application
-      repoRef:
-        name: manifests
-        path: pipeline/pipeline-visualization-service
-    name: pipeline-visualization-service
-  - kustomizeConfig:
-      overlays:
-      - application
-      - istio
-      parameters:
-      - name: admin
-        value: johnDoe@acme.com
-      repoRef:
-        name: manifests
-        path: profiles
-    name: profiles
-  repos:
-  - name: manifests
-    uri: https://github.com/kaizentm/manifests/archive/v1.0.3-kfp-0.5.0.tar.gz
-  version: v1.0.2

setup/kfp/kubeflow-install.sh

@@ -1,21 +1,18 @@
 #!/bin/sh
 
-mkdir -p ~/tmp
-mkdir -p ~/tmp/kfctl
 
-# Linux
-curl -L https://github.com/kubeflow/kfctl/releases/download/v1.0.2/kfctl_v1.0.2-0-ga476281_linux.tar.gz --output ~/tmp/kfctl/kfctl_v1.0.2-0-ga476281_linux.tar.gz
-tar -zxf ~/tmp/kfctl/kfctl_v1.0.2-0-ga476281_linux.tar.gz 
-sudo mv kfctl /usr/local/bin/
+kubectl apply -k github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources
+kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s
 
-# Mac OS X
-# curl -L https://github.com/kubeflow/kfctl/releases/download/v1.0.2/kfctl_v1.0.2-0-ga476281_darwin.tar.gz | tar -xv -C ~/tmp/kfctl/
-# sudo mv ~/tmp/kfctl/kfctl /usr/local/bin/
+# Update Argo workflow config to use "pns" containerRuntimeExecutor instead of default "docker"
+# to support containerd
+# https://github.com/kubeflow/pipelines/issues/1654
+# https://github.com/kubeflow/pipelines/issues/1654#issuecomment-595722994
+kubectl patch configmap/workflow-controller-configmap -n kubeflow --patch  "$(cat workflow-controller-configmap-patch.yaml)"
+kubectl apply -k github.com/kubeflow/pipelines/manifests/kustomize/env/platform-agnostic
+
+# Apply this role and binding to fix the permission issue of cache-deployer-deployment
+# https://github.com/kubeflow/pipelines/pull/4246
+kubectl apply -f crb.yaml
+kubectl apply -f istio-resources.yaml
 
-export BASE_DIR=kfinstall
-export KF_NAME=kubeflow
-export KF_DIR=${BASE_DIR}/${KF_NAME}
-mkdir -p ${KF_DIR}
-cp kfctl_k8s_light.yaml ${KF_DIR}/
-cd ${KF_DIR}
-kfctl apply -V -f kfctl_k8s_light.yaml

setup/kfp/workflow-controller-configmap-patch.yaml

@@ -0,0 +1,25 @@
+data:
+  config: |
+    {
+    namespace: kubeflow,
+    executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
+    containerRuntimeExecutor: pns,
+    artifactRepository:
+    {
+        s3: {
+            bucket: mlpipeline,
+            keyPrefix: artifacts,
+            endpoint: minio-service.kubeflow:9000,
+            insecure: true,
+            accessKeySecret: {
+                name: mlpipeline-minio-artifact,
+                key: accesskey
+            },
+            secretKeySecret: {
+                name: mlpipeline-minio-artifact,
+                key: secretkey
+            }
+        },
+        archiveLogs: true
+    }
+    }

setup/seldon/setup.sh

@@ -1,5 +1,11 @@
 kubectl create namespace seldon-system
 
-helm install seldon-core seldon-core-operator --repo https://storage.googleapis.com/seldon-charts --set istio.enabled=true --set usageMetrics.enabled=true --namespace seldon-system
+# Need to create seldon crd explicitly (not with a helm chart)
+# due to incompatibility issue with K8s v1.18
+# https://github.com/SeldonIO/seldon-core/issues/1675
+
+kubectl apply -f seldon-crd.yaml -n seldon-system
+
+helm install seldon-core seldon-core-operator --repo https://storage.googleapis.com/seldon-charts --set istio.enabled=true --set usageMetrics.enabled=true --set crd.create=false --namespace seldon-system
 
 kubectl apply -f gateway.yaml