From b75f66aa069b24334078187a77010d4a3368467a Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Tue, 3 Dec 2019 16:48:03 +0800 Subject: [PATCH] [kubespray] some script to backup and recover data (#3955) --- contrib/kubespray/namespace_secret_backup.py | 82 ++++++++++++ contrib/kubespray/namespace_secret_recover.py | 122 ++++++++++++++++++ contrib/kubespray/readme.md | 39 +++++- 3 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 contrib/kubespray/namespace_secret_backup.py create mode 100644 contrib/kubespray/namespace_secret_recover.py diff --git a/contrib/kubespray/namespace_secret_backup.py b/contrib/kubespray/namespace_secret_backup.py new file mode 100644 index 000000000..8336ee49e --- /dev/null +++ b/contrib/kubespray/namespace_secret_backup.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import logging +import logging.config +import yaml +import os +import argparse +import sys +import time +from kubernetes import client, config +from kubernetes.client.rest import ApiException + + +def setup_logger_config(logger): + """ + Setup logging configuration. + """ + if len(logger.handlers) == 0: + logger.propagate = False + logger.setLevel(logging.DEBUG) + consoleHandler = logging.StreamHandler() + consoleHandler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s : %(message)s') + consoleHandler.setFormatter(formatter) + logger.addHandler(consoleHandler) + + +logger = logging.getLogger(__name__) +setup_logger_config(logger) + + +def get_namespaced_secret(namespace): + config.load_kube_config() + try: + api_instance = client.CoreV1Api() + api_response = api_instance.list_namespaced_secret(namespace) + return api_response.items + except ApiException as e: + if e.status == 404: + return [] + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--namespace', dest="namespace", required=True, + help="The secret in which namespace should be backup") + parser.add_argument('-o', '--output', dest="output", required=True, help="the output file to store ") + + args = parser.parse_args() + namespaces = args.namespace + output = args.output + data = get_namespaced_secret(namespaces) + output_data = [] + for item in data: + if 'default-token-' in item.metadata.name: + continue + output_data.append({ + 'name': item.metadata.name, + 'data': item.data + }) + with open(output, 'w') as yaml_file: + yaml.dump(output_data, yaml_file, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/contrib/kubespray/namespace_secret_recover.py b/contrib/kubespray/namespace_secret_recover.py new file mode 100644 index 000000000..7d662f8b5 --- /dev/null +++ b/contrib/kubespray/namespace_secret_recover.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import logging +import logging.config +import yaml +import os +import argparse +import sys +import time +from kubernetes import client, config +from kubernetes.client.rest import ApiException + + +def setup_logger_config(logger): + """ + Setup logging configuration. + """ + if len(logger.handlers) == 0: + logger.propagate = False + logger.setLevel(logging.DEBUG) + consoleHandler = logging.StreamHandler() + consoleHandler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s [%(levelname)s] - %(filename)s:%(lineno)s : %(message)s') + consoleHandler.setFormatter(formatter) + logger.addHandler(consoleHandler) + + +logger = logging.getLogger(__name__) +setup_logger_config(logger) + + +def create_namespace_if_not_exist(namespace): + config.load_kube_config() + try: + api_instance = client.CoreV1Api() + api_instance.read_namespace(namespace) + except ApiException as e: + if e.status == 404: + api_instance = client.CoreV1Api() + meta_data = client.V1ObjectMeta() + meta_data.name = namespace + body = client.V1Namespace( + metadata=meta_data + ) + api_instance.create_namespace(body) + return True + logger.error("Failed to create namespace [{0}]".format(namespace)) + sys.exit(1) + return False + + +def create_secret_in_namespace_if_not_exist(namespace, payload): + config.load_kube_config() + try: + api_instance = client.CoreV1Api() + api_instance.read_namespaced_secret(payload['name'], namespace) + except ApiException as e: + if e.status == 404: + try: + api_instance = client.CoreV1Api() + meta_data = client.V1ObjectMeta() + meta_data.name = payload['name'] + body = client.V1Secret( + metadata=meta_data, + data=payload['data'] + ) + api_instance.create_namespaced_secret(namespace, body) + except ApiException as create_e: + logger.error("Exception when calling CoreV1Api->create_namespaced_secret: %s\n" % create_e) + sys.exit(1) + else: + logger.error("Exception when calling CoreV1Api->read_namespaced_secret: %s\n" % e) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--namespace', dest="namespace", required=True, + help="The secret in which namespace should be backup") + parser.add_argument('-i', '--input', dest="input", required=True, help="the input file to restore data ") + parser.add_argument('-d', '--delete-backup', dest="delete", default=False, action='store_true') + + args = parser.parse_args() + namespaces = args.namespace + input = args.input + delete_backup = args.delete + + create_namespace_if_not_exist(namespaces) + + with open(input, "r") as f: + secret_data = yaml.load(f, yaml.SafeLoader) + + for item in secret_data: + create_secret_in_namespace_if_not_exist( + namespaces, + item + ) + + if delete_backup: + try: + os.unlink(input) + except OSError as e: + logger.exception(e) + + +if __name__ == "__main__": + main() diff --git a/contrib/kubespray/readme.md b/contrib/kubespray/readme.md index f78885b03..0761b7016 100644 --- a/contrib/kubespray/readme.md +++ b/contrib/kubespray/readme.md @@ -1,5 +1,24 @@ #### Deploy kubernetes through kubespray. +##### Before deployment +###### Backup data (namespaced_secret) from the k8s deployed by paictl + + +Because the kubespray will cleanup the etcd data in local disk, please backup your data every time you wanna stop your cluster. +If you haven't deploy OpenPAI, you can skip this steps. +If you don't want to keep the data in your cluster, you can skip this steps. + + +``` +cd pai/contrib/kubespray + +# Before backup, please ensure there are datas in the namespaces. +# with the command kubectl get secrets -n namespace-name +python3 namespace_secret_backup.py -n pai-user-v2 -o pai-user-v2 +python3 namespace_secret_backup.py -n pai-group -o pai-group +python3 namespace_secret_backup.py -n pai-storage -o pai-storage +python3 namespace_secret_backup.py -n pai-user-token -o pai-user-token +``` #### Environment Setup @@ -416,4 +435,22 @@ cd kubespray/ ansible-playbook -i inventory/mycluster/hosts.yml upgrade-cluster.yml --become --become-user=root --limit=node7,node8,node9 -e "@inventory/mycluster/openpai.yml" -``` \ No newline at end of file +``` + + +#### After deployment + +###### Recover backup data (namespaced_secret) from the backup data after setuping k8s with kubespray. + +If you backup your data from the k8s deployed by openpai, you could recover the data to a specified namespace after the k8s deployment of kubespray done. + +``` +cd pai/contrib/kubespray + +# Before backup, please ensure there are datas in the namespaces. +# with the command kubectl get secrets -n namespace-name +python3 namespace_secret_recover.py -n pai-user-v2 -i pai-user-v2 +python3 namespace_secret_recover.py -n pai-group -i pai-group +python3 namespace_secret_recover.py -n pai-storage -i pai-storage +python3 namespace_secret_recover.py -n pai-user-token -i pai-user-token +``` \ No newline at end of file