From 2172a06e4ff330b886ea1295b11062ec6dc19caa Mon Sep 17 00:00:00 2001 From: Cristian Balas Date: Mon, 3 May 2021 15:21:37 +0300 Subject: [PATCH] Scripts for DigitalOcean oneclick image (#209) --- .circleci/config.yml | 4 +- packages/frontend/nginx/nginx.conf | 2 +- utils/1click_image_scripts/download.sh | 14 ++ utils/1click_image_scripts/setup.py | 193 ++++++++++++++++++ .../template-docker-compose.yml | 70 +++++++ .../template-nginx-site.conf | 31 +++ .../test-deployment}/install_prerequisites.sh | 0 .../test-deployment}/run_tests.py | 0 8 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 utils/1click_image_scripts/download.sh create mode 100755 utils/1click_image_scripts/setup.py create mode 100644 utils/1click_image_scripts/template-docker-compose.yml create mode 100644 utils/1click_image_scripts/template-nginx-site.conf rename {test-deployment => utils/test-deployment}/install_prerequisites.sh (100%) rename {test-deployment => utils/test-deployment}/run_tests.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5bc4c70f..44ec98450 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,9 +94,9 @@ jobs: - run: name: Test deployment command: | - ./test-deployment/install_prerequisites.sh + ./utils/test-deployment/install_prerequisites.sh SPECKLE_SERVER=https://latest.speckle.dev if [[ "$CIRCLE_TAG" =~ ^v.* ]]; then SPECKLE_SERVER=https://speckle.xyz fi - ./test-deployment/run_tests.py $SPECKLE_SERVER + ./utils/test-deployment/run_tests.py $SPECKLE_SERVER diff --git a/packages/frontend/nginx/nginx.conf b/packages/frontend/nginx/nginx.conf index 8518ece54..11acdac79 100644 --- a/packages/frontend/nginx/nginx.conf +++ b/packages/frontend/nginx/nginx.conf @@ -7,7 +7,7 @@ server { try_files $uri $uri/ /app.html; } - location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)) { + location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) { resolver 127.0.0.11 valid=30s; set $upstream_speckle_server speckle-server; client_max_body_size 100m; diff --git a/utils/1click_image_scripts/download.sh b/utils/1click_image_scripts/download.sh new file mode 100644 index 000000000..ee790569d --- /dev/null +++ b/utils/1click_image_scripts/download.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +echo "* Getting latest version of SpeckleServer Setup files..." + +mkdir -p /opt/speckle-server +cd /opt/speckle-server + +wget https://raw.githubusercontent.com/specklesystems/speckle-server/main/utils/1click_image_scripts/setup.py -O setup.py +wget https://raw.githubusercontent.com/specklesystems/speckle-server/main/utils/1click_image_scripts/template-nginx-site.conf -O template-nginx-site.conf +wget https://raw.githubusercontent.com/specklesystems/speckle-server/main/utils/1click_image_scripts/template-docker-compose.yml -O template-docker-compose.yml + + +echo "* Getting the docker images for the latest SpeckleServer release..." +docker-compose -f template-docker-compose.yml pull diff --git a/utils/1click_image_scripts/setup.py b/utils/1click_image_scripts/setup.py new file mode 100755 index 000000000..abedf7ff0 --- /dev/null +++ b/utils/1click_image_scripts/setup.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 + +import os +import socket +import subprocess +import secrets +import ruamel.yaml # this module preserves yaml comments and whitespaces +from ruamel.yaml.scalarstring import DoubleQuotedScalarString + +FILE_PATH = os.path.dirname(os.path.abspath(__file__)) + +LOGO_STR = ''' + _____ _ _ _____ +/ ___| | | | | / ___| +\ `--. _ __ ___ ___| | _| | ___\ `--. ___ _ ____ _____ _ __ + `--. \ '_ \ / _ \/ __| |/ / |/ _ \`--. \/ _ \ '__\ \ / / _ \ '__| +/\__/ / |_) | __/ (__| <| | __/\__/ / __/ | \ V / __/ | +\____/| .__/ \___|\___|_|\_\_|\___\____/ \___|_| \_/ \___|_| + | | + |_| +''' + +def get_local_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + if not ip: + print("Error: Can't get local IP address") + exit(1) + return ip + + +def read_domain(ip): + print("\nYou can set up a domain name for this Speckle server.") + print("Important: To use a domain name, you must first configure it to point to this VM address (so we can issue the SSL certificate)") + print(f"VM address: {ip}") + while True: + domain = input(f'Domain name (leave blank to use the IP address): ').strip() + if not domain: + return None + try: + domain_ip = socket.gethostbyname(domain.strip()) + except Exception as ex: + print(f"Error: Domain '{domain}' cannot be resolved: {str(ex)}") + continue + + if domain_ip != ip: + print(f"Error: Domain '{domain}' points to {domain_ip} instead of {ip}") + continue + + return domain + + +def read_email_settings(domain): + print("\nYou should configure an email provider to allow the Speckle Server to send emails.") + print("Supported vendors: Any email provider that can provide SMTP connection details (mailjet, mailgun, etc).") + print("Important: If you don't configure email details, some features that require sending emails will not work, nevertheless the server should be functional.") + while True: + enable_email = False + while True: + enable_email = input("Enable emails? [Y/n]: ").strip().lower() + if enable_email in ['n', 'no']: + enable_email = False + break + elif enable_email in ['', 'y', 'yes']: + enable_email = True + break + else: + print("Unrecognized option") + continue + + if not enable_email: + return None + + print("Enter your SMTP connection details offered by your email provider") + smtp_host = input("SMTP server / host: ").strip() + smtp_port = input("SMTP port: ").strip() + try: + int(smtp_port) + except Exception: + print('Error: SMTP port must be a number. Retrying...') + continue + smtp_user = input("SMTP Username: ").strip() + smtp_pass = input("SMTP Password: ").strip() + + if domain: + default_from_email = 'no-reply@' + domain + else: + default_from_email = '' + email_from = input(f"Email address to send email as [{default_from_email}]: ") + if not email_from.strip(): + email_from = default_from_email + + if not smtp_host or not smtp_port or not smtp_user or not smtp_pass or not email_from: + print("Error: One or more fields were empty. Retrying...") + continue + + return { + 'host': smtp_host, + 'port': smtp_port, + 'user': smtp_user, + 'pass': smtp_pass, + 'from': email_from + } + + +def main(): + print(LOGO_STR) + ip = get_local_ip() + + ### + ### Read user input + ######### + domain = read_domain(ip) + if domain: + canonical_url = f'https://{domain}' + else: + canonical_url = f'http://{ip}' + + email = read_email_settings(domain) + + ### + ### Create docker-compose.yml from the template + ######### + print("\nConfiguring docker containers...") + + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + with open(os.path.join(FILE_PATH, 'template-docker-compose.yml'), 'r') as f: + yml_doc = yaml.load(f) + env = yml_doc['services']['speckle-server']['environment'] + env['CANONICAL_URL'] = DoubleQuotedScalarString(canonical_url) + env['SESSION_SECRET'] = DoubleQuotedScalarString(secrets.token_hex(32)) + if email: + env['EMAIL'] = DoubleQuotedScalarString('true') + env['EMAIL_HOST'] = DoubleQuotedScalarString(email['host']) + env['EMAIL_PORT'] = DoubleQuotedScalarString(email['port']) + env['EMAIL_USERNAME'] = DoubleQuotedScalarString(email['user']) + env['EMAIL_PASSWORD'] = DoubleQuotedScalarString(email['pass']) + env['EMAIL_FROM'] = DoubleQuotedScalarString(email['from']) + else: + env['EMAIL'] = DoubleQuotedScalarString('false') + + with open(os.path.join(FILE_PATH, 'docker-compose.yml'), 'w') as f: + f.write('# This file was generated by SpeckleServer setup.\n') + f.write('# If the setup is re-run, this file will be overwritten.\n\n') + yaml.dump(yml_doc, f) + + ### + ### Run the new docker-compose file (will update containers if already running) + ######### + subprocess.run(['bash', '-c', f'cd "{FILE_PATH}"; docker-compose up -d'], check=True) + + + ### + ### Update nginx config and restart nginx + ######### + print("\nConfiguring local nginx...") + + nginx_conf_str = '# This file is managed by SpeckleServer setup script.\n' + nginx_conf_str += '# Any modifications will be removed when the setup script is re-executed\n\n' + with open(os.path.join(FILE_PATH, 'template-nginx-site.conf'), 'r') as f: + nginx_conf_str += f.read() + if domain: + nginx_conf_str = nginx_conf_str.replace('TODO_REPLACE_WITH_SERVER_NAME', domain) + else: + nginx_conf_str = nginx_conf_str.replace('TODO_REPLACE_WITH_SERVER_NAME', '_') + with open('/etc/nginx/sites-available/speckle-server', 'w') as f: + f.write(nginx_conf_str) + subprocess.run(['nginx', '-s', 'reload'], check=True) + + ### + ### Run letsencrypt on new config + ######### + if domain: + print("\n***") + print("*** Will now run LetsEncrypt utility to generate https certificate. Please answer any questions that are presented") + print("*** We highly recommend setting a good email address so that you are notified if there is any action needed to renew certificates") + print("***") + subprocess.run(['certbot', '--nginx', '-d', domain]) + + print("\nConfiguration complete!") + print("You can access your speckle server at: " + canonical_url) + print(LOGO_STR) + print("\nOne more thing and you are ready to roll:") + print(f" - Go to {canonical_url} in your browser and create an account. The first user to register will be granted administrator rights.") + print(" - Fill in information about your server under your profile page (in the lower left corner).") + print("\nHappy Speckling!") + + +if __name__ == '__main__': + main() diff --git a/utils/1click_image_scripts/template-docker-compose.yml b/utils/1click_image_scripts/template-docker-compose.yml new file mode 100644 index 000000000..384019720 --- /dev/null +++ b/utils/1click_image_scripts/template-docker-compose.yml @@ -0,0 +1,70 @@ +version: "3" +services: + #### + # Speckle Server dependencies + ####### + postgres: + image: "postgres:13.1-alpine" + restart: always + environment: + POSTGRES_DB: speckle + POSTGRES_USER: speckle + POSTGRES_PASSWORD: speckle + volumes: + - ./postgres-data:/var/lib/postgresql/data/ + ports: + - "127.0.0.1:5432:5432" + + redis: + image: "redis:6.0-alpine" + restart: always + volumes: + - ./redis-data:/data + ports: + - "127.0.0.1:6379:6379" + + #### + # Speckle Server + ####### + speckle-frontend: + image: speckle/speckle-frontend:2 + restart: always + ports: + - "127.0.0.1:8000:80" + + speckle-server: + image: speckle/speckle-server:2 + restart: always + ports: + - "127.0.0.1:3000:3000" + command: ["bash", "-c", "/wait && node bin/www"] + environment: + CANONICAL_URL: "TODO: change" + SESSION_SECRET: "TODO: change" + + STRATEGY_LOCAL: "true" + DEBUG: "speckle:*" + + POSTGRES_URL: "postgres" + POSTGRES_USER: "speckle" + POSTGRES_PASSWORD: "speckle" + POSTGRES_DB: "speckle" + + REDIS_URL: "redis://redis" + WAIT_HOSTS: "postgres:5432, redis:6379" + + EMAIL: "false" + EMAIL_HOST: "TODO" + EMAIL_PORT: "TODO" + EMAIL_USERNAME: "TODO" + EMAIL_PASSWORD: "TODO" + EMAIL_FROM: "TODO" + + EMAIL_SECURE: "false" + + speckle-preview-service: + image: speckle/speckle-preview-service:2 + restart: always + environment: + DEBUG: "preview-service:*" + PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle" diff --git a/utils/1click_image_scripts/template-nginx-site.conf b/utils/1click_image_scripts/template-nginx-site.conf new file mode 100644 index 000000000..d46fc94d7 --- /dev/null +++ b/utils/1click_image_scripts/template-nginx-site.conf @@ -0,0 +1,31 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + + server_name TODO_REPLACE_WITH_SERVER_NAME; + + location / { + client_max_body_size 100m; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://localhost:8000; + + proxy_buffering off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) { + client_max_body_size 100m; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://localhost:3000; + + proxy_buffering off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} + diff --git a/test-deployment/install_prerequisites.sh b/utils/test-deployment/install_prerequisites.sh similarity index 100% rename from test-deployment/install_prerequisites.sh rename to utils/test-deployment/install_prerequisites.sh diff --git a/test-deployment/run_tests.py b/utils/test-deployment/run_tests.py similarity index 100% rename from test-deployment/run_tests.py rename to utils/test-deployment/run_tests.py