From 79ac767cbe32dc63e386208eb64aae867d29e9bb Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Mon, 24 Jul 2023 15:46:50 +0200 Subject: [PATCH] Migration to Moodle 4 --- azuredeploy.json | 20 +-- scripts/helper_functions.sh | 64 ++++++-- scripts/install_elastic.sh | 44 +++++- scripts/install_gluster.sh | 16 +- scripts/install_moodle.sh | 306 +++++++++++++++++++----------------- scripts/setup_nfs_ha.sh | 4 +- scripts/setup_webserver.sh | 161 +++++++++---------- 7 files changed, 340 insertions(+), 275 deletions(-) diff --git a/azuredeploy.json b/azuredeploy.json index f73beff..ef5a9e8 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -121,10 +121,9 @@ }, "moodleVersion": { "allowedValues": [ - "MOODLE_39_STABLE", - "MOODLE_311_STABLE" + "MOODLE_402_STABLE" ], - "defaultValue": "MOODLE_39_STABLE", + "defaultValue": "MOODLE_402_STABLE", "metadata": { "description": "The Moodle version you want to install." }, @@ -195,11 +194,9 @@ }, "phpVersion": { "allowedValues": [ - "7.2", - "7.3", - "7.4" + "8.1" ], - "defaultValue": "7.4", + "defaultValue": "8.1", "metadata": { "description": "php version" }, @@ -288,10 +285,9 @@ }, "mysqlVersion": { "allowedValues": [ - "5.7", "8.0.21" ], - "defaultValue": "5.7", + "defaultValue": "8.0.21", "metadata": { "description": "Mysql version" }, @@ -799,9 +795,9 @@ "ubuntuVersion": { "type": "string", "allowedValues": [ - "20_04-lts-gen2" + "22_04-lts-gen2" ], - "defaultValue": "20_04-lts-gen2" + "defaultValue": "22_04-lts-gen2" }, "location": { "type": "string", @@ -1372,7 +1368,7 @@ "nfsHaNode1IP": "[concat(variables('subnetSanPrefix'), '.120')]", "osDiskStorageType": "[parameters('osDiskStorageType')]", "osType": { - "offer": "0001-com-ubuntu-server-focal", + "offer": "0001-com-ubuntu-server-jammy", "publisher": "Canonical", "sku": "[parameters('ubuntuVersion')]", "version": "latest" diff --git a/scripts/helper_functions.sh b/scripts/helper_functions.sh index 1004dc9..afc5e08 100644 --- a/scripts/helper_functions.sh +++ b/scripts/helper_functions.sh @@ -2,15 +2,37 @@ # Common functions definitions +function wait_for_process { + until [ -z $(/usr/bin/pgrep ${1}) ]; do + printf '.' + sleep 0.5 + done +} + +function apt_update_noninteractive { + export DEBIAN_FRONTEND='noninteractive' + + # waiting for apt to finish before running any other commands + wait_for_process apt; + + apt --yes -qq -o=Dpkg::Use-Pty=0 update +} + +function apt_install_noninteractive { + export DEBIAN_FRONTEND='noninteractive' + export NEEDRESTART_MODE='a' + export ACCEPT_EULA='Y' + + # waiting for apt to finish before running any other commands + wait_for_process apt; + + apt --yes --no-install-recommends -qq -o=Dpkg::Use-Pty=0 -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install "${@}" +} + function get_setup_params_from_configs_json { local configs_json_path=${1} # E.g., /var/lib/cloud/instance/moodle_on_azure_configs.json - # (dpkg -l jq &> /dev/null) || (apt -y update; apt -y install jq) - # sudo add-apt-repository universe - # sudo apt-get -y update - # sudo apt-get -y install jq - # Added wget command to download jq. wget https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 -O /usr/bin/jq && chmod +x /usr/bin/jq @@ -83,12 +105,22 @@ function get_php_version { function install_php_mssql_driver { # Download and build php/mssql driver - /usr/bin/curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - /usr/bin/curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install msodbcsql mssql-tools unixodbc-dev -y - echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bash_profile - echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc + export AZ_REPO=$(lsb_release -cs) + export DEBIAN_FRONTEND='noninteractive' + export NEEDRESTART_MODE='a' + export ACCEPT_EULA='Y' + + mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/keyrings/microsoft.gpg && chmod go+r /etc/apt/keyrings/microsoft.gpg + + echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/ubuntu/22.04/prod $AZ_REPO main" > /etc/apt/sources.list.d/mssql-release.list + + wait_for_process apt && \ + apt_update_noninteractive && \ + apt --yes --no-install-recommends -qq install msodbcsql18 mssql-tools18 unixodbc-dev >> /tmp/apt.log + + echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bash_profile + echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc source ~/.bashrc #Build mssql driver @@ -234,9 +266,9 @@ EOF # TODO refactor these functions with the same ones in install_gluster.sh function scan_for_new_disks { - local BLACKLIST=${1} # E.g., /dev/sda|/dev/sdb + local ALLOWLIST=${1} # E.g., /dev/sda|/dev/sdb declare -a RET - local DEVS=$(ls -1 /dev/sd*|egrep -v "${BLACKLIST}"|egrep -v "[0-9]$") + local DEVS=$(ls -1 /dev/disk/azure/scsi1/lun*|egrep "${ALLOWLIST}"|egrep "[0-9]$") for DEV in ${DEVS}; do # Check each device if there is a "1" partition. If not, @@ -262,7 +294,7 @@ function create_raid0_ubuntu { if [ $_RET -eq 1 ]; then echo "installing mdadm" - sudo apt-get -y -q install mdadm + apt_install_noninteractive mdadm fi echo "Creating raid0" udevadm control --stop-exec-queue @@ -311,7 +343,7 @@ function setup_raid_disk_and_filesystem { local RAIDPARTITION=${3} # E.g., /dev/md1p1 local CREATE_FILESYSTEM=${4} # E.g., "" (true) or any non-empty string (false) - local DISKS=$(scan_for_new_disks "/dev/sda|/dev/sdb") + local DISKS=$(scan_for_new_disks "/dev/disk/azure/scsi1/lun0|/dev/disk/azure/scsi1/lun1") echo "Disks are ${DISKS}" declare -i DISKCOUNT local DISKCOUNT=$(echo "$DISKS" | wc -w) @@ -353,7 +385,7 @@ function configure_nfs_server_and_export { local MOUNTPOINT=${1} # E.g., /moodle echo "Installing nfs server..." - apt install -y nfs-kernel-server + apt_install_noninteractive nfs-kernel-server echo "Exporting ${MOUNTPOINT}..." grep -q -s "^${MOUNTPOINT}" /etc/exports && _RET=$? || _RET=$? diff --git a/scripts/install_elastic.sh b/scripts/install_elastic.sh index 999a44c..5e064b2 100644 --- a/scripts/install_elastic.sh +++ b/scripts/install_elastic.sh @@ -31,20 +31,48 @@ echo $elasticvm1ip >> /tmp/vars.txt echo $elasticvm2ip >> /tmp/vars.txt echo $elasticvm3ip >> /tmp/vars.txt +function wait_for_process { + until [ -z $(/usr/bin/pgrep ${1}) ]; do + printf '.' + sleep 0.5 + done +} + +function apt_update_noninteractive { + export DEBIAN_FRONTEND='noninteractive' + + # waiting for apt to finish before running any other commands + wait_for_process apt; + + apt --yes -qq -o=Dpkg::Use-Pty=0 update +} + +function apt_install_noninteractive { + export DEBIAN_FRONTEND='noninteractive' + export NEEDRESTART_MODE='a' + export ACCEPT_EULA='Y' + + # waiting for apt to finish before running any other commands + wait_for_process apt; + + apt --yes --no-install-recommends -qq -o=Dpkg::Use-Pty=0 -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install "${@}" +} + { # make sure the system does automatic update - sudo apt-get -y update - sudo apt-get -y install unattended-upgrades + apt_update_noninteractive + apt_install_noninteractive unattended-upgrades apt-transport-https # configure elastic search repository & install elastic search - wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list - sudo apt-get -y update - sudo apt-get -y install elasticsearch=5.5.0 + wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/5.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-5.x.list + + apt_update_noninteractive + apt_install_noninteractive elasticsearch=5.6.16 # install the required packages - sudo apt-get install -y openjdk-8-jre openjdk-8-jdk default-jre default-jdk + apt_install_noninteractive openjdk-8-jre openjdk-8-jdk default-jre default-jdk # Configure elasticsearch cat < /etc/elasticsearch/elasticsearch.yml @@ -70,7 +98,7 @@ cluster.name: ${esClusterName} # # Use a descriptive name for the node: # -node.name: \${HOSTNAME} +node.name: ${HOSTNAME} # # Add custom attributes to the node: # diff --git a/scripts/install_gluster.sh b/scripts/install_gluster.sh index dbb229a..73932e0 100644 --- a/scripts/install_gluster.sh +++ b/scripts/install_gluster.sh @@ -5,7 +5,9 @@ # You can also customize it to work with other Linux flavours and versions. # If you customize it, copy it to either Azure blob storage or Github so that Azure # custom script Linux VM extension can access it, and specify its location in the -# parameters of powershell script or runbook or Azure Resource Manager CRP template. +# parameters of powershell script or runbook or Azure Resource Manager CRP template. + +. ./helper_functions.sh AZUREVMOFFSET=4 @@ -35,8 +37,8 @@ RAIDPARTITION="/dev/md1p1" BLACKLIST="/dev/sda|/dev/sdb" # make sure the system does automatic update -sudo apt-get -y update -sudo apt-get -y install unattended-upgrades +apt_update_noninteractive +apt_install_noninteractive unattended-upgrades { check_os() { @@ -187,13 +189,13 @@ sudo apt-get -y install unattended-upgrades if [ ! -e /etc/apt/sources.list.d/gluster* ]; then echo "adding gluster ppa" - apt-get -y install python-software-properties - apt-add-repository -y ppa:gluster/glusterfs-3.10 - apt-get -y update + apt_install_noninteractive python-software-properties + apt-add-repository ppa:gluster/glusterfs-3.10 --yes + apt_update_noninteractive fi echo "installing gluster" - apt-get -y install glusterfs-server + apt_install_noninteractive glusterfs-server return } diff --git a/scripts/install_moodle.sh b/scripts/install_moodle.sh index eddfe97..89f0617 100644 --- a/scripts/install_moodle.sh +++ b/scripts/install_moodle.sh @@ -30,51 +30,47 @@ set -ex get_setup_params_from_configs_json $moodle_on_azure_configs_json_path || exit 99 - echo $moodleVersion >> /tmp/vars.txt - echo $glusterNode >> /tmp/vars.txt - echo $glusterVolume >> /tmp/vars.txt - echo $siteFQDN >> /tmp/vars.txt - echo $httpsTermination >> /tmp/vars.txt - echo $dbIP >> /tmp/vars.txt - echo $moodledbname >> /tmp/vars.txt - echo $moodledbuser >> /tmp/vars.txt - echo $moodledbpass >> /tmp/vars.txt - echo $adminpass >> /tmp/vars.txt - echo $dbadminlogin >> /tmp/vars.txt - echo $dbadminloginazure >> /tmp/vars.txt - echo $dbadminpass >> /tmp/vars.txt - echo $storageAccountName >> /tmp/vars.txt - echo $storageAccountKey >> /tmp/vars.txt - echo $azuremoodledbuser >> /tmp/vars.txt - echo $redisDns >> /tmp/vars.txt - echo $redisAuth >> /tmp/vars.txt - echo $elasticVm1IP >> /tmp/vars.txt - echo $installO365pluginsSwitch >> /tmp/vars.txt - echo $dbServerType >> /tmp/vars.txt - echo $fileServerType >> /tmp/vars.txt - echo $mssqlDbServiceObjectiveName >> /tmp/vars.txt - echo $mssqlDbEdition >> /tmp/vars.txt - echo $mssqlDbSize >> /tmp/vars.txt - echo $installObjectFsSwitch >> /tmp/vars.txt - echo $installGdprPluginsSwitch >> /tmp/vars.txt - echo $thumbprintSslCert >> /tmp/vars.txt - echo $thumbprintCaCert >> /tmp/vars.txt - echo $searchType >> /tmp/vars.txt - echo $azureSearchKey >> /tmp/vars.txt - echo $azureSearchNameHost >> /tmp/vars.txt - echo $tikaVmIP >> /tmp/vars.txt - echo $nfsByoIpExportPath >> /tmp/vars.txt - echo $storageAccountType >>/tmp/vars.txt - echo $fileServerDiskSize >>/tmp/vars.txt - echo $phpVersion >> /tmp/vars.txt - echo $isMigration >> /tmp/vars.txt + echo $moodleVersion >> /tmp/vars.txt + echo $glusterNode >> /tmp/vars.txt + echo $glusterVolume >> /tmp/vars.txt + echo $siteFQDN >> /tmp/vars.txt + echo $httpsTermination >> /tmp/vars.txt + echo $dbIP >> /tmp/vars.txt + echo $moodledbname >> /tmp/vars.txt + echo $moodledbuser >> /tmp/vars.txt + echo $moodledbpass >> /tmp/vars.txt + echo $adminpass >> /tmp/vars.txt + echo $dbadminlogin >> /tmp/vars.txt + echo $dbadminloginazure >> /tmp/vars.txt + echo $dbadminpass >> /tmp/vars.txt + echo $storageAccountName >> /tmp/vars.txt + echo $storageAccountKey >> /tmp/vars.txt + echo $azuremoodledbuser >> /tmp/vars.txt + echo $redisDns >> /tmp/vars.txt + echo $redisAuth >> /tmp/vars.txt + echo $elasticVm1IP >> /tmp/vars.txt + echo $installO365pluginsSwitch >> /tmp/vars.txt + echo $dbServerType >> /tmp/vars.txt + echo $fileServerType >> /tmp/vars.txt + echo $mssqlDbServiceObjectiveName >> /tmp/vars.txt + echo $mssqlDbEdition >> /tmp/vars.txt + echo $mssqlDbSize >> /tmp/vars.txt + echo $installObjectFsSwitch >> /tmp/vars.txt + echo $installGdprPluginsSwitch >> /tmp/vars.txt + echo $thumbprintSslCert >> /tmp/vars.txt + echo $thumbprintCaCert >> /tmp/vars.txt + echo $searchType >> /tmp/vars.txt + echo $azureSearchKey >> /tmp/vars.txt + echo $azureSearchNameHost >> /tmp/vars.txt + echo $tikaVmIP >> /tmp/vars.txt + echo $nfsByoIpExportPath >> /tmp/vars.txt + echo $storageAccountType >>/tmp/vars.txt + echo $fileServerDiskSize >>/tmp/vars.txt + echo $phpVersion >> /tmp/vars.txt + echo $isMigration >> /tmp/vars.txt check_fileServerType_param $fileServerType - #Updating php sources - sudo add-apt-repository ppa:ondrej/php -y - sudo apt-get update - if [ "$dbServerType" = "mysql" ]; then mysqlIP=$dbIP mysqladminlogin=$dbadminloginazure @@ -83,7 +79,6 @@ set -ex mssqlIP=$dbIP mssqladminlogin=$dbadminloginazure mssqladminpass=$dbadminpass - elif [ "$dbServerType" = "postgres" ]; then postgresIP=$dbIP pgadminlogin=$dbadminloginazure @@ -93,51 +88,53 @@ set -ex exit 1 fi - # make sure system does automatic updates and fail2ban - sudo apt-get -y update - sudo apt-get -y install unattended-upgrades fail2ban + # + # Export apt default settings for this install script + # + + apt_update_noninteractive >> /tmp/apt.log + apt_install_noninteractive fail2ban >> /tmp/apt.log config_fail2ban # create gluster, nfs or Azure Files mount point mkdir -p /moodle - export DEBIAN_FRONTEND=noninteractive - if [ $fileServerType = "gluster" ]; then - # configure gluster repository & install gluster client - sudo add-apt-repository ppa:gluster/glusterfs-3.10 -y >> /tmp/apt1.log + # configure gluster repository & install gluster clientapt + add-apt-repository ppa:gluster/glusterfs-9 --yes >> /tmp/apt.log elif [ $fileServerType = "nfs" ]; then # configure NFS server and export setup_raid_disk_and_filesystem /moodle /dev/md1 /dev/md1p1 configure_nfs_server_and_export /moodle fi - sudo apt-get -y update >> /tmp/apt2.log - sudo apt-get -y --force-yes install rsyslog git >> /tmp/apt3.log + apt_update_noninteractive >> /tmp/apt.log + apt_install_noninteractive rsyslog git >> /tmp/apt.log if [ $fileServerType = "gluster" ]; then - sudo apt-get -y --force-yes install glusterfs-client >> /tmp/apt3.log + apt_install_noninteractive glusterfs-client >> /tmp/apt.log elif [ "$fileServerType" = "azurefiles" ]; then - sudo apt-get -y --force-yes install cifs-utils >> /tmp/apt3.log + apt_install_noninteractive cifs-utils >> /tmp/apt.log fi if [ $dbServerType = "mysql" ]; then - sudo apt-get -y --force-yes install mysql-client >> /tmp/apt3.log + apt_install_noninteractive mysql-client >> /tmp/apt.log elif [ "$dbServerType" = "postgres" ]; then - sudo apt-get -y --force-yes install postgresql-client >> /tmp/apt3.log + apt_install_noninteractive postgresql-client >> /tmp/apt.log fi - - if [ "$installObjectFsSwitch" = "true" -o "$fileServerType" = "azurefiles" ]; then - # install azure cli & setup container - AZ_REPO=$(lsb_release -cs) - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list - curl -L https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - >> /tmp/apt4.log - sudo apt-get -y install apt-transport-https >> /tmp/apt4.log - sudo apt-get -y update > /dev/null - sudo apt-get -y install azure-cli >> /tmp/apt4.log - + if [ "$installObjectFsSwitch" = "true" -o "$fileServerType" = "azurefiles" ]; then + # install azure cli + AZ_REPO=$(lsb_release -cs) + mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/keyrings/microsoft.gpg && chmod go+r /etc/apt/keyrings/microsoft.gpg + echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" > /etc/apt/sources.list.d/azure-cli.list + + apt_update_noninteractive >> /tmp/apt.log + + apt_install_noninteractive apt-transport-https ca-certificates curl apt-transport-https lsb-release gnupg azure-cli >> /tmp/apt.log + # FileStorage accounts can only be used to store Azure file shares; # Premium_LRS will support FileStorage kind # No other storage resources (blob containers, queues, tables, etc.) can be deployed in a FileStorage account. @@ -191,32 +188,47 @@ set -ex echo -e '\n\rMounting NFS export from '$nfsByoIpExportPath' on /moodle\n\r' configure_nfs_client_and_mount0 $nfsByoIpExportPath /moodle fi - - # install pre-requisites - sudo add-apt-repository ppa:ubuntu-toolchain-r/ppa - sudo apt-get -y update > /dev/null 2>&1 - # sudo apt-get install -y --fix-missing python-software-properties unzip - sudo apt-get -y install software-properties-common - sudo apt-get -y install unzip + # install pre-requisites + add-apt-repository ppa:ubuntu-toolchain-r/ppa --yes >> /tmp/apt.log + apt_update_noninteractive >> /tmp/apt.log + apt_install_noninteractive software-properties-common unzip >> /tmp/apt.log # install the entire stack - # passing php versions $phpVersion - sudo apt-get -y --force-yes install nginx php$phpVersion-fpm varnish >> /tmp/apt5a.log - sudo apt-get -y --force-yes install php$phpVersion php$phpVersion-cli php$phpVersion-curl php$phpVersion-zip >> /tmp/apt5b.log + apt_install_noninteractive \ + nginx \ + varnish \ + php \ + php-fpm \ + php-cli \ + php-curl \ + php-zip \ + graphviz \ + aspell \ + php-common \ + php-soap \ + php-json \ + php-redis \ + php-bcmath \ + php-ldap \ + php-gd \ + php-xmlrpc \ + php-intl \ + php-xml \ + php-bz2 \ + php-pear \ + php-mbstring \ + php-dev \ + mcrypt >> /tmp/apt.log - # Moodle requirements - sudo apt-get -y update > /dev/null - sudo apt-get install -y --force-yes graphviz aspell php$phpVersion-common php$phpVersion-soap php$phpVersion-json php$phpVersion-redis > /tmp/apt6.log - sudo apt-get install -y --force-yes php$phpVersion-bcmath php$phpVersion-ldap php$phpVersion-gd php$phpVersion-xmlrpc php$phpVersion-intl php$phpVersion-xml php$phpVersion-bz2 php-pear php$phpVersion-mbstring php$phpVersion-dev mcrypt >> /tmp/apt6.log PhpVer=$(get_php_version) if [ $dbServerType = "mysql" ]; then - sudo apt-get install -y --force-yes php$phpVersion-mysql + apt_install_noninteractive php-mysql elif [ $dbServerType = "mssql" ]; then - sudo apt-get install -y libapache2-mod-php # Need this because install_php_mssql_driver tries to update apache2-mod-php settings always (which will fail without this) + apt_install_noninteractive libapache2-mod-php install_php_mssql_driver else - sudo apt-get install -y --force-yes php-pgsql + apt_install_noninteractive php-pgsql fi # Set up initial moodle dirs @@ -229,68 +241,69 @@ set -ex moodleUnzipDir=$(get_moodle_unzip_dir_from_moodle_version $moodleVersion) # install Moodle - echo '#!/bin/bash - mkdir -p /moodle/tmp - cd /moodle/tmp + cat < /tmp/setup-moodle.sh +#!/bin/bash +mkdir -p /moodle/tmp +cd /moodle/tmp - if [ ! -d /moodle/html/moodle ]; then - # downloading moodle only if /moodle/html/moodle does not exist -- if it exists, user should populate it in advance correctly as below. This is to reduce template deployment time. - /usr/bin/curl -k --max-redirs 10 https://github.com/moodle/moodle/archive/'$moodleVersion'.zip -L -o moodle.zip - /usr/bin/unzip -q moodle.zip - /bin/mv '$moodleUnzipDir' /moodle/html/moodle - fi +if [ ! -d /moodle/html/moodle ]; then + # downloading moodle only if /moodle/html/moodle does not exist -- if it exists, user should populate it in advance correctly as below. This is to reduce template deployment time. + /usr/bin/curl -k --max-redirs 10 https://github.com/moodle/moodle/archive/MOODLE_402_STABLE.zip -L -o moodle.zip + /usr/bin/unzip -q moodle.zip + /bin/mv "$moodleUnzipDir" /moodle/html/moodle +fi - if [ "'$installGdprPluginsSwitch'" = "true" ]; then - # install Moodle GDPR plugins (Note: This is only for Moodle versions 3.4.2+ or 3.3.5+ and will be included in Moodle 3.5, so no need for 3.5) - curl -k --max-redirs 10 https://github.com/moodlehq/moodle-tool_policy/archive/'$moodleStableVersion'.zip -L -o plugin-policy.zip - unzip -q plugin-policy.zip - mv moodle-tool_policy-'$moodleStableVersion' /moodle/html/moodle/admin/tool/policy +if [ "$installGdprPluginsSwitch" = "true" ]; then + # install Moodle GDPR plugins (Note: This is only for Moodle versions 3.4.2+ or 3.3.5+ and will be included in Moodle 3.5, so no need for 3.5) + curl -k --max-redirs 10 https://github.com/moodlehq/moodle-tool_policy/archive/"$moodleStableVersion".zip -L -o plugin-policy.zip + unzip -q plugin-policy.zip + mv moodle-tool_policy-"$moodleStableVersion" /moodle/html/moodle/admin/tool/policy - curl -k --max-redirs 10 https://github.com/moodlehq/moodle-tool_dataprivacy/archive/'$moodleStableVersion'.zip -L -o plugin-dataprivacy.zip - unzip -q plugin-dataprivacy.zip - mv moodle-tool_dataprivacy-'$moodleStableVersion' /moodle/html/moodle/admin/tool/dataprivacy - fi + curl -k --max-redirs 10 https://github.com/moodlehq/moodle-tool_dataprivacy/archive/"$moodleStableVersion".zip -L -o plugin-dataprivacy.zip + unzip -q plugin-dataprivacy.zip + mv moodle-tool_dataprivacy-"$moodleStableVersion" /moodle/html/moodle/admin/tool/dataprivacy +fi - if [ "'$installO365pluginsSwitch'" = "true" ]; then - # install Office 365 plugins - curl -k --max-redirs 10 https://github.com/Microsoft/o365-moodle/archive/'$o365pluginVersion'.zip -L -o o365.zip - unzip -q o365.zip - cp -r o365-moodle-'$o365pluginVersion'/* /moodle/html/moodle - rm -rf o365-moodle-'$o365pluginVersion' - fi +if [ "$installO365pluginsSwitch" = "true" ]; then + # install Office 365 plugins + curl -k --max-redirs 10 https://github.com/Microsoft/o365-moodle/archive/"$o365pluginVersion".zip -L -o o365.zip + unzip -q o365.zip + cp -r o365-moodle-"$o365pluginVersion"/* /moodle/html/moodle + rm -rf o365-moodle-"$o365pluginVersion" +fi - if [ "'$searchType'" = "elastic" ]; then - # Install ElasticSearch plugin - /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-search_elastic/archive/master.zip -L -o plugin-elastic.zip - /usr/bin/unzip -q plugin-elastic.zip - /bin/mv moodle-search_elastic-master /moodle/html/moodle/search/engine/elastic +if [ "$searchType" = "elastic" ]; then + # Install ElasticSearch plugin + /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-search_elastic/archive/master.zip -L -o plugin-elastic.zip + /usr/bin/unzip -q plugin-elastic.zip + /bin/mv moodle-search_elastic-master /moodle/html/moodle/search/engine/elastic - # Install ElasticSearch plugin dependency - /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-local_aws/archive/master.zip -L -o local-aws.zip - /usr/bin/unzip -q local-aws.zip - /bin/mv moodle-local_aws-master /moodle/html/moodle/local/aws + # Install ElasticSearch plugin dependency + /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-local_aws/archive/master.zip -L -o local-aws.zip + /usr/bin/unzip -q local-aws.zip + /bin/mv moodle-local_aws-master /moodle/html/moodle/local/aws - elif [ "'$searchType'" = "azure" ]; then - # Install Azure Search service plugin - /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-search_azure/archive/master.zip -L -o plugin-azure-search.zip - /usr/bin/unzip -q plugin-azure-search.zip - /bin/mv moodle-search_azure-master /moodle/html/moodle/search/engine/azure - fi +elif [ "$searchType" = "azure" ]; then + # Install Azure Search service plugin + /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-search_azure/archive/master.zip -L -o plugin-azure-search.zip + /usr/bin/unzip -q plugin-azure-search.zip + /bin/mv moodle-search_azure-master /moodle/html/moodle/search/engine/azure +fi - if [ "'$installObjectFsSwitch'" = "true" ]; then - # Install the ObjectFS plugin - /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-tool_objectfs/archive/master.zip -L -o plugin-objectfs.zip - /usr/bin/unzip -q plugin-objectfs.zip - /bin/mv moodle-tool_objectfs-master /moodle/html/moodle/admin/tool/objectfs +if [ "$installObjectFsSwitch" = "true" ]; then + # Install the ObjectFS plugin + /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-tool_objectfs/archive/master.zip -L -o plugin-objectfs.zip + /usr/bin/unzip -q plugin-objectfs.zip + /bin/mv moodle-tool_objectfs-master /moodle/html/moodle/admin/tool/objectfs - # Install the ObjectFS Azure library - /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-local_azure_storage/archive/master.zip -L -o plugin-azurelibrary.zip - /usr/bin/unzip -q plugin-azurelibrary.zip - /bin/mv moodle-local_azure_storage-master /moodle/html/moodle/local/azure_storage - fi - cd /moodle - rm -rf /moodle/tmp - ' > /tmp/setup-moodle.sh + # Install the ObjectFS Azure library + /usr/bin/curl -k --max-redirs 10 https://github.com/catalyst/moodle-local_azure_storage/archive/master.zip -L -o plugin-azurelibrary.zip + /usr/bin/unzip -q plugin-azurelibrary.zip + /bin/mv moodle-local_azure_storage-master /moodle/html/moodle/local/azure_storage +fi +cd /moodle +rm -rf /moodle/tmp +EOF chmod 755 /tmp/setup-moodle.sh /tmp/setup-moodle.sh >> /tmp/setupmoodle.log @@ -473,12 +486,12 @@ EOF chmod 0400 /moodle/certs/nginx.* fi - # php config + # PHP 8 fpm config PhpVer=$(get_php_version) PhpIni=/etc/php/${PhpVer}/fpm/php.ini sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni - sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni + sed -i "s/;max_input_vars.*/max_input_vars = 100000/" $PhpIni sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni @@ -490,6 +503,10 @@ EOF sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni + # required for PHP8 cli at install time + PhpIniCli=/etc/php/${PhpVer}/cli/php.ini + sed -i "s/;max_input_vars.*/max_input_vars = 100000/" $PhpIniCli + # fpm config - overload this cat < /etc/php/${PhpVer}/fpm/pool.d/www.conf [www] @@ -764,11 +781,10 @@ EOF service varnish restart if [ $dbServerType = "mysql" ]; then - mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "CREATE DATABASE ${moodledbname} CHARACTER SET utf8;" - mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "GRANT ALL ON ${moodledbname}.* TO ${moodledbuser} IDENTIFIED BY '${moodledbpass}';" + mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "CREATE DATABASE ${moodledbname} DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" | tee -a /tmp/debug + mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "CREATE USER ${moodledbuser}@'%' IDENTIFIED BY '${moodledbpass}';" | tee -a /tmp/debug + mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e "GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,CREATE TEMPORARY TABLES,DROP,INDEX,ALTER ON ${moodledbname}.* TO ${moodledbuser}@'%';" | tee -a /tmp/debug - echo "mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e \"CREATE DATABASE ${moodledbname};\"" >> /tmp/debug - echo "mysql -h $mysqlIP -u $mysqladminlogin -p${mysqladminpass} -e \"GRANT ALL ON ${moodledbname}.* TO ${moodledbuser} IDENTIFIED BY '${moodledbpass}';\"" >> /tmp/debug elif [ $dbServerType = "mssql" ]; then /opt/mssql-tools/bin/sqlcmd -S $mssqlIP -U $mssqladminlogin -P ${mssqladminpass} -Q "CREATE DATABASE ${moodledbname} ( MAXSIZE = $mssqlDbSize, EDITION = '$mssqlDbEdition', SERVICE_OBJECTIVE = '$mssqlDbServiceObjectiveName' )" /opt/mssql-tools/bin/sqlcmd -S $mssqlIP -U $mssqladminlogin -P ${mssqladminpass} -Q "CREATE LOGIN ${moodledbuser} with password = '${moodledbpass}'" @@ -912,7 +928,7 @@ EOF sed -i "23 a \$CFG->searchengine = 'elastic';" /moodle/html/moodle/config.php sed -i "23 a \$CFG->enableglobalsearch = 'true';" /moodle/html/moodle/config.php # create index - php /moodle/html/moodle/search/cli/indexer.php --force --reindex + php /moodle/html/moodle/search/cli/indexer.php --force --reindex || true elif [ "$searchType" = "azure" ]; then # Set up Azure Search service plugin @@ -925,7 +941,7 @@ EOF sed -i "23 a \$CFG->searchengine = 'azure';" /moodle/html/moodle/config.php sed -i "23 a \$CFG->enableglobalsearch = 'true';" /moodle/html/moodle/config.php # create index - php /moodle/html/moodle/search/cli/indexer.php --force --reindex + php /moodle/html/moodle/search/cli/indexer.php --force --reindex || true fi @@ -937,7 +953,7 @@ EOF if [ "$dbServerType" = "postgres" ]; then # Get a new version of Postgres to match Azure version add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE='a' apt -key add - apt-get update apt-get install -y postgresql-client-9.6 fi diff --git a/scripts/setup_nfs_ha.sh b/scripts/setup_nfs_ha.sh index d879d02..38b8686 100644 --- a/scripts/setup_nfs_ha.sh +++ b/scripts/setup_nfs_ha.sh @@ -20,8 +20,8 @@ MY_IP=$(hostname -i) function setup_required_packages { - apt update - apt -y install build-essential autoconf flex nfs-kernel-server corosync pacemaker resource-agents + apt_update_noninteractive + apt_install_noninteractive build-essential autoconf flex nfs-kernel-server corosync pacemaker resource-agents # Shouldn't let systemd start nfs-kernel-server (Pacemaker should do that) systemctl stop nfs-kernel-server diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index e495bcb..74d7731 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -1,6 +1,5 @@ -# Custom Script for Linux - #!/bin/bash +# Custom Script for Linux # The MIT License (MIT) # @@ -22,55 +21,48 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. set -ex -echo "### Script Start `date`###" +echo "### Script Start $(date)###" moodle_on_azure_configs_json_path=${1} . ./helper_functions.sh -get_setup_params_from_configs_json $moodle_on_azure_configs_json_path || exit 99 +get_setup_params_from_configs_json "$moodle_on_azure_configs_json_path" || exit 99 -echo $glusterNode >> /tmp/vars.txt -echo $glusterVolume >> /tmp/vars.txt -echo $siteFQDN >> /tmp/vars.txt -echo $httpsTermination >> /tmp/vars.txt -echo $syslogServer >> /tmp/vars.txt -echo $webServerType >> /tmp/vars.txt -echo $dbServerType >> /tmp/vars.txt -echo $fileServerType >> /tmp/vars.txt -echo $storageAccountName >> /tmp/vars.txt -echo $storageAccountKey >> /tmp/vars.txt -echo $nfsVmName >> /tmp/vars.txt -echo $nfsByoIpExportPath >> /tmp/vars.txt -echo $htmlLocalCopySwitch >> /tmp/vars.txt -echo $phpVersion >> /tmp/vars.txt +echo "$glusterNode" >> /tmp/vars.txt +echo "$glusterVolume" >> /tmp/vars.txt +echo "$siteFQDN" >> /tmp/vars.txt +echo "$httpsTermination" >> /tmp/vars.txt +echo "$syslogServer" >> /tmp/vars.txt +echo "$webServerType" >> /tmp/vars.txt +echo "$dbServerType" >> /tmp/vars.txt +echo "$fileServerType" >> /tmp/vars.txt +echo "$storageAccountName" >> /tmp/vars.txt +echo "$storageAccountKey" >> /tmp/vars.txt +echo "$nfsVmName" >> /tmp/vars.txt +echo "$nfsByoIpExportPath" >> /tmp/vars.txt +echo "$htmlLocalCopySwitch" >> /tmp/vars.txt +echo "$phpVersion" >> /tmp/vars.txt -check_fileServerType_param $fileServerType +check_fileServerType_param "$fileServerType" { set -ex - echo "### Function Start `date`###" + echo "### Function Start $(date)###" - # add azure-cli repository - curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null + # add azure-cli repository && install azure cli AZ_REPO=$(lsb_release -cs) - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list - - # add PHP-FPM repository - add-apt-repository ppa:ondrej/php -y > /dev/null 2>&1 + mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/keyrings/microsoft.gpg && chmod go+r /etc/apt/keyrings/microsoft.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" > /etc/apt/sources.list.d/azure-cli.list - apt-get -qq -o=Dpkg::Use-Pty=0 update + apt_update_noninteractive # install pre-requisites including VARNISH and PHP-FPM - export DEBIAN_FRONTEND=noninteractive - apt-get --yes \ - --no-install-recommends \ - -qq -o=Dpkg::Use-Pty=0 \ - -o Dpkg::Options::="--force-confdef" \ - -o Dpkg::Options::="--force-confold" \ - install \ + + apt_install_noninteractive \ azure-cli \ ca-certificates \ curl \ @@ -85,28 +77,28 @@ check_fileServerType_param $fileServerType unattended-upgrades \ tuned \ varnish \ - php$phpVersion \ - php$phpVersion-cli \ - php$phpVersion-curl \ - php$phpVersion-zip \ + php \ + php-cli \ + php-curl \ + php-zip \ php-pear \ - php$phpVersion-mbstring \ + php-mbstring \ mcrypt \ - php$phpVersion-dev \ + php-dev \ graphviz \ aspell \ - php$phpVersion-soap \ - php$phpVersion-json \ - php$phpVersion-redis \ - php$phpVersion-bcmath \ - php$phpVersion-ldap \ - php$phpVersion-gd \ - php$phpVersion-pgsql \ - php$phpVersion-mysql \ - php$phpVersion-xmlrpc \ - php$phpVersion-intl \ - php$phpVersion-xml \ - php$phpVersion-bz2 + php-soap \ + php-json \ + php-redis \ + php-bcmath \ + php-ldap \ + php-gd \ + php-pgsql \ + php-mysql \ + php-xmlrpc \ + php-intl \ + php-xml \ + php-bz2 # install azcopy wget -q -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 && mv ./azcopy /usr/bin/ @@ -138,42 +130,42 @@ EOF systemctl enable tuned tuned-adm profile throughput-performance - if [ $fileServerType = "gluster" ]; then + if [ "$fileServerType" = "gluster" ]; then #configure gluster repository & install gluster client - add-apt-repository ppa:gluster/glusterfs-3.10 -y - apt-get -y update - apt-get -y -qq -o=Dpkg::Use-Pty=0 install glusterfs-client + add-apt-repository ppa:gluster/glusterfs-9 --yes + apt_update_noninteractive + apt_install_noninteractive glusterfs-client elif [ "$fileServerType" = "azurefiles" ]; then - apt-get -y -qq -o=Dpkg::Use-Pty=0 install cifs-utils + apt_install_noninteractive cifs-utils fi if [ "$webServerType" = "nginx" -o "$httpsTermination" = "VMSS" ]; then - apt-get --yes -qq -o=Dpkg::Use-Pty=0 install nginx + apt_install_noninteractive nginx fi if [ "$webServerType" = "apache" ]; then # install apache pacakges - apt-get --yes -qq -o=Dpkg::Use-Pty=0 install apache2 libapache2-mod-php + apt_install_noninteractive apache2 libapache2-mod-php else # for nginx-only option - apt-get --yes -qq -o=Dpkg::Use-Pty=0 install php$phpVersion-fpm + apt_install_noninteractive php-fpm fi - + # Moodle requirements if [ "$dbServerType" = "mssql" ]; then install_php_mssql_driver fi - + # PHP Version PhpVer=$(get_php_version) - if [ $fileServerType = "gluster" ]; then + if [ "$fileServerType" = "gluster" ]; then # Mount gluster fs for /moodle sudo mkdir -p /moodle sudo chown www-data /moodle sudo chmod 770 /moodle sudo echo -e 'Adding Gluster FS to /etc/fstab and mounting it' - setup_and_mount_gluster_moodle_share $glusterNode $glusterVolume + setup_and_mount_gluster_moodle_share "$glusterNode" "$glusterVolume" elif [ $fileServerType = "nfs" ]; then # mount NFS export (set up on controller VM--No HA) echo -e '\n\rMounting NFS export from '$nfsVmName':/moodle on /moodle and adding it to /etc/fstab\n\r' @@ -209,7 +201,7 @@ worker_processes auto; pid /run/nginx.pid; events { - worker_connections 8192; + worker_connections 8192; multi_accept on; use epoll; } @@ -286,7 +278,7 @@ EOF mkdir -p /var/www/html ACCOUNT_KEY="$storageAccountKey" NAME="$storageAccountName" - END=`date -u -d "60 minutes" '+%Y-%m-%dT%H:%M:00Z'` + END=$(date -u -d "60 minutes" '+%Y-%m-%dT%H:%M:00Z') htmlRootDir="/var/www/html/moodle" sas=$(az storage share generate-sas \ @@ -320,7 +312,6 @@ server { root ${htmlRootDir}; index index.php index.html index.htm; - ssl on; ssl_certificate /moodle/certs/nginx.crt; ssl_certificate_key /moodle/certs/nginx.key; ssl_session_timeout 1d; @@ -365,7 +356,7 @@ server { listen 81 default; server_name ${siteFQDN}; root ${htmlRootDir}; - index index.php index.html index.htm; + index index.php index.html index.htm; # Log to syslog error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; @@ -394,9 +385,9 @@ EOF return 404; } - location / { - try_files \$uri \$uri/index.php?\$query_string; - } + location / { + try_files \$uri \$uri/index.php?\$query_string; + } location ~ [^/]\.php(/|$) { fastcgi_split_path_info ^(.+?\.php)(/.*)$; @@ -418,7 +409,7 @@ EOF upstream backend { server unix:/run/php/php${PhpVer}-fpm.sock fail_timeout=1s; server unix:/run/php/php${PhpVer}-fpm-backup.sock backup; -} +} EOF fi @@ -430,16 +421,16 @@ EOF cat <> /etc/apache2/sites-enabled/${siteFQDN}.conf - ServerName ${siteFQDN} + ServerName ${siteFQDN} - ServerAdmin webmaster@localhost - DocumentRoot ${htmlRootDir} + ServerAdmin webmaster@localhost + DocumentRoot ${htmlRootDir} - - Options FollowSymLinks - AllowOverride All - Require all granted - + + Options FollowSymLinks + AllowOverride All + Require all granted + EOF if [ "$httpsTermination" != "None" ]; then cat <> /etc/apache2/sites-enabled/${siteFQDN}.conf @@ -456,7 +447,7 @@ EOF SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" forwarded - ErrorLog "|/usr/bin/logger -t moodle -p local1.error" + ErrorLog "|/usr/bin/logger -t moodle -p local1.error" CustomLog "|/usr/bin/logger -t moodle -p local1.notice" combined env=!forwarded CustomLog "|/usr/bin/logger -t moodle -p local1.notice" forwarded env=forwarded @@ -472,7 +463,7 @@ EOF fi sed -i "s/memory_limit.*/memory_limit = 512M/" $PhpIni sed -i "s/max_execution_time.*/max_execution_time = 18000/" $PhpIni - sed -i "s/max_input_vars.*/max_input_vars = 100000/" $PhpIni + sed -i "s/;max_input_vars.*/max_input_vars = 100000/" $PhpIni sed -i "s/max_input_time.*/max_input_time = 600/" $PhpIni sed -i "s/upload_max_filesize.*/upload_max_filesize = 1024M/" $PhpIni sed -i "s/post_max_size.*/post_max_size = 1056M/" $PhpIni @@ -483,7 +474,7 @@ EOF sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 512/" $PhpIni sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 20000/" $PhpIni - + # Remove the default site. Moodle is the only site we want rm -f /etc/nginx/sites-enabled/default if [ "$webServerType" = "apache" ]; then @@ -750,7 +741,7 @@ EOF if [ "$webServerType" = "nginx" ]; then if pgrep -x "$service" >/dev/null then - echo “Stop the $service!!!” + echo "Stop the $service!!!" systemctl stop $service else systemctl mask $service @@ -799,5 +790,5 @@ EOF service apache2 restart fi - echo "### Script End `date`###" + echo "### Script End $(date)###" } 2>&1 | tee /tmp/setup.log