257 строки
8.2 KiB
Bash
257 строки
8.2 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Script to set up highly available NFS server on an Ubuntu 16.04 (or higher) VM
|
|
# that should be used on Azure with the custom script extension (which runs this script as root)
|
|
#
|
|
|
|
set -e
|
|
|
|
# Parameters
|
|
NODE1NAME=$1
|
|
NODE1IP=$2
|
|
NODE2NAME=$3
|
|
NODE2IP=$4
|
|
NFS_CLIENTS_IP_RANGE=$5 # E.g., "10.0.0.0/24". Can be "*", but strongly discouraged
|
|
|
|
# This VM's IP address, to detect if this VM should be the master (Node 1 is the initial master)
|
|
MY_IP=$(hostname -i)
|
|
|
|
. ./helper_functions.sh
|
|
|
|
function setup_required_packages
|
|
{
|
|
apt update
|
|
apt -y install build-essential autoconf flex nfs-kernel-server corosync pacemaker resource-agents
|
|
|
|
# Shouldn't let systemd start nfs-kernel-server (Pacemaker should do that)
|
|
systemctl stop nfs-kernel-server
|
|
systemctl disable nfs-kernel-server
|
|
|
|
# Setup static port assignments for mountd, statd, quotad, nlm (tcp), and nlm (udp) respectively:
|
|
sed -i 's/^\(RPCMOUNTDOPTS="--manage-gids\)"/\1 -p 2000"/g' /etc/default/nfs-kernel-server
|
|
sed -i 's/^STATDOPTS=.*$/STATDOPTS="--port 2001 --outgoing-port 2002"/' /etc/default/nfs-common
|
|
if [ -f /etc/default/quota ]; then
|
|
sed -i 's/^RPCQUOTADOPTS=.*$/RPCQUOTADOPTS="-p 2003"/' /etc/default/quota
|
|
fi
|
|
cat <<EOF > /etc/modprobe.d/azmdl-nfs-ports.conf
|
|
options lockd nlm_udpport=2004 nlm_tcpport=2004
|
|
options nfs callback_tcpport=2005
|
|
EOF
|
|
|
|
cat <<EOF > /etc/sysctl.d/30-azmdl-nfs-ports.conf
|
|
fs.nfs.nlm_tcpport=2004
|
|
fs.nfs.nlm_udpport=2004
|
|
EOF
|
|
# Reread modified sysctl settings for modified NFS static ports
|
|
sysctl --system
|
|
|
|
# Above alone still doesn't work for static ports. Try restarting related services.
|
|
systemctl try-restart nfs-config.service rpcbind.service rpc-statd.service nfs-server.service
|
|
|
|
# We need to install the "azure-lb" command separately if the resource-agents package didn't have it.
|
|
pushd /usr/lib/ocf/resource.d/heartbeat
|
|
if [ ! -e azure-lb ]; then
|
|
curl -LO https://raw.githubusercontent.com/ClusterLabs/resource-agents/master/heartbeat/azure-lb
|
|
chmod +x ./azure-lb
|
|
fi
|
|
popd
|
|
}
|
|
|
|
function setup_drbd_module_and_tools
|
|
{
|
|
# We currently have to build the DRBD kernel module, as it's not included
|
|
# in the default linux-azure kernels or in any Azure extra packages.
|
|
# We should use the packaged DRBD module once Azure starts releasing
|
|
# extra modules packages that include DRBD module.
|
|
|
|
pushd /tmp
|
|
git clone http://github.com/LINBIT/drbd-9.0
|
|
git clone http://github.com/LINBIT/drbd-utils
|
|
cd drbd-9.0
|
|
make && make install
|
|
modprobe drbd
|
|
cd ../drbd-utils
|
|
./autogen.sh
|
|
./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc
|
|
make tools && make install-tools
|
|
cd ..
|
|
rm -rf drbd-9.0 drbd-utils
|
|
popd
|
|
}
|
|
|
|
function setup_drbd_with_disk
|
|
{
|
|
local disk=$1
|
|
local node1name=$2
|
|
local node1ip=$3
|
|
local node2name=$4
|
|
local node2ip=$5
|
|
local drbd_resource_name=$6
|
|
local drbd_device_path=$7
|
|
local drbd_device_mount_point=$8
|
|
|
|
# Put LVM (to gain more flexibility) on the whole disk
|
|
local vgname=drbdvg
|
|
local lvname=drbdlv
|
|
wipefs -af $disk
|
|
pvcreate $disk
|
|
vgcreate $vgname $disk
|
|
lvcreate -n $lvname -l 95%VG $vgname
|
|
|
|
# Set up DRBD config
|
|
cat <<EOF > /etc/drbd.d/${drbd_resource_name}.res
|
|
resource $drbd_resource_name {
|
|
device ${drbd_device_path};
|
|
disk /dev/${vgname}/${lvname};
|
|
meta-disk internal;
|
|
disk {
|
|
c-fill-target 1M;
|
|
c-max-rate 110M;
|
|
c-min-rate 120K;
|
|
}
|
|
net {
|
|
max-buffers 20k;
|
|
}
|
|
on ${node1name} {
|
|
address ${node1ip}:7789;
|
|
}
|
|
on ${node2name} {
|
|
address ${node2ip}:7789;
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Initialize DRBD's metadata and bring up the device on both nodes
|
|
drbdadm create-md $drbd_resource_name
|
|
drbdadm up $drbd_resource_name
|
|
drbdadm status
|
|
|
|
# On the master (initially node 1) only, force DRBD into Primary and create an ext4 file system on the DRBD device
|
|
if [ "$MY_IP" = "$NODE1IP" ]; then
|
|
drbdadm primary $drbd_resource_name --force
|
|
mkfs.ext4 $drbd_device_path
|
|
mkdir -p $drbd_device_mount_point && mount $drbd_device_path $drbd_device_mount_point
|
|
fi
|
|
}
|
|
|
|
function setup_corosync_and_pacemaker_for_nfs
|
|
{
|
|
local node1ip=$1
|
|
local node2ip=$2
|
|
local drbd_resource_name=$3 # E.g., azmdlr0
|
|
local drbd_device_path=$4 # E.g., /dev/drbd0
|
|
local drbd_mount_point=$5 # E.g., /drbd
|
|
local nfs_export_path=$6 # E.g., /drbd/lamp
|
|
local nfs_client_spec=$7 # E.g., * or 10.11.22.0/24
|
|
|
|
mv /etc/corosync/corosync.conf /etc/corosync/corosync.conf.orig || true
|
|
|
|
local cluster_name=azmdl-cluster
|
|
|
|
cat <<EOF > /etc/corosync/corosync.conf
|
|
totem {
|
|
version: 2
|
|
secauth: off
|
|
cluster_name: ${cluster_name}
|
|
transport: udpu
|
|
}
|
|
|
|
nodelist {
|
|
node {
|
|
ring0_addr: ${node1ip}
|
|
nodeid: 1
|
|
}
|
|
node {
|
|
ring0_addr: ${node2ip}
|
|
nodeid: 2
|
|
}
|
|
}
|
|
|
|
quorum {
|
|
provider: corosync_votequorum
|
|
two_node:1
|
|
}
|
|
|
|
logging {
|
|
to_syslog: yes
|
|
}
|
|
EOF
|
|
|
|
systemctl enable corosync pacemaker
|
|
systemctl restart corosync pacemaker
|
|
|
|
# TODO Should confirm if 'corosync-cfgtool -s' gives a non-loopback IP address (not 127.0.0.1), e.g.:
|
|
# $ corosync-cfgtool -s
|
|
# Printing ring status.
|
|
# Local node ID 2
|
|
# RING ID 0
|
|
# id = 10.0.0.5
|
|
# status = ring 0 active with no faults
|
|
|
|
# Finally, configure Pacemaker cluster resources, only on the initial master
|
|
if [ "$MY_IP" = "$node1ip" ]; then
|
|
mkdir -p ${nfs_export_path}
|
|
crm configure <<EOF
|
|
primitive p_drbd_r0 ocf:linbit:drbd \
|
|
params drbd_resource=${drbd_resource_name} \
|
|
op monitor interval=29s role=Master \
|
|
op monitor interval=31s role=Slave \
|
|
op start interval=0s timeout=240s \
|
|
op stop interval=0s timeout=100s
|
|
ms ms_drbd_r0 p_drbd_r0 \
|
|
meta master-max=1 master-node-max=1 clone-node-max=1 clone-max=2 notify=true
|
|
primitive p_fs_data Filesystem \
|
|
params device="${drbd_device_path}" directory="${drbd_mount_point}" \
|
|
fstype=ext4 options=noatime,nodiratime \
|
|
op start interval=0s timeout=100s \
|
|
op stop interval=0s timeout=100s \
|
|
op monitor interval=10s timeout=100s
|
|
primitive p_nfsserver ocf:heartbeat:nfsserver \
|
|
params nfs_shared_infodir="${drbd_mount_point}/nfs_shared_infodir" \
|
|
op start interval=0s timeout=40s \
|
|
op stop interval=0s timeout=20s \
|
|
op monitor interval=10s timeout=20s
|
|
primitive p_exportfs ocf:heartbeat:exportfs \
|
|
params clientspec="${nfs_client_spec}" directory="${nfs_export_path}" fsid=1 \
|
|
unlock_on_stop=1 options=rw,sync,no_root_squash \
|
|
op start interval=0s timeout=40s \
|
|
op stop interval=0s timeout=120s \
|
|
op monitor interval=10s timeout=20s
|
|
primitive p_azure-lb azure-lb \
|
|
params nc="/bin/nc" port="61000" \
|
|
op start interval=0s timeout=20s \
|
|
op stop interval=0s timeout=20s \
|
|
op monitor interval=10s timeout=20s
|
|
group g_services p_fs_data p_nfsserver p_exportfs p_azure-lb
|
|
colocation cl-g_services-with-ms_drbd_r0 inf: g_services ms_drbd_r0:Master
|
|
order o-ms_drbd_r0-before-g_services inf: ms_drbd_r0:promote g_services:start
|
|
property stonith-enabled=false
|
|
EOF
|
|
fi
|
|
# TODO STONITH is disabled for now (two lines above). Should enable it soon.
|
|
|
|
# TODO 'crm status' should show the correctly configured/started cluster resources.
|
|
}
|
|
|
|
# Main
|
|
|
|
setup_required_packages
|
|
|
|
setup_drbd_module_and_tools
|
|
|
|
# Don't create a file system, but just set up a disk (RAID if multiple unpartitioned disks)
|
|
# AZMDL_DISK env var is set by the function as the discovered/initialized disk
|
|
setup_raid_disk_and_filesystem None /dev/md0 None False
|
|
|
|
DRBD_RESOURCE_NAME=azmdlr0 # TODO Avoid hard-coded value
|
|
DRBD_DEVICE_PATH=/dev/drbd0
|
|
DRBD_MOUNT_POINT=/drbd # TODO Avoid hard-coded value
|
|
|
|
setup_drbd_with_disk $AZMDL_DISK $NODE1NAME $NODE1IP $NODE2NAME $NODE2IP $DRBD_RESOURCE_NAME $DRBD_DEVICE_PATH $DRBD_MOUNT_POINT
|
|
|
|
NFS_EXPORT_PATH=${DRBD_MOUNT_POINT}/data # TODO Allow different export dir name -- This requires changes all along the pipeline, from the top-level template to this script...
|
|
|
|
setup_corosync_and_pacemaker_for_nfs $NODE1IP $NODE2IP $DRBD_RESOURCE_NAME $DRBD_DEVICE_PATH $DRBD_MOUNT_POINT $NFS_EXPORT_PATH "$NFS_CLIENTS_IP_RANGE"
|
|
|
|
echo "NFS-HA setup succeeded. NFS_EXPORT_PATH=${NFS_EXPORT_PATH}, NFS_CLIENT_SPEC=${NFS_CLIENT_SPEC}"
|