Page MenuHomePhabricator

Ganeti
Updated 2,664 Days AgoPublic

Install Ganeti

Debian 9

Install packages

apt-get install drbd-utils ganeti2 ganeti-instance-debootstrap qemu-kvm resolvconf lvm2 bridge-utils lzop

PCI Passthrough

/etc/modules

drbd
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd

/etc/modprobe.d/vfio.conf
GTX1080

options vfio-pci ids=10de:1b80,10de:10f0

Networking

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto eth0
iface eth0 inet manual

auto br-lan
iface br-lan inet static
        address 134.197.X.X
        netmask 255.255.255.0
        gateway 134.197.X.X
        bridge_ports    eth0
        bridge_stp      off
        bridge_fd       0
        bridge_maxwait  0

# Replication network DRBD
auto eth1
iface eth1 inet manual

auto br-rep
iface br-rep inet static
        address 172.X.X.X
        netmask 255.255.255.0
        bridge_ports    eth1
        bridge_stp      off
        bridge_fd       0
        bridge_maxwait  0

DRBD Setup

echo "options drbd minor_count=128 usermode_helper=/bin/true" >/etc/modprobe.d/drbd.conf
echo "drbd" >>/etc/modules
rmmod drbd      # ignore error if the module isn't already loaded
modprobe drbd
update-initramfs -u

Initialize Master Node


gnt-cluster init \
--master-netdev=br-lan \
--enabled-hypervisors=kvm \
-s 172.20.128.11 \
--vg-name vg-ganeti \
--compression-tools gzip,gzip-fast,gzip-slow,lzop \
$HOSTNAME


gnt-cluster modify -H kvm:kernel_path=,initrd_path=

#Fix DF logjam
openssl dhparam -out dhparams.pem 2048
cat dhparams.pem >> /var/lib/ganeti/server.pem

#Change default link
gnt-cluster modify --nic-parameters link=br-lan

Add Node

gnt-node add -s 172.X.X.X cnode#.cse.unr.edu

Configuration

GFX Passthrough

unbind the PCI devices

lspci -nn | grep $YOUR_PCI_DEVICE


echo > 1 /sys/bus/pci/devices/0000:42:00.0/remove
echo > 1 /sys/bus/pci/devices/0000:42:00.1/remove
echo > 1 /sys/bus/pci/devices/0000:04:00.1/remove
echo > 1 /sys/bus/pci/devices/0000:04:00.0/remove


echo 10de 1b80 | tee /sys/bus/pci/drivers/vfio-pci/new_id
echo 10de 10f0 | tee /sys/bus/pci/drivers/vfio-pci/new_id

Modify Ganeti after installing Ubuntu\CentOS\RHEL


gnt-instance modify -H kvm_extra="\
-drive if=pflash\,format=raw\,readonly\,file=/usr/share/OVMF/OVMF_CODE.fd \
-drive if=pflash\,format=raw\,file=/var/local/ovmf/$HOST \
-device vfio-pci\,host=04:00.0\,addr=0x9 \
-device vfio-pci\,host=04:00.1\,addr=0x10 \
-cpu host\,kvm=off" $HOST

gnt-instance modify -H vga=none $HOST

Grub Changes for booting

GRUB_DEFAULT=0
GRUB_HIDDEN_TIMEOUT_QUIET=true
GRUB_TIMEOUT=20
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT=""
GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200n8"

GRUB_TERMINAL=serial
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"

Clusters

g.engr.unr.edu

Engineering Cluster

g.engr.unr.edu
g1.engr.unr.edu

Commands

#Add an instance
gnt-instance add -t plain -o debootstrap+stretch --disk 0:size=200G -B vcpus=4,minmem=256M,maxmem=8192M --no-name-check --no-ip-check --no-start $HOSTNAME

#Boot from host kernel
gnt-instance start -H kernel_path=/var/tmp/vmlinuz,initrd_path=/var/tmp/initrd.img,kernel_args="" $HOSTNAME

#Import A Backup
gnt-backup import --src-node=g1.engr.unr.edu --src-dir=$SRC_DIR -t plain --no-name-check --no-ip-check $HOSTNAME
Last Author
newellz2
Last Edited
Jun 22 2017, 1:30 PM

Event Timeline

newellz2 edited the content of this document. (Show Details)
newellz2 changed the visibility from "All Users" to "Public (No Login Required)".
newellz2 edited the content of this document. (Show Details)