Merge pull request virtuozzo#12 from wongma7/grace

Ganesha grace period & recovery

deploy/docker/Dockerfile

@@ -1,19 +1,25 @@
 # Modified from https://github.com/rootfs/nfs-ganesha-docker by Huamin Chen
 FROM fedora:24
 
+# Build ganesha from source, installing deps and removing them in one line.
+# Why?
+# 1. Root_Id_Squash, only present in >= 2.4.0.3 which is not yet packaged
+# 2. Set NFS_V4_RECOV_ROOT to /export
+
 RUN dnf install -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel dbus-x11 rpcbind hostname nfs-utils && dnf clean all \
 	&& curl -L https://github.com/nfs-ganesha/nfs-ganesha/archive/V2.4.0.3.tar.gz | tar zx \
 	&& curl -L https://github.com/nfs-ganesha/ntirpc/archive/v1.4.1.tar.gz | tar zx \
 	&& rm -r nfs-ganesha-2.4.0.3/src/libntirpc \
 	&& mv ntirpc-1.4.1 nfs-ganesha-2.4.0.3/src/libntirpc \
 	&& cd nfs-ganesha-2.4.0.3 \
 	&& cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CONFIG=vfs_only src/ \
+	&& sed -i 's|@SYSSTATEDIR@/lib/nfs/ganesha|/export|' src/include/config-h.in.cmake \
 	&& make \
 	&& make install \
 	&& cp src/scripts/ganeshactl/org.ganesha.nfsd.conf /etc/dbus-1/system.d/ \
 	&& dnf remove -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel && dnf clean all
 
-RUN mkdir /var/run/dbus
+RUN mkdir -p /var/run/dbus
 RUN mkdir -p /export
 
 # expose mountd 20048/tcp and nfsd 2049/tcp and rpcbind 111/tcp 111/udp

deploy/docker/vfs.conf

@@ -33,3 +33,8 @@ NFS_Core_Param
 {
 	MNT_Port = 20048;
 }
+
+NFSV4
+{
+	Grace_Period = 90;
+}

deploy/kube-config/pod.yaml

@@ -22,6 +22,7 @@ spec:
             - DAC_READ_SEARCH
       args:
         - "-provisioner=matthew/nfs"
+        - "-grace-period=0"
       env:
         - name: POD_IP
           valueFrom:

deploy/kube-config/pod_emptydir.yaml

@@ -22,6 +22,7 @@ spec:
             - DAC_READ_SEARCH
       args:
         - "-provisioner=matthew/nfs"
+        - "-grace-period=0"
       env:
         - name: POD_IP
           valueFrom:

docs/deployment.md

@@ -173,3 +173,4 @@ The pod requires authorization to `list` all `StorageClasses`, `PersistentVolume
 * `kubeconfig` - Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.
 * `run-server` - If the provisioner is responsible for running the NFS server, i.e. starting and stopping NFS Ganesha. Default true.
 * `use-ganesha` - If the provisioner will create volumes using NFS Ganesha (D-Bus method calls) as opposed to using the kernel NFS server ('exportfs'). If run-server is true, this must be true. Default true.
+* `grace-period` - NFS Ganesha grace period to use in seconds, from 0-180. If the server is not expected to survive restarts, i.e. it is running as a pod & its export directory is not persisted, this can be set to 0. Can only be set if both run-server and use-ganesha are true. Default 90.

main.go

@@ -39,6 +39,7 @@ var (
 	kubeconfig  = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Either this or master needs to be set if the provisioner is being run out of cluster.")
 	runServer   = flag.Bool("run-server", true, "If the provisioner is responsible for running the NFS server, i.e. starting and stopping NFS Ganesha. Default true.")
 	useGanesha  = flag.Bool("use-ganesha", true, "If the provisioner will create volumes using NFS Ganesha (D-Bus method calls) as opposed to using the kernel NFS server ('exportfs'). If run-server is true, this must be true. Default true.")
+	gracePeriod = flag.Uint("grace-period", 90, "NFS Ganesha grace period to use in seconds, from 0-180. If the server is not expected to survive restarts, i.e. it is running as a pod & its export directory is not persisted, this can be set to 0. Can only be set if both run-server and use-ganesha are true. Default 90.")
 )
 
 const ganeshaConfig = "/export/vfs.conf"
@@ -56,9 +57,15 @@ func main() {
 		glog.Fatalf("Invalid flags specified: if run-server is true, use-ganesha must also be true.")
 	}
 
+	if *gracePeriod != 90 && (!*runServer || !*useGanesha) {
+		glog.Fatalf("Invalid flags specified: custom grace period can only be set if both run-server and use-ganesha are true.")
+	} else if *gracePeriod > 180 && *runServer && *useGanesha {
+		glog.Fatalf("Invalid flags specified: custom grace period must be in the range 0-180")
+	}
+
 	if *runServer {
 		glog.Infof("Starting NFS server!")
-		err := server.Start(ganeshaConfig)
+		err := server.Start(ganeshaConfig, *gracePeriod)
 		if err != nil {
 			glog.Fatalf("Error starting NFS server: %v", err)
 		}

server/server.go

@@ -21,12 +21,14 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"regexp"
+	"strings"
 )
 
 const defaultGaneshaConfig = "/vfs.conf"
 
 // Start starts the NFS server. If an error is encountered at any point it returns it instantly
-func Start(ganeshaConfig string) error {
+func Start(ganeshaConfig string, gracePeriod uint) error {
 	// Start rpcbind if it is not started yet
 	cmd := exec.Command("/usr/sbin/rpcinfo", "127.0.0.1")
 	if err := cmd.Run(); err != nil {
@@ -58,6 +60,10 @@ func Start(ganeshaConfig string) error {
 			return fmt.Errorf("error writing ganesha config: %v", err)
 		}
 	}
+	err := setGracePeriod(ganeshaConfig, gracePeriod)
+	if err != nil {
+		return fmt.Errorf("error setting grace period to ganesha config: %v", err)
+	}
 	// Start ganesha.nfsd
 	cmd = exec.Command("ganesha.nfsd", "-L", "/var/log/ganesha.log", "-f", ganeshaConfig)
 	if out, err := cmd.CombinedOutput(); err != nil {
@@ -67,6 +73,50 @@ func Start(ganeshaConfig string) error {
 	return nil
 }
 
+func setGracePeriod(ganeshaConfig string, gracePeriod uint) error {
+	if gracePeriod > 180 {
+		return fmt.Errorf("grace period cannot be greater than 180")
+	}
+
+	newLine := fmt.Sprintf("Grace_Period = %d;", gracePeriod)
+
+	re := regexp.MustCompile("Grace_Period = [0-9]+;")
+
+	read, err := ioutil.ReadFile(ganeshaConfig)
+	if err != nil {
+		return err
+	}
+
+	old := re.Find(read)
+
+	if old == nil {
+		// Grace_Period line not there, append the whole NFSV4 block.
+		file, err := os.OpenFile(ganeshaConfig, os.O_APPEND|os.O_WRONLY, 0600)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		block := "\nNFSV4\n{\n" +
+			"\t" + newLine + "\n" +
+			"}\n"
+
+		if _, err = file.WriteString(block); err != nil {
+			return err
+		}
+		file.Sync()
+	} else {
+		// Grace_Period line there, just replace it
+		replaced := strings.Replace(string(read), string(old), newLine, -1)
+		err = ioutil.WriteFile(ganeshaConfig, []byte(replaced), 0)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Stop stops the NFS server.
 func Stop() {
 	// /bin/dbus-send --system   --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown