From 1921330f45487532ccaf1e37e18d51e83896f949 Mon Sep 17 00:00:00 2001 From: Varun Sharma Date: Fri, 17 Nov 2023 11:18:33 +0000 Subject: [PATCH] ISSUE-1847: Adding scripts and documenting the steps for neo4j backup/restore --- deepfence_neo4j/Dockerfile | 2 + deepfence_neo4j/backup_neo4j.sh | 24 +++++++++ deepfence_neo4j/load_backup_neo4j.sh | 26 +++++++++ docs/docs/console/database-export-import.md | 59 ++++++++++++++++++++- 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100755 deepfence_neo4j/backup_neo4j.sh create mode 100755 deepfence_neo4j/load_backup_neo4j.sh diff --git a/deepfence_neo4j/Dockerfile b/deepfence_neo4j/Dockerfile index 81f66a6fea..25eb7c6757 100644 --- a/deepfence_neo4j/Dockerfile +++ b/deepfence_neo4j/Dockerfile @@ -1,4 +1,6 @@ FROM neo4j:4.4.25 RUN apt update && apt install rclone -y COPY df.sh /startup +COPY backup_neo4j.sh /usr/local/bin/ +COPY load_backup_neo4j.sh /usr/local/bin/ ENTRYPOINT ["tini", "-g", "--", "/startup/df.sh"] diff --git a/deepfence_neo4j/backup_neo4j.sh b/deepfence_neo4j/backup_neo4j.sh new file mode 100755 index 0000000000..8fbad242e7 --- /dev/null +++ b/deepfence_neo4j/backup_neo4j.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +dt=$(date '+%Y-%m-%d_%H-%M-%S'); +BACKUP_FILE="/backups/neo4j_backup_"$dt + +echo "Backup file is:$BACKUP_FILE" + +mkdir -p /backups/ + +neo4j stop +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Failed to stop the neo4j db" + exit +fi + +neo4j-admin dump --database='neo4j' --to=$BACKUP_FILE +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Failed to create the backup file" +fi + +sleep 2s +/startup/docker-entrypoint.sh neo4j >& /dev/null& diff --git a/deepfence_neo4j/load_backup_neo4j.sh b/deepfence_neo4j/load_backup_neo4j.sh new file mode 100755 index 0000000000..8f149731aa --- /dev/null +++ b/deepfence_neo4j/load_backup_neo4j.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + echo "Missing input parameters" + echo "Correct usage:"$0" " + exit +fi + +BACKUP_FILE=$1 +echo "Using file for db restore: $BACKUP_FILE" + +neo4j stop +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Failed to stop the neo4j db" + exit +fi + +neo4j-admin load --from=$BACKUP_FILE --database='neo4j' --force +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Failed to load the db file" +fi + +/startup/docker-entrypoint.sh neo4j >& /dev/null& +sleep 2s diff --git a/docs/docs/console/database-export-import.md b/docs/docs/console/database-export-import.md index e3de3fedb0..feb4da176e 100644 --- a/docs/docs/console/database-export-import.md +++ b/docs/docs/console/database-export-import.md @@ -2,9 +2,11 @@ title: Database Export and Import --- +## Postgres DB Export and Import + Export PostgreSQL data from one management console and import in another console -## Export +### Export Connect to old management console / database, run following commands to export @@ -23,7 +25,7 @@ docker run --net=host --rm=true --name=postgresql-backup \ deepfenceio/deepfence_backup:2.0.1 ``` -## Import +### Import Connect to new management console / database, run following commands to import @@ -39,3 +41,56 @@ docker run --net=host --rm=true --name=postgresql-restore \ deepfenceio/deepfence_backup:2.0.1 ``` - Restart management console once + + +## Neo4J Graph Database Export and Import + +Export Neo4J data from one management console and Import data in another console + +### Export + +* Step 1: Login to the host running the neo4j docker instance. +* Step 2: Docker exec into the neo4j instance using the below command: + + ```shell + docker exec -it deepfence-neo4j /bin/bash + ``` +* Step 3: Run the backup script from inside the neo4j docker instance as follows: + + ```shell + /usr/local/bin/backup_neo4j.sh + ``` + This will create a backup file inside the container. + The name of the file will be of the format: `neo4j_backup_` + Also, the script will print the name of the file on the stdout. +* Step 4: Copy the neo4j backup file created above to host or any intermediate location + +### Import + +* Step 1: Copy the backup file from intermediate location to the target machine using scp (or similar commands) +* Step 2: Login to the target machine and copy the backup file in to the running neo4j container using below command: + + ```shell + docker cp deepfence-neo4j:/ + ``` +* Step 3: Take a bash session of the running neo4j container using the below command: + + ```shell + docker exec -it deepfence-neo4j /bin/bash + ``` +* Step 4: Run the restore script from inside the neo4j docker instance as follows: + + ```shell + /usr/local/bin/load_backup_neo4j.sh / + ``` + e.g.: + ```shell + /usr/local/bin/load_backup_neo4j.sh /neo4j_backup_2023-11-17_10-25-28 + ``` + +### Steps for Kubernetes: + +The steps for kubernetes remains similar to the above. +For Kubernetes, we will have to use `kubectl` utility to: +* Copy the file from and to the pod. +* Take a bash session of the pod