Merge pull request #57 from galaxyproject/backup-remote-fixes

Fixes for remote rsync + connection options

README.md

@@ -91,9 +91,21 @@ Role Variables
 
 ### Backups ###
 
-- `postgresql_backup_dir`: If set, enables [PITR][postgresql_pitr] backups. Set this to a directory where your database
-  will be backed up (this can be any format supported by rsync, e.g. `user@host:/path`). The most recent backup will be
-  in a subdirectory named `current`.
+This role can deploy and schedule the configuration and scripts to maintain Postgresql [PITR][postgresql_pitr] backups.
+
+Full backups will be made on the configured interval, whereas write-ahead-log (WAL) segments between full backups will
+be archived to `{{ postgresql_backup_dir }}/wal_archive/` when instructed by the PostgreSQL server. WAL segments can be
+removed from this directory once the oldest backup referencing them has been removed. This is done automatically for you
+by the backup script if `postgresql_backup_dir` is mounted locally.
+
+When `postgresql_backup_dir` is a remote rsync path (containing a "`:`"), the backup script will still maintain backups
+(including deleting older full backups) but cannot prune the `wal_archive/` directory automatically. If you are able to
+install the standard `pg_archivecleanup` utility from the PostgreSQL client package on your backup server, you can run
+this role's backup script with the `--clean-archive` option directly on the backup server instead.
+
+- `postgresql_backup_dir`: If set, enables PITR backups. Set this to a directory where your database will be backed up
+  (this can be any format supported by rsync, e.g. `user@host:/path`). The most recent backup will be in a subdirectory
+  named `current`.
 
 - `postgresql_backup_local_dir`: Filesystem path on the PostgreSQL server where backup scripts will be placed.
 
@@ -122,9 +134,8 @@ Standard install: Default `postgresql.conf`, `pg_hba.conf` and default version f
 ---
 
 - hosts: dbservers
-  remote_user: root
   roles:
-    - postgresql
+    - galaxyproject.postgresql
 ```
 
 Use the pgdg packages on a Debian-based host:
@@ -133,11 +144,10 @@ Use the pgdg packages on a Debian-based host:
 ---
 
 - hosts: dbservers
-  remote_user: root
   vars:
     postgresql_flavor: pgdg
   roles:
-    - postgresql
+    - galaxyproject.postgresql
 ```
 
 Use the PostgreSQL 9.5 packages and set some `postgresql.conf` options and `pg_hba.conf` entries:
@@ -146,7 +156,6 @@ Use the PostgreSQL 9.5 packages and set some `postgresql.conf` options and `pg_h
 ---
 
 - hosts: dbservers
-  remote_user: root
   vars:
     postgresql_version: 9.5
     postgresql_conf:
@@ -155,18 +164,44 @@ Use the PostgreSQL 9.5 packages and set some `postgresql.conf` options and `pg_h
     postgresql_pg_hba_conf:
       - host all all 10.0.0.0/8 md5
   roles:
-    - postgresql
+    - galaxyproject.postgresql
 ```
 
-Enable backups to /archive
+Enable backups to /archive:
 
 ```yaml
 - hosts: all
-  remote_user: root
   vars:
     postgresql_backup_dir: /archive
   roles:
-    - postgresql
+    - galaxyproject.postgresql
+```
+
+Enable backups to /archive on a remote server:
+
+```yaml
+- hosts: dbservers
+  vars:
+    postgresql_backup_dir: backup.example.org:/archive
+  roles:
+    - galaxyproject.postgresql
+
+- hosts: backupservers
+  tasks:
+    - name: Install PostgreSQL scripts
+      ansible.builtin.apt:
+        name: postgresql-common
+    - name: Copy backup script
+      ansible.builtin.copy:
+        src: roles/galaxyproject.postgresql/files/backup.py
+        dest: /usr/local/bin/pgbackup.py
+        mode: "0755"
+    - name: Schedule WAL pruning
+      ansible.builtin.cron:
+        name: Prune PostgreSQL Archived WALs
+        hour: 22
+        minute: 0
+        job: /usr/local/bin/pgbackup.py --clean-archive /archive
 ```
 
 License

defaults/main.yml

@@ -17,6 +17,9 @@ postgresql_backup_local_dir: >-
       '/var/lib/postgresql' if ansible_os_family == 'Debian' else '~postgres') }}/backup
 postgresql_create_backup_dir: true
 
+# Controls whether the wal_archive directory is cleaned and whether postgresql_create_backup_dir can be used
+postgresql_backup_dir_is_remote: "{{ postgresql_backup_dir[0] != '/' }}"
+
 # Options used for the WAL archive command - do not change this unless you have read the PITR documentation and
 # understand how this command must work.
 postgresql_archive_wal_rsync_args: '--ignore-existing -ptg --info=skip1'
@@ -31,11 +34,13 @@ postgresql_backup_keep: 30
 
 __postgresql_pgdg_bin_dir: "{{ '/usr/pgsql-' ~ (postgresql_version | replace('.', '')) ~ '/bin' }}"
 postgresql_backup_command: >-
-  {{ postgresql_backup_local_dir | quote }}/bin/backup.py
+  {{ postgresql_backup_python_executable }} {{ postgresql_backup_local_dir | quote }}/bin/backup.py
   {{ '--rsync-connect-opts ' ~ (postgresql_backup_rsync_connect_opts | quote) if postgresql_backup_rsync_connect_opts else '' }}
   --rsync-backup-opts {{ postgresql_backup_rsync_backup_opts | regex_replace('^-', '\-') | quote }}
   --keep {{ postgresql_backup_keep | quote }}
   {{ '--pg-bin-dir ' ~ __postgresql_pgdg_bin_dir if ansible_os_family == 'RedHat' else '' }}
-  --backup --clean-archive {{ postgresql_backup_dir | quote }}
+  --backup {{ postgresql_backup_dir_is_remote | ternary('', '--clean-archive') }} {{ postgresql_backup_dir | quote }}
+
+postgresql_backup_python_executable: "python"
 
 postgresql_default_auth_method: "{{ (postgresql_version is version('13', '>')) | ternary('scram-sha-256', 'md5') }}"

files/backup.py

@@ -30,7 +30,10 @@
 except ImportError:
     from pipes import quote as shlex_quote
 
-import psycopg2
+try:
+    import psycopg2
+except ImportError:
+    psycopg2 = None
 
 
 RSYNC_EXCLUDES = (
@@ -109,7 +112,7 @@ def pg_major_version(self):
     def rsync_cmd(self):
         cmd = ['rsync']
         if self._rsync_opts:
-            cmd.extend(shlex.split(rsync_opts))
+            cmd.extend(shlex.split(self._rsync_opts))
         return cmd
 
     @property
@@ -147,6 +150,8 @@ def parse_args(argv):
     parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output')
     parser.add_argument('backup_path', help='Backup to location (rsync-compatible string)')
     args = parser.parse_args(argv)
+    if args.backup and psycopg2 is None:
+        parser.error('--backup specified but psycopg2 could not be imported')
     if args.clean_archive and ':' in args.backup_path:
         parser.error('--clean-archive cannot be used with remote backup directories')
     return args

tasks/backup.yml

@@ -18,7 +18,7 @@
     mode: 0750
     state: directory
     path: "{{ postgresql_backup_dir }}"
-  when: postgresql_backup_dir[0] == '/' and postgresql_create_backup_dir
+  when: not postgresql_backup_dir_is_remote and postgresql_create_backup_dir
 
 - name: Install backup script templates
   template:

templates/archive_wal.sh.j2

@@ -13,10 +13,16 @@ wal_archive_dir={{ (postgresql_backup_dir ~ '/wal_archive') | quote }}
 file_path="$1"
 file_name="$2"
 
+{% if postgresql_backup_dir_is_remote %}
+empty=$(mktemp -d -t ansible-postgresql-empty.XXXXXX)
+rsync {{ postgresql_backup_rsync_connect_opts }} "${empty}/" "$wal_archive_dir"
+rmdir "$empty"
+{% else %}
 mkdir -p "$wal_archive_dir"
+{% endif %}
 
 # If rsync outputs anything to stdout, the destination already existed, which should not happen
-if [ -n "$(rsync {{ postgresql_archive_wal_rsync_args }} "$file_path" "$wal_archive_dir")" ]; then
+if [ -n "$(rsync {{ postgresql_backup_rsync_connect_opts }} {{ postgresql_archive_wal_rsync_args }} "$file_path" "$wal_archive_dir")" ]; then
     echo "ERROR: ${wal_archive_dir}/${file_name} already exists, overwriting is not allowed!"
     exit 1
 fi