Skip to content

Commit

Permalink
refactored versions add (#7)
Browse files Browse the repository at this point in the history
  • Loading branch information
alexanderbazhenoff authored Jan 6, 2024
1 parent 9174b12 commit 8e39e18
Show file tree
Hide file tree
Showing 40 changed files with 595 additions and 328 deletions.
53 changes: 53 additions & 0 deletions .github/linters/.groovylintrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"extends": "recommended",
"rules": {
"comments.ClassJavadoc": "off",
"formatting.Indentation": {
"spacesPerIndentLevel": 4,
"severity": "info"
},
"braces.ElseBlockBraces": {
"enabled": false
},
"braces.ForStatementBraces": {
"enabled": false
},
"braces.IfStatementBraces": {
"enabled": false
},
"braces.WhileStatementBraces": {
"enabled": false
},
"unused.UnusedVariable": {
"ignoreVariableNames": "__"
},
"size.NestedBlockDepth": {
"maxNestedBlockDepth": 10
},
"size.ParameterCount": {
"enabled": false
},
"naming.VariableName": {
"enabled": false
},
"ConsecutiveBlankLines": {
"enabled": false
},
"CatchException": {
"enabled": false
},
"convention.CompileStatic": {
"enabled": false
},
"BlockStartsWithBlankLine": {
"enabled": false
},
"BlockEndsWithBlankLine": {
"enabled": false
},
"DuplicateStringLiteral": {
"enabled": false
},
"UnnecessaryReturnKeyword": "error"
}
}
37 changes: 37 additions & 0 deletions .github/linters/.markdown-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
---

# Default state for all rules
default: true

# Path to configuration file to extend
extends: null

# MD009/no-trailing-spaces : Trailing spaces : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md009.md
MD009:
# Spaces for line break
br_spaces: 2
# Allow spaces for empty lines in list items
list_item_empty_lines: false
# Include unnecessary breaks
strict: false

# MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md
MD013:
# Number of characters
line_length: 120
# Number of characters for headings
heading_line_length: 80
# Number of characters for code blocks
code_block_line_length: 120
# Include tables
tables: false
# Include headings
headings: false

# MD011/no-reversed-links : Reversed link syntax : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md011.md
MD011: true

# MD012/no-multiple-blanks : Multiple consecutive blank lines
MD012:
# Consecutive blank lines
maximum: 2
40 changes: 40 additions & 0 deletions .github/linters/.yaml-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
---

extends: default
ignore: |
README.md
LICENSE
rules:
braces:
max-spaces-inside: 1
level: error
brackets:
max-spaces-inside: 1
level: error
colons:
max-spaces-after: -1
level: error
commas:
max-spaces-after: -1
level: error
comments: disable
comments-indentation: disable
document-start: disable
empty-lines:
max: 2
level: error
hyphens:
level: error
indentation: disable
key-duplicates: enable
line-length:
max: 120
level: error
new-line-at-end-of-file: disable
new-lines:
type: unix
trailing-spaces: enable
truthy:
allowed-values: ['true', 'false'] # yamllint disable-line rule:truthy
check-keys: true
26 changes: 26 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
---
name: lint

on: # yamllint disable-line rule:truthy
push: null
pull_request: null

jobs:
build:
name: Lint
runs-on: ubuntu-latest

permissions:
contents: read
packages: read
statuses: write

steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Super-linter
uses: super-linter/super-linter@v5
env:
DEFAULT_BRANCH: main
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
BSD 3-Clause License

Copyright (c) 2014-2023, Aleksandr Bazhenov
Copyright (c) 2014-2024, Aleksandr Bazhenov

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Expand Down
24 changes: 15 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
# Scripts and tools

![lint](https://github.com/alexanderbazhenoff/scripts-and-tools/actions/workflows/lint.yml/badge.svg?branch=master)

System administration and engineering scripts and tiny tools.

## Backup scripts

A tiny infrastructure-ready tools for backup, restore and maintenance your data.
An infrastructure-ready scripts for backup, restore and maintenance your data.

- [**backup_btrfs_filesystem_using_snapshots**](backup/backup_btrfs_using_snapshot/README.md) - example how to backups
btrfs filesystems using snapshots.
btrfs filesystems using snapshots.
- [**backup_lxc_and_filesystem_path**](backup/backup_lxc_and_filesystem_path/README.md) - backup, encrypt, and restore a
filesystem path to remote drives with UID/GUI storing (especially for containers) on Linux systems.
- [**bareos_pool_operations**](backup/bareos_pool_operations/README.md) - batch process to remove and clean-up
[Bareos](https://www.bareos.com/) volumes in the pool.
- [**calculate_and_check_md5**](backup/calculate_and_check_md5) - calculate and check md5 of required folder.
- [**gitlab_dump_and_restore**](backup/gitlab_dump_and_restore/README.md) - create GitLab dump and restore gitlab from
a dump.
- [**gitlab_dump_and_restore**](backup/gitlab_dump_and_restore/README.md) - create GitLab dump and restore gitlab from a
dump.
- [**kvm_vm_backup**](backup/kvm_vm_backup/README.md) - backup running or stopped
[Kernel Virtual Machine (KVM)](https://www.linux-kvm.org/page/Main_Page) instance(s) on Linux systems.
- [**mysql_dump**](backup/mysql_dump/README.md) - bash and batch command file examples of MySQL dump.
Expand All @@ -19,19 +25,19 @@ a dump.

A set of scripts to benchmarks and performance stats.

- [**filesystems_benchmarks**](benchmark/filesystems_benchmarks/README.md) - a set of filesystem benchmark and
- [**filesystems_benchmarks**](benchmark/filesystems_benchmarks/README.md) - a set of filesystem benchmark and
performance comparison scripts.

## Network scripts

Configure and collect network settings and software on operating systems.

- [**get_dhcp_leases**](network/get_dhcpd_leases/README.md) - get leases info from isc-dhcp server including client,
expiration and client vendor.
- [**get_dhcp_leases**](network/get_dhcpd_leases/README.md) - get leases info from isc-dhcp server including client
expiration and vendor.
- [**inner_vps_bridge**](network/inner_vps_bridge/README.md) - example how to organize the network inside your VPS by
linux bridge.
- [**ixnetwork_related_scripts**](network/ixnetwork_related_scripts/README.md) -
[IxNetwork server](https://support.ixiacom.com/version/ixnetwork-916) related scripts: automation, interaction, etc..
- [**ixnetwork_related_scripts**](network/ixnetwork_related_scripts/README.md) -
[IxNetwork server](https://support.ixiacom.com/version/ixnetwork-916) related scripts: automation, interaction, etc.
- [**vlan_filtered_bridges**](network/vlan_filtered_bridges/README.md) - systemd unit and script to create bridge with
VLAN filtering to prevent MAC-table overflow on the host. Actually this is an example how to organize linux bridges by
script.
9 changes: 4 additions & 5 deletions backup/README.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
Data and backup scripts
-----------------------
# Data and backup scripts

A tiny infrastructure-ready tools for backup, restore and maintenance your data:

- [**backup_btrfs_filesystem_using_snapshots**](backup_btrfs_using_snapshot/README.md) - example how to backups btrfs
- filesystems using snapshots.
- [**backup_lxc_and_filesystem_path**](backup_lxc_and_filesystem_path/README.md) - backup, encrypt and restore
filesystem path to remote drives with UID/GUI storing (especially for containers) on Linux systems.
- filesystems using snapshots.
- [**backup_lxc_and_filesystem_path**](backup_lxc_and_filesystem_path/README.md) - backup, encrypt, and restore
a filesystem path to remote drives with UID/GUI storing (especially for containers) on Linux systems.
- [**bareos_pool_operations**](bareos_pool_operations/README.md) - batch process to remove and clean-up
[Bareos](https://www.bareos.com/) volumes in the pool.
- [**calculate_and_check_md5**](calculate_and_check_md5) - calculate and check md5 of required folder.
Expand Down
11 changes: 5 additions & 6 deletions backup/backup_btrfs_using_snapshot/README.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
backup btrfs filesystem using snapshot
======================================
# backup btrfs filesystem using snapshot

**WARNING! Running this file, you accept that you know what you're doing. All actions with this script are at your own
**WARNING! Running this file, you accept that you know what you're doing. All actions with this script are at your own
risk.**

Example of how to
[perform backup of btrfs filesystem using snapshot(s)](https://archive.kernel.org/oldwiki/btrfs.wiki.kernel.org/index.php/Incremental_Backup.html).
Example of how to perform backup of btrfs filesystem using
[snapshot(s)](https://archive.kernel.org/oldwiki/btrfs.wiki.kernel.org/index.php/Incremental_Backup.html).
Before you begin to check your kernel version,
[btrfs status](https://archive.kernel.org/oldwiki/btrfs.wiki.kernel.org/index.php/Status.html) and
[btrfs changelog](https://archive.kernel.org/oldwiki/btrfs.wiki.kernel.org/index.php/Changelog.html).
Expand All @@ -14,6 +13,6 @@ Some old versions may have a non-working functional.
Variables inside a script:

- **SOURCE_FILESYSTEM_PATH** (e.g. `"/mnt/data/ssd/folder"`): Source btrfs filesystem to create snapshot from.
- **BACKUP_FILESYSTEM_PATH** (e.g `"/mnt/data/backup"`): Destination filesystem to send created snapshot. In general
- **BACKUP_FILESYSTEM_PATH** (e.g `"/mnt/data/backup"`): Destination filesystem to send created snapshot. In general
this is a backup path.
- **SNAPSHOTS_PATH** (e.g. `"/mnt/data/ssd/.snapshots"`): Temporary filesystem to create snapshot.
Empty file.
37 changes: 18 additions & 19 deletions backup/backup_lxc_and_filesystem_path/README.md
Original file line number Diff line number Diff line change
@@ -1,32 +1,31 @@
Backup LXC and filesystem path
==============================
# Backup LXC and filesystem path

These scripts perform backing up filesystem path with UID/GID saving using tar and gpg. Possible to save and upload the
whole containers.
These scripts perform backing up a filesystem path with UID/GID saving using tar and gpg. Possible to save and upload
the whole containers.

**WARNING! Running this file you accept that you know what you're doing. All actions with this script are at your own
**WARNING! Running this file, you accept that you know what you're doing. All actions with this script are at your own
risk.**

Usage
-----
## Usage

1. Upload this folder content to your Linux system, e.g., inside of `/opt/scripts` folder.
2. (Optional) Set up your [rclone](https://rclone.org/drive/) to mount destination drive, e.g. Google Drive, Dropbox,
Amazon Drive, Amazon S3 Compliant Storage Providers (including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS,
Minio, SeaweedFS, and Tencent COS), Google Cloud Storage, Hadoop distributed file system, Mail.Ru Cloud, Mega,
Microsoft Azure Blob Storage, Microsoft OneDrive, OpenStack Swift, QingCloud Object Storage, Yandex Disk, Uptobox, Zoho
Microsoft Azure Blob Storage, Microsoft OneDrive, OpenStack Swift, QingCloud Object Storage, Yandex Disk, Uptobox, Zoho
or another connection method(s) like ftp, ssh, sftp, http. Or mount your backup path manually, e.g. `/mnt/backup`.
3. Fill your `empty_password.txt` with gpg key if you wish to encrypt your files or leave them empty.
4. Edit `lxc_exclude` list of excluded path.
5. (Optional) Put your error notification commands to `error_notification.sh` if you would like to recieve error
messages.
4. Edit `lxc_exclude` list of an excluded path.
5. (Optional) Put your error notification commands to `error_notification.sh` if you would like to receive error
messages.
6. Edit your `backup_container.sh` with appropriate params of rclone mount (optional) and `backup_path_tar_gpg.sh` with
appropriate backup and path params.
7. Run:

```bash
sudo /./opt/scripts/backup_conatainer.sh -b -p /opt/scripts/empty_password.txt
```

with predefined destination path, or:

```bash
Expand All @@ -40,8 +39,7 @@ Restore files:
sudo /./opt/scripts/backup_conatainer.sh --source /mnt/backup --password-file /opt/scripts/empty_password.txt --restore
```

Usage examples
--------------
## Usage examples

Mount your remote drive to `/mnt/backup`, e.g. rclone pre-configured Google Drive mount. Add to `backup_container.sh`
the code below before backup block:
Expand All @@ -51,11 +49,13 @@ rm -rf /mnt/backup/*
sync; echo 3 > /proc/sys/vm/drop_caches; sync
rclone mount googledrive:vps0 /mnt/backup --drive-use-trash=false --daemon --allow-non-empty; sleep 5
```
Optionally remove an old backups, e.g. older than 60 days:

Optionally remove old backups, e.g., older than 60 days:

```bash
cd "$BACKUP_DESTINATION" && find . -mtime +60 -exec rm -rf {} \;
```

Encrypted and tar.gz packed backup of LXC-container called **container_name** placed in default path should look like:

```bash
Expand All @@ -73,6 +73,7 @@ task_error $? '/var/lib/lxc/container_name'
# You can also disable compression, but leave encryption enabled. In this way you'll get encrypted gpg file with .enc
# extension.
```

At the end of the script unmount backup path:

```bash
Expand All @@ -81,13 +82,11 @@ sleep 60
fusermount -uz /mnt/backup
```

Advanced usage
--------------
## Advanced usage

You can use `backup_path_tar_gpg.sh` directly by separate call of each backup path. Run
You can use `backup_path_tar_gpg.sh` directly by separate call of each backup path. Run
`./backup_path_tar_gpg.sh --help` for the help.

License
-------
## License

[BSD 3-Clause License](../../LICENSE)
8 changes: 4 additions & 4 deletions backup/backup_lxc_and_filesystem_path/backup_container.sh
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ process_path(){
ARGS+="--debug"
fi
# set your params for chrt command to change process priority.
chrt -i 0 "/.$BACKUP_SCRIPT_PATH" $ARGS && return 0 || return 34
chrt -i 0 "/.$BACKUP_SCRIPT_PATH" "$ARGS" && return 0 || return 34
}

BACKUP_MODE=false
Expand Down Expand Up @@ -214,12 +214,12 @@ sleep 5

# Backup all these scripts from /opt/script
process_path "/opt" "$BACKUP_DESTINATION" "$ACTION" \
"server3_scripts_$(date +%y%m%d).tar.gz" "$PASSWORD" false true true ""
"server3_scripts_$(date +%y%m%d).tar.gz" "$PASSWORD" false true true ""
task_error $? '/opt/scripts'

# Backup LXC container from /var/lib/lxc/lxc_container_name
process_path "/var/lib/lxc/bareos.emzior" "$BACKUP_DESTINATION" "$ACTION" \
"bareos_lxc_$(date +%y%m%d).tar.gz" "$PASSWORD" false true true "/opt/scripts/lxc_exclude"
echo $?; task_error $? '/var/lib/lxc/lxc_container_name'
"bareos_lxc_$(date +%y%m%d).tar.gz" "$PASSWORD" false true true "/opt/scripts/lxc_exclude"
task_error $? '/var/lib/lxc/lxc_container_name'

sync; echo 3 > /proc/sys/vm/drop_caches; sync
Loading

0 comments on commit 8e39e18

Please sign in to comment.