Skip to content
This repository has been archived by the owner on Jun 9, 2022. It is now read-only.

Updating Build/Deploy automations #243

Merged
merged 10 commits into from
May 22, 2019
Merged

Updating Build/Deploy automations #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7d5ac7e
Adding checkout flag to reset SSOT branch to directory default
May 20, 2019
9c935c7
Adding conditional to build script to remove existing pyenv
May 20, 2019
f500f1c
Adding Python2 flag to build script
May 20, 2019
b0afe15
Abstracting scripts to build and deploy SSOT separately, adding scrip…
May 21, 2019
599fb50
Amending commented documentation, removing reference to unused variable
May 21, 2019
e95b7a0
Adding symlinks to replace after hard reset of all directories
May 21, 2019
41f0758
Removing unrequired symlink
May 21, 2019
6c6412e
Updating with new server credentials
May 21, 2019
b3c08ee
Adding conditional to remove backup
May 22, 2019
32ff94c
Moving backup script to backup dir
May 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions deployment/build_and_deploy_iati_standard.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

# Script to automate the build and deployment of the IATI SSOT reference site.
# Builds base version branches using default directory and host configuration.


./build_iati_standard.sh -c
./deploy_iati_standard.sh
75 changes: 58 additions & 17 deletions deployment/build_iati_standard.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,35 @@
# Assumes running on webserver3 and the documentation for versions 1.04,
# 1.05, 2.01 & 2.02 are all to be regenerated.

# Maintained under the `iati-bot` gist account.

# Software dependencies must already be installed.

# Script must be run as user 'ssot'. This ensures user 'ssot' is the owner
# Script must be run as user 'iati'. This ensures user 'iati' is the owner
# of all files generated:
# $ sudo -u ssot ./deploy_iati_standard.sh

# The public key for webserver3 must be added to the 'iatiuser' account on
# webserver5.
# $ sudo -u iati ./build_iati_standard.sh

OPTIND=1 # Reset in case getopts has been used previously in the shell.

DEFAULT_DIR=/home/ssot/live
DEFAULT_DIR=/var/www/dev.reference.iatistandard.org/html

while getopts "d:" opt; do
while getopts "d:ch" opt; do
echo $OPTARG
case "$opt" in
d) DIRECTORY=$OPTARG
;;
c) CHECKOUT=true
;;
h) echo "Usage:"
echo " build_iati_standard.sh [-d [path/to/dir]] [-h] [-c] [-p]."
echo " -d [path/to/dir] specify a build path for the SSOT"
echo " -c force check out base branches (version-2.03 etc) for build"
echo " -p use Python 2.x virtual environment"
echo " -h display this menu"
exit 0
;;
\? )
echo "Option invalid: -$OPTARG" 1>&2
exit 1
;;
esac
done

Expand All @@ -37,11 +46,11 @@ cd ${DIRECTORY:-$DEFAULT_DIR}
# Ensure documentation and template dependencies are on the 'live' branches and up-to-date
for f in IATI-Developer-Documentation/ IATI-Guidance/ IATI-Websites/; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"
echo -e "NOW IN FOLDER: $PWD \n\n"

git checkout live
git pull
echo -e "DONE CODE PULLS: $f \n\n"
echo -e "DONE CODE PULLS: $PWD \n\n"

cd ..
done
Expand All @@ -51,24 +60,56 @@ echo -e "UPDATED DOCS & TEMPLATES \n\n"
# Regenerate all versions of the sites, saving HTML outputs in '_build/dirhtml'
for f in 2.01 2.02 2.03 1.05 1.04; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"
site_folder="${f//.}"
echo -e "NOW IN FOLDER: $PWD \n\n"



# Ensure that directories are up to date and clean of changes
git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
git fetch origin

git reset --hard HEAD^1
git clean -fd

# Ensure code and submodules are up-to-date with origin
git pull
git submodule update
if [ "$CHECKOUT" = true ] ; then
git checkout version-$f
git pull origin version-$f
else
git pull
fi

git submodule deinit --all -f
git submodule update --init --recursive
echo -e "DONE CODE PULLS: $f \n\n"

# Set-up the environment and repository dependencies
# Remove existing pyenv to replace with new
if [ -d "pyenv/" ]; then
echo "REMOVING EXISTING PYENV"
rm -Rf pyenv/
fi


python3 -m venv pyenv
source pyenv/bin/activate
pip install -r requirements.txt
echo -e "DONE VIRTUALENV AND PIP: $f \n\n"

# Symlink styles
echo "SYMLINKING STYLING"
cd IATI-Extra-Documentation/en
ln -s ../../../IATI-Websites/iatistandard/_templates/ ./
ln -s ../../../IATI-Websites/iatistandard/_static/ ./

# Generate a version of the documentation
cd ../../

# Run script that creates the static text of the SSOT (using the codelists to generate tables etc.)
./combined_gen.sh
deactivate
echo -e "DONE GENERATING SITE: $f \n\n"

site_folder="${f//.}"

cd ..
done
done
143 changes: 55 additions & 88 deletions deployment/deploy_iati_standard.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,112 +4,79 @@
# Assumes running on webserver3 and the documentation for versions 1.04,
# 1.05, 2.01 & 2.02 are all to be regenerated.

# Maintained under the `iati-bot` gist account.

# Software dependencies must already be installed.

# Script must be run as user 'ssot'. This ensures user 'ssot' is the owner
# Script must be run as user 'iati'. This ensures user 'iati' is the owner
# of all files generated:
# $ sudo -u ssot ./deploy_iati_standard.sh

# The public key for webserver3 must be added to the 'iatiuser' account on
# webserver5.

cd /home/ssot/live


# Ensure documentation and template dependencies are on the 'live' branches and up-to-date
for f in IATI-Developer-Documentation/ IATI-Guidance/ IATI-Websites/; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"

git checkout live
git pull
echo -e "DONE CODE PULLS: $f \n\n"

cd ..
done
echo -e "UPDATED DOCS & TEMPLATES \n\n"


# Regenerate all versions of the sites, saving HTML outputs in '_build/dirhtml'
for f in 1.04 1.05 2.01 2.02 2.03; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"

# Ensure code and submodules are up-to-date with origin
git pull
git submodule update
echo -e "DONE CODE PULLS: $f \n\n"

# Set-up the environment and repository dependencies
source pyenv/bin/activate
pip install -r requirements.txt
echo -e "DONE VIRTUALENV AND PIP: $f \n\n"

# Run script that creates the static text of the SSOT (using the codelists to generate tables etc.)
./combined_gen.sh
deactivate
echo -e "DONE GENERATING SITE: $f \n\n"

site_folder="${f//.}"

# Copy the output files to the live webserver
scp -r docs-copy/en/_build/dirhtml [email protected]:~/ssot/${site_folder}-new
# Real live folder is '/home/iatiuser/ssot/'

# Make a backup version of the current site, and make the new version live
ssh [email protected] "cd ~/ssot/;rm -rf ${site_folder}.bak;mv ${site_folder} ${site_folder}.bak;mv ${site_folder}-new ${site_folder}"

cd ..
# $ sudo -u iati ./deploy_iati_standard.sh

# The public key for the user that this script is to be run on must
# be an authorised key of the user of the live instance.

DEFAULT_DIR=/var/www/dev.reference.iatistandard.org/html
DEFAULT_SSH_USER=iati
DEFAULT_SSH_HOST=134.209.18.180
DEFAULT_BUILD_DIR=docs-copy/en/_build/dirhtml
DEFAULT_REMOTE_DIR=/var/www/reference.iatistandard.org/html
DEFAULT_BACKUP_DIR=/var/www/reference.iatistandard.org/html/backups

while getopts "d:s:i:b:r:ht:" opt; do
echo $OPTARG
case "$opt" in
d) DIRECTORY_INPUT=$OPTARG
;;
s) SSH_USER_INPUT=$OPTARG
;;
i) SSH_HOST_INPUT=$OPTARG
;;
b) BUILD_DIR_INPUT=$OPTARG
;;
r) REMOTE_DIR_INPUT=$OPTARG
;;
t) BACKUP_DIR_INPUT=$OPTARG
;;
h) echo "Usage:"
echo " deploy_iati_standard.sh [-d [path/to/dir]] [-s [ssh_user] [-i [ssh.host] -b [path/to/dir] [-r [path/to/dir]]]]."
echo " -d [path/to/dir] specify a path for the SSOT"
echo " -s specify a ssh user for the remote host"
echo " -i specify a hostname or IP for the remote host"
echo " -b [path/to/dir] specify a local build directory for the HTML output"
echo " -r [path/to/dir] specify a remote path for the SSOT"
echo " -t [path/to/dir] specify a directory for backups"
exit 0
;;
\? )
echo "Option invalid: -$OPTARG" 1>&2
exit 1
;;
esac
done


cd /home/ssot/live
DIRECTORY=${DIRECTORY_INPUT:-$DEFAULT_DIR}
SSH_USER=${SSH_USER_INPUT:-$DEFAULT_SSH_USER}
SSH_HOST=${SSH_HOST_INPUT:-$DEFAULT_SSH_HOST}
BUILD_DIR=${BUILD_DIR_INPUT:-$DEFAULT_BUILD_DIR}
REMOTE_DIR=${REMOTE_DIR_INPUT:-$DEFAULT_REMOTE_DIR}
BACKUP_DIR=${BACKUP_DIR_INPUT:-$DEFAULT_BACKUP_DIR}


# Ensure documentation and template dependencies are on the 'live' branches and up-to-date
for f in IATI-Developer-Documentation/ IATI-Guidance/ IATI-Websites/; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"

git checkout live
git pull
echo -e "DONE CODE PULLS: $f \n\n"

cd ..
done
echo -e "UPDATED DOCS & TEMPLATES \n\n"
cd $DIRECTORY


# Regenerate all versions of the sites, saving HTML outputs in '_build/dirhtml'
for f in 1.04 1.05 2.01 2.02 2.03; do
cd $f
echo -e "NOW IN FOLDER: $f \n\n"

# Ensure code and submodules are up-to-date with origin
git pull
git submodule update
echo -e "DONE CODE PULLS: $f \n\n"

# Set-up the environment and repository dependencies
source pyenv/bin/activate
pip install -r requirements.txt
echo -e "DONE VIRTUALENV AND PIP: $f \n\n"

# Run script that creates the static text of the SSOT (using the codelists to generate tables etc.)
./combined_gen.sh
deactivate
echo -e "DONE GENERATING SITE: $f \n\n"
echo -e "NOW IN FOLDER: $PWD \n\n"

site_folder="${f//.}"

# Copy the output files to the live webserver
scp -r docs-copy/en/_build/dirhtml [email protected]:~/ssot/${site_folder}-new
# Real live folder is '/home/iatiuser/ssot/'
echo "COPYING BUILD DIRECTORY TO REMOTE AS NEW VERSION"

rsync -ai ${BUILD_DIR} ${SSH_USER}@${SSH_HOST}:${REMOTE_DIR}/${site_folder}-new

# Make a backup version of the current site, and make the new version live
ssh [email protected] "cd ~/ssot/;rm -rf ${site_folder}.bak;mv ${site_folder} ${site_folder}.bak;mv ${site_folder}-new ${site_folder}"
ssh ${SSH_USER}@${SSH_HOST} "cd ${REMOTE_DIR};if [ -d ${BACKUP_DIR}/${site_folder}.bak ]; then rm -rf ${site_folder}.bak; fi;mv $f ${BACKUP_DIR}/${site_folder}.bak;mv ${site_folder}-new $f;"

cd ..
done