Add email capability, unattended option, general tidyup

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - sat_import now checks for exports that have not been imported (missed/skipped)
 - --fixhistory option in sat_import to align import/export histories
+- Email output capability for notifications when automating scripts
+- Add unattended option to allow scripts to be automated
 
 ### Changed
 - --notar export saved in /cdn_export dir rather than /export to prevent it being deleted

README.md

@@ -68,6 +68,11 @@ logging:
   dir: /var/log/sat6-scripts     (Directory to use for logging)
   debug: [True|False]
 
+email:
+  mailout: True
+  mailfrom: Satellite 6 <[email protected]>
+  mailto: [email protected]
+
 export:
   dir: /var/sat-export           (Directory to export content to - Connected Satellite)
 
@@ -199,6 +204,7 @@ optional arguments:
   -l, --last            Display time of last export
   -L, --list            List all successfully completed exports
   --nogpg               Skip GPG checking
+  -u, --unattended      Answer any prompts safely, allowing automated usage
   -r, --repodata        Include repodata for repos with no incremental content
   -p, --puppetforge     Include puppet-forge-server format Puppet Forge repo
   --notar               Do not archive the extracted content
@@ -271,6 +277,7 @@ optional arguments:
   -L, --list            List all successfully completed imports
   -c, --count           Display all package counts after import
   -f, --force           Force import of data if it has previously been done  
+  -u, --unattended      Answer any prompts safely, allowing automated usage
   --fixhistory          Force import history to match export history  
 ```
 

config/config.yml.example

@@ -11,6 +11,11 @@ logging:
   dir: /var/log/satellite
   debug: False
 
+email:
+  mailout: True
+  mailfrom: Satellite 6 <[email protected]>
+  mailto: [email protected]
+
 export:
   dir: /var/sat-export
 

helpers.py

@@ -10,9 +10,10 @@
 """Functions common to various Satellite 6 scripts"""
 
 import sys, os, time, datetime, argparse
-import logging
+import logging, tempfile
 from time import sleep
 from hashlib import sha256
+import smtplib
 
 try:
     import requests
@@ -59,6 +60,14 @@
     PROMOTEBATCH = CONFIG['promotion']['batch']
 else:
     PROMOTEBATCH = 255
+if 'mailout' in CONFIG['email']:
+    MAILOUT = CONFIG['email']['mailout']
+else:
+    MAILOUT = False
+if 'mailfrom' in CONFIG['email']:
+    MAILFROM = CONFIG['email']['mailfrom']
+if 'mailto' in CONFIG['email']:
+    MAILTO = CONFIG['email']['mailto']
 if 'hostname' in CONFIG['puppet-forge-server']:
     PFSERVER = CONFIG['puppet-forge-server']['hostname']
 
@@ -85,6 +94,11 @@
 BOLD = '\033[1m'
 UNDERLINE = '\033[4m'
 
+# Mailout pre-canned subjects
+MAILSUBJ_FI = "Satellite 6 import failure"
+MAILSUBJ_SI = "Satellite 6 import successful"
+MAILSUBJ_FP = "Satellite 6 publish failure"
+MAILSUBJ_SP = "Satellite 6 publish successful"
 
 def who_is_running():
     """ Return the OS user that is running the script """
@@ -435,6 +449,20 @@ def query_yes_no(question, default="yes"):
                              "(or 'y' or 'n').\n")
 
 
+def mailout(subject, message):
+    """
+    Function to handle simple SMTP mailouts for alerting.
+    Assumes localhost is configured for SMTP forwarding (postfix)
+    """
+    sender = MAILFROM
+    receivers = [MAILTO]
+
+    body = 'From: {}\nSubject: {}\n\n{}'.format(sender, subject, message)
+
+    smtpObj = smtplib.SMTP('localhost')
+    smtpObj.sendmail(sender, receivers, body)
+
+
 #-----------------------
 # Configure logging
 if not os.path.exists(LOGDIR):
@@ -453,6 +481,9 @@ def query_yes_no(question, default="yes"):
 logging.getLogger("requests").setLevel(logging.WARNING)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
+# Open a temp file to hold the email output
+tf = tempfile.NamedTemporaryFile()
+
 def log_msg(msg, level):
     """Write message to logfile"""
 
@@ -463,10 +494,13 @@ def log_msg(msg, level):
             print BOLD + "DEBUG: " + msg + ENDC
     elif level == 'ERROR':
         logging.error(msg)
+        tf.write('ERROR:' + msg + '\n')
         print ERROR + "ERROR: " + msg + ENDC
     elif level == 'WARNING':
         logging.warning(msg)
+        tf.write('WARNING:' + msg + '\n')
         print WARNING + "WARNING: " + msg + ENDC
     # Otherwise if we ARE in debug, write everything to the log AND stdout
     else:
         logging.info(msg)
+        tf.write(msg + '\n')

import_auto.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+import sys, os, glob
+import subprocess
+import argparse
+import helpers
+
+
+def run_imports(dryrun):
+    print "Processing Imports..."
+
+    # Find any sha256 files in the import dir
+    infiles = glob.glob(helpers.IMPORTDIR + '/*.sha256')
+
+    # Extract the dataset timestamp/name from the filename and add to a new list
+    # Assumes naming standard   sat6_export_YYYYMMDD-HHMM_NAME.sha256
+    # 'sorted' function should result in imports being done in correct order by filename
+    tslist = []
+    for f in sorted(infiles):
+        dstime = f.split('_')[-2]
+        dsname = (f.split('_')[-1]).split('.')[-2]
+        tslist.append(dstime + '_' + dsname)
+
+    if tslist:
+        msg = 'Found import datasets on disk...\n' + '\n'.join(tslist)
+    else:
+        msg = 'No import datasets to process'
+    helpers.log_msg(msg, 'INFO')
+    print msg
+
+    # Now for each import file in the list, run the import script in unattended mode:-)
+    if tslist:
+        if not dryrun:
+            for dataset in tslist:
+                rc = subprocess.call(['/usr/local/bin/sat_import', '-u', '-r', '-d', dataset])
+                print rc
+        else:
+            msg = "Dry run - not actually performing import"
+            helpers.log_msg(msg, 'WARNING')
+
+
+def main(args):
+
+    ### Run import/publish on scheduled day
+
+    # Check for sane input
+    parser = argparse.ArgumentParser(
+        description='Imports, Publishes and Promotes content views.')
+    parser.add_argument('-d', '--dryrun', help='Dry Run - Only show what will be done',
+        required=False, action="store_true")
+
+    args = parser.parse_args()
+
+    if args.dryrun:
+        dryrun = True
+    else:
+        dryrun = False
+
+
+    # Check if there are any imports in our input dir and import them
+    run_imports(dryrun)
+
+    # If all imports successful run publish
+
+
+    ### Run promote on scheduled display
+
+
+
+    ### Run cleanup on scheduled day
+
+
+
+
+
+
+if __name__ == "__main__":
+    try:
+        main(sys.argv[1:])
+    except KeyboardInterrupt, e:
+        print >> sys.stderr, ("\n\nExiting on user cancel.")
+        sys.exit(1)

sat_export.py

@@ -11,7 +11,7 @@
 """
 
 import sys, argparse, datetime, os, shutil, pickle, re
-import fnmatch, subprocess, tarfile
+import fnmatch, subprocess, tarfile, tempfile
 import simplejson as json
 from glob import glob
 from distutils.dir_util import copy_tree
@@ -82,12 +82,22 @@ def export_cv(dov_ver, last_export, export_type):
     except: # pylint: disable-msg=W0702
         msg = "Unable to start export - Conflicting Sync or Export already in progress"
         helpers.log_msg(msg, 'ERROR')
+        if helpers.MAILOUT:
+            helpers.tf.seek(0)
+            output = "{}".format(helpers.tf.read())
+            subject = "Satellite 6 export failure"
+            helpers.mailout(subject, output)
         sys.exit(1)
 
     # Trap some other error conditions
     if "Required lock is already taken" in str(task_id):
         msg = "Unable to start export - Sync in progress"
         helpers.log_msg(msg, 'ERROR')
+        if helpers.MAILOUT:
+            helpers.tf.seek(0)
+            output = "{}".format(helpers.tf.read())
+            subject = "Satellite 6 export failure"
+            helpers.mailout(subject, output)
         sys.exit(1)
 
     msg = "Export started, task_id = " + str(task_id)
@@ -105,7 +115,7 @@ def export_repo(repo_id, last_export, export_type):
         msg = "Exporting repository id " + str(repo_id)
     else:
         msg = "Exporting repository id " + str(repo_id) + " from start date " + last_export
-    helpers.log_msg(msg, 'INFO')
+    helpers.log_msg(msg, 'DEBUG')
 
     try:
         if export_type == 'full':
@@ -126,12 +136,22 @@ def export_repo(repo_id, last_export, export_type):
     except: # pylint: disable-msg=W0702
         msg = "Unable to start export - Conflicting Sync or Export already in progress"
         helpers.log_msg(msg, 'ERROR')
+        if helpers.MAILOUT:
+            helpers.tf.seek(0)
+            output = "{}".format(helpers.tf.read())
+            subject = "Satellite 6 export failure"
+            helpers.mailout(subject, output)
         sys.exit(1)
 
     # Trap some other error conditions
     if "Required lock is already taken" in str(task_id):
         msg = "Unable to start export - Sync in progress"
         helpers.log_msg(msg, 'ERROR')
+        if helpers.MAILOUT:
+            helpers.tf.seek(0)
+            output = "{}".format(helpers.tf.read())
+            subject = "Satellite 6 export failure"
+            helpers.mailout(subject, output)
         sys.exit(1)
 
     msg = "Export started, task_id = " + str(task_id)
@@ -404,14 +424,19 @@ def check_incomplete_sync():
     if incomplete_sync:
         msg = "Incomplete sync jobs detected"
         helpers.log_msg(msg, 'WARNING')
-        answer = helpers.query_yes_no("Continue with export?", "no")
-        if not answer:
+        if not args.unattended:
+            answer = helpers.query_yes_no("Continue with export?", "no")
+            if not answer:
+                msg = "Export Aborted"
+                helpers.log_msg(msg, 'ERROR')
+                sys.exit(3)
+            else:
+                msg = "Export continued by user"
+                helpers.log_msg(msg, 'INFO')
+        else:
             msg = "Export Aborted"
             helpers.log_msg(msg, 'ERROR')
-            sys.exit(1)
-        else:
-            msg = "Export continued by user"
-            helpers.log_msg(msg, 'INFO')
+            sys.exit(3)
 
 
 def check_disk_space(export_type):
@@ -423,14 +448,19 @@ def check_disk_space(export_type):
     if export_type == 'full' and int(float(pulp_used)) > 50:
         msg = "Insufficient space in /var/lib/pulp for a full export. >50% free space is required."
         helpers.log_msg(msg, 'WARNING')
-        answer = helpers.query_yes_no("Continue with export?", "no")
-        if not answer:
+        if not args.unattended:
+            answer = helpers.query_yes_no("Continue with export?", "no")
+            if not answer:
+                msg = "Export Aborted"
+                helpers.log_msg(msg, 'ERROR')
+                sys.exit(3)
+            else:
+                msg = "Export continued by user"
+                helpers.log_msg(msg, 'INFO')
+        else:
             msg = "Export Aborted"
             helpers.log_msg(msg, 'ERROR')
-            sys.exit(1)
-        else:
-            msg = "Export continued by user"
-            helpers.log_msg(msg, 'INFO')
+            sys.exit(3)
 
 
 def locate(pattern, root=os.curdir):
@@ -473,6 +503,12 @@ def do_gpg_check(export_dir):
             helpers.log_msg(msg, 'ERROR')
         msg = "------ Export Aborted ------"
         helpers.log_msg(msg, 'INFO')
+        if helpers.MAILOUT:
+            helpers.tf.seek(0)
+            output = "{}".format(helpers.tf.read())
+            subject = "Satellite 6 export failure - GPG checksum failure"
+            message = "GPG check of exported RPMs failed. Check logs for details\n\n" + output
+            helpers.mailout(subject, message)
         sys.exit(1)
     else:
         msg = "GPG check completed successfully"
@@ -651,6 +687,8 @@ def main(args):
         action="store_true")
     parser.add_argument('-n', '--nogpg', help='Skip GPG checking', required=False,
         action="store_true")
+    parser.add_argument('-u', '--unattended', help='Answer any prompts safely, allowing automated usage',
+        required=False, action="store_true")
     parser.add_argument('--notar', help='Skip TAR creation', required=False,
         action="store_true")
     parser.add_argument('--forcexport', help='Force export on import-only satellite', required=False,
@@ -775,6 +813,7 @@ def main(args):
         msg = "------ " + ename + " Content export started by " + runuser + " ---------"
     helpers.log_msg(msg, 'INFO')
 
+
     # Get the current time - this will be the 'last export' time if the export is OK
     start_time = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
     print "START: " + start_time + " (" + ename + " export)"
@@ -922,6 +961,11 @@ def main(args):
                             if not os.path.exists(exportpath):
                                 msg = exportpath + " was not created.\nCheck permissions/SELinux on export dir"
                                 helpers.log_msg(msg, 'ERROR')
+                                if helpers.MAILOUT:
+                                    helpers.tf.seek(0)
+                                    output = "{}".format(helpers.tf.read())
+                                    subject = "Satellite 6 export failure"
+                                    helpers.mailout(subject, output)
                                 sys.exit(1)
 
                             os.chdir(exportpath)
@@ -1105,6 +1149,13 @@ def main(args):
     msg = "Export complete"
     helpers.log_msg(msg, 'INFO')
 
+    if helpers.MAILOUT:
+        helpers.tf.seek(0)
+        output = "{}".format(helpers.tf.read())
+        subject = "Satellite 6 export complete"
+        message = "Export of " + ename + " successfully completed\n\n" + output
+        helpers.mailout(subject, message)
+
     # Exit cleanly
     sys.exit(0)