#!/usr/bin/python #-------------------------------------------------------------------------------------------------------------# # Author: Cliff Cogdill # Description: Review the logs for disk alerts and open a servicenow ticket if one exists. # # #-------------------------------------------------------------------------------------------------------------# import sys,argparse,json,time,smtplib from email.message import EmailMessage sys.path.insert(0, './classes/') import cohesityAPI as cohesity import serviceNowAPI as snow import automationsAPI as dashboard def GetArgs(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--cluster', '-c', type=str, action='store') parser.add_argument('--debugMode', '-d', action='store_true') parser.add_argument('--days', '-t', type=int, action='store') parser.add_argument('--help', '-h', action='store_true') return (parser.parse_args()) def PrintHelp(): print("\nBasic Usage:") print("\n python3 diskAlerts.py -c cluster1.domain.tld" ) print("\t -c FQDN of Cohesity cluster address") print("\t -t time duration to look back in days") print("\t -h Prints this help message") def SendEmail(body, cluster_name): if debugMode: recipients = ['cecogdill@nd.gov'] else: recipients = ['zmeier@nd.gov', 'cecogdill@nd.gov'] email = EmailMessage() email['Subject'] = "Cohesity drive failure on {0}".format(cluster_name) email['From'] = "No-Reply@nd.gov" email['To'] = ", ".join(recipients) email.set_content(body) with smtplib.SMTP('apprelay.nd.gov') as smtp: smtp.send_message(email) print("Sent email") # Define variables args = GetArgs() if args.debugMode: debugMode = True else: debugMode = False # Check for arguments and act accoringly if args.help: PrintHelp() exit(1) # Establish a connection to ServiceNow if debugMode: ticketSystem = snow.SnowAPI("northdakotadev.service-now.com") else: ticketSystem = snow.SnowAPI("northdakota.service-now.com") # Establish a connection to Cohesity mycluster = cohesity.API(args.cluster) authToken = mycluster.GetAuthToken() mycluster.UpdateHeaders(authToken['accessToken']) # Set lookback period if args.days: d = args.days if d > 0: d = d * -1 else: d = -1 # Get yesterday's start time in unixEpoch: prevWindow = mycluster.GetRelativeTimestamp(d, 0, 0, 0) print(prevWindow) # Pull a list of protection runs from yesterday's backup window. diskAlerts = mycluster.GetDiskAlerts(prevWindow) print(diskAlerts) # Track disk id so we don't open multiple tickets for the same drive disks = [] for alert in diskAlerts: alert_name = alert['alertDocument']['alertName'] # Valid alertNames: NewDiskFound, DriveRemoved, DiskOkToRemove, DiskNotHealthy, MarkDiskForRemoval if alert_name == 'DiskNotHealthy' or alert_name == 'MarkDiskForRemoval': for index in alert['propertyList']: if index['key'] == 'disk_id': disk_num = index['value'] # Ticket has already been opened for this disk if disk_num in disks: continue else: # Create the incident incident = ticketSystem.submitTicket("svccohesityadm", "Cohesity Drive Failure: " + args.cluster , alert['alertDocument']['alertCause']) incidentID = incident['result']['sys_id'] # Assign the incident to storage ticketSystem.assignTicketToGroup(incidentID, 'NDIT-Cloud Platforms') message = "Cohesity Drive Failure" body = "Incident: " + incident['result']['number'] body = body + "\nFollow Procedure:" body = body + "\n\thttps://northdakota.service-now.com/kb_view.do?sysparm_article=KB0014369" # Send the email to the list of recipients in the local SendEmail function SendEmail(body, args.cluster) for index in alert['propertyList']: if index['key'] == 'disk_id': disks.append(index['value']) dashboard.send_automation({'AutomationName': 'Infra-Cohesity', 'Action': 'Maintenance', 'Platform': 'Python-dailyErrors.py', 'Units': 60})