Zack Meier
125 lines
4.1 KiB
Python
125 lines
4.1 KiB
Python
#!/usr/bin/python
|
|
#-------------------------------------------------------------------------------------------------------------#
|
|
# Author: Cliff Cogdill
|
|
# Description: Review the logs for disk alerts and open a servicenow ticket if one exists.
|
|
#
|
|
#
|
|
#-------------------------------------------------------------------------------------------------------------#
|
|
import sys,argparse,json,time,smtplib
|
|
from email.message import EmailMessage
|
|
sys.path.insert(0, './classes/')
|
|
|
|
import cohesityAPI as cohesity
|
|
import serviceNowAPI as snow
|
|
import automationsAPI as dashboard
|
|
|
|
def GetArgs():
|
|
parser = argparse.ArgumentParser(add_help=False)
|
|
parser.add_argument('--cluster', '-c', type=str, action='store')
|
|
parser.add_argument('--debugMode', '-d', action='store_true')
|
|
parser.add_argument('--days', '-t', type=int, action='store')
|
|
parser.add_argument('--help', '-h', action='store_true')
|
|
return (parser.parse_args())
|
|
|
|
def PrintHelp():
|
|
print("\nBasic Usage:")
|
|
print("\n python3 diskAlerts.py -c cluster1.domain.tld" )
|
|
print("\t -c FQDN of Cohesity cluster address")
|
|
print("\t -t time duration to look back in days")
|
|
print("\t -h Prints this help message")
|
|
|
|
def SendEmail(body, cluster_name):
|
|
if debugMode:
|
|
recipients = ['cecogdill@nd.gov']
|
|
else:
|
|
recipients = ['zmeier@nd.gov', 'cecogdill@nd.gov']
|
|
|
|
email = EmailMessage()
|
|
email['Subject'] = "Cohesity drive failure on {0}".format(cluster_name)
|
|
email['From'] = "No-Reply@nd.gov"
|
|
email['To'] = ", ".join(recipients)
|
|
email.set_content(body)
|
|
|
|
with smtplib.SMTP('apprelay.nd.gov') as smtp:
|
|
smtp.send_message(email)
|
|
|
|
print("Sent email")
|
|
|
|
# Define variables
|
|
args = GetArgs()
|
|
|
|
if args.debugMode:
|
|
debugMode = True
|
|
else:
|
|
debugMode = False
|
|
|
|
# Check for arguments and act accoringly
|
|
if args.help:
|
|
PrintHelp()
|
|
exit(1)
|
|
|
|
# Establish a connection to ServiceNow
|
|
if debugMode:
|
|
ticketSystem = snow.SnowAPI("northdakotadev.service-now.com")
|
|
else:
|
|
ticketSystem = snow.SnowAPI("northdakota.service-now.com")
|
|
|
|
# Establish a connection to Cohesity
|
|
mycluster = cohesity.API(args.cluster)
|
|
authToken = mycluster.GetAuthToken()
|
|
mycluster.UpdateHeaders(authToken['accessToken'])
|
|
|
|
# Set lookback period
|
|
if args.days:
|
|
d = args.days
|
|
if d > 0:
|
|
d = d * -1
|
|
else:
|
|
d = -1
|
|
|
|
# Get yesterday's start time in unixEpoch:
|
|
prevWindow = mycluster.GetRelativeTimestamp(d, 0, 0, 0)
|
|
print(prevWindow)
|
|
|
|
# Pull a list of protection runs from yesterday's backup window.
|
|
diskAlerts = mycluster.GetDiskAlerts(prevWindow)
|
|
print(diskAlerts)
|
|
|
|
# Track disk id so we don't open multiple tickets for the same drive
|
|
disks = []
|
|
|
|
for alert in diskAlerts:
|
|
|
|
alert_name = alert['alertDocument']['alertName']
|
|
|
|
# Valid alertNames: NewDiskFound, DriveRemoved, DiskOkToRemove, DiskNotHealthy, MarkDiskForRemoval
|
|
if alert_name == 'DiskNotHealthy' or alert_name == 'MarkDiskForRemoval':
|
|
for index in alert['propertyList']:
|
|
if index['key'] == 'disk_id':
|
|
disk_num = index['value']
|
|
|
|
# Ticket has already been opened for this disk
|
|
if disk_num in disks:
|
|
continue
|
|
else:
|
|
# Create the incident
|
|
incident = ticketSystem.submitTicket("svccohesityadm", "Cohesity Drive Failure: " + args.cluster , alert['alertDocument']['alertCause'])
|
|
incidentID = incident['result']['sys_id']
|
|
|
|
# Assign the incident to storage
|
|
ticketSystem.assignTicketToGroup(incidentID, 'NDIT-Cloud Platforms')
|
|
|
|
message = "Cohesity Drive Failure"
|
|
body = "Incident: " + incident['result']['number']
|
|
body = body + "\nFollow Procedure:"
|
|
body = body + "\n\thttps://northdakota.service-now.com/kb_view.do?sysparm_article=KB0014369"
|
|
|
|
# Send the email to the list of recipients in the local SendEmail function
|
|
SendEmail(body, args.cluster)
|
|
|
|
for index in alert['propertyList']:
|
|
if index['key'] == 'disk_id':
|
|
disks.append(index['value'])
|
|
|
|
dashboard.send_automation({'AutomationName': 'Infra-Cohesity', 'Action': 'Maintenance', 'Platform': 'Python-dailyErrors.py', 'Units': 60})
|