#!/usr/bin/python #-------------------------------------------------------------------------------------------------# # Author: Cliff Cogdill # Description: This script accomplishes a few things: # - Reviews all assets in the CMDB to determine if they have been setup in backups. # - If not, identify them and send create a ticket for investigation. # - Reviews paused protection jobs to determine if the job is still empty. # - If not empty, resume the job. # - Send summary report of unprotected servers and VM jobs to the storage team. # ------------------------------------------------------------------------------------------------# import sys,argparse,json,time,yaml,re,smtplib from email.message import EmailMessage sys.path.insert(0, './classes/') import cohesityAPI as cohesity import sharePointAPI as sharePoint import automationsAPI as dashboard #import serviceNowAPI as snow import nditServiceNow as snow import logging #import pymsteams def GetArgs(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('--cluster', '-c', type=str, action='store') parser.add_argument('--debug', '-d', action='store_true') parser.add_argument('--excludeTickets', '-x', action='store_true') parser.add_argument('--refresh', '-r', action='store_true') parser.add_argument('--type', '-t', type=str, action='store') parser.add_argument('--vCenter', '-v', type=str, action='store') parser.add_argument('--help', '-h', action='store_true') return (parser.parse_args()) def SendEmail(body, cluster): if debug: recipients = ['itbackups@nd.gov'] else: recipients = ['zmeier@nd.gov', 'cecogdill@nd.gov','khellman@nd.gov','mlaverdure@nd.gov','ndheck@nd.gov'] email = EmailMessage() email['Subject'] = "Backup Review for {0}".format(cluster) email['From'] = "No-Reply@nd.gov" email['To'] = ", ".join(recipients) email.set_content(body) with smtplib.SMTP('apprelay.nd.gov') as smtp: smtp.send_message(email) print("Sent email") def PrintHelp(): print("\nBasic Usage:") print("\n python3 protectedItems.py -c cluster1.domain.tld [ -v vCenter.domain.tld ] [ -j protectionJobName ]") print("\t -d debug: execute in debug mode to expand summary and do not open tickets") print("\t -x excludeTickets: run against prod serivce now but do not open tickets or changes") print("\t -r refresh: force source refresh in debug mode") print("\t -c FQDN of Cohesity cluster address") print("\t -t type of protection source") print("\t -h Prints this help message") #itdTeamsMessage = pymsteams.connectorcard("https://ndgov.webhook.office.com/webhookb2/aad89030-8f69-4a4d-a853-c882cbb01a10@2dea0464-da51-4a88-bae2-b3db94bc0c54/IncomingWebhook/d3c07921419c4920810d0c32558c05e3/edcc1502-1ff0-4c3e-9fe2-1ce3f9f7981a") args = GetArgs() if args.help: PrintHelp() exit() logging.basicConfig(filename="log/" + args.cluster + '.log', filemode='w', format='%(asctime)s %(message)s', datefmt='%Y-%m-%d %I:%M:%S %p', level=logging.DEBUG) # Define empty lists that will hold JSON dictionaries vmObjects = [] protectedObjects = [] vCenters = [] protectedVM = [] unprotectedVM = [] undocumentedVM = [] populatedJobs = [] populatedPausedJobs = [] emptyJobs = [] # Default to the main cluster on premise if the cluster argument was not specified if not args.cluster: args.cluster = "itdmdndpc01.nd.gov" args.type = "kVMware" if not args.type: args.type = "kVMware" if args.cluster == "itdazdpc01.nd.gov": args.type = "kAzure" if args.cluster == "itdazdpc02.nd.gov": args.type = "kAzure" logging.info("Creating cluster object for {0} with type {1}".format(args.cluster, args.type)) # Setup the token for cluster API usage cluster = cohesity.API(args.cluster) authToken = cluster.GetAuthToken() cluster.UpdateHeaders(authToken['accessToken']) # Lookup the AppNames and vCenter for unprotected VMs #logging.info("Createing cmdb object for SharePoint list") #cmdb = sharePoint.API() # When true, tickets will only be created in ServiceNow dev and JSON output may be expanded to stdout if args.debug: debug = True else: debug = False if args.excludeTickets: excludeTickets = True else: excludeTickets = False # Create the connection to ServiceNow logging.info("Creating ticketSystem object for ServiceNow") if debug: ticketSystem = snow.SnowAPI("northdakotadev.service-now.com") else: ticketSystem = snow.SnowAPI("northdakota.service-now.com") if args.type == "kPhysical": physicalJobs = cluster.GetPhysicalJobs() sys.exit("Warning: Type not implmented for this script.") elif args.type == "kSQL": sys.exit("Warning: Type not implmented for this script.") elif args.type == "kAzure": # Get all registered subscription sources so we can use the source ids for other REST calls logging.info("Collecting all registered Azure sources") nodes = cluster.GetAZsources() # Iterate through all the Azure subscriptions to create one large dataset for all VMs logging.info("Creating dataset for all VMs in our Azure subscriptions") logging.info("Creating dataset for all protected VMs in our Azure subscriptions") for node in nodes: hypervisorId = node['protectionSource']['id'] #Refresh the sources so we ensure we are getting valid information if debug: print("Debug enabled, not refreshing source use -r option to override") else: cluster.RefreshSource(node['protectionSource']['id']) if args.refresh: cluster.RefreshSource(node['protectionSource']['id']) myObjects = cluster.GetAZObjects(hypervisorId) myProtectedObjects = cluster.GetProtectedAZobjects(hypervisorId) # Combine all VM objects into one dictionary for val in myObjects: vmObjects.append(val) # Combine all protected objects into one dictionary for val in myProtectedObjects: protectedObjects.append(val) # Get all the native snapshot protection jobs that exist in the cluster logging.info("Gathering all current protection jobs from {0}".format(args.cluster)) sourceJobs = cluster.GetAZNativeJobs() elif args.type == "kVMware": # Get all registered vCenter sources so we can use the source ids for other REST calls logging.info("Collecting all registered vmWare sources") nodes = cluster.GetVMsources() # Iterate through all the vCenter servers to create one large dataset for all VMs logging.info("Creating dataset for all VMs in our Azure subscriptions") logging.info("Creating dataset for all protected VMs in our Azure subscriptions") for node in nodes: hypervisorId = node['protectionSource']['id'] #Refresh the sources so we ensure we are getting valid information cluster.RefreshSource(node['protectionSource']['id']) myObjects = cluster.GetVMObjects(hypervisorId) myProtectedObjects = cluster.GetProtectedVMobjects(hypervisorId) # Combine all VM objects into one dictionary for val in myObjects: vmObjects.append(val) # Combine all protected objects into one dictionary for val in myProtectedObjects: protectedObjects.append(val) # At this point we have two sets of data # 1. All vms objects whether they are protected or not ( vmObjects ) # 2. All protected vm objects ( protectedObjects ) # Get all the protection jobs that exist for vmware in the cluster logging.info("Gathering all current protection jobs from {0}".format(args.cluster)) sourceJobs = cluster.GetVMjobs() if args.type == "kVMware" or args.type == "kAzure": ############################## BEGIN: IDENTIFY EMPTY & POPULATE PAUSED JOBS ####################################### # Loop through the sourceJobs to see if they show up in at least once in the list of protected objects. # if they don't show up, we can assume the job is created but empty, which is usually the case for # placeholder VMs # In certain circumstances, jobs can be defined but exempt from normal processing. These jobs are normally used from time to time for POC work and do not necessarily need to be protected, here we will load them into the exempt jobs array from the exceptions.yml file. logging.info("Opening exemptions file: vars/exemptions.yml") with open("./vars/exemptions.yml", "r") as exemptionJobData: try: exemptionJobData = yaml.load(exemptionJobData, Loader=yaml.FullLoader) except: print("Unable to load exemptions file.") exemptJobs = [] logging.info("Parsing exemptions YAML") for i in exemptionJobData: for e in exemptionJobData[i]['jobs']: #exemptJobs.append(e.lower()) exemptJobs.append([exemptionJobData[i]['id'],e.lower()]) # Loop through all the jobs for logging.info("Iterating through all protection jobs") for sourceJob in sourceJobs: #Set default control variables for each job isJobExempt = False isJobEmpty = True # The REST GetVMjobs function returns unfiltered jobs, so lets get rid of the deleted ones if 'isDeleted' in sourceJob: if sourceJob['isDeleted'] == True: logging.info("--Job {0} is marked for deletion, skipping review".format(sourceJob['name'])) continue # Look through the exemptJobs listing for entry in exemptJobs: parent = entry[0] pattern = entry[1] if parent == sourceJob['parentSourceId']: if re.search(pattern, sourceJob['name'].lower()): isJobExempt = True break if isJobExempt == True: if debug == True: logging.info("--Job {0} has been exempt from backups, skipping review".format(sourceJob['name'])) print("Job {0} is exempt from tickets".format(sourceJob['name'])) continue # The protection jobs only show tags, they do not show VM ojbects; therefore, to determine if the job is still valid, we have to iterate through all protected objects to see if the job name shows up or not. If it does not show up, then it is assumed to be empty and paused to prevent backup errors. Also, there is usually only one job listed per protected VM; however, there could be cases where a VM is in two or more jobs so we need to check each job listed. # Sample dataset can be seen by using the following url: # https://itdmdndpc01.nd.gov/irisservices/api/v1/public/protectionSources/protectedObjects?environment=kVMware&id=3156 logging.info("--Iterating through all protected objects to verify if job {0} is empty".format(sourceJob['name'])) for item in protectedObjects: for pJob in item['protectionJobs']: if pJob['name'] == sourceJob['name']: isJobEmpty = False continue # Seperate empty from non-empty jobs if isJobEmpty: logging.info("--{0} is empty, pausing future runs".format(sourceJob['name'])) emptyJobs.append(sourceJob) # Pause the job so we don't generate erroneous errors. cluster.PauseJob(sourceJob['id']) else: # The job is not empty, lets check to see if it is paused so we can create an alert. # First we have to check if the isPaused key exists, if it has never been set, it will not. if 'isPaused' in sourceJob: # Now we check to see if isPaused is set to True if sourceJob['isPaused']: logging.info("----{0} is paused but contains VM objects, adding to populatedPausedJobs dataset".format(sourceJob['name'])) populatedPausedJobs.append(sourceJob) # isPaused might be defined but it is set to False else: populatedJobs.append(sourceJob) # If the job has never been paused, isPaused may not be defined else: populatedJobs.append(sourceJob) ############################## END: IDENTIFY EMPTY & POPULATE JOBS ####################################### logging.info("Iteration of all protection jobs is complete") ######################### BEGIN: OPEN TICKETS FOR POPULATED PAUSED JOBS ################################## # This block of code is duplicate logic from above, the populatedPausedJobs array should not contain exempt jobs, we already checked for that. - CC 20230928 # if debug: # print("Exempt jobs:\n{0}".format(exemptJobs)) # # for job in populatedPausedJobs: # isJobExempt = False # # for entry in exemptJobs: # parent = entry[0] # pattern = entry[1] # # if parent == job['parentSourceId']: # if re.search(pattern, job['name'].lower()): # isJobExempt = True # break # # if isJobExempt: # continue # else: logging.info("Opening tickets for paused jobs that contain VM objects") for job in populatedPausedJobs: if not excludeTickets: subject = "Backup job " + job['name'] + " is paused" message = "This backup job contains VMs and should not be paused per SLA. Either fix the issue and resume the job, exempt the VM(s) from backups, or decommission the VMs." incident = ticketSystem.submitTicket("svccohesityadm", subject, message) incidentID = incident['result']['sys_id'] ticketSystem.assignTicketToGroup(incidentID, "NDIT-Cloud Platforms") logging.info("--{0} created for {1}".format(incident['result']['number'],job['name'])) ######################### END: OPEN TICKETS FOR POPULATED PAUSED JOBS ################################## ############################## BEGIN: IDENTIFY UNPROTECTED VMS ################################# # RFE: 002 # Open the list of documented exceptions, in the long term this may be in the ServiceNow CMDB # if it is, we would have to look it up using the SNOW API with open("./vars/exemptions.yml", "r") as exemptionData: try: exemptionData = yaml.load(exemptionData, Loader=yaml.FullLoader) except: print("Unable to load exemptions file.") exemptions = [] for i in exemptionData: for e in exemptionData[i]['vms']: #exemptions.append([exemptionData[i]['id'],e.lower()]) exemptions.append([exemptionData[i]['id'],e]) # Find protected / unprotected vms logging.info("Iterating through all vmObjects found under sources registered to {0}".format(args.cluster)) for vm in vmObjects: # We don't need to monitor VMs that are marked as a template VM if vm['environment'] == "kVMware": if 'isVmTemplate' in vm['vmWareProtectionSource']: if vm['vmWareProtectionSource']['isVmTemplate'] == True: continue # Assume all VM objects are not protected until we prove otherwise isProtected = False # Use flag to determine if VM was found or not for pVM in protectedObjects: if vm['name'] == pVM['protectionSource']['name']: isProtected = True protectedVM.append(vm) break # The Azure naming convention is different from the vmware naming convention, we need to normalize the names for sharepoint loookups # and exemption lookups # SharePoint Removal #if vm['environment'] == "kAzure": # if "-" in vm['name']: # nameParts = vm['name'].split("-") # hostname= cmdb.find_VM(nameParts[1]) # if nameParts[1] == hostname: # hostname = cmdb.find_VM(vm['name']) # if hostname == "vm-lcsdev1-1": # hostname = cmdb.get_Node("vm-lcsdev1dev.centralus.cloudapp.azure.com") # else: # hostname = cmdb.find_VM(vm['name']) # vm.update({"name": hostname}) # This part needs some significant improvement on handling the exceptions. Ideally we would want to ensure the # hostname that is excluded belongs to the same parent in the exemption file and in Cohesity. if not isProtected: isExempted = False for entry in exemptions: parent = entry[0] pattern = entry[1] if parent == vm['parentId']: if re.search(pattern, vm['name'].lower()): isExempted = True break else: continue if isExempted == False: unprotectedVM.append(vm) ############################## END: IDENTIFY UNPROTECTED VMS ################################# ###################### BEGIN: OPEN TICKETS FOR UNPROTECTED & UNEXEMPTED VMS ######################## # Setup an array to hold unique AppNames unProtectedApps = [] for vm in unprotectedVM: #RFE 001: CMDB reference lookups will need to be updated to the SeriveNow CMDB once it is ready for production, this is slated for October 2022. ###### sharepoint ##### #cmdbRecord = cmdb.get_Node(vm['name']) ###### ServiceNow ##### cmdbRecord = ticketSystem.getCMDBItemByFQDN(vm['name']) vcenterSource = cluster.GetFilteredRequest("/public/protectionSources", "?id=" + str(vm['parentId'])) vcenterName = vcenterSource[0]['protectionSource']['name'].split(".")[0] if args.type == "kAzure": if vcenterName == "437b2bfa-850e-4464-b6c2-38a68cda7c69": vcenterName = "prd01" policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-PRD") for pol in policyData: if pol['name'] == "ITD-Bronze-PRD": jobPolicy=pol['id'] elif vcenterName == "76297098-764c-43de-8525-c9fda1b237be": vcenterName = "npd01" policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-NPD") for pol in policyData: if pol['name'] == "ITD-Bronze-NPD": jobPolicy=pol['id'] elif vcenterName == "e53aa0c7-824d-40a2-b420-4ab77b1051d2": vcenterName = "infra01" policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-INFRA") for pol in policyData: if pol['name'] == "ITD-Bronze-INFRA": jobPolicy=pol['id'] elif vcenterName == "cb39dbef-0ecf-434d-a8b9-444cef8f390a": vcenterName = "cares01" policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-CARES") for pol in policyData: if pol['name'] == "ITD-Bronze-CARES": jobPolicy=pol['id'] elif vcenterName == "54048952-c2b2-4c65-ab44-157750caa8e6": vcenterName = "NDIT-PaaS" policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-NDIT-PaaS") for pol in policyData: if pol['name'] == "ITD-Bronze-NDIT-PaaS": jobPolicy=pol['id'] # Trim off .nd.gov from the name vcenterName = vcenterName.split(".")[0] ########################################################################################################################## # # This was part of the SharePoint & ServiceNow cutover. Since we already pull the snow record above now, we don't # # need to try to look for it again. # if len(cmdbRecord) == 0: # # Handle cases where the record doesn't exist in Sharepoitn yet, but might be going through # # the build process in ServiceNow # try: # print("Trying to find: {}".format(vm['name'])) # cmdbData = ticketSystem.getCMDBItemByFQDN(vm['name']) # # print("ServiceNow: {0}, vmware: {1}".format(cmdbData[0]['fqdn'],vm['name'])) # # if cmdbData[0]['fqdn'] == vm['name']: # print("Found in Snow CMDB") # inSnow = True # except: # inSnow = False # ######################################################################################################################## # Open a ticket for unprotected VMs that are not in the CMDB if not excludeTickets: undocumentedVM.append(vm['name'] + " on " + vcenterName) #12/11/2023: Removed tickets for undocumented VMs until the CMDB is fixed. print("Undocumented VM: {0} on {1}".format(vm['name'],vcenterName)) print("Ticket suppressed") ''' subject = "Backup job for " + vm['name'] + " on " + vcenterName + " was not found." message = "This VM was not found in the CMDB and is not being backed up.\nInvestigate why the object is not documented in the vmware list, and not being backed up. If necessary have the backup admins exclude the item in the exemptions.yml file" 12/11/2023: Removed tickets for undocumented VMs until the CMDB is fixed. incident = ticketSystem.submitTicket("svccohesityadm", subject, message) jincidentID = incident['result']['sys_id'] ticketSystem.assignTicketToGroup(incidentID, "NDIT-Cloud Platforms") print("Ticket {} created for {}".format(incident['result']['number'], vm['name'])) ''' continue else: continue ######################################################################################################################## # There is a record for the VM in the CMDB so lets construct the appropriate protection group name else: #RFE 001: CMDB reference lookups will need to be updated to the SeriveNow CMDB once it is ready for production, this is slated for October 2022. ###### SharePoint ##### #vmAppName = cmdb.get_AppName(cmdbRecord[0]['AppNameId']) #vmEnvironment = cmdbRecord[0]['Environment'] ###### ServiceNow ##### vmAppName = cmdbRecord[0]['u_nd_application_svc']['name'] vmEnvironment = cmdbRecord[0]['environment'] if args.type == "kVMware": if vmEnvironment == "Production": jobName = vmAppName + "@" + "PRD-" + vcenterName policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-PRD") for pol in policyData: if pol['name'] == "ITD-Bronze-PRD": jobPolicy=pol['id'] #jobPolicy="4847477517838800:1610060943809:72255100" elif vmEnvironment == "Test": jobName = vmAppName + "@" + "NPD-" + vcenterName policyData = cluster.GetProtectionPolicyByName("ITD-Bronze-NPD") for pol in policyData: if pol['name'] == "ITD-Bronze-NPD": jobPolicy=pol['id'] #jobPolicy="4847477517838800:1610060943809:72255060" elif args.type == "kAzure": jobName = "{0}@{1}".format(vmAppName, vcenterName) # Don't add the job name to the unprotectedApps array more than once if jobName in unProtectedApps: continue else: # Verify that the job does not already exist in Cohesity verify = cluster.GetFilteredRequest("/public/protectionJobs", "?names=" + jobName ) if len(verify) > 0: for record in verify: if record['name'] != jobName: jobExists = False else: jobExists = True break if (len(verify) == 0) or (jobExists == False): unProtectedApps.append(jobName) if args.type == "kVMware": tagId = cluster.GetVcenterTagId(vcenterSource, vmAppName) if vmEnvironment == "Test": excludeTag = cluster.GetVcenterTagId(vcenterSource, "Production") elif vmEnvironment == "Production": excludeTag = cluster.GetVcenterTagId(vcenterSource, "Test") if tagId is not None: confirm = "yes" #confirm = input("About to create a protection job for {0}. Are you sure? [yes|no]".format(jobName)) if (confirm.lower() == "yes") and (not excludeTickets): print("Creating standard change request for {0}.".format(jobName)) changeID = ticketSystem.CreateStandardChange("NDIT-SPS-Cohesity Data Protection", "NDIT-Cloud Platforms", "svccohesityadm", "ndheck", "mlaverdure", "Systems Platform - Systems", "Backup/Restore", "Automated Change - Create protection job {0}".format(jobName), "Initial protection group creation.") changeSysID = changeID['result']['sys_id']['value'] ticketSystem.scheduleStandardChange(changeSysID) ticketSystem.implementStandardChange(changeSysID) ticketSystem.reviewStandardChange(changeSysID) resp = cluster.CreateVMProtectionJob(jobName, vcenterSource[0]['protectionSource']['id'], tagId, excludeTag,jobPolicy) postChangeNotes = resp.text ticketSystem.addStandardChangeNotes(changeSysID, postChangeNotes) ticketSystem.closeStandardChange(changeSysID, "Successful", "Change Complete") print("Job {} created, {} complete".format(jobName, changeID['result']['number']['value'])) if args.type == "kAzure": print("Create backup job for job {0}".format(jobName)) print("Looking up Tag ID for {0}".format(vmAppName)) tagId = cluster.GetAzureTagId(vcenterSource, vmAppName) if (tagId is not None) and (not excludeTickets): changeID = ticketSystem.CreateStandardChange("NDIT-SPS-Cohesity Data Protection", "NDIT-Cloud Platforms", "svccohesityadm", "ndheck", "mlaverdure", "Systems Platforms - Systems", "Backup/Restore", "Automated Change - Create protection job {0}".format(jobName), "Initial protection group creation.") changeSysID = changeID['result']['sys_id']['value'] ticketSystem.scheduleStandardChange(changeSysID) ticketSystem.implementStandardChange(changeSysID) ticketSystem.reviewStandardChange(changeSysID) resp = cluster.CreateAZProtectionJob(jobName, vcenterSource[0]['protectionSource']['id'], tagId) postChangeNotes = resp.text ticketSystem.addStandardChangeNotes(changeSysID, postChangeNotes) ticketSystem.closeStandardChange(changeSysID, "Successful", "Change Complete") print("Created {0} for {1}".format(changeID,jobName)) else: print("Tag not found in Azure: {}".format(vmAppName)) subject = "Tag {} was not found in the {} virtualization platform".format(vmAppName, args.type) message = "The application name tag {0} used to create backup job {1} was not found. The ApplicationName tag for {2} must be a case sensitive match with the Sharepoint record.".format(vmAppName, jobName, vm['name']) incident = ticketSystem.submitTicket("svccohesityadm", subject, message) incidentID = incident['result']['sys_id'] ticketSystem.assignTicketToGroup(incidentID, "NDIT-Cloud Platforms") else: if not excludeTickets: subject = "Backup job for " + vm['name'] + " was not found in " + jobName + "." message = "Assign ticket to vmWare or Storage to investigate vm tagging issues." incident = ticketSystem.submitTicket("svccohesityadm", subject, message) incidentID = incident['result']['sys_id'] ticketSystem.assignTicketToGroup(incidentID, "NDIT-Cloud Platforms") print("Ticket {} created for {}".format(incident['result']['number'], vm['name'])) #itdTeamsMessage.text("Empty Jobs: {0}\nPopulated Jobs: {1}\nPopulated Paused Jobs: {2}\nUnprotected VMs: {3}".format(str(len(emptyJobs)),str(len(populatedJobs)),str(len(populatedPausedJobs)),str(len(unprotectedVM)))) #itdTeamsMessage.send() body = "Backup Jobs: " + str(len(populatedJobs)) body = body + "\nPaused Jobs: " + str(len(populatedPausedJobs)) body = body + "\nUnprotected VMs: " + str(len(undocumentedVM)) for vm in undocumentedVM: body = body + "\n\t" + vm SendEmail(body, args.cluster) dashboard.send_automation({'AutomationName': 'Infra-Cohesity', 'Action': 'Provisioning', 'Platform': 'Python-dailyProtectionReview.py', 'Units': 120}) if debug: array1=[] print("Empty Jobs: " + str(len(emptyJobs))) for value in emptyJobs: print("\t" + value['name']) print("Populated Jobs: " + str(len(populatedJobs))) print("Paused Jobs: " + str(len(populatedPausedJobs))) for value in populatedPausedJobs: print("\t" + value['name']) print("Unprotected VMs: " + str(len(unprotectedVM))) for value in unprotectedVM: array1.append(value['name'] + ", " + str(value['parentId'])) print("Sorted Unprotected VMs") array1.sort() for i in array1: print(str(i)) print("Unprotected Apps") unProtectedApps.sort() for i in unProtectedApps: print(str(i))