Posted by Step 1: Prepare python environment:
- Download Python 3 available at https://www.python.org/downloads/
- Run the installer and make sure to install pip
- Using CLI with Root/Administrative privileges run following commands
- pip install –upgrade pip
- pip install pandas
- pip install matplotlib
- pip install numpy
- pip install ipython
Step 2: The code
The code is written for a Linux system. Hence, the console output will have a few leading special characters. These characters control font color in Linux.
#!/usr/bin/env python
# coding: utf-8
# importing Modules
import os
import pandas as pd
import numpy as np
pd.set_option('max_colwidth', -1)
pd.set_option('display.max_rows',3000)
import matplotlib.pyplot as plt
import time
import tarfile
import sys
from termcolor import cprint
loop = True
cprint ("The program has the ability of plotting graphs. To efficiently use this feature, filter the data for specific objects. This will reduce the options that the program will present you. Once the system is done processing the file, it will give you an option to input your filters.",'blue',attrs=['bold'],file=sys.stderr)
cprint ("The system Truncates the option list to 3000 entries",'blue',attrs=['bold'],file=sys.stderr)
cprint("Enter the SR number you are working on:",'blue',attrs=['bold'],file=sys.stdout)
filepath = input()
if not (os.path.exists(os.path.join('./',filepath))):
os.makedirs(os.path.join('./',filepath))
def tarit(SR_Num,FileName):
tarfileName = os.path.join('./',SR_Num,FileName)
tar = tarfile.open('{}.tar.gz'.format(tarfileName), 'w:gz')
for file_n in os.listdir(os.path.join('./',SR_Num)):
file_n = str(file_n)
if '.png' in file_n:
tar.add(os.path.join('./',SR_Num, file_n))
os.remove(os.path.join('./',SR_Num, file_n))
tar.close()
cprint ("Archive for Plots selected generated in your workspace!",'red',attrs=['bold'],file=sys.stderr)
return
def plotit(fname):
CGname = fname.split(".")
CGname = str(CGname[0])
fname = os.path.join('./',filepath,fname)
Plotdf = pd.DataFrame(pd.read_csv(fname))
Clist = pd.DataFrame(Plotdf.columns)
Xdata = Clist[0][0]
Clist = Clist.drop([0])
Clist.columns = ['Plottable Counters']
print (Clist)
cprint ("What Counters Would you like to Plot from above?",'blue',attrs=['bold'],file=sys.stdout)
val_in = False
while val_in == False:
cprint ("Enter your Choice [Counter1,Counter2,..,n][List of Number][Default:None]:",'blue',attrs=['bold'],file=sys.stdout)
CSel=input()
CSel=list(CSel.split(","))
try:
for i in CSel:
int(i)
val_in = True
except ValueError:
if CSel[0] =='':
val_in = True
else:
cprint ("Input Error Integer expected",'red',attrs=['bold'],file=sys.stderr)
cprint ("Example Input: 1,2,3",'blue',attrs=['bold'],file=sys.stdout)
if CSel[0] =='':
return
for i in CSel:
i = int(i)
pname = str('Plot-'+str(i)+'-'+CGname+'-'+str(int(time.time())))
pname = os.path.join('./',filepath,pname)
Ydata=Clist['Plottable Counters'][i]
Fig_len = int(len(Plotdf[Xdata]))*0.24
Fig_Wid = Fig_len * 0.5625
if Fig_Wid > 5:
Fig_Wid = 5
plt.figure(figsize=(Fig_len,Fig_Wid))
plt.autoscale(enable=True, axis='both', tight=False)
plt.xlabel('Time')
plt.ylabel('Value')
plt.title(Ydata)
plt.xticks(rotation='vertical')
plt.plot(Plotdf[Xdata],Plotdf[Ydata]);
plt.savefig(pname,bbox_inches="tight")
plt.show();
plt.close();
tarit(filepath,str('Plots-'+CGname+'-'+str(int(time.time()))))
return
cprint ("Please upload the Esxtop CSV data in your workspace under directory",'red',attrs=['bold'],file=sys.stderr)
cprint (filepath,'red',attrs=['bold'],file=sys.stderr)
cprint ("Hit enter when upload is done",'red',attrs=['bold'],file=sys.stderr)
# Reading input file
val_in = False
while val_in == False:
cprint ("Enter name of file with Esxtop CSV data:",'blue',attrs=['bold'],file=sys.stdout)
filename=input()
cprint ("Processing the CSV Please wait, this may take a while",'red',attrs=['bold'],file=sys.stderr)
try:
tdf = pd.DataFrame(pd.read_csv(os.path.join('./',filepath,filename)))
val_in = True
except:
cprint ("Input Error",'red',attrs=['bold'],file=sys.stderr)
cprint ("Check the file name provided and Make sure file upload is complete",'red',attrs=['bold'],file=sys.stderr)
# user input1
cprint ("Do you know a name of object(VM name, Naa ID etc) you would like to process for?",'blue',attrs=['bold'],file=sys.stdout)
val_in = False
while val_in == False:
cprint ("In case you do not know the name of object just procced by hitting ENTER",'blue',attrs=['bold'],file=sys.stdout)
cprint ("Enter your Choice[Object1,Object2,..,Object(n)][List of String]:",'blue',attrs=['bold'],file=sys.stdout)
ObjectSlection=input()
ObjectSlection=list(ObjectSlection.split(","))
input_error=0
for i in ObjectSlection:
try:
int(i)
input_error=input_error+1
except ValueError:
input_error=input_error
if input_error > 0:
val_in = False
cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
cprint ("Example Input: TestVM1,naa.123456789",'blue',attrs=['bold'],file=sys.stdout)
else:
val_in = True
cdf = pd.DataFrame(tdf.columns)
cdf.columns = ['Objects']
# Processing data based on user input1
if ObjectSlection[0] !='':
ColumnList=[]
for Object in ObjectSlection:
ColumnList.extend(cdf.index[cdf['Objects'].str.contains(Object)].tolist())
ColumnList.sort(reverse = False)
ColName=[]
ColName.insert(0,cdf['Objects'][0])
i=1
for Col in ColumnList:
ColName.insert(i,cdf['Objects'][Col])
i=i+1
OutDf_Of=tdf[ColName]
else:
OutDf_Of=tdf
#Genrating user input3 and Processing data for input2 and input3
while loop == True:
OutDf = OutDf_Of
ColNameDf = pd.DataFrame(OutDf.columns)
# Validating data in Object filtred output
if ColNameDf[0].count() <= 1:
loop = False
cprint("Unable to find the objects specified",'red',attrs=['bold'],file=sys.stderr)
cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)
else:
# Genrating user input2 List and filter
loop = True
CGnC_Df =ColNameDf[0].str.split(("\\"),expand = True)
CG_Df= CGnC_Df[3].str.split(("\("),expand = True)
CGnC_Df[3] = CG_Df[0]
CounterGroups = pd.DataFrame(CG_Df[0].unique())
CounterGroups.columns = ['Counter Groups']
CounterGroups=CounterGroups.replace(to_replace='None', value=np.nan).dropna()
print (CounterGroups)
val_in = False
while val_in == False:
cprint ("What Counter Group Would you like to process from the above list?",'blue',attrs=['bold'],file=sys.stdout)
cprint ("Enter your Choice [Default:ALL][Exit:X][String]:",'blue',attrs=['bold'],file=sys.stdout)
CGSlection=str(input())
try:
int(CGSlection)
cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
cprint ("Example Input: Physical Disk SCSI Device",'blue',attrs=['bold'],file=sys.stdout)
except ValueError:
innum=len(list(CGSlection.split(",")))
if innum <= 1:
val_in = True
else:
cprint ("Input Error only 1 input expected",'red',attrs=['bold'],file=sys.stderr)
cprint ("Example Input: Physical Disk SCSI Device",'blue',attrs=['bold'],file=sys.stdout)
if CGSlection =='':
if ObjectSlection[0] =='':
# Validating filtrer input
loop = False
cprint ("Both Object and Counter Group filter are blank. The output will be same as input.",'red',attrs=['bold'],file=sys.stderr)
cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)
else:
outfile=str(ObjectSlection[0]+str(int(time.time()))+".csv")
OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
plotit(outfile)
else:
if CGSlection =='X' or CGSlection =='x':
loop = False
cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)
else:
ColumnList=[0]
ColumnList.extend(CGnC_Df.index[CGnC_Df[3]== CGSlection].tolist())
ColumnList.sort(reverse = False)
ColName = []
i=0
for Col in ColumnList:
ColName.insert(i,ColNameDf[0][Col])
i=i+1
OutDf=tdf[ColName]
ColNameDf = pd.DataFrame(OutDf.columns)
CGnC_Df =ColNameDf[0].str.split(("\\"),expand = True)
CG_Df= CGnC_Df[3].str.split(("\("),expand = True)
CGnC_Df[3] = CG_Df[0]
Counters = CGnC_Df.loc[CGnC_Df[3] == CGSlection]
Counters = pd.DataFrame(Counters[4].unique())
Counters.columns = ['Counters']
print(Counters)
val_in = False
while val_in == False:
cprint ("What Counters Would you like to process from the above list?",'blue',attrs=['bold'],file=sys.stdout)
cprint ("Enter your Choice [Counter1,Counter2,..,n][List of String][Default:ALL]:",'blue',attrs=['bold'],file=sys.stdout)
CounterSlection=str(input())
CounterSlection=list(CounterSlection.split(","))
input_error=0
for i in CounterSlection:
try:
int(i)
input_error=input_error+1
except ValueError:
input_error=input_error
if input_error > 0:
val_in = False
cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
cprint ("Example Input: Reads/sec,Writes/sec,Commands/sec",'blue',attrs=['bold'],file=sys.stdout)
else:
val_in = True
if CounterSlection[0] =='':
outfile=str(CGSlection+str(int(time.time()))+".csv")
OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
plotit(outfile)
else:
cprint ("Processing counter Group:",'red',attrs=['bold'],file=sys.stderr)
cprint (CGSlection,'red',attrs=['bold'],file=sys.stderr)
cprint ('For Counters:','red',attrs=['bold'],file=sys.stderr)
cprint (CounterSlection,'red',attrs=['bold'],file=sys.stderr)
ColumnList=[0]
for Counter in CounterSlection:
ColumnList.extend(CGnC_Df.index[CGnC_Df[4]== Counter].tolist())
ColumnList.sort(reverse = False)
ColName = []
i=0
for Col in ColumnList:
ColName.insert(i,ColNameDf[0][Col])
i=i+1
OutDf=OutDf[ColName]
outfile=str(CGSlection+str(int(time.time()))+".csv")
OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
plotit(outfile)Step 3: Execution
- Save the above code as .py file
- Using CLI execute the script and follow the instructions
- You may run the Script using “jupyter notebook” as well