Using python to review esxtop -b output

Posted by

Step 1: Prepare python environment:

  • Download Python 3 available at https://www.python.org/downloads/
  • Run the installer and make sure to install pip
  • Using CLI with Root/Administrative privileges run following commands
    • pip install –upgrade pip
    • pip install pandas
    • pip install matplotlib
    • pip install numpy
    • pip install ipython

Step 2: The code

The code is written for a Linux system. Hence, the console output will have a few leading special characters. These characters control font color in Linux.

#!/usr/bin/env python
# coding: utf-8
# importing Modules
import os
import pandas as pd
import numpy as np
pd.set_option('max_colwidth', -1)
pd.set_option('display.max_rows',3000)
import matplotlib.pyplot as plt
import time
import tarfile
import sys
from termcolor import cprint
loop = True
cprint ("The program has the ability of plotting graphs. To efficiently use this feature, filter the data for specific objects. This will reduce the options that the program will present you. Once the system is done processing the file, it will give you an option to input your filters.",'blue',attrs=['bold'],file=sys.stderr)
cprint ("The system Truncates the option list to 3000 entries",'blue',attrs=['bold'],file=sys.stderr)
cprint("Enter the SR number you are working on:",'blue',attrs=['bold'],file=sys.stdout)
filepath = input()
if not (os.path.exists(os.path.join('./',filepath))):
    os.makedirs(os.path.join('./',filepath))

def tarit(SR_Num,FileName):
    tarfileName = os.path.join('./',SR_Num,FileName)
    tar = tarfile.open('{}.tar.gz'.format(tarfileName), 'w:gz')
    for file_n in os.listdir(os.path.join('./',SR_Num)):
        file_n = str(file_n)
        if '.png' in file_n:
             tar.add(os.path.join('./',SR_Num, file_n))
             os.remove(os.path.join('./',SR_Num, file_n))
    tar.close()
    cprint ("Archive for Plots selected generated in your workspace!",'red',attrs=['bold'],file=sys.stderr)    
    return

def plotit(fname):
    CGname = fname.split(".")
    CGname = str(CGname[0])
    fname = os.path.join('./',filepath,fname)
    Plotdf = pd.DataFrame(pd.read_csv(fname))
    Clist = pd.DataFrame(Plotdf.columns)
    Xdata = Clist[0][0]
    Clist = Clist.drop([0])
    Clist.columns = ['Plottable Counters']
    print (Clist)
    cprint ("What Counters Would you like to Plot from above?",'blue',attrs=['bold'],file=sys.stdout)
    val_in = False
    while val_in == False:
        cprint ("Enter your Choice [Counter1,Counter2,..,n][List of Number][Default:None]:",'blue',attrs=['bold'],file=sys.stdout)
        CSel=input()    
        CSel=list(CSel.split(",")) 
        try:
            for i in CSel: 
                int(i)
            val_in = True
        except ValueError:
            if CSel[0] =='':
                val_in = True
            else:
                cprint ("Input Error Integer expected",'red',attrs=['bold'],file=sys.stderr)
                cprint ("Example Input: 1,2,3",'blue',attrs=['bold'],file=sys.stdout)
            
    if CSel[0] =='':
        return     
    for i in CSel:
        i = int(i)
        pname = str('Plot-'+str(i)+'-'+CGname+'-'+str(int(time.time())))
        pname = os.path.join('./',filepath,pname)
        Ydata=Clist['Plottable Counters'][i]
        Fig_len = int(len(Plotdf[Xdata]))*0.24
        Fig_Wid = Fig_len * 0.5625
        if Fig_Wid > 5:
            Fig_Wid = 5
        plt.figure(figsize=(Fig_len,Fig_Wid))
        plt.autoscale(enable=True, axis='both', tight=False)
        plt.xlabel('Time')
        plt.ylabel('Value')
        plt.title(Ydata)
        plt.xticks(rotation='vertical')
        plt.plot(Plotdf[Xdata],Plotdf[Ydata]);
        plt.savefig(pname,bbox_inches="tight")
        plt.show();
        plt.close();
    tarit(filepath,str('Plots-'+CGname+'-'+str(int(time.time()))))
    return

cprint ("Please upload the Esxtop CSV data in your workspace under directory",'red',attrs=['bold'],file=sys.stderr)
cprint (filepath,'red',attrs=['bold'],file=sys.stderr)
cprint ("Hit enter when upload is done",'red',attrs=['bold'],file=sys.stderr)
# Reading input file 
val_in = False
while val_in == False:
    cprint ("Enter name of file with Esxtop CSV data:",'blue',attrs=['bold'],file=sys.stdout)
    filename=input()
    cprint ("Processing the CSV Please wait, this may take a while",'red',attrs=['bold'],file=sys.stderr)
    try:
        tdf = pd.DataFrame(pd.read_csv(os.path.join('./',filepath,filename)))
        val_in = True
    except:
        cprint ("Input Error",'red',attrs=['bold'],file=sys.stderr)
        cprint ("Check the file name provided and Make sure file upload is complete",'red',attrs=['bold'],file=sys.stderr)
# user input1 
cprint ("Do you know a name of object(VM name, Naa ID etc) you would like to process for?",'blue',attrs=['bold'],file=sys.stdout)
val_in = False
while val_in == False:
    cprint ("In case you do not know the name of object just procced by hitting ENTER",'blue',attrs=['bold'],file=sys.stdout)
    cprint ("Enter your Choice[Object1,Object2,..,Object(n)][List of String]:",'blue',attrs=['bold'],file=sys.stdout)
    ObjectSlection=input()
    ObjectSlection=list(ObjectSlection.split(","))
    input_error=0
    for i in ObjectSlection:
        try:
            int(i)
            input_error=input_error+1
        except ValueError:
            input_error=input_error
    if input_error > 0:
        val_in = False
        cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
        cprint ("Example Input: TestVM1,naa.123456789",'blue',attrs=['bold'],file=sys.stdout)
    else:
        val_in = True
cdf = pd.DataFrame(tdf.columns)
cdf.columns = ['Objects']
# Processing data based on user input1
if ObjectSlection[0] !='':
    ColumnList=[]
    for Object in ObjectSlection:
        ColumnList.extend(cdf.index[cdf['Objects'].str.contains(Object)].tolist())
        ColumnList.sort(reverse = False)
    ColName=[]
    ColName.insert(0,cdf['Objects'][0])
    i=1
    for Col in ColumnList:
        ColName.insert(i,cdf['Objects'][Col])
        i=i+1
        OutDf_Of=tdf[ColName]
else:
    OutDf_Of=tdf
#Genrating user input3 and Processing data for input2 and input3 
while loop == True:       
    OutDf = OutDf_Of
    ColNameDf = pd.DataFrame(OutDf.columns)
    # Validating data in Object filtred output 
    if ColNameDf[0].count() <= 1:
        loop = False
        cprint("Unable to find the objects specified",'red',attrs=['bold'],file=sys.stderr)
        cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)     
       
    else:
        # Genrating user input2 List and filter
        loop = True
        CGnC_Df =ColNameDf[0].str.split(("\\"),expand = True)
        CG_Df= CGnC_Df[3].str.split(("\("),expand = True)
        CGnC_Df[3] = CG_Df[0]
        CounterGroups = pd.DataFrame(CG_Df[0].unique())
        CounterGroups.columns = ['Counter Groups']
        CounterGroups=CounterGroups.replace(to_replace='None', value=np.nan).dropna()
        print (CounterGroups)
        val_in = False
        while val_in == False:
            cprint ("What Counter Group Would you like to process from the above list?",'blue',attrs=['bold'],file=sys.stdout)
            cprint ("Enter your Choice [Default:ALL][Exit:X][String]:",'blue',attrs=['bold'],file=sys.stdout)
            CGSlection=str(input())
            try:
                int(CGSlection)
                cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
                cprint ("Example Input: Physical Disk SCSI Device",'blue',attrs=['bold'],file=sys.stdout)
            except ValueError:
                innum=len(list(CGSlection.split(",")))
                if innum <= 1:
                    val_in = True
                else:
                    cprint ("Input Error only 1 input expected",'red',attrs=['bold'],file=sys.stderr)
                    cprint ("Example Input: Physical Disk SCSI Device",'blue',attrs=['bold'],file=sys.stdout)                    
                    
        if CGSlection =='':
            if ObjectSlection[0] =='':
                # Validating filtrer input 
                loop = False
                cprint ("Both Object and Counter Group filter are blank. The output will be same as input.",'red',attrs=['bold'],file=sys.stderr)
                cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)                
            else:
                outfile=str(ObjectSlection[0]+str(int(time.time()))+".csv")
                OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
                cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
                cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
                plotit(outfile)                    

        else:    
            if CGSlection =='X' or CGSlection =='x':
                loop = False
                cprint ("Exiting the program",'red',attrs=['bold'],file=sys.stderr)                
            else:
                ColumnList=[0]
                ColumnList.extend(CGnC_Df.index[CGnC_Df[3]== CGSlection].tolist())
                ColumnList.sort(reverse = False)
                ColName = []
                i=0
                for Col in ColumnList:
                    ColName.insert(i,ColNameDf[0][Col])
                    i=i+1
                OutDf=tdf[ColName]
                ColNameDf = pd.DataFrame(OutDf.columns)
                CGnC_Df =ColNameDf[0].str.split(("\\"),expand = True)
                CG_Df= CGnC_Df[3].str.split(("\("),expand = True)
                CGnC_Df[3] = CG_Df[0]
                Counters = CGnC_Df.loc[CGnC_Df[3] == CGSlection]
                Counters = pd.DataFrame(Counters[4].unique())
                Counters.columns = ['Counters']
                print(Counters)
                val_in = False
                while val_in == False:
                    cprint ("What Counters Would you like to process from the above list?",'blue',attrs=['bold'],file=sys.stdout)
                    cprint ("Enter your Choice [Counter1,Counter2,..,n][List of String][Default:ALL]:",'blue',attrs=['bold'],file=sys.stdout)
                    CounterSlection=str(input())
                    CounterSlection=list(CounterSlection.split(",")) 
                    input_error=0
                    for i in CounterSlection:
                        try:
                            int(i)
                            input_error=input_error+1
                        except ValueError:
                            input_error=input_error
                    if input_error > 0:
                        val_in = False
                        cprint ("Input Error String expected",'red',attrs=['bold'],file=sys.stderr)
                        cprint ("Example Input: Reads/sec,Writes/sec,Commands/sec",'blue',attrs=['bold'],file=sys.stdout)
                    else:
                        val_in = True                           
                
                if CounterSlection[0] =='':
                    outfile=str(CGSlection+str(int(time.time()))+".csv")
                    OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
                    cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
                    cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
                    plotit(outfile)                
                else:    
                    cprint ("Processing counter Group:",'red',attrs=['bold'],file=sys.stderr)
                    cprint (CGSlection,'red',attrs=['bold'],file=sys.stderr)
                    cprint ('For Counters:','red',attrs=['bold'],file=sys.stderr)
                    cprint (CounterSlection,'red',attrs=['bold'],file=sys.stderr)
                    ColumnList=[0]
                    for Counter in CounterSlection:
                        ColumnList.extend(CGnC_Df.index[CGnC_Df[4]== Counter].tolist())
                        ColumnList.sort(reverse = False)
                    ColName = []
                    i=0
                    for Col in ColumnList:
                        ColName.insert(i,ColNameDf[0][Col])
                        i=i+1
                    OutDf=OutDf[ColName]
                    outfile=str(CGSlection+str(int(time.time()))+".csv")
                    OutDf.to_csv(os.path.join('./',filepath,outfile),index=False)
                    cprint ("Generated output in your workspace:",'red',attrs=['bold'],file=sys.stderr)
                    cprint (outfile,'red',attrs=['bold'],file=sys.stderr)
                    plotit(outfile)

Step 3: Execution

  • Save the above code as .py file
  • Using CLI execute the script and follow the instructions
    • You may run the Script using “jupyter notebook” as well

Advertisements

Leave a Reply

This site uses Akismet to reduce spam. Learn how your comment data is processed.