import os,csv
import numpy as np
class loader(object):
def __init__(self,excel,error):
self.x=excel
self.e=error
self.passfail='Pass/Fail'
def load_data(self,csv_data_list):
#This is the first function that loads all needed CSV files
#load_csv(filename) function used to actually load data from CSV files
#Loaded data written to excell worksheet with write_data_to_current_ws() function
print "Loading data files"
for data_set in csv_data_list:#for each CSV file
sheet_name=data_set['sheet']#Destination sheet name
filename=data_set['file']#Source file
add_corners=data_set['add_corners']
ignore_columns=data_set['ignore']
#print "Loading data from",filename
if not self.x.sheet_exists(sheet_name):#If destination sheet not exists
data=self.load_csv(filename,add_corners,ignore_columns)#Load data from CSV
self.x.add_sheet(sheet_name)#Create destination sheet
else:#If destination sheet exists
self.x.activate_sheet(sheet_name)#Make it active
sheet_header=self.x.get_column_names_active_sheet()
data=self.load_csv(filename,add_corners,ignore_columns,sheet_header)#Load data from CSV
self.write_data_to_current_ws(data)#Write CSV data to current worksheet
def load_csv(self,filename,add_corners=[],ignore_col_list=[],col_list=[]):
#Load ADE-XL simulation results data from CSV file
#Function returns header (list with names of columns) and data (list of lists)
if not os.path.isfile(filename):#Check if file exists
self.e.print_error("Source CSV file %s not found"%(filename))
print "\tLoading CSV file %s ..."%(filename),
data=[]
ignore_col_nums=[]
if len(add_corners)>0:
additional_corners_values=[]
for item in add_corners:
additional_corners_values.append(item[1].strip())
with open(filename, 'rb') as csvfile:#Open source CSV file
freader = csv.reader(csvfile, delimiter=',')#use csv module
line_num=0#Initiate lines number counter
for row in freader:#for each row
if line_num==0:#If first line it is header line
line_num+=1#Increment line number
header=[]
if len(add_corners)>0:
row=[item[0] for item in add_corners]+row
original_header=zip(range(len(row)),row)
for col_num,col_name in original_header:#Put header line in separate variable
item_clean=col_name.strip()
if item_clean not in ignore_col_list:
header.append(item_clean)
else:
ignore_col_nums.append(col_num)
if len(col_list)!=0:
for col_name in col_list:
if col_name not in header:
self.e.print_error("Column '%s' not in sheet header"%(col_name))
else:
col_list=header
passfail_header_index=[item[1] for item in original_header].index(self.passfail)
passfail_collist_index=col_list.index(self.passfail)
measures_itarator=[]
corners_iterator=[]
for col_num in range(len(col_list)):
if col_num>=passfail_collist_index:
measures_itarator.append(col_num)
else:
corners_iterator.append(col_num)
elif len(row)>0:#If not header line and line not empty
vardict={'c':{},'m':{}}
line_num+=1#Increment line number
if len(add_corners)>0:#If we need to add additional values to each data row
newrow=map(str.strip,additional_corners_values+row)
else:
newrow=map(str.strip,row)
for col_num,col_name in original_header:
if col_num in ignore_col_nums:
continue
try:
val=float(newrow[col_num])
except:
val=newrow[col_num]
if col_num>=passfail_header_index:
vardict['m'][col_name]=val
else:
vardict['c'][col_name]=val
corners_values=[vardict['c'][col_list[col_num]] for col_num in corners_iterator]
measures_values=[vardict['m'][col_list[col_num]] for col_num in measures_itarator]
data.append(corners_values+measures_values)#Append line data to data array
print "%s lines ... done"%(line_num)
if line_num==0:
self.e.print_error("Empty input file",curinfo())
if not self.x.is_valid_header(col_list):
self.e.print_error("Invalid header. Possible not transposed format")
return {'header':col_list,'data':data,'linenum':line_num}
def write_data_to_current_ws(self,csv_file_data,add_corners=[],corner_columns=None):
#This wunction writes data ov ADE-XL CSV file to current excell worksheet
header=[col_name.strip() for col_name in csv_file_data['header']]#List with header names
if len(add_corners)>0:
data=[]
for row in csv_file_data['data']:
data.append([item[1] for item in add_corners]+row)
else:
data=csv_file_data['data']#Array with CSV file data
print "\tWriting data to sheet '%s'... "%(self.x.get_active_sheet()),
#Take care about header
final_header=[item[0] for item in add_corners]+header
if not self.x.is_header_set_active_sheet():#If starting from start row so new sheet. Header need to be printed
#Construct new header from new columns and original columns
if not self.x.set_active_sheet_header(final_header,corner_columns):
self.e.print_error('Duplicated column names in header')
self.x.set_first_data_row_to_current_active_sheet()#Save number of first row with data
#End of new sheet part
else:#Else the sheet already exists and need to check data consistency
if not self.x.is_same_header_active_sheet(final_header):
sheet_header=self.x.get_column_names_active_sheet()
self.compare_headers(sheet_header,final_header)
#---- Header part completed
#--- Write all data
self.x.add_data_block_active_sheet(data)
self.x.set_last_data_row_to_current_active_sheet()#Save number of last row with data
self.x.next_line_active_sheet()
#print self.x.get_last_data_row_active_sheet()
print "done"
def compare_headers(self,old,new):
final=[]
if len(old)!=len(new):
self.print_headers_compare(old,new)
print "Existing header have %d columns"%(len(old))
print "New header have %d columns"%(len(new))
self.e.print_error('Headers not match, different number of columns')
else:
self.print_headers_compare(old,new)
self.e.print_error('Headers not match, different order of columns or column names')
def print_headers_compare(self,old,new):
max_old=len('Existing')
max_new=len('New')
compare=[]
maxcol=max(len(old),len(new))
for n in range(maxcol):
compare.append(['',''])
n=0
for col_name in old:
compare[n][0]=col_name
n+=1
max_old=max(max_old,len(col_name))
n=0
for col_name in new:
compare[n][1]=col_name
n+=1
max_new=max(max_new,len(col_name))
print "%s %s"%('Existing'.ljust(max_old),'New'.ljust(max_new))
for item in compare:
print "%s %s"%(item[0].ljust(max_old),item[1].ljust(max_new))
