from google.colab import drive
drive.mount('/content/drive')cd 'drive/My Drive/vitalSigns/vs_acs'cd ../cd vs_acsls! cd VSScript && lscd VSScriptimport sys
print(sys.path)
Import Modules & Construct Path Handlers
import os
import sys
sys.path.insert(0,'./VSScript')
import pandas as pd
pd.set_option('display.expand_frame_repr', False)
pd.set_option('display.precision', 2)# Find Relative Path to Files
def findFile(root, file):
for d, subD, f in os.walk(root):
if file in f:
return "{1}/{0}".format(file, d)
break
# To 'import' a script you wrote, map its filepath into the sys
def addPath(root, file): sys.path.append(os.path.abspath( findFile( './', file) ))cd ../Get Vital Signs Reference Table
lsfile = 'VitalSignsCensus_ACS_Tables.xlsx'
xls = pd.ExcelFile(findFile('./', file))
indicators = pd.read_excel(xls, sheet_name='indicators', index_col=0 )indicators.head(30)ACS TOOL STEP 2 -> Execute :
Create ACS Indicators
Settings/ Get Data
flag = True;
year = '19'
vsTbl = pd.read_excel(xls, sheet_name=str('vs'+year), index_col=0 )
# Prepare the Compare Historic Data
file = 'VitalSignsCensus_ACS_compare_data.xlsm'
compare_table = pd.read_excel(findFile('./', file), None);
comparable = False
if( str('VS'+year) in compare_table.keys() ):
compare_table = compare_table[str('VS'+year)]
comparable = True
columnsNames = compare_table.iloc[0]
compare_table = compare_table.drop(compare_table.index[0])
compare_table.set_index(['CSA2010'], drop = True, inplace = True)Create Indicators
# For Each Indicator
for x, row in indicators.iterrows():
# Grab its Meta Data
shortSource = str(indicators.loc[x, 'Short Source'])
shortName = str(indicators.loc[x, 'ShortName'])[:-2]
yearExists = int(float(indicators.loc[x, year+'_exists']))
indicator = str(indicators.loc[x, 'Indicator'])
indicator_number = str(indicators.index.tolist().index(x)+1 )
fileLocation = str(findFile( './', shortName+'.py') )
# If the Indicator is valid for the year, and uses ACS Data, and method exists
flag = True if fileLocation != str('None') else False
flag = True if flag and yearExists else False
flag = True if flag and shortSource in ['ACS', 'Census'] else False
if flag:
print(shortSource, shortName, yearExists, indicator, fileLocation, indicator_number )
# retrieve the Python ACS indicator
module = __import__( shortName )
result = getattr( module, shortName )( year )
# Put Baltimore City at the bottom of the list
idx = result.index.tolist()
idx.pop(idx.index('Baltimore City'))
result = result.reindex(idx+['Baltimore City'])
# Write the results back into the XL dataframe
vsTbl[ str(indicator_number + '_' +shortName ) ] = result
# Save the Data
result.to_csv('./VSData/vs'+ str(year)+'_'+shortName+'.csv')
# drop columns with any empty values
vsTbl = vsTbl.dropna(axis=1, how='any')
# Save the Data
file = 'VS'+str(year)+'_indicators.xlsx'
file = findFile( 'VSData', file)
# writer = pd.ExcelWriter(file)
#vsTbl.to_excel(writer, str(year+'New_VS_Values') )
# Save the Data
vsTbl.to_csv('./VSData/vs'+str(year+'_New_VS_Values')+'.csv')
# Include Historic Data if exist
if( comparable ):
# add historic indicator to excel doc
# compare_table.to_excel(writer,sheet_name = str(year+'Original_VS_Values') )
# compare sets
info = pd.DataFrame()
diff = pd.DataFrame()
simi = pd.DataFrame()
for column in vsTbl:
number = ''
plchld = ''
if str(column[0:3]).isdigit(): plchld = 3
elif str(column[0:2]).isdigit(): plchld = 2
else: number = plchld = 1
number = int(column[0:plchld])
if number == 98: twoNotThree = False;
new = pd.to_numeric(vsTbl[column], downcast='float')
old = pd.to_numeric(compare_table[number], downcast='float', errors='coerce')
info[str(number)+'_Error#_'] = old - new
diff[str(number)+'_Error#_'] = old - new
info[str(number)+'_Error%_'] = old / new
simi[str(number)+'_Error%_'] = old / new
info[str(number)+'_new_'+column[plchld:]] = vsTbl[column]
info[str(number)+'_old_'+columnsNames[number]] = compare_table[number]
#info.to_csv('./VSData/vs_comparisons_'+ str(year)+'.csv')
#diff.to_csv('./VSData/vs_differences_'+ str(year)+'.csv')
# Save the info dataframe
#info.to_excel(writer, str(year+'_ExpandedView') )
# Save the diff dataframe
#diff.to_excel(writer,sheet_name = str(year+'_Error') )
# Save the diff dataframe
#simi.to_excel(writer,sheet_name = str(year+'_Similarity_Ratio') )
info.to_csv('./VSData/vs'+str(year+'_ExpandedView')+'.csv')
diff.to_csv('./VSData/vs'+str(year+'_Error')+'.csv')
simi.to_csv('./VSData/vs'+str(year+'_Similarity_Ratio')+'.csv')
# writer.save()
Compare Historic Indicators
ls# Quick test
shortName = str('hh25inc')
year = 19
# retrieve the Python ACS indicator
module = __import__( shortName )
result = getattr( module, shortName )( year )
result
# Delete Unassigned--Jail
df = df[df.index != 'Unassigned--Jail']
# Move Baltimore to Bottom
bc = df.loc[ 'Baltimore City' ]
df = df.drop( df.index[1] )
df.loc[ 'Baltimore City' ] = bcvsTbl['18_fam']