In [2]:
#importing datasets function using pandas, parameters: string name of dataset, dataset should be .csv file
import pandas as pd
def import_data(dataName):
csv_path_full = dataName
print (csv_path_full)
return pd.read_csv(csv_path_full)
In [3]:
#calling function
econData = import_data("QOL_data_Economy.csv")
edData = import_data("QOL_data_education.csv")
tranData = import_data("QOL_data_transportation.csv")
engData = import_data("QOL_data_engagement.csv")
houseData = import_data("QOL_data_Housing.csv")
houseData.head()
Out[3]:
In [4]:
#econData.describe()
#econData.info()
#engData.info()
houseData.info()
In [5]:
%matplotlib inline
import matplotlib.pyplot as plt
tranData.hist(bins=50, figsize=(20,15))
plt.show()
In [22]:
totalSet["Board_Committee_Participation_2015"]
Out[22]:
In [9]:
frames = [econData, edData, tranData, engData, houseData]
totalSet = pd.concat(frames, axis = 1)
totalSet.head()
Out[9]:
In [21]:
list(totalSet)
Out[21]:
In [17]:
houseChange= totalSet["Home_Sales_Price_2015"] - totalSet["Home_Sales_Price_2013"]
totalSet['houseChange'] = houseChange
totalSetc = totalSet
constructChange= totalSet["Commercial_Construction_2015"] - totalSet["Commercial_Construction_2011"]
totalSetc['constructChange'] = constructChange
In [27]:
totalSetH = totalSetc[["Home_Sales_Price_2013", "Housing_Density_2013"]].copy()
totalSetH
Out[27]:
In [99]:
median = totalSetc["Home_Sales_Price_2013"].median()
median2 = totalSetc["Home_Sales_Price_2015"].median()
median, median2
totalSetc["Lower_Homes_2013"]=totalSetc["Home_Sales_Price_2013"].where(totalSetc["Home_Sales_Price_2013"] < median, inplace=False)
totalSetc["Lower_Homes_2015"]=totalSetc["Home_Sales_Price_2015"].where(totalSetc["Home_Sales_Price_2015"] < median2, inplace=False)
In [242]:
#totalSetc["Lower_Homes_2015"]
lowerHouseChange = totalSetc["Lower_Homes_2015"]- totalSetc["Lower_Homes_2013"]
lowerHouseChange
totalSetc["lowerHouseChange"]= lowerHouseChange
totalSetS = totalSetc[["lowerHouseChange","Commercial_Construction_2015","Household_Income_2015" "Employment_Rate_2015","Arts_Participation_2013","Arts_Participating_Households_2013", "Neighborhood_Organizations_2014", "Neighborhood_Organizations_2015", "Board_Committee_Participation_2015", "Housing_Density_2011", "Housing_Density_2013", "Housing_Density_2015", "Single_Family_Housing_2015","Single_Family_Housing_2013", "Single_Family_Housing_2011", "Single_Family_Units_2015", "Single_Family_Units_2013", "Single_Family_Units_2011", "311_Requests_2015","311_Requests_2013","311_Calls_2015", "311_Calls_2013", "Voter_Participation_2015", "Voter_Participation_2014", "Voter_Participation_2012", "Voter_Participation_2010", "Job_Density_2013", "Job_Density_2012", "New_Residential_2015", "New_Residential_2014", "New_Residential_2013", "New_Residential_2012", "New_Residential_2011", 'Transit_Proximity_2015','Transit_Proximate_Units_2015','Transit_Proximity_2013','Transit_Proximate_Units_2013','Transit_Proximity_2011','Transit_Proximate_Units_2011', 'Bicycle_Friendliness_2015','Bicycle_Friendliness_2013','Bicycle_Friendliness_2011',
'Street_Connectivity_2015',
'Street_Connectivity_2013',
'Street_Connectivity_2011',
'High_School_Diploma_2015',
'Early_Care_Proximity_2015',
'Early_Care_Proximate_Units_2015',
'Early_Care_Proximity_2013',
'Early_Care_Proximate_Units_2013',
'Early_Care_Proximity_2011',
'Early_Care_Proximate_Units_2011',
'SchoolAge_Proximity_2015',
'SchoolAge_Proximate_Units_2015',
'SchoolAge_Proximity_2013',
'SchoolAge_Proximate_Units_2013',
'SchoolAge_Proximity_2011',
'SchoolAge_Proximate_Units_2011',
'Proficiency_Elementary_School_2013',
'Proficiency_Middle_School_2014',
'Proficiency_Middle_School_2013',
'Proficiency_High_School_2014',
'Proficiency_High_School_2013',
'Neighborhood_School_Attendance_2014',
'Neighborhood_School_Attendance_2013',
'Neighborhood_School_Attendance_2011',
'Job_Density_2014',
'Commercial_Construction_2014',
'Commercial_Construction_Permitted_Units_2014',
'Commercial_Construction_2013',
'Commercial_Construction_Permitted_Units_2013',
'Commercial_Construction_2012',
'Commercial_Construction_Permitted_Units_2012',
'Commercial_Construction_2011',
'Commercial_Construction_Permitted_Units_2011',
'Bachelors_Degree_2015',
'Home_Sales_Price_2015',
'Home_Sales_Price_2013',
'Housing_Size_2015',
'Housing_Size_2013']].copy()
In [240]:
totalSetS = totalSetS[np.isfinite(totalSetS['lowerHouseChange'])]
list(totalSetS)
Out[240]:
In [241]:
totalSetS.to_csv("trimmed_QOL_set")
In [233]:
corr_matrix = totalSetc.corr()
corr_matrix["Household_Income_2015"].sort_values(ascending=False)
Out[233]:
In [103]:
import numpy as np
def split_train_test(data, test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size= int(len(data) * test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices], data.iloc[test_indices]
In [104]:
train_setS, test_setS = split_train_test(totalSetS, 0.2)
In [105]:
train_setS.info()
In [106]:
trainData = train_setS[np.isfinite(train_setS['lowerHouseChange'])]
trainData.head()
Out[106]:
In [223]:
totalform = totalSetS[np.isfinite(totalSetS['lowerHouseChange'])]
totalform.to_csv("trimmed_QOL_dataset")
In [129]:
from sklearn.preprocessing import Imputer
imputer = Imputer(missing_values='NaN', strategy='median')
#imputer.fit(trainData)
#trainDataPrepared = imputer.transform(trainData)
In [130]:
imputer.fit(trainData)
Out[130]:
In [131]:
trainData.isnull()
Out[131]:
In [132]:
trainDataPrepared = imputer.transform(trainData)
In [136]:
trainDataPrepared
Out[136]:
In [144]:
import numpy
numpy.isnan(trainDataPrepared).any()
Out[144]:
In [149]:
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
In [150]:
scaler.fit(trainDataPrepared)
Out[150]:
In [151]:
trainPrep= scaler.transform(trainDataPrepared)
In [153]:
numpy.isnan(trainPrep).any()
Out[153]:
In [192]:
testData = test_setS[np.isfinite(test_setS['lowerHouseChange'])]
imputer4 = Imputer(missing_values='NaN', strategy='median')
imputer4.fit(testData)
Out[192]:
In [193]:
testDataPrepared = imputer.transform(testData)
In [194]:
scaler2 =StandardScaler()
scaler.fit(testDataPrepared)
Out[194]:
In [154]:
trainPrep.shape
Out[154]:
In [195]:
testPrep = scaler.transform(testDataPrepared)
In [156]:
imputer2 = Imputer()
imputer2.fit(trainPrep)
trainImp= imputer.transform(trainPrep)
In [176]:
#find_labels = trainData[["lowerHouseChange", "Commercial_Construction_2015","Household_Income_2015","Employment_Rate_2015" ]].copy()
find_labels = trainData[["lowerHouseChange"]].copy()
In [177]:
imputer3 = Imputer()
In [178]:
imputer3.fit(find_labels)
Out[178]:
In [179]:
labelImp=imputer3.transform(find_labels)
In [231]:
trainImp
Out[231]:
In [180]:
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(trainImp, labelImp)
Out[180]:
In [186]:
def display_scores(scores):
print("Scores:", scores)
print("Mean:", scores.mean())
print("Standard Deviation:", scores.std())
In [187]:
display_scores(forest_rmse_scores)
In [184]:
trainData["lowerHouseChange"]
Out[184]:
In [226]:
test = testPrep[22]
trainPredict = forest_reg.predict(trainImp)
In [251]:
from sklearn import cross_validation
model = RandomForestRegressor(n_estimators=100)
#Simple K-Fold cross validation. 10 folds.
cv = cross_validation.KFold(len(trainImp), n_folds=10)
results = []
# "Error_function" can be replaced by the error function of your analysis
for traincv, testcv in cv:
model.fit(trainImp, labelImp)
In [257]:
from sklearn.svm import LinearSVR
svm_reg = LinearSVR(epsilon=1.5, random_state=42)
svm_reg.fit(trainImp, labelImp)
Out[257]:
In [260]:
svm_reg1 = LinearSVR(epsilon=1.5, random_state=42)
svm_reg2 = LinearSVR(epsilon=0.5, random_state=42)
svm_reg1.fit(trainImp, labelImp)
svm_reg2.fit(trainImp, labelImp)
def find_support_vectors(svm_reg, trainImp, labelImp):
y_pred = svm_reg.predict(testPrep)
off_margin = (np.abs(labelImp - y_pred) >= svm_reg.epsilon)
return np.argwhere(off_margin)
svm_reg1.support_ = find_support_vectors(svm_reg1, trainImp, labelImp)
svm_reg2.support_ = find_support_vectors(svm_reg2, trainImp, labelImp)
eps_x1 = 1
eps_y_pred = svm_reg1.predict(testPrep)
In [261]:
eps_y_pred
Out[261]:
In [202]:
test_labels = testData[["lowerHouseChange"]].copy()
In [203]:
imputer4 = Imputer()
imputer4.fit(test_labels)
Out[203]:
In [204]:
imputer4.transform(test_labels)
Out[204]:
In [1]:
#from sklearn.model_selection import GridSearchCV
#param_grid = [
# {'n_estimators': [3,10,30, 40, 50, 60 , 70, 80 , 90, 100], 'max_features':[2,4,6,8, 10, 12, 14, 16, 18, 20]},
#{'bootstrap': [False], 'n_estimators': [3,10], 'max_features':[2,3,4]},
#]
#forest_reg2 = RandomForestRegressor()
#grid_search = GridSearchCV(forest_reg2, param_grid, cv=5, scoring='neg_mean_squared_error')
#grid_search.fit(trainImp, labelImp)
In [211]:
model = grid_search.best_estimator_
In [218]:
testPred = model.predict(testPrep)
In [219]:
testPred
Out[219]:
In [263]:
test_set.head()