Equipment Success prediction

from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Input, advanced_activations
from keras import optimizers
from keras import regularizers

keras.__version__

equipment_success_valid = scale(equipment_success_valid, ['freq_250', 'freq_500', 'freq_750', 'freq_1000', 
                                                          'freq_1500', 'freq_2000', 'freq_3000', 'freq_4000',
                                                          'freq_6000', 'freq_8000', 'years', 'decades_old', 'age_in_months_at_equipped_start'])


equipment_success_all = equipment_success_valid.copy()
#[(equipment_success_valid['isFemale'] == 1)
#                                            & (equipment_success_valid['decades_old'] == 8)
#                                           ].copy()

equipment_success = equipment_success_all.sample(frac=1)
 
equipment_success

features = [
   "years_scaled",
    "isMale",
    "isFemale",
    #"isLeftEar",
    #"isRightEar",
   # 'freq_250_scaled',
    'freq_500_scaled',
    #'freq_750_scaled',
    'freq_1000_scaled',
    #'freq_1500_scaled',
    'freq_2000_scaled',
    #'freq_3000_scaled', 
    'freq_4000_scaled',
    #'freq_6000_scaled',
    'freq_8000_scaled',
    #'freq_500_over_freq_1000',
    #'freq_2000_over_freq_1000',
    #'freq_4000_over_freq_1000'
    
]

data = (equipment_success[features].values)
labels = (equipment_success['eq_freq_1000'].values)

print("original good prediction ratio: ", model_good_prediction_ratio_array(labels, np.average(labels), 5))
print("original rmse: " , np.std(labels - np.average(labels)))

original_data = data.copy()
original_labels = labels.copy()

unscaled_features = [
   "age_in_months_at_equipped_start",
    "isMale",
    "isFemale",
    'freq_250',
    'freq_500',
    'freq_750',
    'freq_1000',
    'freq_1500',
    'freq_2000',
    'freq_3000', 
    'freq_4000',
    'freq_6000',
    'freq_8000',
    
]

ff = equipment_success[unscaled_features].values
labels = equipment_success['eq_freq_1000'].values
[ff.shape, labels.shape]

len(ff)

bad = {}
for i in range(len(ff)):
    if i % 100 == 0:
        print(i)
        print(bad)
    for j in range(i + 1, len(ff)):
        if ff[i][1] == ff[j][1] and ff[i][2] == ff[j][2] and abs(ff[i][0] - ff[j][0]) < 120 and np.linalg.norm(ff[i][3:] - ff[j][3:]) < 10 and abs(labels[i] - labels[j]) >= 5:
        #if ff[i][1] == ff[j][1] and ff[i][2] == ff[j][2] and abs(ff[i][0] - ff[j][0]) < 120 and abs(ff[i][6] - ff[j][6]) <= 5 and abs(labels[i] - labels[j]) > 5:
        #if abs(ff[i][6] - ff[j][6]) <= 5 and abs(labels[i] - labels[j]) > 5:


            if bad.get(i) == None:
                bad[i] = []
            bad[i].append(j)
bad

[ff[0], labels[0]]

[ff[26], labels[26]]

list(bad.keys())

good_data = np.delete(data, list(bad.keys()), axis=0)
good_labels = np.delete(labels, list(bad.keys()), axis=0)

[good_data.shape, good_labels.shape]

from sklearn import svm

my_features = [
    'center_id',
   'years',
 'genderNum',
#'freq_250',
 'freq_500',
# 'freq_750',
 'freq_1000',
# 'freq_1500',
 'freq_2000',
# 'freq_3000',
 'freq_4000',
# 'freq_6000',
 'freq_8000'
              ]
my_equipment_success = equipment_success.copy()
my_equipment_success['valid'] = (
  my_equipment_success['eq_freq_1000'].apply(lambda x : 20 <= x <= 60) 
)

#my_equipment_success = my_equipment_success.query(' valid == True').copy()
my_equipment_success = my_equipment_success.sample(frac=1)
data = my_equipment_success[my_features].values
labels = (my_equipment_success['eq_freq_1000'].values)
[data.shape, labels.shape]

n = 10000
clf = svm.SVR(gamma='scale', kernel='rbf', C=4, epsilon=4, shrinking=True)
clf.fit(data[0:n], labels[0:n])

predictions = clf.predict(data)
print("training")

print(np.std(labels[0:n]-np.mean(labels)))
print(np.std(labels[0:n]-predictions[0:n]))
print(np.std(labels[0:n]-predictions[0:n]) ** 2)
print(model_good_prediction_ratio_array(labels[0:n], np.mean(labels), 5))
print(model_good_prediction_ratio_array(labels[0:n], predictions[0:n], 5))
print("validation")
print(np.std(labels[n+1:]-np.mean(labels)))
print(np.std(labels[n+1:]-predictions[n+1:]))
print(np.std(labels[n+1:]-predictions[n+1:]) ** 2)
print(model_good_prediction_ratio_array(labels[n+1:], np.mean(labels), 5))
print(model_good_prediction_ratio_array(labels[n+1:], predictions[n+1:], 5))

from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.svm import SVR

X_train, X_test, y_train, y_test = train_test_split(
    data, labels, test_size=0.2, random_state=0)
tuned_parameters = [{'kernel': ['rbf'], 
                     'gamma': ['scale'],
                     'C': [4, 6,10],
                     'epsilon': [ 4, 6, 10],
                     'shrinking': [True]
                    }
                   ]

print("Generating the models")
clf = GridSearchCV(SVR(), tuned_parameters, cv=5, scoring = "neg_mean_squared_error")

clf.fit(X_train, y_train)

print("Best parameters set found on development set:")
print()
print(clf.best_params_)
print()
print("Grid scores on development set:")
print()
means = clf.cv_results_['mean_test_score']
stds = clf.cv_results_['std_test_score']

for mean, std, params in zip(means, stds, clf.cv_results_['params']):
    print("%0.3f (+/-%0.03f) for %r"
          % (mean, std * 2, params))
print()

clf.get_params()

import lightgbm as lgb

def model_good_prediction_ratio_diffs_array(diffs, max_distance):
    predictionsDiff = list(map(lambda x: abs(x), list(diffs)))
    return len(list(filter(lambda x: x < max_distance, predictionsDiff)))/len(predictionsDiff)

my_equipment_success = equipment_success.copy()

#my_equipment_success = my_equipment_success.query(' valid == True').copy()
my_equipment_success = my_equipment_success.sample(frac=1)

my_features = [
   'center_id',
     'freq_1000',
   'decades_old',
   # 'years',
   'genderNum',
#'freq_250',
 'freq_500',
 #'freq_750',
 #'freq_1500',
 'freq_2000',
 #'freq_3000',
 'freq_4000',
 #'freq_6000',
 #'freq_8000'
              ]

data = my_equipment_success[my_features].values
labels = (my_equipment_success['eq_freq_1000'].values)
losses = (my_equipment_success['freq_1000'].values)
labels_ratio = (my_equipment_success['eq_freq_1000'].values)/losses

[data.shape, labels.shape]

center_ids = my_equipment_success['center_id'].values
tests_per_center = {}
for i in range(len(data)):
    center_id = center_ids[i]
    tests_per_center[center_id] = safe_inc(tests_per_center.get(center_id))
    
labels_per_center = {}
for i in range(len(data)):
    center_id = center_ids[i]
    labels_per_center[center_id] = safe_append(labels_per_center.get(center_id), labels[i])
    
mean_labels_per_center = {k: np.mean(v) for k, v in labels_per_center.items()}

labels_ratio_per_center = {}
for i in range(len(data)):
    center_id = center_ids[i]
    labels_ratio_per_center[center_id] = safe_append(labels_ratio_per_center.get(center_id),
                                                          labels[i]/data[i][1])
    
mean_labels_ratio_per_center = {k: np.mean(v) for k, v in labels_ratio_per_center.items()}

labels_ratio_scaled_per_center = []
for i in range(len(data)):
    center_id = center_ids[i]
    labels_ratio_scaled_per_center.append(labels_ratio[i]/mean_labels_ratio_per_center[center_id])

min_test_per_center = 200
good_indexes = []
for i in range(len(data)):
    center_id = center_ids[i]
    if tests_per_center[center_id] > min_test_per_center:
        good_indexes.append(i)
    
data = np.asarray([data[index] for index in good_indexes])
labels = np.asarray([labels[index] for index in good_indexes])
losses = np.asarray([losses[index] for index in good_indexes])
labels_ratio = np.asarray([labels_ratio[index] for index in good_indexes])
labels_ratio_scaled_per_center = np.asarray([labels_ratio_scaled_per_center[index] for index in good_indexes])
center_ids = np.asarray([center_ids[index] for index in good_indexes])

n = 4000


[data.shape, labels_ratio.shape]

{k:[v,tests_per_center.get(k)] for k,v in mean_labels_ratio_per_center.items() if tests_per_center.get(k) >= min_test_per_center}

train_data = lgb.Dataset(data[0:n], label=labels[0:n],
                         feature_name=my_features, categorical_feature=['genderNum', 'center_id'])
param = {'num_leaves':37,
        # 'max_depth':20,
   # 'min_gain_to_split': 1000,
        'num_trees':85, 
#          'min_data_in_leaf': 100,
#          'max_bin': 100,
         'objective':'regression'}
num_round = 30
bst = lgb.train(param, train_data, num_round)
predictions = bst.predict(data)#*losses

# for i in range(len(data)):
#     center_id = center_ids[i]
#     predictions[i] = predictions[i]*mean_labels_ratio_per_center[center_id]

delta = 5

print("training")
print(np.std(labels[0:n]-np.mean(labels)))
print(np.std(labels[0:n]-predictions[0:n]))
print(model_good_prediction_ratio_array(labels[0:n], np.mean(labels), delta))
print(model_good_prediction_ratio_array(labels[0:n], predictions[0:n], delta))

print("validation")
print(np.std(labels[n+1:]-np.mean(labels)))
print(np.std(labels[n+1:]-predictions[n+1:]))
print(model_good_prediction_ratio_array(labels[n+1:], np.mean(labels), delta))
print(model_good_prediction_ratio_array(labels[n+1:], predictions[n+1:], delta))

center_ids = my_equipment_success['center_id'].values
diff_per_center = {}
low_baseline_diff_per_center = {}
baseline_diff_per_center = {}
for i in range(0,n):#,len(data)):
    center_id = data[i][0]
    diff = (predictions - labels)[i]
    baseline_diff = labels[i] - mean_labels_per_center[center_id]
    low_baseline_diff = labels[i] - np.mean(labels)
    diff_per_center[center_id] = safe_append(diff_per_center.get(center_id), diff)
    baseline_diff_per_center[center_id] = safe_append(baseline_diff_per_center.get(center_id), baseline_diff)
    low_baseline_diff_per_center[center_id] = safe_append(low_baseline_diff_per_center.get(center_id), low_baseline_diff)

my_dictionary = {k: [np.std(v),
                     model_good_prediction_ratio_diffs_array(v, delta),
                     len(v)] for k, v in diff_per_center.items()}
good_centers = {k: 1 for k,v in my_dictionary.items() if v[1] >= 0}
good_centers = {k:v  for k,v in my_dictionary.items() if good_centers.get(k) }
[sum(list(map(lambda x: x[2], list(my_dictionary.values())))),
 sum(list(map(lambda x: x[2], list(good_centers.values()))))]

{k:v for k,v in mean_labels_ratio_per_center.items() if tests_per_center.get(k) >= 100}

print("validation on good centers")
validation_labels = labels[n+1:]
validation_predictions = predictions[n+1:]
validation_data = data[n+1:]
validataion_center_ids = center_ids[n+1:]



validation_predictions_good = []
validation_labels_good = []
for i in range(len(validation_data)):
    center_id = validataion_center_ids[i]
    if good_centers.get(center_id):
        validation_labels_good.append(validation_labels[i])
        validation_predictions_good.append(validation_predictions[i])

validation_labels_good = np.asarray(validation_labels_good)
validation_predictions_good = np.asarray(validation_predictions_good)


print(np.std(validation_labels_good - np.mean(labels)))
print(np.std(validation_labels_good - validation_predictions_good))
print(model_good_prediction_ratio_array(validation_labels_good, np.mean(labels), delta))
print(model_good_prediction_ratio_array(validation_labels_good, validation_predictions_good, delta))

my_dictionary = {k: [np.std(v),
                     model_good_prediction_ratio_diffs_array(v, delta),
                     len(v)] for k, v in baseline_diff_per_center.items()}
my_dictionary

my_dictionary = {k: [np.std(v),
                     model_good_prediction_ratio_diffs_array(v, delta),
                     len(v)] for k, v in low_baseline_diff_per_center.items()}
my_dictionary

predictions = bst.predict(data)*(my_equipment_success['freq_1000'].values)

delta = 5

print("training")
print(np.std(labels[0:n]-np.mean(labels)))
print(np.std(labels[0:n]-predictions[0:n]))
print(model_good_prediction_ratio_array(labels[0:n], np.mean(labels), delta))
print(model_good_prediction_ratio_array(labels[0:n], predictions[0:n], delta))

print("validation")
print(np.std(labels[n+1:]-np.mean(labels)))
print(np.std(labels[n+1:]-predictions[n+1:]))
print(model_good_prediction_ratio_array(labels[n+1:], np.mean(labels), delta))
print(model_good_prediction_ratio_array(labels[n+1:], predictions[n+1:], delta))

from sklearn import tree
import graphviz 

my_features = [
    'center_id',
   'decades_old',
 'genderNum',
#'freq_250',
 'freq_500',
# 'freq_750',
 'freq_1000',
# 'freq_1500',
 'freq_2000',
# 'freq_3000',
 'freq_4000',
# 'freq_6000',
 'freq_8000'
              ]
my_equipment_success = equipment_success.copy()


#my_equipment_success = my_equipment_success.query(' valid == True').copy()
my_equipment_success = my_equipment_success.sample(frac=1)
data = my_equipment_success[my_features].values
labels = (my_equipment_success['eq_freq_1000'].values)
[data.shape, labels.shape]

def good_prediction_error(y_true, y_pred):
    g = tf.subtract(y_true, y_pred)
    g = tf.cast(g < 5.0, tf.float32)
    return g
    
    
#good_prediction_error(np.array([1, 20]), np.array([0, 0]))

features

eq = equipment_success.sample(frac=1)
data = (equipment_success[features].values)
labels = (equipment_success['eq_freq_1000'].values)

data = original_data.copy()
np.random.shuffle(data)
data.shape

model = Sequential([
    Dense(300, input_dim=len(features)),
    advanced_activations.LeakyReLU(alpha=0.3),
    Dense(100),
    advanced_activations.LeakyReLU(alpha=0.3),
    Dense(100),
    advanced_activations.LeakyReLU(alpha=0.3),
    Dense(1),
])


model.compile(optimizer=optimizers.Adam(lr=0.01),
              loss='mean_squared_error',       
              metrics=[ 'mean_squared_error'])  

validation_split=0.2
training_samples = int(len(data)*(1-validation_split))
history = model.fit(data, labels, epochs=200, steps_per_epoch=100,
                   validation_split=validation_split, 
                    validation_steps=100,
                    verbose=1)

from operator import itemgetter
# ע
a, b = min(enumerate(history.history['mean_squared_error']), key=itemgetter(1)) 
[a, math.sqrt(b)]

relevant_data = data[0:training_samples]
relevant_labels = labels[0:training_samples]
predictions = model.predict(relevant_data).transpose()[0]
[model_good_prediction_ratio_array(relevant_labels, predictions, 5),
 model_good_prediction_ratio_array(relevant_labels, np.average(relevant_labels), 5),
 np.std(relevant_labels - predictions),
 np.std(relevant_labels - np.average(relevant_labels))
]

relevant_data = data[training_samples:]
relevant_labels = labels[training_samples:]
predictions = model.predict(relevant_data).transpose()[0]
[model_good_prediction_ratio_array(relevant_labels, predictions, 5),
 model_good_prediction_ratio_array(relevant_labels, np.average(relevant_labels), 5),
 np.std(relevant_labels - predictions),
 np.std(relevant_labels - np.average(relevant_labels))
]

# Plot training & validation loss values
plt.plot(history.history['mean_squared_error'])
plt.plot(history.history['val_mean_squared_error'])
plt.title('Model mean_squared_error')
plt.ylabel('mean_squared_error')
plt.ylim(0, 100)
plt.xlabel('Epoch')
plt.legend(['Train', 'Test'], loc='upper right')
plt.show()

test_dataset = equipment_success_all.copy() #[(equipment_success_valid['isMale'] == 0)
                                           # & (equipment_success_valid['decades_old'] == 7)
                                           #].copy()
        
test_dataset = test_dataset.sample(frac=1)
test_dataset = test_dataset.tail(1000)
test_data = (test_dataset[features].values)
test_labels = (test_dataset['eq_freq_2000'].values)

test_predictions = model.predict(test_data).transpose()[0]
[model_good_prediction_ratio_array(test_labels, test_predictions, 5),
 model_good_prediction_ratio_array(test_labels, np.average(test_labels), 5)]

test_labels - test_predictions