The data consists of 48x48 pixel grayscale images of faces. The faces have been automatically registered so that the face is more or less centered and occupies about the same amount of space in each image. The task is to categorize each face based on the emotion shown in the facial expression in to one of seven categories (0=Angry, 1=Disgust, 2=Fear, 3=Happy, 4=Sad, 5=Surprise, 6=Neutral).
train.csv contains two columns, "emotion" and "pixels". The "emotion" column contains a numeric code ranging from 0 to 6, inclusive, for the emotion that is present in the image. The "pixels" column contains a string surrounded in quotes for each image. The contents of this string a space-separated pixel values in row major order. test.csv contains only the "pixels" column and your task is to predict the emotion column.
The training set consists of 28,709 examples. The public test set used for the leaderboard consists of 3,589 examples. The final test set, which was used to determine the winner of the competition, consists of another 3,589 examples.
This dataset was prepared by Pierre-Luc Carrier and Aaron Courville, as part of an ongoing research project. They have graciously provided the workshop organizers with a preliminary version of their dataset to use for this contest.
%matplotlib inline
import graphlab as gl
import matplotlib.pyplot as plt
import numpy as np
from timeit import default_timer as timer
import os
#cnn_testpred = np.load('GraphLabOutput/testpred_GraphLab_CNN.npy')
cnn_testpred = np.load('GraphLabOutput/testpred_GraphLab_CNN_48.npy')
knn_testpred = np.load('SFrame/test_37percent_120cos11.npy')
cnn_pritestpred = np.load('GraphLabOutput/pri_testpred_GraphLab_CNN_48.npy')
knn_pritestpred = np.load('SFrame/pritest_37percent_120cos11.npy')
# CNN predictions are shuffled - have to reshuffle them to be able to compare with KNN
# Load the random permutations that were used on CNN's test data
cnn_testidx = np.load('GraphLabOutput/test_idx.npy')
cnn_pri_testidx = np.load('GraphLabOutput/pri_test_idx.npy')
cnn_testpred = cnn_testpred.astype(int)
knn_testpred = knn_testpred.astype(int)
cnn_pritestpred = cnn_pritestpred.astype(int)
knn_pritestpred = knn_pritestpred.astype(int)
# Read true public test labels
testlabels = []
testfile = 'test.txt'
with open(testfile, 'r') as f:
for line in f:
currfile, currlabel = line.split()
testlabels.append(currlabel)
# Read true private test labels
pri_testlabels = []
pri_testfile = 'pri_test.txt'
with open(pri_testfile, 'r') as f:
for line in f:
currfile, currlabel = line.split()
pri_testlabels.append(currlabel)
# Convert the labels to int
testlabels = np.asarray(testlabels).astype(int)
pri_testlabels = np.asarray(pri_testlabels).astype(int)
# Re-permute cnn predictions based on the originally used random indices
cnn_testpred_perm = [cnn_testpred[i] for i in cnn_testidx]
cnn_testpred_perm = np.asarray(cnn_testpred_perm)
cnn_pri_testpred_perm = [cnn_pritestpred[i] for i in cnn_pri_testidx]
cnn_pri_testpred_perm = np.asarray(cnn_pri_testpred_perm)
# Verify shapes of the datasets
print cnn_testpred.shape, knn_testpred.shape, testlabels.shape, cnn_testidx.shape, cnn_pri_testidx.shape, \
cnn_testpred_perm.shape, cnn_pri_testpred_perm.shape
# In what % of instances are both knn and cnn's predictions the same - approx 35%
print float(np.sum(cnn_testpred_perm == knn_testpred))/len(knn_testpred)
print float(np.sum(cnn_pri_testpred_perm == knn_pritestpred))/len(knn_pritestpred)
# Verify original models' accuracies
print "CNN Public Test Accuracy: ", float(np.sum(cnn_testpred_perm == testlabels))/len(testlabels)
print "CNN Private Test Accuracy: ", float(np.sum(cnn_pri_testpred_perm == pri_testlabels))/len(pri_testlabels)
print "KNN Public Test Accuracy: ", float(np.sum(knn_testpred == testlabels))/len(testlabels)
print "KNN Private Test Accuracy: ", float(np.sum(knn_pritestpred == pri_testlabels))/len(pri_testlabels)
# Stack all labels column wise and create two datasets - one for public
# and one for private
data = np.column_stack([cnn_testpred_perm, knn_testpred]).astype(int)
print data.shape
pri_data = np.column_stack([cnn_pri_testpred_perm, knn_pritestpred]).astype(int)
print pri_data.shape
from sklearn.naive_bayes import GaussianNB
# Experiment with uniform priors
numlabels = len(np.unique(testlabels))
priors = np.asarray([1./numlabels]*numlabels)
priors[6] += 1 - np.sum(priors) # To counter precision errors (not summing to 1)
print priors
# Fit a Naive Bayes model
#gnb = GaussianNB(priors=priors) # With uniform priors
gnb = GaussianNB() # With priors based on the data
gnb.fit(data, testlabels)
# Make predictions on the public test dataset
gnb_data_pred = gnb.predict(data)
print "Public Test Accuracy: ", float(np.sum(gnb_data_pred == testlabels))/len(testlabels)
# Make predictions on the private test dataset
gnb_data_pred = gnb.predict(pri_data)
print "Private Test Accuracy: ", float(np.sum(gnb_data_pred == pri_testlabels))/len(pri_testlabels)
from sklearn.linear_model import LogisticRegression
logistic = LogisticRegression(max_iter=1000, solver = 'sag', tol = 1e-10)
logistic.fit(data, testlabels)
# Make predictions on the public test dataset
logistic_data_pred = logistic.predict(data)
print "Public Test Accuracy: ", float(np.sum(logistic_data_pred == testlabels))/len(testlabels)
# Make predictions on the private test dataset
logistic_data_pred = logistic.predict(pri_data)
print "Private Test Accuracy: ", float(np.sum(logistic_data_pred == pri_testlabels))/len(pri_testlabels)
from sklearn.ensemble import GradientBoostingClassifier
boost = GradientBoostingClassifier(n_estimators=50, learning_rate=0.1, max_depth=10, max_leaf_nodes=10,
min_samples_split=5)
#boost = GradientBoostingClassifier()
boost.fit(data, testlabels)
# Make predictions on the public test dataset
boost_data_pred = boost.predict(data)
print "Public Test Accuracy: ", float(np.sum(boost_data_pred == testlabels))/len(testlabels)
# Save the predictions
np.save("GraphLabOutput/boost_data_publicpred_49_v2%.npy", boost_data_pred)
# Make predictions on the private test dataset
boost_data_pred = boost.predict(pri_data)
print "Private Test Accuracy: ", float(np.sum(boost_data_pred == pri_testlabels))/len(pri_testlabels)
np.save("GraphLabOutput/boost_data_privatepred_49_v2%.npy", boost_data_pred)
# In what % instances are both boostpred and cnn/knn's predictions the same
print float(np.sum(cnn_pri_testpred_perm == boost_data_pred))/len(boost_data_pred)
print float(np.sum(knn_pritestpred == boost_data_pred))/len(boost_data_pred)
from sklearn.metrics import confusion_matrix
import pylab as pl
# Create a confusion matrix from the predictions
cm = confusion_matrix(pri_testlabels, boost_data_pred)
print cm
cm_norm = cm.astype(float)/cm.sum(axis = 1, keepdims=True) # Standardized confusion matrix
# Plot the confusion matrix
labels = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
# With help from stackoverflow
fig = plt.figure(figsize=(12, 12))
ax = fig.add_subplot(111)
cax = ax.matshow(cm_norm)
pl.title('Confusion matrix')
fig.colorbar(cax)
ax.set_xticklabels([''] + labels)
ax.set_yticklabels([''] + labels)
pl.xlabel('Predicted')
pl.ylabel('True')
pl.show()
'Disgust' is one of the hardest facial expressions to predict. Unsurprising - it has the least number of training data. Even fear is pretty hard - model gets confused between 'fear' and 'sad'.
from os import listdir
from os.path import isfile, join
# Verify ensemble model's accuracy
num_correct = np.count_nonzero(boost_data_pred == pri_testlabels)
num_not_correct = len(pri_testlabels) - num_correct
print num_correct, num_not_correct
print float(num_correct)/len(pri_testlabels), float(num_not_correct)/len(pri_testlabels)
# Get correct predictions
correct = np.where((boost_data_pred == pri_testlabels)==True)
print correct[0].shape
# Get incorrect predictions
incorrect = np.where((boost_data_pred == pri_testlabels)==False)
print incorrect[0].shape
# Randomly sample indices of a few correct and incorrect predictions
np.random.seed(40)
num_samples = 4
rand_correct = np.random.randint(0, num_correct, num_samples)
rand_incorrect = np.random.randint(0, num_correct, num_samples)
# Print the numbers
print rand_correct
print rand_incorrect
# Get the prediction indices corresponding to the random numbers (and sort them)
sorted_idx_correct = sorted(correct[0][rand_correct])
sorted_idx_incorrect = sorted(incorrect[0][rand_incorrect])
# Note: (0=Angry, 1=Disgust, 2=Fear, 3=Happy, 4=Sad, 5=Surprise, 6=Neutral)
# Get the labels of these correct predictions
print boost_data_pred[sorted_idx_correct]
print pri_testlabels[sorted_idx_correct]
# Get the labels of these incorrect predictions
print boost_data_pred[sorted_idx_incorrect]
print pri_testlabels[sorted_idx_incorrect]
# Read the corresponding images from the private data
overall_ctr = 0
correct_imgs = []
incorrect_imgs = []
with open('pri_test.txt','w') as f:
for i in range(7):
path = 'PrivateTest/'+str(i)
onlyfiles = [fl for fl in listdir(path) if isfile(join(path, fl))]
print len(onlyfiles), onlyfiles[0]
for curr_file in onlyfiles:
overall_ctr += 1
# To avoid reading unwanted files
if curr_file.find(".DS_Store") != -1:
continue
fname = path + '/' + curr_file + ' ' + str(i) + '\n'
if overall_ctr in sorted_idx_correct:
correct_imgs.append(fname)
if overall_ctr in sorted_idx_incorrect:
incorrect_imgs.append(fname)
print overall_ctr
# print correct_imgs
# print incorrect_imgs
# Correctly predicted
emotions = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
plt.figure(figsize=(15, 10))
for i in range(len(correct_imgs)):
img_path, img_label = correct_imgs[i].split()
print i, "Actual vs. Predicted:", emotions[np.int(img_label)], "vs.",\
emotions[boost_data_pred[sorted_idx_correct[i]]]
plt.subplot('1'+str(num_samples)+str(i+1))
curr_img = plt.imread(img_path)
plt.imshow(curr_img, cmap='gray')
plt.title(i)
plt.axis('off')
plt.show()
# Inorrectly predicted
emotions = ['Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral']
plt.figure(figsize=(15, 10))
for i in range(len(incorrect_imgs)):
img_path, img_label = incorrect_imgs[i].split()
print i, "Actual vs. Predicted:", emotions[np.int(img_label)], "vs.", \
emotions[boost_data_pred[sorted_idx_incorrect[i]]]
plt.subplot('1'+str(num_samples)+str(i+1))
curr_img = plt.imread(img_path)
plt.imshow(curr_img, cmap='gray')
plt.title(i)
plt.axis('off')
plt.show()