Finetuning - AlexNet

Applying transfer learning to the facial expression recognition problem

In [1]:
%matplotlib inline
In [2]:
import os
In [3]:
import tensorflow as tf
import numpy as np
from timeit import default_timer as timer 
from AlexNet.alexnet import AlexNet
from matplotlib import pyplot as plt
import os
import cv2
from datagenerator import ImageDataGenerator
import time

Finetuning AlexNet

In [4]:
# Load mean
train_mean = np.load('Training_Img_Mean.npy')
print train_mean.shape
#plt.imshow(train_mean, cmap='gray')
train_mean = train_mean.astype(np.float32) # cv2.cvtColor requires this dtype
train_mean = cv2.cvtColor(train_mean, cv2.COLOR_GRAY2RGB)
train_mean = cv2.resize(train_mean, (227, 227))
print train_mean.shape
plt.imshow(train_mean)
(48, 48)
(227, 227, 3)
Out[4]:
<matplotlib.image.AxesImage at 0x115fa0f50>
In [5]:
# stacked_img = np.stack((train_mean,)*3).reshape((48, 48, 3))
# print stacked_img.shape
# plt.imshow(stacked_img)
# plt.imshow(train_mean)
In [6]:
# With new layers
n_classes = 7
x = tf.placeholder(tf.float32, (None, 227, 227, 3))
#x = tf.placeholder(tf.float32, (None, 227, 227, 1))
y = tf.placeholder(tf.float32, (None, n_classes))

fc6 = AlexNet(x, feature_extract=True)
fc6 = tf.stop_gradient(fc6)

shape = (fc6.get_shape().as_list()[-1], 4096)
fc7W = tf.Variable(tf.truncated_normal(shape, stddev=0.001))
fc7b = tf.Variable(tf.zeros(4096))
fc7 = tf.nn.relu(tf.matmul(fc6, fc7W) + fc7b)

shape2 = (fc7.get_shape().as_list()[-1], 2048)
fc8W = tf.Variable(tf.truncated_normal(shape2, stddev=0.001))
fc8b = tf.Variable(tf.zeros(2048))
fc8 = tf.nn.relu(tf.matmul(fc7, fc8W) + fc8b)

shape3 = (fc8.get_shape().as_list()[-1], 512)
fc9W = tf.Variable(tf.truncated_normal(shape3, stddev=0.001))
fc9b = tf.Variable(tf.zeros(512))
fc9 = tf.nn.relu(tf.matmul(fc8, fc9W) + fc9b)

shape4 = (fc9.get_shape().as_list()[-1], n_classes)
fc10W = tf.Variable(tf.truncated_normal(shape4, stddev=0.001))
fc10b = tf.Variable(tf.zeros(n_classes))
logits = tf.matmul(fc9, fc10W) + fc10b
#probs = tf.nn.softmax(logits)

cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss_op = tf.reduce_mean(cross_entropy)
opt = tf.train.AdamOptimizer()
train_op = opt.minimize(loss_op, var_list=[fc7W, fc7b, fc8W, fc8b, fc9W, fc9b, fc10W, fc10b])

#preds = tf.arg_max(logits, 1)
#accuracy_op = tf.reduce_mean(tf.cast(tf.equal(preds, labels), tf.float32))
correct_pred = tf.equal(tf.arg_max(logits, 1), tf.arg_max(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

init = tf.global_variables_initializer()
Returning fc6
In [7]:
# Hyperparameters
start = timer()
training_iters = 200000
batch_size = 128
display_step = 10
train_file = 'train.txt'
test_file = 'test.txt'

train_generator = ImageDataGenerator(train_file, horizontal_flip=False, shuffle=True, \
                                                   mean=train_mean, scale_size_alex=(227, 227, 3), \
                                     nb_classes=7, alex=True)
test_generator = ImageDataGenerator(test_file, horizontal_flip=False, shuffle=False, \
                                                  mean=train_mean, scale_size_alex=(227, 227, 3), nb_classes=7, \
                                    alex=True)

# Number of batches per epoch for each dataset
train_batches_per_epoch = np.floor(train_generator.data_size/batch_size).astype(np.int16) 
test_batches_per_epoch = np.floor(test_generator.data_size/batch_size).astype(np.int16)

# Train the network
with tf.Session() as sess: 
    sess.run(init)
    step = 1
    while step*batch_size < training_iters:
        batch_x, batch_y = train_generator.next_batch(batch_size)
        sess.run(train_op, feed_dict = {x:batch_x, y:batch_y})
        if step%display_step == 0:
            loss, acc = sess.run([loss_op, accuracy], feed_dict={x:batch_x, y:batch_y})
            print "\nIter " + str(step*batch_size) + " Loss=", loss, "Accuracy=", acc
        #print step,
        step += 1
    print "-----Optimization Done-----"

    # Test accuracy - sum up accuracies on each of the test batches
    test_acc = 0.
    test_count = 0
    print "Testing the model on batch: "
    for _ in range(test_batches_per_epoch):
        batch_tx, batch_ty = test_generator.next_batch(batch_size) 
        acc = sess.run(accuracy, feed_dict={x: batch_tx, y: batch_ty})
        test_acc += acc
        test_count += 1
        print test_count, 
        
    test_acc /= test_count
    print("Test Accuracy = {:.7f}".format(test_acc))


os.system('say "Your program has finished"') 
end = timer()
print "\nTime elapsed = ", end-start, " seconds"
Iter 1280 Loss= 1.94004 Accuracy= 0.210938

Iter 2560 Loss= 1.84533 Accuracy= 0.21875

Iter 3840 Loss= 1.83232 Accuracy= 0.210938

Iter 5120 Loss= 1.8105 Accuracy= 0.242188

Iter 6400 Loss= 1.71722 Accuracy= 0.304688

Iter 7680 Loss= 1.77944 Accuracy= 0.234375

Iter 8960 Loss= 1.76442 Accuracy= 0.210938

Iter 10240 Loss= 1.77979 Accuracy= 0.234375

Iter 11520 Loss= 1.75749 Accuracy= 0.25

Iter 12800 Loss= 1.76835 Accuracy= 0.210938

Iter 14080 Loss= 1.57577 Accuracy= 0.40625

Iter 15360 Loss= 1.62068 Accuracy= 0.375

Iter 16640 Loss= 1.53608 Accuracy= 0.382812

Iter 17920 Loss= 1.69437 Accuracy= 0.34375

Iter 19200 Loss= 1.68515 Accuracy= 0.28125

Iter 20480 Loss= 1.70408 Accuracy= 0.3125

Iter 21760 Loss= 1.72275 Accuracy= 0.320312

Iter 23040 Loss= 1.70466 Accuracy= 0.28125

Iter 24320 Loss= 1.58744 Accuracy= 0.367188

Iter 25600 Loss= 1.50983 Accuracy= 0.398438

Iter 26880 Loss= 1.68081 Accuracy= 0.28125

Iter 28160 Loss= 1.59946 Accuracy= 0.34375

Iter 29440 Loss= 1.64091 Accuracy= 0.335938

Iter 30720 Loss= 1.72315 Accuracy= 0.289062

Iter 32000 Loss= 1.65467 Accuracy= 0.390625

Iter 33280 Loss= 1.47172 Accuracy= 0.429688

Iter 34560 Loss= 1.62763 Accuracy= 0.398438

Iter 35840 Loss= 1.60918 Accuracy= 0.34375

Iter 37120 Loss= 1.66392 Accuracy= 0.40625

Iter 38400 Loss= 1.55973 Accuracy= 0.382812

Iter 39680 Loss= 1.57517 Accuracy= 0.34375

Iter 40960 Loss= 1.68858 Accuracy= 0.3125

Iter 42240 Loss= 1.66743 Accuracy= 0.320312

Iter 43520 Loss= 1.59606 Accuracy= 0.3125

Iter 44800 Loss= 1.5878 Accuracy= 0.335938

Iter 46080 Loss= 1.59299 Accuracy= 0.367188

Iter 47360 Loss= 1.63054 Accuracy= 0.367188

Iter 48640 Loss= 1.54307 Accuracy= 0.453125

Iter 49920 Loss= 1.49004 Accuracy= 0.390625

Iter 51200 Loss= 1.52401 Accuracy= 0.351562

Iter 52480 Loss= 1.60148 Accuracy= 0.335938

Iter 53760 Loss= 1.49802 Accuracy= 0.398438

Iter 55040 Loss= 1.65124 Accuracy= 0.3125

Iter 56320 Loss= 1.60244 Accuracy= 0.320312

Iter 57600 Loss= 1.54589 Accuracy= 0.398438

Iter 58880 Loss= 1.68927 Accuracy= 0.28125

Iter 60160 Loss= 1.54936 Accuracy= 0.359375

Iter 61440 Loss= 1.6024 Accuracy= 0.34375

Iter 62720 Loss= 1.58445 Accuracy= 0.375

Iter 64000 Loss= 1.45088 Accuracy= 0.4375

Iter 65280 Loss= 1.48862 Accuracy= 0.453125

Iter 66560 Loss= 1.7006 Accuracy= 0.273438

Iter 67840 Loss= 1.56688 Accuracy= 0.421875

Iter 69120 Loss= 1.40765 Accuracy= 0.429688

Iter 70400 Loss= 1.63092 Accuracy= 0.359375

Iter 71680 Loss= 1.40531 Accuracy= 0.414062

Iter 72960 Loss= 1.57897 Accuracy= 0.375

Iter 74240 Loss= 1.70175 Accuracy= 0.3125

Iter 75520 Loss= 1.61078 Accuracy= 0.34375

Iter 76800 Loss= 1.50716 Accuracy= 0.390625

Iter 78080 Loss= 1.47247 Accuracy= 0.414062

Iter 79360 Loss= 1.53252 Accuracy= 0.34375

Iter 80640 Loss= 1.4507 Accuracy= 0.421875

Iter 81920 Loss= 1.65584 Accuracy= 0.304688

Iter 83200 Loss= 1.66649 Accuracy= 0.359375

Iter 84480 Loss= 1.55055 Accuracy= 0.40625

Iter 85760 Loss= 1.59846 Accuracy= 0.375

Iter 87040 Loss= 1.53267 Accuracy= 0.453125

Iter 88320 Loss= 1.50796 Accuracy= 0.398438

Iter 89600 Loss= 1.55378 Accuracy= 0.351562

Iter 90880 Loss= 1.6141 Accuracy= 0.359375

Iter 92160 Loss= 1.47656 Accuracy= 0.34375

Iter 93440 Loss= 1.54646 Accuracy= 0.421875

Iter 94720 Loss= 1.57131 Accuracy= 0.351562

Iter 96000 Loss= 1.33659 Accuracy= 0.492188

Iter 97280 Loss= 1.45626 Accuracy= 0.421875

Iter 98560 Loss= 1.52032 Accuracy= 0.4375

Iter 99840 Loss= 1.53346 Accuracy= 0.398438

Iter 101120 Loss= 1.51464 Accuracy= 0.382812

Iter 102400 Loss= 1.56517 Accuracy= 0.367188

Iter 103680 Loss= 1.50627 Accuracy= 0.398438

Iter 104960 Loss= 1.55258 Accuracy= 0.328125

Iter 106240 Loss= 1.59755 Accuracy= 0.359375

Iter 107520 Loss= 1.4444 Accuracy= 0.4375

Iter 108800 Loss= 1.34088 Accuracy= 0.53125

Iter 110080 Loss= 1.39672 Accuracy= 0.492188

Iter 111360 Loss= 1.58181 Accuracy= 0.398438

Iter 112640 Loss= 1.52383 Accuracy= 0.359375

Iter 113920 Loss= 1.53599 Accuracy= 0.390625

Iter 115200 Loss= 1.54004 Accuracy= 0.40625

Iter 116480 Loss= 1.47502 Accuracy= 0.453125

Iter 117760 Loss= 1.59527 Accuracy= 0.359375

Iter 119040 Loss= 1.3464 Accuracy= 0.554688

Iter 120320 Loss= 1.35642 Accuracy= 0.46875

Iter 121600 Loss= 1.54799 Accuracy= 0.390625

Iter 122880 Loss= 1.52898 Accuracy= 0.367188

Iter 124160 Loss= 1.42564 Accuracy= 0.375

Iter 125440 Loss= 1.41111 Accuracy= 0.429688

Iter 126720 Loss= 1.4441 Accuracy= 0.429688

Iter 128000 Loss= 1.48114 Accuracy= 0.4375

Iter 129280 Loss= 1.40878 Accuracy= 0.4375

Iter 130560 Loss= 1.47505 Accuracy= 0.421875

Iter 131840 Loss= 1.39698 Accuracy= 0.484375

Iter 133120 Loss= 1.39627 Accuracy= 0.453125

Iter 134400 Loss= 1.43877 Accuracy= 0.414062

Iter 135680 Loss= 1.44277 Accuracy= 0.453125

Iter 136960 Loss= 1.3659 Accuracy= 0.46875

Iter 138240 Loss= 1.51683 Accuracy= 0.40625

Iter 139520 Loss= 1.48111 Accuracy= 0.421875

Iter 140800 Loss= 1.48633 Accuracy= 0.398438

Iter 142080 Loss= 1.45707 Accuracy= 0.4375

Iter 143360 Loss= 1.58149 Accuracy= 0.351562

Iter 144640 Loss= 1.52732 Accuracy= 0.398438

Iter 145920 Loss= 1.36251 Accuracy= 0.445312

Iter 147200 Loss= 1.50967 Accuracy= 0.476562

Iter 148480 Loss= 1.42844 Accuracy= 0.40625

Iter 149760 Loss= 1.33798 Accuracy= 0.4375

Iter 151040 Loss= 1.4797 Accuracy= 0.40625

Iter 152320 Loss= 1.36009 Accuracy= 0.507812

Iter 153600 Loss= 1.51846 Accuracy= 0.382812

Iter 154880 Loss= 1.47853 Accuracy= 0.4375

Iter 156160 Loss= 1.52384 Accuracy= 0.4375

Iter 157440 Loss= 1.34364 Accuracy= 0.492188

Iter 158720 Loss= 1.61808 Accuracy= 0.375

Iter 160000 Loss= 1.47643 Accuracy= 0.398438

Iter 161280 Loss= 1.36074 Accuracy= 0.515625

Iter 162560 Loss= 1.40639 Accuracy= 0.476562

Iter 163840 Loss= 1.41435 Accuracy= 0.453125

Iter 165120 Loss= 1.41706 Accuracy= 0.421875

Iter 166400 Loss= 1.38485 Accuracy= 0.445312

Iter 167680 Loss= 1.62195 Accuracy= 0.328125

Iter 168960 Loss= 1.40624 Accuracy= 0.421875

Iter 170240 Loss= 1.37098 Accuracy= 0.453125

Iter 171520 Loss= 1.51764 Accuracy= 0.414062

Iter 172800 Loss= 1.41531 Accuracy= 0.4375

Iter 174080 Loss= 1.3111 Accuracy= 0.460938

Iter 175360 Loss= 1.44415 Accuracy= 0.40625

Iter 176640 Loss= 1.21074 Accuracy= 0.5625

Iter 177920 Loss= 1.33451 Accuracy= 0.453125

Iter 179200 Loss= 1.52357 Accuracy= 0.421875

Iter 180480 Loss= 1.33343 Accuracy= 0.492188

Iter 181760 Loss= 1.41813 Accuracy= 0.484375

Iter 183040 Loss= 1.39446 Accuracy= 0.460938

Iter 184320 Loss= 1.29041 Accuracy= 0.546875

Iter 185600 Loss= 1.39572 Accuracy= 0.460938

Iter 186880 Loss= 1.48529 Accuracy= 0.382812

Iter 188160 Loss= 1.34073 Accuracy= 0.507812

Iter 189440 Loss= 1.37457 Accuracy= 0.460938

Iter 190720 Loss= 1.36389 Accuracy= 0.515625

Iter 192000 Loss= 1.3549 Accuracy= 0.484375

Iter 193280 Loss= 1.3277 Accuracy= 0.546875

Iter 194560 Loss= 1.47235 Accuracy= 0.429688

Iter 195840 Loss= 1.42402 Accuracy= 0.40625

Iter 197120 Loss= 1.40838 Accuracy= 0.453125

Iter 198400 Loss= 1.36032 Accuracy= 0.445312

Iter 199680 Loss= 1.33222 Accuracy= 0.492188
-----Optimization Done-----
Testing the model on batch: 
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 Test Accuracy = 0.4400112

Time elapsed =  18168.9822328  seconds
In [ ]: