From 2eb13d63b15a8ac908d6fa324c7f3d19141ca570 Mon Sep 17 00:00:00 2001
From: natanielruiz <nataniel777@hotmail.com>
Date: 星期六, 12 八月 2017 08:57:15 +0800
Subject: [PATCH] Temperature softmax and 10 shape PCA regression.
---
code/hopenet.py | 26 +++
code/test_resnet_shape.py | 145 ++++++++++++++++++++
code/test_resnet_bins.py | 16 -
code/train_resnet_shape.py | 53 ++++--
code/utils.py | 5
practice/aflw_example.py | 133 +++++++++++++++++++
6 files changed, 345 insertions(+), 33 deletions(-)
diff --git a/code/hopenet.py b/code/hopenet.py
index 9ba8f04..1b94fa1 100644
--- a/code/hopenet.py
+++ b/code/hopenet.py
@@ -106,7 +106,7 @@
# This is just Hopenet with 3 output layers for yaw, pitch and roll.
def __init__(self, block, layers, num_bins, shape_bins):
self.inplanes = 64
- super(Hopenet, self).__init__()
+ super(Hopenet_shape, self).__init__()
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
@@ -120,7 +120,16 @@
self.fc_yaw = nn.Linear(512 * block.expansion, num_bins)
self.fc_pitch = nn.Linear(512 * block.expansion, num_bins)
self.fc_roll = nn.Linear(512 * block.expansion, num_bins)
+ self.fc_shape_0 = nn.Linear(512 * block.expansion, shape_bins)
self.fc_shape_1 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_2 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_3 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_4 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_5 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_6 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_7 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_8 = nn.Linear(512 * block.expansion, shape_bins)
+ self.fc_shape_9 = nn.Linear(512 * block.expansion, shape_bins)
for m in self.modules():
if isinstance(m, nn.Conv2d):
@@ -163,6 +172,17 @@
yaw = self.fc_yaw(x)
pitch = self.fc_pitch(x)
roll = self.fc_roll(x)
- shape_1 = self.fc_shape_1(x)
- return yaw, pitch, roll, shape_1
+ shape = []
+ shape.append(self.fc_shape_0(x))
+ shape.append(self.fc_shape_1(x))
+ shape.append(self.fc_shape_2(x))
+ shape.append(self.fc_shape_3(x))
+ shape.append(self.fc_shape_4(x))
+ shape.append(self.fc_shape_5(x))
+ shape.append(self.fc_shape_6(x))
+ shape.append(self.fc_shape_7(x))
+ shape.append(self.fc_shape_8(x))
+ shape.append(self.fc_shape_9(x))
+
+ return yaw, pitch, roll, shape
diff --git a/code/test_resnet_bins.py b/code/test_resnet_bins.py
index 699c9c9..4b1a655 100644
--- a/code/test_resnet_bins.py
+++ b/code/test_resnet_bins.py
@@ -103,18 +103,14 @@
_, pitch_bpred = torch.max(pitch.data, 1)
_, roll_bpred = torch.max(roll.data, 1)
- yaw_predicted = F.softmax(yaw)
- pitch_predicted = F.softmax(pitch)
- roll_predicted = F.softmax(roll)
-
# Continuous predictions
- yaw_predicted = torch.sum(yaw_predicted.data * idx_tensor, 1)
- pitch_predicted = torch.sum(pitch_predicted.data * idx_tensor, 1)
- roll_predicted = torch.sum(roll_predicted.data * idx_tensor, 1)
+ yaw_predicted = utils.softmax_temperature(yaw.data, 1)
+ pitch_predicted = utils.softmax_temperature(pitch.data, 1)
+ roll_predicted = utils.softmax_temperature(roll.data, 1)
- yaw_predicted = yaw_predicted.cpu()
- pitch_predicted = pitch_predicted.cpu()
- roll_predicted = roll_predicted.cpu()
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1).cpu()
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1).cpu()
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1).cpu()
# Mean absolute error
yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw) * 3)
diff --git a/code/test_resnet_shape.py b/code/test_resnet_shape.py
new file mode 100644
index 0000000..b35c64f
--- /dev/null
+++ b/code/test_resnet_shape.py
@@ -0,0 +1,145 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torch.backends.cudnn as cudnn
+import torchvision
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import utils
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--snapshot', dest='snapshot', help='Name of model snapshot.',
+ default='', type=str)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=1, type=int)
+ parser.add_argument('--save_viz', dest='save_viz', help='Save images with pose cube.',
+ default=False, type=bool)
+
+ args = parser.parse_args()
+
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ gpu = args.gpu_id
+ snapshot_path = os.path.join('output/snapshots', args.snapshot + '.pkl')
+
+ # ResNet101 with 3 outputs.
+ # model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 23, 3], 66)
+ # ResNet50
+ model = hopenet.Hopenet_shape(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66, 60)
+ # ResNet18
+ # model = hopenet.Hopenet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2], 66)
+
+ print 'Loading snapshot.'
+ # Load snapshot
+ saved_state_dict = torch.load(snapshot_path)
+ model.load_state_dict(saved_state_dict)
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(224),
+ transforms.RandomCrop(224), transforms.ToTensor()])
+
+ pose_dataset = datasets.AFLW2000_binned(args.data_dir, args.filename_list,
+ transformations)
+ test_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=args.batch_size,
+ num_workers=2)
+
+ model.cuda(gpu)
+
+ print 'Ready to test network.'
+
+ # Test the Model
+ model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
+ total = 0
+ n_margins = 20
+ yaw_correct = np.zeros(n_margins)
+ pitch_correct = np.zeros(n_margins)
+ roll_correct = np.zeros(n_margins)
+
+ idx_tensor = [idx for idx in xrange(66)]
+ idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
+
+ yaw_error = .0
+ pitch_error = .0
+ roll_error = .0
+
+ l1loss = torch.nn.L1Loss(size_average=False)
+
+ for i, (images, labels, name) in enumerate(test_loader):
+ images = Variable(images).cuda(gpu)
+ total += labels.size(0)
+ label_yaw = labels[:,0].float()
+ label_pitch = labels[:,1].float()
+ label_roll = labels[:,2].float()
+
+ yaw, pitch, roll, shape = model(images)
+
+ # Binned predictions
+ _, yaw_bpred = torch.max(yaw.data, 1)
+ _, pitch_bpred = torch.max(pitch.data, 1)
+ _, roll_bpred = torch.max(roll.data, 1)
+
+ # Continuous predictions
+ yaw_predicted = utils.softmax_temperature(yaw.data, 1)
+ pitch_predicted = utils.softmax_temperature(pitch.data, 1)
+ roll_predicted = utils.softmax_temperature(roll.data, 1)
+
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1).cpu()
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1).cpu()
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1).cpu()
+
+ # Mean absolute error
+ yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw) * 3)
+ pitch_error += torch.sum(torch.abs(pitch_predicted - label_pitch) * 3)
+ roll_error += torch.sum(torch.abs(roll_predicted - label_roll) * 3)
+
+ # Binned Accuracy
+ # for er in xrange(n_margins):
+ # yaw_bpred[er] += (label_yaw[0] in range(yaw_bpred[0,0] - er, yaw_bpred[0,0] + er + 1))
+ # pitch_bpred[er] += (label_pitch[0] in range(pitch_bpred[0,0] - er, pitch_bpred[0,0] + er + 1))
+ # roll_bpred[er] += (label_roll[0] in range(roll_bpred[0,0] - er, roll_bpred[0,0] + er + 1))
+
+ # print label_yaw[0], yaw_bpred[0,0]
+
+ # Save images with pose cube.
+ # TODO: fix for larger batch size
+ if args.save_viz:
+ name = name[0]
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '.jpg'))
+ #print os.path.join('output/images', name + '.jpg')
+ #print label_yaw[0] * 3 - 99, label_pitch[0] * 3 - 99, label_roll[0] * 3 - 99
+ #print yaw_predicted * 3 - 99, pitch_predicted * 3 - 99, roll_predicted * 3 - 99
+ utils.plot_pose_cube(cv2_img, yaw_predicted[0] * 3 - 99, pitch_predicted[0] * 3 - 99, roll_predicted[0] * 3 - 99)
+ cv2.imwrite(os.path.join('output/images', name + '.jpg'), cv2_img)
+
+ print('Test error in degrees of the model on the ' + str(total) +
+ ' test images. Yaw: %.4f, Pitch: %.4f, Roll: %.4f' % (yaw_error / total,
+ pitch_error / total, roll_error / total))
+
+ # Binned accuracy
+ # for idx in xrange(len(yaw_correct)):
+ # print yaw_correct[idx] / total, pitch_correct[idx] / total, roll_correct[idx] / total
diff --git a/code/train_resnet_shape.py b/code/train_resnet_shape.py
index c874fae..f6baddf 100644
--- a/code/train_resnet_shape.py
+++ b/code/train_resnet_shape.py
@@ -66,7 +66,17 @@
b.append(model.fc_yaw)
b.append(model.fc_pitch)
b.append(model.fc_roll)
+ b.append(model.fc_shape_0)
b.append(model.fc_shape_1)
+ b.append(model.fc_shape_2)
+ b.append(model.fc_shape_3)
+ b.append(model.fc_shape_4)
+ b.append(model.fc_shape_5)
+ b.append(model.fc_shape_6)
+ b.append(model.fc_shape_7)
+ b.append(model.fc_shape_8)
+ b.append(model.fc_shape_9)
+
for i in range(len(b)):
for j in b[i].modules():
for k in j.parameters():
@@ -96,7 +106,7 @@
# ResNet101 with 3 outputs
# model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 23, 3], 66)
# ResNet50
- model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66)
+ model = hopenet.Hopenet_shape(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66, 60)
# ResNet18
# model = hopenet.Hopenet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2], 66)
load_filtered_state_dict(model, model_zoo.load_url(model_urls['resnet50']))
@@ -114,8 +124,8 @@
num_workers=2)
model.cuda(gpu)
- criterion = nn.CrossEntropyLoss()
- reg_criterion = nn.MSELoss()
+ criterion = nn.CrossEntropyLoss().cuda(gpu)
+ reg_criterion = nn.MSELoss().cuda(gpu)
# Regression loss coefficient
alpha = 0.1
lsm = nn.Softmax()
@@ -124,21 +134,23 @@
idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
optimizer = torch.optim.Adam([{'params': get_ignored_params(model), 'lr': args.lr},
- {'params': get_non_ignored_params(model), 'lr': args.lr * 10}],
+ {'params': get_non_ignored_params(model), 'lr': args.lr}],
lr = args.lr)
print 'Ready to train network.'
for epoch in range(num_epochs):
for i, (images, labels, name) in enumerate(train_loader):
- images = Variable(images).cuda(gpu)
- label_yaw = Variable(labels[:,0]).cuda(gpu)
- label_pitch = Variable(labels[:,1]).cuda(gpu)
- label_roll = Variable(labels[:,2]).cuda(gpu)
- label_shape_1 = Variable(labels[:,3]).cuda(gpu)
+ images = Variable(images.cuda(gpu))
+ label_yaw = Variable(labels[:,0].cuda(gpu))
+ label_pitch = Variable(labels[:,1].cuda(gpu))
+ label_roll = Variable(labels[:,2].cuda(gpu))
+ label_shape = Variable(labels[:,3:].cuda(gpu))
optimizer.zero_grad()
- yaw, pitch, roll, shape_1 = model(images)
+ model.zero_grad()
+
+ yaw, pitch, roll, shape = model(images)
# Cross entropy loss
loss_yaw = criterion(yaw, label_yaw)
@@ -158,17 +170,18 @@
loss_reg_pitch = reg_criterion(pitch_predicted, label_pitch.float())
loss_reg_roll = reg_criterion(roll_predicted, label_roll.float())
- # Shape space loss
- loss_shape_1 = criterion(shape_1, label_shape_1)
-
# Total loss
loss_yaw += alpha * loss_reg_yaw
loss_pitch += alpha * loss_reg_pitch
loss_roll += alpha * loss_reg_roll
- loss_seq = [loss_yaw, loss_pitch, loss_roll, loss_shape_1]
+ loss_seq = [loss_yaw, loss_pitch, loss_roll]
+
+ # Shape space loss
+ for idx in xrange(len(shape)):
+ loss_seq.append(criterion(shape[idx], label_shape[:,idx]))
+
grad_seq = [torch.Tensor(1).cuda(gpu) for _ in range(len(loss_seq))]
- model.zero_grad()
torch.autograd.backward(loss_seq, grad_seq)
optimizer.step()
@@ -176,17 +189,17 @@
# %(epoch+1, num_epochs, i+1, len(pose_dataset)//batch_size, loss_yaw.data[0], loss_pitch.data[0], loss_roll.data[0]))
if (i+1) % 100 == 0:
- print ('Epoch [%d/%d], Iter [%d/%d] Losses: Yaw %.4f, Pitch %.4f, Roll %.4f'
- %(epoch+1, num_epochs, i+1, len(pose_dataset)//batch_size, loss_yaw.data[0], loss_pitch.data[0], loss_roll.data[0]))
+ print ('Epoch [%d/%d], Iter [%d/%d] Losses: Yaw %.4f, Pitch %.4f, Roll %.4f, Shape %.4f'
+ %(epoch+1, num_epochs, i+1, len(pose_dataset)//batch_size, loss_yaw.data[0], loss_pitch.data[0], loss_roll.data[0], loss_seq[3].data[0]))
if epoch == 0:
torch.save(model.state_dict(),
- 'output/snapshots/resnet50_iter_'+ str(i+1) + '.pkl')
+ 'output/snapshots/resnet50_shape_iter_'+ str(i+1) + '.pkl')
# Save models at numbered epochs.
if epoch % 1 == 0 and epoch < num_epochs - 1:
print 'Taking snapshot...'
torch.save(model.state_dict(),
- 'output/snapshots/resnet50_epoch_'+ str(epoch+1) + '.pkl')
+ 'output/snapshots/resnet50_shape_epoch_'+ str(epoch+1) + '.pkl')
# Save the final Trained Model
- torch.save(model.state_dict(), 'output/snapshots/resnet50_epoch_' + str(epoch+1) + '.pkl')
+ torch.save(model.state_dict(), 'output/snapshots/resnet50_shape_epoch_' + str(epoch+1) + '.pkl')
diff --git a/code/utils.py b/code/utils.py
index 09a47a8..01710b2 100644
--- a/code/utils.py
+++ b/code/utils.py
@@ -7,6 +7,11 @@
import math
from math import cos, sin
+def softmax_temperature(tensor, temperature):
+ result = torch.exp(tensor / temperature)
+ result = torch.div(result, torch.sum(result, 1).unsqueeze(1).expand_as(result))
+ return result
+
def get_pose_params_from_mat(mat_path):
# This functions gets the pose parameters from the .mat
# Annotations that come with the 300W_LP dataset.
diff --git a/practice/aflw_example.py b/practice/aflw_example.py
new file mode 100644
index 0000000..f81f333
--- /dev/null
+++ b/practice/aflw_example.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+##
+# Massimiliano Patacchiola, Plymouth University 2016
+# website: http://mpatacchiola.github.io/
+# email: massimiliano.patacchiola@plymouth.ac.uk
+# Python code for information retrieval from the Annotated Facial Landmarks in the Wild (AFLW) dataset.
+# In this example the faces are isolated and saved in a specified output folder.
+# Some information (roll, pitch, yaw) are returned, they can be used to filter the images.
+# This code requires OpenCV and Numpy. You can easily bypass the OpenCV calls if you want to use
+# a different library. In order to use the code you have to unzip the images and store them in
+# the directory "flickr" mantaining the original folders name (0, 2, 3).
+#
+# The following are the database properties available (last updated version 2012-11-28):
+#
+# databases: db_id, path, description
+# faceellipse: face_id, x, y, ra, rb, theta, annot_type_id, upsidedown
+# faceimages: image_id, db_id, file_id, filepath, bw, widht, height
+# facemetadata: face_id, sex, occluded, glasses, bw, annot_type_id
+# facepose: face_id, roll, pitch, yaw, annot_type_id
+# facerect: face_id, x, y, w, h, annot_type_id
+# faces: face_id, file_id, db_id
+# featurecoords: face_id, feature_id, x, y
+# featurecoordtype: feature_id, descr, code, x, y, z
+
+import sqlite3
+import cv2
+import os.path
+import numpy as np
+
+#Change this paths according to your directories
+images_path = "./flickr/"
+storing_path = "./output/"
+
+def main():
+
+ #Image counter
+ counter = 1
+
+ #Open the sqlite database
+ conn = sqlite3.connect('aflw.sqlite')
+ c = conn.cursor()
+
+ #Creating the query string for retriving: roll, pitch, yaw and faces position
+ #Change it according to what you want to retrieve
+ select_string = "faceimages.filepath, faces.face_id, facepose.roll, facepose.pitch, facepose.yaw, facerect.x, facerect.y, facerect.w, facerect.h"
+ from_string = "faceimages, faces, facepose, facerect"
+ where_string = "faces.face_id = facepose.face_id and faces.file_id = faceimages.file_id and faces.face_id = facerect.face_id"
+ query_string = "SELECT " + select_string + " FROM " + from_string + " WHERE " + where_string
+
+ #It iterates through the rows returned from the query
+ for row in c.execute(query_string):
+
+ #Using our specific query_string, the "row" variable will contain:
+ # row[0] = image path
+ # row[1] = face id
+ # row[2] = roll
+ # row[3] = pitch
+ # row[4] = yaw
+ # row[5] = face coord x
+ # row[6] = face coord y
+ # row[7] = face width
+ # row[8] = face heigh
+
+ #Creating the full path names for input and output
+ input_path = images_path + str(row[0])
+ output_path = storing_path + str(row[0])
+
+ #If the file exist then open it
+ if(os.path.isfile(input_path) == True):
+ #image = cv2.imread(input_path, 0) #load in grayscale
+ image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) #load the colour version
+
+ #Image dimensions
+ image_h, image_w = image.shape
+ #Roll, pitch and yaw
+ roll = row[2]
+ pitch = row[3]
+ yaw = row[4]
+ #Face rectangle coords
+ face_x = row[5]
+ face_y = row[6]
+ face_w = row[7]
+ face_h = row[8]
+
+ #Error correction
+ if(face_x < 0): face_x = 0
+ if(face_y < 0): face_y = 0
+ if(face_w > image_w):
+ face_w = image_w
+ face_h = image_w
+ if(face_h > image_h):
+ face_h = image_h
+ face_w = image_h
+
+ #Crop the face from the image
+ image_cropped = np.copy(image[face_y:face_y+face_h, face_x:face_x+face_w])
+ #Uncomment the lines below if you want to rescale the image to a particular size
+ #to_size = 64
+ #image_rescaled = cv2.resize(image_cropped, (to_size,to_size), interpolation = cv2.INTER_AREA)
+ #Uncomment the line below if you want to use adaptive histogram normalisation
+ #clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(5,5))
+ #image_normalised = clahe.apply(image_rescaled)
+ #Save the image
+ #change "image_cropped" with the last uncommented variable name above
+ cv2.imwrite(output_path, image_cropped)
+
+ #Printing the information
+ print "Counter: " + str(counter)
+ print "iPath: " + input_path
+ print "oPath: " + output_path
+ print "Roll: " + str(roll)
+ print "Pitch: " + str(pitch)
+ print "Yaw: " + str(yaw)
+ print "x: " + str(face_x)
+ print "y: " + str(face_y)
+ print "w: " + str(face_w)
+ print "h: " + str(face_h)
+ print ""
+
+ #Increasing the counter
+ counter = counter + 1
+
+ #if the file does not exits it return an exception
+ else:
+ raise ValueError('Error: I cannot find the file specified: ' + str(input_path))
+
+ #Once finished the iteration it closes the database
+ c.close()
+
+if __name__ == "__main__":
+ main()
+
--
Gitblit v1.8.0