From 31fc66b795c0a57b8009d7b03f49f6cd099ceb29 Mon Sep 17 00:00:00 2001
From: natanielruiz <nataniel777@hotmail.com>
Date: 星期六, 23 九月 2017 12:07:48 +0800
Subject: [PATCH] Trying superres
---
code/test_alexnet.py | 144 ++++++++
code/test_resnet50_regression.py | 3
code/datasets.py | 65 +++
code/test_resnet50_regression_extreme.py | 132 +++++++
code/test_preangles_extreme.py | 151 ++++++++
code/train_preangles_focalloss.py | 224 +++++++++++++
code/vdsr.py | 40 ++
code/test_preangles_superres.py | 182 ++++++++++
code/loss.py | 37 ++
code/test_preangles.py | 3
10 files changed, 978 insertions(+), 3 deletions(-)
diff --git a/code/datasets.py b/code/datasets.py
index da0603f..17f1899 100644
--- a/code/datasets.py
+++ b/code/datasets.py
@@ -106,15 +106,16 @@
# Crop the face
pt2d = utils.get_pt2d_from_mat(mat_path)
+
x_min = min(pt2d[0,:])
y_min = min(pt2d[1,:])
x_max = max(pt2d[0,:])
y_max = max(pt2d[1,:])
- k = 0.15
- x_min -= 0.6 * k * abs(x_max - x_min)
+ k = 0.20
+ x_min -= 2 * k * abs(x_max - x_min)
y_min -= 2 * k * abs(y_max - y_min)
- x_max += 0.6 * k * abs(x_max - x_min)
+ x_max += 2 * k * abs(x_max - x_min)
y_max += 0.6 * k * abs(y_max - y_min)
img = img.crop((int(x_min), int(y_min), int(x_max), int(y_max)))
@@ -138,6 +139,64 @@
# 2,000
return self.length
+class AFLW2000_ds(Dataset):
+ def __init__(self, data_dir, filename_path, transform, img_ext='.jpg', annot_ext='.mat', image_mode='RGB'):
+ self.data_dir = data_dir
+ self.transform = transform
+ self.img_ext = img_ext
+ self.annot_ext = annot_ext
+
+ filename_list = get_list_from_filenames(filename_path)
+
+ self.X_train = filename_list
+ self.y_train = filename_list
+ self.image_mode = image_mode
+ self.length = len(filename_list)
+
+ def __getitem__(self, index):
+ img = Image.open(os.path.join(self.data_dir, self.X_train[index] + self.img_ext))
+ img = img.convert(self.image_mode)
+ mat_path = os.path.join(self.data_dir, self.y_train[index] + self.annot_ext)
+
+ # Crop the face
+ pt2d = utils.get_pt2d_from_mat(mat_path)
+ x_min = min(pt2d[0,:])
+ y_min = min(pt2d[1,:])
+ x_max = max(pt2d[0,:])
+ y_max = max(pt2d[1,:])
+
+ k = 0.20
+ x_min -= 2 * k * abs(x_max - x_min)
+ y_min -= 2 * k * abs(y_max - y_min)
+ x_max += 2 * k * abs(x_max - x_min)
+ y_max += 0.6 * k * abs(y_max - y_min)
+ img = img.crop((int(x_min), int(y_min), int(x_max), int(y_max)))
+
+ ds = 5
+ original_size = img.size
+ img = img.resize((img.size[0] / ds, img.size[1] / ds), resample=0)
+ img = img.resize((original_size[0], original_size[1]), resample=0)
+
+ # We get the pose in radians
+ pose = utils.get_ypr_from_mat(mat_path)
+ # And convert to degrees.
+ pitch = pose[0] * 180 / np.pi
+ yaw = pose[1] * 180 / np.pi
+ roll = pose[2] * 180 / np.pi
+ # Bin values
+ bins = np.array(range(-99, 102, 3))
+ labels = torch.LongTensor(np.digitize([yaw, pitch, roll], bins) - 1)
+ cont_labels = torch.FloatTensor([yaw, pitch, roll])
+
+ if self.transform is not None:
+ img = self.transform(img)
+
+ return img, labels, cont_labels, self.X_train[index]
+
+ def __len__(self):
+ # 2,000
+ return self.length
+
class AFLW_aug(Dataset):
def __init__(self, data_dir, filename_path, transform, img_ext='.jpg', annot_ext='.txt', image_mode='RGB'):
self.data_dir = data_dir
diff --git a/code/loss.py b/code/loss.py
new file mode 100644
index 0000000..805731b
--- /dev/null
+++ b/code/loss.py
@@ -0,0 +1,37 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+
+def one_hot(index, classes):
+ size = index.size() + (classes,)
+ view = index.size() + (1,)
+
+ mask = torch.Tensor(*size).fill_(0)
+ index = index.view(*view)
+ ones = 1.
+
+ if isinstance(index, Variable):
+ ones = Variable(torch.Tensor(index.size()).fill_(1))
+ mask = Variable(mask, volatile=index.volatile)
+
+ return mask.scatter_(1, index, ones)
+
+
+class FocalLoss(nn.Module):
+
+ def __init__(self, gamma=0, eps=1e-7):
+ super(FocalLoss, self).__init__()
+ self.gamma = gamma
+ self.eps = eps
+
+ def forward(self, input, target):
+ y = one_hot(target, input.size(-1))
+ logit = F.softmax(input)
+ logit = logit.clamp(self.eps, 1. - self.eps)
+
+ loss = -1 * y * torch.log(logit) # cross entropy
+ loss = loss * (1 - logit) ** self.gamma # focal loss
+
+ return loss.sum()
diff --git a/code/test_alexnet.py b/code/test_alexnet.py
new file mode 100644
index 0000000..d9cc0a3
--- /dev/null
+++ b/code/test_alexnet.py
@@ -0,0 +1,144 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torch.backends.cudnn as cudnn
+import torchvision
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import utils
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--snapshot', dest='snapshot', help='Name of model snapshot.',
+ default='', type=str)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=1, type=int)
+ parser.add_argument('--save_viz', dest='save_viz', help='Save images with pose cube.',
+ default=False, type=bool)
+ parser.add_argument('--dataset', dest='dataset', help='Dataset type.', default='AFLW2000', type=str)
+
+ args = parser.parse_args()
+
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ gpu = args.gpu_id
+ snapshot_path = args.snapshot
+
+ model = hopenet.AlexNet(66)
+
+ print 'Loading snapshot.'
+ # Load snapshot
+ saved_state_dict = torch.load(snapshot_path)
+ model.load_state_dict(saved_state_dict)
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(224),
+ transforms.CenterCrop(224), transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
+
+ if args.dataset == 'AFLW2000':
+ pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'BIWI':
+ pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW':
+ pose_dataset = datasets.AFLW(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'Pose_300W_LP':
+ pose_dataset = datasets.Pose_300W_LP(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFW':
+ pose_dataset = datasets.AFW(args.data_dir, args.filename_list, transformations)
+ else:
+ print 'Error: not a valid dataset name'
+ sys.exit()
+ test_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=args.batch_size,
+ num_workers=2)
+
+ model.cuda(gpu)
+
+ print 'Ready to test network.'
+
+ # Test the Model
+ model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
+ total = 0
+
+ idx_tensor = [idx for idx in xrange(66)]
+ idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
+
+ yaw_error = .0
+ pitch_error = .0
+ roll_error = .0
+
+ l1loss = torch.nn.L1Loss(size_average=False)
+
+ for i, (images, labels, cont_labels, name) in enumerate(test_loader):
+ images = Variable(images).cuda(gpu)
+ total += cont_labels.size(0)
+ label_yaw = cont_labels[:,0].float()
+ label_pitch = cont_labels[:,1].float()
+ label_roll = cont_labels[:,2].float()
+
+ yaw, pitch, roll = model(images)
+
+ # Binned predictions
+ _, yaw_bpred = torch.max(yaw.data, 1)
+ _, pitch_bpred = torch.max(pitch.data, 1)
+ _, roll_bpred = torch.max(roll.data, 1)
+
+ # Continuous predictions
+ yaw_predicted = utils.softmax_temperature(yaw.data, 1)
+ pitch_predicted = utils.softmax_temperature(pitch.data, 1)
+ roll_predicted = utils.softmax_temperature(roll.data, 1)
+
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1).cpu() * 3 - 99
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1).cpu() * 3 - 99
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1).cpu() * 3 - 99
+
+ # Mean absolute error
+ yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw))
+ pitch_error += torch.sum(torch.abs(pitch_predicted - label_pitch))
+ roll_error += torch.sum(torch.abs(roll_predicted - label_roll))
+
+ # Save images with pose cube.
+ # TODO: fix for larger batch size
+ if args.save_viz:
+ name = name[0]
+ if args.dataset == 'BIWI':
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '_rgb.png'))
+ else:
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '.jpg'))
+ if args.batch_size == 1:
+ error_string = 'y %.2f, p %.2f, r %.2f' % (torch.sum(torch.abs(yaw_predicted - label_yaw)), torch.sum(torch.abs(pitch_predicted - label_pitch)), torch.sum(torch.abs(roll_predicted - label_roll)))
+ cv2.putText(cv2_img, error_string, (30, cv2_img.shape[0]- 30), fontFace=1, fontScale=1, color=(0,0,255), thickness=1)
+ utils.plot_pose_cube(cv2_img, yaw_predicted[0], pitch_predicted[0], roll_predicted[0])
+ cv2.imwrite(os.path.join('output/images', name + '.jpg'), cv2_img)
+
+ print('Test error in degrees of the model on the ' + str(total) +
+ ' test images. Yaw: %.4f, Pitch: %.4f, Roll: %.4f' % (yaw_error / total,
+ pitch_error / total, roll_error / total))
diff --git a/code/test_preangles.py b/code/test_preangles.py
index d5c9ec7..b742195 100644
--- a/code/test_preangles.py
+++ b/code/test_preangles.py
@@ -67,6 +67,9 @@
if args.dataset == 'AFLW2000':
pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
elif args.dataset == 'BIWI':
pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
elif args.dataset == 'AFLW':
diff --git a/code/test_preangles_extreme.py b/code/test_preangles_extreme.py
new file mode 100644
index 0000000..d82577b
--- /dev/null
+++ b/code/test_preangles_extreme.py
@@ -0,0 +1,151 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torch.backends.cudnn as cudnn
+import torchvision
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import utils
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--snapshot', dest='snapshot', help='Name of model snapshot.',
+ default='', type=str)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=1, type=int)
+ parser.add_argument('--save_viz', dest='save_viz', help='Save images with pose cube.',
+ default=False, type=bool)
+ parser.add_argument('--dataset', dest='dataset', help='Dataset type.', default='AFLW2000', type=str)
+ parser.add_argument('--min_yaw', dest='min_yaw', type=float)
+
+ args = parser.parse_args()
+
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ gpu = args.gpu_id
+ snapshot_path = args.snapshot
+
+ # ResNet101 with 3 outputs.
+ # model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 23, 3], 66)
+ # ResNet50
+ model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66, 0)
+ # ResNet18
+ # model = hopenet.Hopenet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2], 66)
+
+ print 'Loading snapshot.'
+ # Load snapshot
+ saved_state_dict = torch.load(snapshot_path)
+ model.load_state_dict(saved_state_dict)
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(224),
+ transforms.CenterCrop(224), transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
+
+ if args.dataset == 'AFLW2000':
+ pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'BIWI':
+ pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW':
+ pose_dataset = datasets.AFLW(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'Pose_300W_LP':
+ pose_dataset = datasets.Pose_300W_LP(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFW':
+ pose_dataset = datasets.AFW(args.data_dir, args.filename_list, transformations)
+ else:
+ print 'Error: not a valid dataset name'
+ sys.exit()
+ test_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=args.batch_size,
+ num_workers=2)
+
+ model.cuda(gpu)
+
+ print 'Ready to test network.'
+
+ # Test the Model
+ model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
+ total = 0
+
+ idx_tensor = [idx for idx in xrange(66)]
+ idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
+
+ yaw_error = .0
+ pitch_error = .0
+ roll_error = .0
+
+ l1loss = torch.nn.L1Loss(size_average=False)
+
+ for i, (images, labels, cont_labels, name) in enumerate(test_loader):
+ images = Variable(images).cuda(gpu)
+ label_yaw = cont_labels[:,0].float()
+ label_pitch = cont_labels[:,1].float()
+ label_roll = cont_labels[:,2].float()
+
+ yaw, pitch, roll, angles = model(images)
+
+ # Binned predictions
+ _, yaw_bpred = torch.max(yaw.data, 1)
+ _, pitch_bpred = torch.max(pitch.data, 1)
+ _, roll_bpred = torch.max(roll.data, 1)
+
+ # Continuous predictions
+ yaw_predicted = utils.softmax_temperature(yaw.data, 1)
+ pitch_predicted = utils.softmax_temperature(pitch.data, 1)
+ roll_predicted = utils.softmax_temperature(roll.data, 1)
+
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1).cpu() * 3 - 99
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1).cpu() * 3 - 99
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1).cpu() * 3 - 99
+
+ # Mean absolute error
+ if args.min_yaw <= label_yaw[0]:
+ yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw))
+ pitch_error += torch.sum(torch.abs(pitch_predicted - label_pitch))
+ roll_error += torch.sum(torch.abs(roll_predicted - label_roll))
+ total += 1
+
+ # Save images with pose cube.
+ # TODO: fix for larger batch size
+ if args.save_viz:
+ name = name[0]
+ if args.dataset == 'BIWI':
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '_rgb.png'))
+ else:
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '.jpg'))
+ if args.batch_size == 1:
+ error_string = 'y %.2f, p %.2f, r %.2f' % (torch.sum(torch.abs(yaw_predicted - label_yaw)), torch.sum(torch.abs(pitch_predicted - label_pitch)), torch.sum(torch.abs(roll_predicted - label_roll)))
+ cv2.putText(cv2_img, error_string, (30, cv2_img.shape[0]- 30), fontFace=1, fontScale=1, color=(0,0,255), thickness=1)
+ utils.plot_pose_cube(cv2_img, yaw_predicted[0], pitch_predicted[0], roll_predicted[0])
+ cv2.imwrite(os.path.join('output/images', name + '.jpg'), cv2_img)
+
+ print('Test error in degrees of the model on the ' + str(total) +
+ ' test images. Yaw: %.4f, Pitch: %.4f, Roll: %.4f' % (yaw_error / total,
+ pitch_error / total, roll_error / total))
diff --git a/code/test_preangles_superres.py b/code/test_preangles_superres.py
new file mode 100644
index 0000000..faf73d3
--- /dev/null
+++ b/code/test_preangles_superres.py
@@ -0,0 +1,182 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torch.backends.cudnn as cudnn
+import torchvision
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import utils
+
+from PIL import Image
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--snapshot', dest='snapshot', help='Name of model snapshot.',
+ default='', type=str)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=1, type=int)
+ parser.add_argument('--save_viz', dest='save_viz', help='Save images with pose cube.',
+ default=False, type=bool)
+ parser.add_argument('--dataset', dest='dataset', help='Dataset type.', default='AFLW2000', type=str)
+
+ args = parser.parse_args()
+
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ gpu = args.gpu_id
+ snapshot_path = args.snapshot
+
+ model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66, 0)
+
+ print 'Loading snapshot.'
+ # Load snapshot
+ saved_state_dict = torch.load(snapshot_path)
+ model.load_state_dict(saved_state_dict)
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(224),
+ transforms.CenterCrop(224), transforms.ToTensor()])
+
+ if args.dataset == 'AFLW2000':
+ pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'BIWI':
+ pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW':
+ pose_dataset = datasets.AFLW(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'Pose_300W_LP':
+ pose_dataset = datasets.Pose_300W_LP(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFW':
+ pose_dataset = datasets.AFW(args.data_dir, args.filename_list, transformations)
+ else:
+ print 'Error: not a valid dataset name'
+ sys.exit()
+ test_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=args.batch_size,
+ num_workers=2)
+
+ model.cuda(gpu)
+
+ print 'Ready to test network.'
+
+ # Super-resolution model
+ sr_model = torch.load('data/sr_model/model_epoch_50.pth')["model"]
+ sr_model = sr_model.cuda(gpu)
+
+ # Test the Model
+ model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
+ total = 0
+
+ idx_tensor = [idx for idx in xrange(66)]
+ idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
+
+ yaw_error = .0
+ pitch_error = .0
+ roll_error = .0
+
+ l1loss = torch.nn.L1Loss(size_average=False)
+
+ for i, (images, labels, cont_labels, name) in enumerate(test_loader):
+
+ ### START Super-resolution ###
+ # To new color space
+ img = transforms.ToPILImage()(images[0])
+ print img
+ img = img.convert('YCbCr')
+ img_y, img_cb, img_cr = img.split()
+
+ # Super-resolution
+ img_y_var = Variable(transforms.ToTensor()(img_y)).view(1, -1, img_y.size[0], img_y.size[1]).cuda(gpu) / 255.
+ out_sr = sr_model(img_y_var)
+
+ img_h_y = out_sr.data[0].cpu().numpy().astype(np.float32)
+
+ img_h_y = img_h_y * 255
+ img_h_y[img_h_y<0] = 0
+ img_h_y[img_h_y>255.] = 255.
+ img_h_y = img_h_y[0]
+
+ img_new = np.zeros((img_h_y.shape[0], img_h_y.shape[1], 3), np.uint8)
+ img_new[:,:,0] = img_h_y
+ # img_new[:,:,0] = np.asarray(img_y)
+ img_new[:,:,1] = np.asarray(img_cb)
+ img_new[:,:,2] = np.asarray(img_cr)
+ img_new = Image.fromarray(img_new, "YCbCr").convert("RGB")
+
+ # To tensor and normalize
+ img_new.save('output/test_superres/' + name[0] + '.jpg', "JPEG")
+ img = transforms.ToTensor()(img_new)
+ img = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])(img)
+ images = Variable(img.view(1,-1,img.shape[1],img.shape[2])).cuda(gpu)
+
+ ### END Super-resolution ###
+
+ total += cont_labels.size(0)
+ label_yaw = cont_labels[:,0].float()
+ label_pitch = cont_labels[:,1].float()
+ label_roll = cont_labels[:,2].float()
+
+ yaw, pitch, roll, angles = model(images)
+
+ # Binned predictions
+ _, yaw_bpred = torch.max(yaw.data, 1)
+ _, pitch_bpred = torch.max(pitch.data, 1)
+ _, roll_bpred = torch.max(roll.data, 1)
+
+ # Continuous predictions
+ yaw_predicted = utils.softmax_temperature(yaw.data, 1)
+ pitch_predicted = utils.softmax_temperature(pitch.data, 1)
+ roll_predicted = utils.softmax_temperature(roll.data, 1)
+
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1).cpu() * 3 - 99
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1).cpu() * 3 - 99
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1).cpu() * 3 - 99
+
+ # Mean absolute error
+ yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw))
+ pitch_error += torch.sum(torch.abs(pitch_predicted - label_pitch))
+ roll_error += torch.sum(torch.abs(roll_predicted - label_roll))
+
+ # Save images with pose cube.
+ # TODO: fix for larger batch size
+ if args.save_viz:
+ name = name[0]
+ if args.dataset == 'BIWI':
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '_rgb.png'))
+ else:
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '.jpg'))
+ if args.batch_size == 1:
+ error_string = 'y %.2f, p %.2f, r %.2f' % (torch.sum(torch.abs(yaw_predicted - label_yaw)), torch.sum(torch.abs(pitch_predicted - label_pitch)), torch.sum(torch.abs(roll_predicted - label_roll)))
+ cv2.putText(cv2_img, error_string, (30, cv2_img.shape[0]- 30), fontFace=1, fontScale=1, color=(0,0,255), thickness=1)
+ utils.plot_pose_cube(cv2_img, yaw_predicted[0], pitch_predicted[0], roll_predicted[0])
+ cv2.imwrite(os.path.join('output/images', name + '.jpg'), cv2_img)
+
+ print('Test error in degrees of the model on the ' + str(total) +
+ ' test images. Yaw: %.4f, Pitch: %.4f, Roll: %.4f' % (yaw_error / total,
+ pitch_error / total, roll_error / total))
diff --git a/code/test_resnet50_regression.py b/code/test_resnet50_regression.py
index 553b439..85207f8 100644
--- a/code/test_resnet50_regression.py
+++ b/code/test_resnet50_regression.py
@@ -62,6 +62,9 @@
if args.dataset == 'AFLW2000':
pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
elif args.dataset == 'BIWI':
pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
elif args.dataset == 'AFLW':
diff --git a/code/test_resnet50_regression_extreme.py b/code/test_resnet50_regression_extreme.py
new file mode 100644
index 0000000..656cda5
--- /dev/null
+++ b/code/test_resnet50_regression_extreme.py
@@ -0,0 +1,132 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torch.backends.cudnn as cudnn
+import torchvision
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import utils
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--snapshot', dest='snapshot', help='Name of model snapshot.',
+ default='', type=str)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=1, type=int)
+ parser.add_argument('--save_viz', dest='save_viz', help='Save images with pose cube.',
+ default=False, type=bool)
+ parser.add_argument('--dataset', dest='dataset', help='Dataset type.', default='AFLW2000', type=str)
+ parser.add_argument('--min_yaw', dest='min_yaw', type=float)
+
+ args = parser.parse_args()
+
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ gpu = args.gpu_id
+ snapshot_path = args.snapshot
+
+ model = hopenet.ResNet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 3)
+
+ print 'Loading snapshot.'
+ # Load snapshot
+ saved_state_dict = torch.load(snapshot_path)
+ model.load_state_dict(saved_state_dict)
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(224),
+ transforms.CenterCrop(224), transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
+
+ if args.dataset == 'AFLW2000':
+ pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'AFLW2000_ds':
+ pose_dataset = datasets.AFLW2000_ds(args.data_dir, args.filename_list,
+ transformations)
+ elif args.dataset == 'BIWI':
+ pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW':
+ pose_dataset = datasets.AFLW(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'Pose_300W_LP':
+ pose_dataset = datasets.Pose_300W_LP(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFW':
+ pose_dataset = datasets.AFW(args.data_dir, args.filename_list, transformations)
+ else:
+ print 'Error: not a valid dataset name'
+ sys.exit()
+ test_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=args.batch_size,
+ num_workers=2)
+
+ model.cuda(gpu)
+
+ print 'Ready to test network.'
+
+ # Test the Model
+ model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
+ total = 0
+
+ yaw_error = .0
+ pitch_error = .0
+ roll_error = .0
+
+ l1loss = torch.nn.L1Loss(size_average=False)
+
+ for i, (images, labels, cont_labels, name) in enumerate(test_loader):
+ images = Variable(images).cuda(gpu)
+ label_yaw = cont_labels[:,0].float()
+ label_pitch = cont_labels[:,1].float()
+ label_roll = cont_labels[:,2].float()
+
+ angles = model(images)
+ yaw_predicted = angles[:,0].data.cpu()
+ pitch_predicted = angles[:,1].data.cpu()
+ roll_predicted = angles[:,2].data.cpu()
+
+ # Mean absolute error
+ if args.min_yaw <= label_yaw[0]:
+ yaw_error += torch.sum(torch.abs(yaw_predicted - label_yaw))
+ pitch_error += torch.sum(torch.abs(pitch_predicted - label_pitch))
+ roll_error += torch.sum(torch.abs(roll_predicted - label_roll))
+ total += 1
+
+ # Save images with pose cube.
+ # TODO: fix for larger batch size
+ if args.save_viz:
+ name = name[0]
+ if args.dataset == 'BIWI':
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '_rgb.png'))
+ else:
+ cv2_img = cv2.imread(os.path.join(args.data_dir, name + '.jpg'))
+ if args.batch_size == 1:
+ error_string = 'y %.2f, p %.2f, r %.2f' % (torch.sum(torch.abs(yaw_predicted - label_yaw)), torch.sum(torch.abs(pitch_predicted - label_pitch)), torch.sum(torch.abs(roll_predicted - label_roll)))
+ cv2.putText(cv2_img, error_string, (30, cv2_img.shape[0]- 30), fontFace=1, fontScale=1, color=(0,0,255), thickness=1)
+ utils.plot_pose_cube(cv2_img, yaw_predicted[0], pitch_predicted[0], roll_predicted[0])
+ cv2.imwrite(os.path.join('output/images', name + '.jpg'), cv2_img)
+
+ print('Test error in degrees of the model on the ' + str(total) +
+ ' test images. Yaw: %.4f, Pitch: %.4f, Roll: %.4f' % (yaw_error / total,
+ pitch_error / total, roll_error / total))
diff --git a/code/train_preangles_focalloss.py b/code/train_preangles_focalloss.py
new file mode 100644
index 0000000..64b42d6
--- /dev/null
+++ b/code/train_preangles_focalloss.py
@@ -0,0 +1,224 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from torch.utils.data import DataLoader
+from torchvision import transforms
+import torchvision
+import torch.backends.cudnn as cudnn
+import torch.nn.functional as F
+
+import cv2
+import matplotlib.pyplot as plt
+import sys
+import os
+import argparse
+
+import datasets
+import hopenet
+import torch.utils.model_zoo as model_zoo
+
+import time
+import loss
+
+model_urls = {
+ 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
+ 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
+ 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
+ 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
+ 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
+}
+
+def parse_args():
+ """Parse input arguments."""
+ parser = argparse.ArgumentParser(description='Head pose estimation using the Hopenet network.')
+ parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use [0]',
+ default=0, type=int)
+ parser.add_argument('--num_epochs', dest='num_epochs', help='Maximum number of training epochs.',
+ default=5, type=int)
+ parser.add_argument('--batch_size', dest='batch_size', help='Batch size.',
+ default=16, type=int)
+ parser.add_argument('--lr', dest='lr', help='Base learning rate.',
+ default=0.001, type=float)
+ parser.add_argument('--data_dir', dest='data_dir', help='Directory path for data.',
+ default='', type=str)
+ parser.add_argument('--filename_list', dest='filename_list', help='Path to text file containing relative paths for every example.',
+ default='', type=str)
+ parser.add_argument('--output_string', dest='output_string', help='String appended to output snapshots.', default = '', type=str)
+ parser.add_argument('--alpha', dest='alpha', help='Regression loss coefficient.',
+ default=0.001, type=float)
+ parser.add_argument('--dataset', dest='dataset', help='Dataset type.', default='Pose_300W_LP', type=str)
+
+ args = parser.parse_args()
+ return args
+
+def get_ignored_params(model):
+ # Generator function that yields ignored params.
+ b = []
+ b.append(model.conv1)
+ b.append(model.bn1)
+ b.append(model.fc_finetune)
+ for i in range(len(b)):
+ for module_name, module in b[i].named_modules():
+ if 'bn' in module_name:
+ module.eval()
+ for name, param in module.named_parameters():
+ yield param
+
+def get_non_ignored_params(model):
+ # Generator function that yields params that will be optimized.
+ b = []
+ b.append(model.layer1)
+ b.append(model.layer2)
+ b.append(model.layer3)
+ b.append(model.layer4)
+ for i in range(len(b)):
+ for module_name, module in b[i].named_modules():
+ if 'bn' in module_name:
+ module.eval()
+ for name, param in module.named_parameters():
+ yield param
+
+def get_fc_params(model):
+ b = []
+ b.append(model.fc_yaw)
+ b.append(model.fc_pitch)
+ b.append(model.fc_roll)
+ for i in range(len(b)):
+ for module_name, module in b[i].named_modules():
+ for name, param in module.named_parameters():
+ yield param
+
+def load_filtered_state_dict(model, snapshot):
+ # By user apaszke from discuss.pytorch.org
+ model_dict = model.state_dict()
+ # 1. filter out unnecessary keys
+ snapshot = {k: v for k, v in snapshot.items() if k in model_dict}
+ # 2. overwrite entries in the existing state dict
+ model_dict.update(snapshot)
+ # 3. load the new state dict
+ model.load_state_dict(model_dict)
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ cudnn.enabled = True
+ num_epochs = args.num_epochs
+ batch_size = args.batch_size
+ gpu = args.gpu_id
+
+ if not os.path.exists('output/snapshots'):
+ os.makedirs('output/snapshots')
+
+ # ResNet101 with 3 outputs
+ # model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 23, 3], 66)
+ # ResNet50
+ model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66, 0)
+ # ResNet18
+ # model = hopenet.Hopenet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2], 66)
+ load_filtered_state_dict(model, model_zoo.load_url(model_urls['resnet50']))
+
+ print 'Loading data.'
+
+ transformations = transforms.Compose([transforms.Scale(240),
+ transforms.RandomCrop(224), transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
+
+ if args.dataset == 'Pose_300W_LP':
+ pose_dataset = datasets.Pose_300W_LP(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW2000':
+ pose_dataset = datasets.AFLW2000(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'BIWI':
+ pose_dataset = datasets.BIWI(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW':
+ pose_dataset = datasets.AFLW(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFLW_aug':
+ pose_dataset = datasets.AFLW_aug(args.data_dir, args.filename_list, transformations)
+ elif args.dataset == 'AFW':
+ pose_dataset = datasets.AFW(args.data_dir, args.filename_list, transformations)
+ else:
+ print 'Error: not a valid dataset name'
+ sys.exit()
+ train_loader = torch.utils.data.DataLoader(dataset=pose_dataset,
+ batch_size=batch_size,
+ shuffle=True,
+ num_workers=2)
+
+ model.cuda(gpu)
+ softmax = nn.Softmax().cuda(gpu)
+ criterion = loss.FocalLoss()
+ reg_criterion = nn.MSELoss().cuda(gpu)
+ # Regression loss coefficient
+ alpha = args.alpha
+
+ idx_tensor = [idx for idx in xrange(66)]
+ idx_tensor = Variable(torch.FloatTensor(idx_tensor)).cuda(gpu)
+
+ optimizer = torch.optim.Adam([{'params': get_ignored_params(model), 'lr': 0},
+ {'params': get_non_ignored_params(model), 'lr': args.lr},
+ {'params': get_fc_params(model), 'lr': args.lr * 5}],
+ lr = args.lr)
+
+ print 'Ready to train network.'
+ print 'First phase of training.'
+ for epoch in range(num_epochs):
+ # start = time.time()
+ for i, (images, labels, cont_labels, name) in enumerate(train_loader):
+ # print i
+ # print 'start: ', time.time() - start
+ images = Variable(images).cuda(gpu)
+ label_yaw = Variable(labels[:,0].contiguous())
+ label_pitch = Variable(labels[:,1].contiguous())
+ label_roll = Variable(labels[:,2].contiguous())
+
+ label_angles = Variable(cont_labels[:,:3]).cuda(gpu)
+ label_yaw_cont = Variable(cont_labels[:,0]).cuda(gpu)
+ label_pitch_cont = Variable(cont_labels[:,1]).cuda(gpu)
+ label_roll_cont = Variable(cont_labels[:,2]).cuda(gpu)
+
+ optimizer.zero_grad()
+ model.zero_grad()
+
+ pre_yaw, pre_pitch, pre_roll, angles = model(images)
+ # Cross entropy loss
+ loss_yaw = criterion(pre_yaw.cpu(), label_yaw).cuda(gpu)
+ loss_pitch = criterion(pre_pitch.cpu(), label_pitch).cuda(gpu)
+ loss_roll = criterion(pre_roll.cpu(), label_roll).cuda(gpu)
+
+ # MSE loss
+ yaw_predicted = softmax(pre_yaw)
+ pitch_predicted = softmax(pre_pitch)
+ roll_predicted = softmax(pre_roll)
+
+ yaw_predicted = torch.sum(yaw_predicted * idx_tensor, 1) * 3 - 99
+ pitch_predicted = torch.sum(pitch_predicted * idx_tensor, 1) * 3 - 99
+ roll_predicted = torch.sum(roll_predicted * idx_tensor, 1) * 3 - 99
+
+ loss_reg_yaw = reg_criterion(yaw_predicted, label_yaw_cont)
+ loss_reg_pitch = reg_criterion(pitch_predicted, label_pitch_cont)
+ loss_reg_roll = reg_criterion(roll_predicted, label_roll_cont)
+
+ # Total loss
+ loss_yaw += alpha * loss_reg_yaw
+ loss_pitch += alpha * loss_reg_pitch
+ loss_roll += alpha * loss_reg_roll
+
+ loss_seq = [loss_yaw, loss_pitch, loss_roll]
+ grad_seq = [torch.Tensor(1).cuda(gpu) for _ in range(len(loss_seq))]
+ torch.autograd.backward(loss_seq, grad_seq)
+ optimizer.step()
+
+ # print 'end: ', time.time() - start
+
+ if (i+1) % 100 == 0:
+ print ('Epoch [%d/%d], Iter [%d/%d] Losses: Yaw %.4f, Pitch %.4f, Roll %.4f'
+ %(epoch+1, num_epochs, i+1, len(pose_dataset)//batch_size, loss_yaw.data[0], loss_pitch.data[0], loss_roll.data[0]))
+ # if epoch == 0:
+ # torch.save(model.state_dict(),
+ # 'output/snapshots/' + args.output_string + '_iter_'+ str(i+1) + '.pkl')
+
+ # Save models at numbered epochs.
+ if epoch % 1 == 0 and epoch < num_epochs:
+ print 'Taking snapshot...'
+ torch.save(model.state_dict(),
+ 'output/snapshots/' + args.output_string + '_epoch_'+ str(epoch+1) + '.pkl')
diff --git a/code/vdsr.py b/code/vdsr.py
new file mode 100755
index 0000000..1c4f163
--- /dev/null
+++ b/code/vdsr.py
@@ -0,0 +1,40 @@
+import torch
+import torch.nn as nn
+from math import sqrt
+
+class Conv_ReLU_Block(nn.Module):
+ def __init__(self):
+ super(Conv_ReLU_Block, self).__init__()
+ self.conv = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1, bias=False)
+ self.relu = nn.ReLU(inplace=True)
+
+ def forward(self, x):
+ return self.relu(self.conv(x))
+
+class Net(nn.Module):
+ def __init__(self):
+ super(Net, self).__init__()
+ self.residual_layer = self.make_layer(Conv_ReLU_Block, 18)
+ self.input = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, stride=1, padding=1, bias=False)
+ self.output = nn.Conv2d(in_channels=64, out_channels=1, kernel_size=3, stride=1, padding=1, bias=False)
+ self.relu = nn.ReLU(inplace=True)
+
+ for m in self.modules():
+ if isinstance(m, nn.Conv2d):
+ n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+ m.weight.data.normal_(0, sqrt(2. / n))
+
+ def make_layer(self, block, num_of_layer):
+ layers = []
+ for _ in range(num_of_layer):
+ layers.append(block())
+ return nn.Sequential(*layers)
+
+ def forward(self, x):
+ residual = x
+ out = self.relu(self.input(x))
+ out = self.residual_layer(out)
+ out = self.output(out)
+ out = torch.add(out,residual)
+ return out
+
\ No newline at end of file
--
Gitblit v1.8.0