vgg_layer_analysis.py

import keras
from keras.applications import VGG19
from keras.applications.vgg19 import preprocess_input
from keras import Input
from keras.models import Model
from keras.optimizers import Adam
import numpy as np
from scipy.misc import imresize, imread
from glob import glob
import matplotlib.pyplot as plt
from keras import backend as K
from keras import Sequential
from keras.layers import Conv2D, Dense


def sample_images(data_dir, batch_size, high_resolution_shape, low_resolution_shape):
    # Make a list of all images inside the data directory
    all_images = data_dir

    # Choose a random batch of images
    images_batch = np.random.choice(all_images, size=batch_size)

    low_resolution_images = []
    high_resolution_images = []

    for img in images_batch:
        # Get an ndarray of the current image
        img1 = imread(img, mode='RGB')
        img1 = img1.astype(np.float32)

        # Resize the image
        img1_high_resolution = imresize(img1, high_resolution_shape)
        img1_low_resolution = imresize(img1, low_resolution_shape)

        # Do a random horizontal flip
        if np.random.random() < 0.5:
            img1_high_resolution = np.fliplr(img1_high_resolution)
            img1_low_resolution = np.fliplr(img1_low_resolution)

        high_resolution_images.append(img1_high_resolution)
        low_resolution_images.append(img1_low_resolution)

    # Convert the lists to Numpy NDArrays
    return np.array(high_resolution_images), np.array(low_resolution_images)


def build_vgg_1():
    """
    Builds a pre-trained VGG19 model that outputs image features extracted at the
    third block of the model
    """
    input_shape = (256, 256, 3)
    vgg = VGG19(weights="imagenet")
    print(vgg.summary())
    # Set the outputs to outputs of last conv. layer in block 3
    # See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
    vgg.outputs = [vgg.layers[9].output]

    print(vgg.layers[11])
    img = Input(shape=input_shape)

    # Extract the image features
    img_features = vgg(img)

    return Model(inputs=[img], outputs=[img_features], name='vgg')


vgg_1 = build_vgg_1()
vgg_1.trainable = False
print(vgg_1.summary())
vgg_1.compile(optimizer=Adam(lr=0.0002, beta_1=0.9), loss='mse', metrics=['accuracy'])


def build_vgg_2():
    """
    Builds a pre-trained VGG19 model that outputs image features extracted at the
    third block of the model
    """
    input_shape = (256, 256, 3)
    vgg = VGG19(weights="imagenet")
    # Set the outputs to outputs of last conv. layer in block 3
    # See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
    vgg.outputs = [vgg.layers[10].output]

    img = Input(shape=input_shape)

    # Extract the image features
    img_features = vgg(img)

    return Model(inputs=[img], outputs=[img_features], name='vgg')


vgg_2 = build_vgg_2()
vgg_2.trainable = False
vgg_2.compile(optimizer=Adam(lr=0.0002, beta_1=0.9), loss='mse', metrics=['accuracy'])


def build_vgg_3():
    """
    Builds a pre-trained VGG19 model that outputs image features extracted at the
    third block of the model
    """
    input_shape = (256, 256, 3)
    vgg = VGG19(weights="imagenet")
    # Set the outputs to outputs of last conv. layer in block 3
    # See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
    vgg.outputs = [vgg.layers[8].output]

    img = Input(shape=input_shape)

    # Extract the image features
    img_features = vgg(img)

    return Model(inputs=[img], outputs=[img_features], name='vgg')


vgg_3 = build_vgg_3()
vgg_3.trainable = False
vgg_3.compile(optimizer=Adam(lr=0.0002, beta_1=0.9), loss='mse', metrics=['accuracy'])


def build_vgg_4():
    """
    Builds a pre-trained VGG19 model that outputs image features extracted at the
    third block of the model
    """
    input_shape = (256, 256, 3)
    vgg = VGG19(weights="imagenet")
    # Set the outputs to outputs of last conv. layer in block 3
    # See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
    vgg.outputs = [vgg.layers[7].output]

    img = Input(shape=input_shape)

    # Extract the image features
    img_features = vgg(img)

    return Model(inputs=[img], outputs=[img_features], name='vgg')


vgg_4 = build_vgg_4()
vgg_4.trainable = False
vgg_4.compile(optimizer=Adam(lr=0.0002, beta_1=0.9), loss='mse', metrics=['accuracy'])

# Shape of low-resolution and high-resolution images
low_resolution_shape = (64, 64, 3)
high_resolution_shape = (256, 256, 3)

# High and Low resolution inputs to the network
input_high_resolution = Input(shape=high_resolution_shape)
input_low_resolution = Input(shape=low_resolution_shape)

data_dir = glob('./Training_data/*')
batch_size = 1
high_resolution_images, low_resolution_images = sample_images(data_dir=data_dir, batch_size=batch_size,
                                                              high_resolution_shape=high_resolution_shape,
                                                              low_resolution_shape=low_resolution_shape)
high_resolution_images = high_resolution_images / 127.5 - 1
low_resolution_images = low_resolution_images / 127.5 - 1

features_1 = vgg_1.predict(high_resolution_images)
features_2 = vgg_2.predict(high_resolution_images)
features_3 = vgg_3.predict(high_resolution_images)
features_4 = vgg_4.predict(high_resolution_images)

### NEW METHOD ###
high_resolution_images = high_resolution_images.reshape(256, 256, 3)
high_resolution_images = 0.5 * high_resolution_images + 0.5

fig = plt.figure()
ax = fig.add_subplot(3, 2, 1)
plt.imshow(high_resolution_images)
plt.axis("off")
ax.set_title("High-resolution")

ax = fig.add_subplot(3, 2, 2)
ax.imshow(features_4[0, :, :, 1], cmap='viridis')
ax.axis("off")
ax.set_title("Features Layer 7")

ax = fig.add_subplot(3, 2, 3)
ax.imshow(features_3[0, :, :, 1], cmap='viridis')
ax.axis("off")
ax.set_title("Features Layer 8")

ax = fig.add_subplot(3, 2, 4)
ax.imshow(features_1[0, :, :, 1], cmap='viridis')
ax.axis("off")
ax.set_title("Features Layer 9")

ax = fig.add_subplot(3, 2, 5)
ax.imshow(features_2[0, :, :, 1], cmap='viridis')
ax.axis("off")
ax.set_title("Features Layer 10")

plt.show()