Version ready to train

Author: capimx

tutorials/03-advanced/image_captioning/READ ME.txt

@@ -0,0 +1,60 @@
+{\rtf1\ansi\ansicpg1252\cocoartf1671\cocoasubrtf200
+{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
+{\colortbl;\red255\green255\blue255;}
+{\*\expandedcolortbl;;}
+\paperw11900\paperh16840\margl1440\margr1440\vieww10800\viewh8400\viewkind0
+\pard\tx566\tx1133\tx1700\tx2267\tx2834\tx3401\tx3968\tx4535\tx5102\tx5669\tx6236\tx6803\pardirnatural\partightenfactor0
+
+\f0\fs24 \cf0 import os\
+os.chdir('pytorch-tutorial/tutorials/03-advanced/image_captioning/')\
+\
+\
+import nltk\
+import gensim\
+nltk.download('punkt')\
+\
+\
+!pip install bert-embedding\
+!pip install https://github.com/dmlc/gluon-nlp/tarball/master\
+!pip install mxnet-cu100\
+\
+\
+os.chdir('/content')\
+\
+\
+%%shell\
+git clone https://github.com/pdollar/coco.git\
+cd coco/PythonAPI/\
+make\
+python setup.py build\
+python setup.py install\
+cd ../../\
+git clone https://github.com/capimx/pytorch-tutorial.git\
+\
+\
+#Changing directories\
+os.chdir('pytorch-tutorial/tutorials/03-advanced/image_captioning/')\
+!sed -i 's/unzip /unzip -q /g' download.sh  #Make quiet unzip\
+\
+\
+%%time\
+%%shell\
+pip install -r requirements.txt\
+chmod +x download.sh\
+./download.sh\
+\
+\
+########### REPLACE ALL THE DOWNLOADED FILES WITH OURS ###########\
+############### UPLOAD IN THE SAME FOLDER BERT EMBS ###############\
+\
+\
+%%time\
+!python build_vocab.py\
+\
+\
+%%time\
+!python resize.py\
+\
+\
+%%time\
+!python train.py}

tutorials/03-advanced/image_captioning/build_vocab.py

@@ -6,26 +6,32 @@
 
 
 class Vocabulary(object):
-    """Simple vocabulary wrapper."""
+    
     def __init__(self):
         self.word2idx = {}
         self.idx2word = {}
         self.idx = 0
-
+        
     def add_word(self, word):
         if not word in self.word2idx:
             self.word2idx[word] = self.idx
             self.idx2word[self.idx] = word
             self.idx += 1
-
+            
     def __call__(self, word):
         if not word in self.word2idx:
             return self.word2idx['<unk>']
         return self.word2idx[word]
-
+        
     def __len__(self):
         return len(self.word2idx)
-
+        
+    def __keys__(self):
+        iterable = []
+        for key in self.word2idx:
+            iterable.append(key)
+        return iterable
+        
 def build_vocab(json, threshold):
     """Build a simple vocabulary wrapper."""
     coco = COCO(json)
@@ -73,4 +79,4 @@ def main(args):
     parser.add_argument('--threshold', type=int, default=4, 
                         help='minimum word count threshold')
     args = parser.parse_args()
-    main(args)
+    main(args)

tutorials/03-advanced/image_captioning/data_loader.py

@@ -102,4 +102,4 @@ def get_loader(root, json, vocab, transform, batch_size, shuffle, num_workers):
                                               shuffle=shuffle,
                                               num_workers=num_workers,
                                               collate_fn=collate_fn)
-    return data_loader
+    return data_loader

tutorials/03-advanced/image_captioning/model.py

@@ -1,9 +1,11 @@
 import torch
+import gensim
+import numpy as np
 import torch.nn as nn
 import torchvision.models as models
+from bert_embedding import BertEmbedding
 from torch.nn.utils.rnn import pack_padded_sequence
 
-
 class EncoderCNN(nn.Module):
     def __init__(self, embed_size):
         """Load the pretrained ResNet-152 and replace top fc layer."""
@@ -24,12 +26,32 @@ def forward(self, images):
 
 
 class DecoderRNN(nn.Module):
-    def __init__(self, embed_size, hidden_size, vocab_size, num_layers, max_seq_length=20):
+    def __init__(self, embed_size, hidden_size, vocab, num_layers, max_seq_length=20):
         """Set the hyper-parameters and build the layers."""
         super(DecoderRNN, self).__init__()
-        self.embed = nn.Embedding(vocab_size, embed_size)
+        Bert_file = "bert-base-uncased.30522.768d.vec"
+        Lookup = gensim.models.KeyedVectors.load_word2vec_format(Bert_file, binary=False)
+        
+        Embed = np.zeros((len(vocab), embed_size))
+        Embed[vocab('<pad>'),:] = np.random.normal(0, 1, embed_size)
+        Embed[vocab('<start>'),:] = np.random.normal(0, 1, embed_size)
+        Embed[vocab('<end>'),:] = np.random.normal(0, 1, embed_size)
+        Embed[vocab('<unk>'),:] = np.random.normal(0, 1, embed_size)
+
+        for word in vocab.__keys__():
+            try:
+                Embed[vocab(word),:] = Lookup[word]
+            except:
+                bert_word = word
+                token = bert_word.split('\n')
+                bert_embedding = BertEmbedding()
+                pred = bert_embedding(token)
+                Embed[vocab(word),:] = pred[0][1][0]
+               
+        self.embed = nn.Embedding(len(vocab), embed_size)
+        self.embed.weight.data.copy_(torch.FloatTensor(Embed))
         self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
-        self.linear = nn.Linear(hidden_size, vocab_size)
+        self.linear = nn.Linear(hidden_size, len(vocab))
         self.max_seg_length = max_seq_length
 
     def forward(self, features, captions, lengths):
@@ -47,10 +69,10 @@ def sample(self, features, states=None):
         inputs = features.unsqueeze(1)
         for i in range(self.max_seg_length):
             hiddens, states = self.lstm(inputs, states)          # hiddens: (batch_size, 1, hidden_size)
-            outputs = self.linear(hiddens.squeeze(1))            # outputs:  (batch_size, vocab_size)
+            outputs = self.linear(hiddens.squeeze(1))            # outputs:  (batch_size, len(vocab))
             _, predicted = outputs.max(1)                        # predicted: (batch_size)
             sampled_ids.append(predicted)
             inputs = self.embed(predicted)                       # inputs: (batch_size, embed_size)
             inputs = inputs.unsqueeze(1)                         # inputs: (batch_size, 1, embed_size)
         sampled_ids = torch.stack(sampled_ids, 1)                # sampled_ids: (batch_size, max_seq_length)
-        return sampled_ids
+        return sampled_ids

tutorials/03-advanced/image_captioning/resize.py

@@ -39,4 +39,4 @@ def main(args):
     parser.add_argument('--image_size', type=int, default=256,
                         help='size for image after processing')
     args = parser.parse_args()
-    main(args)
+    main(args)

tutorials/03-advanced/image_captioning/train.py

@@ -1,21 +1,22 @@
-import argparse
-import torch
-import torch.nn as nn
-import numpy as np
 import os
+import torch
 import pickle
+import argparse
+import numpy as np
+import torch.nn as nn
 from data_loader import get_loader 
 from build_vocab import Vocabulary
+from torchvision import transforms
 from model import EncoderCNN, DecoderRNN
 from torch.nn.utils.rnn import pack_padded_sequence
-from torchvision import transforms
 import datetime
 
 # Device configuration
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 now = datetime.datetime.now()
 dir = '/content/drive/My Drive/NLPResults/'
 
+
 def append_progress(line):  
   filename = "Progress" + str(now.day) +'-'+ str(now.hour) + str(now.minute) + str(now.second) + ".txt"
   with open(dir + filename, 'a') as f:
@@ -48,10 +49,10 @@ def main(args):
     data_loader = get_loader(args.image_dir, args.caption_path, vocab, 
                              transform, args.batch_size,
                              shuffle=True, num_workers=args.num_workers) 
-
+    
     # Build the models
     encoder = EncoderCNN(args.embed_size).to(device)
-    decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers).to(device)
+    decoder = DecoderRNN(args.embed_size, args.hidden_size, vocab, args.num_layers).to(device)
 
     # Loss and optimizer
     criterion = nn.CrossEntropyLoss()
@@ -80,20 +81,20 @@ def main(args):
             # Print log info
             if i % args.log_step == 0:
                 log_info = 'Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Perplexity: {:5.4f}'.format(epoch, args.num_epochs, i, total_step, loss.item(), np.exp(loss.item()))
+                print(log_info)
                 append_progress(log_info)
-                print(log_info) 
 
             # Save the model checkpoints
             if (i+1) % args.save_step == 0:
                 torch.save(decoder.state_dict(), os.path.join(
                     args.model_path, 'decoder-{}-{}.ckpt'.format(epoch+1, i+1)))
                 torch.save(encoder.state_dict(), os.path.join(
                     args.model_path, 'encoder-{}-{}.ckpt'.format(epoch+1, i+1)))
-                #Save to DRive
+                #Save to Drive
                 torch.save(decoder.state_dict(), os.path.join(
-                    dir, 'decoder-{}-{}-{}.ckpt'.format(epoch+1, i+1,now)))
+                    dir, 'decoder-{}-{}-{}.ckpt'.format(epoch+1, i+1, now)))
                 torch.save(encoder.state_dict(), os.path.join(
-                    dir, 'encoder-{}-{}-{}.ckpt'.format(epoch+1, i+1,now)))
+                    dir, 'encoder-{}-{}-{}.ckpt'.format(epoch+1, i+1, now)))
 
 
 if __name__ == '__main__':
@@ -107,8 +108,8 @@ def main(args):
     parser.add_argument('--save_step', type=int , default=1000, help='step size for saving trained models')
 
     # Model parameters
-    parser.add_argument('--embed_size', type=int , default=256, help='dimension of word embedding vectors')
-    parser.add_argument('--hidden_size', type=int , default=512, help='dimension of lstm hidden states')
+    parser.add_argument('--embed_size', type=int , default=768, help='dimension of word embedding vectors')
+    parser.add_argument('--hidden_size', type=int , default=1536, help='dimension of lstm hidden states')
     parser.add_argument('--num_layers', type=int , default=1, help='number of layers in lstm')
 
     parser.add_argument('--num_epochs', type=int, default=5)
@@ -117,4 +118,4 @@ def main(args):
     parser.add_argument('--learning_rate', type=float, default=0.001)
     args = parser.parse_args()
     print(args)
-    main(args)
+    main(args)