Resume-Parser-using-spaCy-deployment-using-Paperclip

This project uses Python’s library, SpaCy to implement various NLP (natural language processing) techniques like tokenization, lemmatization, parts of speech tagging, etc., for building a resume parser in Python. Considering all the resumes are submitted in PDF format, we’ll how to implement optical character recognition (OCR) for extracting textual data from the documents.

Get Saksham Saxena’s stories in your inbox

Join Medium for free to get updates from this writer.

Remember me for faster sign in

Importing the train and test dataset in the form of json file:

import utils

# function to read the data
def read_data():
    train=utils.convert_json_to_spacy("D:\\ProjectProd\\Resume_Parser\\input\\train_1.json") # train file

    test=utils.convert_json_to_spacy("D:\\ProjectProd\\Resume_Parser\\input\\test.json")     # test file

    return train,test

Converting Json format to Spacy format

import json
import os
import json
import random
import logging
from sklearn.metrics import classification_report
from sklearn.metrics import precision_recall_fscore_support
from spacy.gold import GoldParse
from spacy.scorer import Scorer

# function to convert data 
def convert_data_to_spacy(JSON_FilePath):
    try:
        training_data = [] # create an empty list for training data
        lines=[]
        with open(JSON_FilePath, 'r',encoding='utf-8') as f: # open the json file
            lines = f.readlines()

        for line in lines:
            data = json.loads(line)
            text = data['content']
            entities = []
            for annotation in data['annotation']:
                #only a single point in text annotation.
                point = annotation['points'][0]
                labels = annotation['label']
                # handle both list of labels or a single label.
                if not isinstance(labels, list):
                    labels = [labels]

                for label in labels:
                    
                    entities.append((point['start'], point['end'] + 1 ,label))


            training_data.append((text, {"entities" : entities}))

        return training_data
    except Exception as e: # in case of exception print-
        logging.exception("Unable to process " + JSON_FilePath + "\n" + "error = " + str(e))
        return None

Building a function to extract Keywords from the Dataset(Entity Recognition)

import spacy
import re

#Function to extract names from the string using spacy
def extract_name(string):
    r1 = unicode(string)
    nlp = spacy.load('xx_ent_wiki_sm') # load the model
    doc = nlp(r1)
    for ent in doc.ents:
        if(ent.label_ == 'PER'):
            print(ent.text)
            break
#Function to extract Phone Numbers from string using regular expressions
def extract_phone_numbers(string):
    r = re.compile(r'(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-\.\s]??\d{4}|\d{3}[-\.\s]??\d{4})')
    phone_numbers = r.findall(string)
    return [re.sub(r'\D', '', number) for number in phone_numbers]


#Function to extract Email address from a string using regular expressions
def extract_email_addresses(string):
    r = re.compile(r'[\w\.-]+@[\w\.-]+')
    return r.findall(string)

Building a function to extract the text under the keywords extracted in the previous section

from tika import parser
import os

# function for text convertion 
def convert_pdf_to_text(dir):
    output=[]
    for root, dirs, files in os.walk(dir):
        print(files)
        for file in files:
            path_to_pdf = os.path.join(root, file)
            #print(path_to_pdf)
            [stem, ext] = os.path.splitext(path_to_pdf)
            if ext == '.pdf':
                print("Processing " + path_to_pdf)
                pdf_contents = parser.from_file(path_to_pdf,service='text')
                path_to_txt = stem + '.txt'
                # with open(path_to_txt, 'w',encoding='utf-8') as txt_file:
                #     print("Writing contents to " + path_to_txt)
                #     txt_file.write(pdf_contents['content'])
                output.append(pdf_contents['content'])
    return output

Training the model on the dataset

import spacy
import random
from spacy.util import minibatch, compounding

# function to train the model
def build_spacy_model(train,model):

    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    TRAIN_DATA =train
    #nlp = spacy.blank('en')  # create blank Language class
    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last=True)
    else:
        ner = nlp.get_pipe("ner")
       

    # add labels
    for _, annotations in TRAIN_DATA:
         for ent in annotations.get('entities'):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  # only train NER
        if model is None:
            optimizer = nlp.begin_training()
        for itn in range(2):
            print("Starting iteration " + str(itn))
            # random.shuffle(TRAIN_DATA)
            # losses = {}
            # batches = minibatch(TRAIN_DATA, size=compounding(8., 32., 1.001))
            # for batch in batches:
            #     texts, annotations = zip(*batch)
            #     nlp.update(texts, annotations, sgd=optimizer, 
            #                losses=losses)
            # print('Losses', losses)
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                    try:
                        nlp.update(
                            [text],  # batch of texts
                            [annotations],  # batch of annotations
                            drop=0.2,  # dropout - make it harder to memorise data
                            sgd=optimizer,  # callable to update weights
                            losses=losses)
                    except Exception as e:
                        pass
            print(losses)
    
    nlp.to_disk("model")
    return nlp

Testing the model on testing dataset

import spacy
import text_extractor

# function for prediction
def predict(path):
    #output={}
    nlp=spacy.load("model") # load the model
    test_text=text_extractor.convert_pdf_to_text(path) # convert
    for text in test_text:
        text=text.replace('\n',' ') # replace
        doc = nlp(text)
        for ent in doc.ents:
            print(f'{ent.label_.upper():{30}}-{ent.text}')
            #output[ent.label_.upper()]=ent.text
    #return output

Templating all the functions thru a class ML_Pipeline

from ML_Pipeline import json_spacy
from ML_Pipeline import train_model
from ML_Pipeline import predict_model
from ML_Pipeline import text_extractor
from ML_Pipeline import utils

#####First lets create training data out of the tagged data############

train= json_spacy.convert_data_to_spacy("D:\\ProjectProd\\Resume_Parser\\input\\training\\Entity Recognition in Resumes.json")

#print(train[0])
print("Done. Converted into spacy format")

print("Checking if previously built spacy model exists. If not, we will train a new one")

model=utils.check_existing_model("nlp_model")

model=train_model.build_spacy_model(train,model)

#test_text=text_extractor.convert_pdf_to_text("D:\\ProjectProd\\Resume_Parser\\output\\")


predict_model.predict("D:\\ProjectProd\\Resume_Parser\\output\\")


#mine = text_extractor.convert_pdf_to_text('D:\\ProjectProd\\Resume_Parser\\input\\resumes\\resumes')

#print(test.head())

The resulting application requires minimum human intervention to extract crucial information from a resume, such as an work experience, name, skills, etc.