How I scrapped 2000+ UPSC MAINS questions using JavaScript.

I am developing an application which uses AI and helps UPSC aspirants improve their answer writing, where I wanted some demo questions for testing.

This is my database schema. I am going to scrape some data from web and fill out these tables.

// schema.prisma
enum Papers {
  GS1
  GS2
  GS3
  GS4
  ESSAY
  OPTIONAL
}
model Topics {
  id        String     @id @default(cuid())
  name      String     @unique // name of the topic e.g Socialogy
  // Questions on that topic.
  questions Question[]

  createdAt DateTime @default(now()) @map("created_at")
  updatedAt DateTime @updatedAt @map("updated_at")

  @@map("topics")
}

model Question {
  id        String    @id @default(cuid())
  text      String // question as plain text
  published Boolean   @default(false) // field to determine weather the question is published or not
  paper     Papers? // Which paper this question is asked. There might be a case the question is not yet asked in any paper.
  askedDate DateTime? @map("asked_date") // date of paper conducted when this question was asked in that paper. If not asked in any paper then date is the date published
  words     Int? // Answer to be written in words
  marks     Int? // Maximum marks for the answer
  topics    Topics[] // Topics related to that question
  answers   Answer[] // Answers written for the question

  createdAt DateTime @default(now()) @map("created_at")
  updatedAt DateTime @updatedAt @map("updated_at")
}

After hunting some websites, I found a perfect website which was best suitable for scrapping for the following reasons.

Check out this URL’s we are going to scrap all the questions for GS1, GS2, GS3, GS4, ESSAY one by one navigating on the URL.

Get Aayush Pagare’s stories in your inbox

Join Medium for free to get updates from this writer.

Remember me for faster sign in

To scrape all the questions of GS-1 category we are going to hit a get request on /mains-practice-questions/gs-paper-1?page=1, /mains-practice-questions/gs-paper-1?page=2 and so on and save the html in a file using the fs module.

async function scrape() { 
    for( let i = 1; i <= 20; i++) { 
        const resp = await axios.get(`https://www.drishtiias.com/mains-practice-question/gs-paper-1?page=${i}`) 
        const html = resp.data;
        fs.writeFileSync(path.join(__dirname, `/questions/drishti/GS1/set${i}-questions.html`), html)
    }
}
scrape();

After completion, the scrapped data looked like.

Now, we need to extract some useful information from this html i.e.

1. question’s text
2. asked date
3. paper
4. related subjects

Now, to parse the HTML we can use a powerful package called as cheerio.

To parse the HTML, find patterns in html like I, found out that the list of questions lies inside the div with id accordion, and all the useful information about each question is stored inside the <a> tag.

const axios = require('axios');
const fs = require('fs');
const path = require('path')
// Import cheerio
const cheerio = require('cheerio');

async function parse() { 
    // read the scraped html file
    const html = fs.readFileSync(path.join(__dirname, `/questions/drishti/GS1/set1-questions.html`), 'utf8');
    // load the html
    const $ = cheerio.load(html);
    // selected the div with id accordian and all a tags inside it.
    // that will return a string 
    // convert that into an array with deimeter as new line.
    const parsedArray = $('#accordion a').text().split('\n')
    console.log(parsedArray)
}
                        
// output
["'Q. Discuss the concept of ocean thermohaline circulation and its role in global climate regulation. How could changes in this system impact Earth's climate? (250 words)'",
  "'26 Aug, 2024'",
  "'GS Paper 1'",
  "'Indian Heritage & Culture'",
  ...more items
 ]                            

We have extracted all the useful information. we can tweak this as we need now.

I have parsed and cleaned the data and then used Prisma to store the data in my postgreSQL database.

/* eslint-disable @typescript-eslint/no-require-imports */
const fs = require('fs');
const path = require('path');
const cheerio = require('cheerio');
const _ = require('lodash');
const { PrismaClient } = require('@prisma/client');
const { DateTime } = require('luxon');
// const { text } = require('stream/consumers');


// Enum mapping
const Papers = {
    GS1: "GS1",
    GS2: "GS2",
    GS3: "GS3",
    GS4: "GS4",
    ESSAY: "ESSAY",
    OPTIONAL: "OPTIONAL"
};

// Helper function to map paper string to enum
function mapPaperToEnum(paperString) {
    const paperMap = {
        'GS Paper 1': Papers.GS1,
        'GS Paper 2': Papers.GS2,
        'GS Paper 3': Papers.GS3,
        'GS Paper 4': Papers.GS4,
        'Essay': Papers.ESSAY,
        'Optional': Papers.OPTIONAL
    };
    
    return paperMap[paperString] || paperString;
}

const chunkArray = (arr, chunkSize) => {
    return Array.from(
        { length: Math.ceil(arr.length / chunkSize) },
        (_, index) => arr.slice(index * chunkSize, (index + 1) * chunkSize)
    );
};

function convertToObject(arr) {
    // Parse the date using Luxon
    const parsedDate = DateTime.fromFormat(arr[1], 'dd MMM, yyyy').toUTC().toISO();
    
    const questionObject = {
        question: arr[0]
            .replace(/^Q\.\s*/, '')
            .replace(/^\"|\"$/, '')
            .replace(/\s*\(\d+\s*words\)$/, '')
            .trim(),
        askedDate: parsedDate,
        paper: mapPaperToEnum(arr[2]),
        topics: arr[3].split(','),
        words: parseInt(arr[0].match(/\((\d+)\s*words\)/)?.[1] || '0')
    };
    return questionObject;
}

const prisma = new PrismaClient();
async function parse(index) {
    const html = fs.readFileSync(path.join(__dirname, `/public/questions/drishti/GS1/set${index}-questions.html`), 'utf8');
    const $ = cheerio.load(html);
    
    const parsedArray = $('#accordion a')
        .map((_, el) => $(el).text())
        .get()
        .map(text => _.trim(text))
        .filter(text => text.length > 0);

    const chunkedArrays = chunkArray(parsedArray, 4);
    const questionsObjects = chunkedArrays.map(arr => convertToObject(arr));
    
    try {
        await Promise.all(
            questionsObjects.map(async questionObject => {
                // Create connectOrCreate objects for each topic
                const topicConnections = questionObject.topics.map(topic => ({
                    where: { name: _.trim(topic) },
                    create: { name: _.trim(topic) }
                }));

                await prisma.question.create({
                    data: {
                        text: questionObject.question,
                        askedDate: questionObject.askedDate,
                        published: true,
                        paper: questionObject.paper,
                        topics: {
                            connectOrCreate: topicConnections
                        },
                        words: questionObject.words || null
                    }
                });
            })
        );
        console.log('Successfully created questions with topics');
    } catch (error) {
        console.error('Error creating questions:', error);
    } finally {
        await prisma.$disconnect();
    }
}

parse(3);  // Call the parse function for each set of questions`

This is the simplest way to scrap web using axios and cheerio. Also, this website was really easy to scrape as it allowed me to continuously hit get requests on it. Normally my IP would have got blocked. We would look in some other blog that how to use proxies and browser automation for scrapping.

Thanks :)