path: root/api_review_GPT.js

// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

const config = require('./config.json');
const { OpenAIClient, AzureKeyCredential } = require("@azure/openai");
const express = require('express');
const bodyParser = require('body-parser');
const safeJsonStringify = require('safe-json-stringify');
const https = require('https');
const path = require('path');

const client = new OpenAIClient(
  "https://qttesteu.openai.azure.com/",
  new AzureKeyCredential(config.azureKey)
);

const deploymentId = config.deploymentId;
const deploymentId35 = config.deploymentId35;

const usage = {
  completionTokens: 0,
  promptTokens: 0,
  totalTokens: 0
};

async function evaluateDiff(diff, commitMessage) {
  const instructions = "Task: Classify the change in a public header file as significant to the"
  + " behavior and usage of the API or not. Consider the content of the commit message in"
  + " the evaluation."
  + " Additional qualifications: changes to 'private:' sections of public headers are not significant;"
  + " whitespace-only and comment-only changes are not significant;"
  + " additions of new and removal of old APIs are significant unless otherwise excluded."
  + " Only include real additions, not user comments about additions.";

  try {
    const result = await client.getChatCompletions(deploymentId, [
      { role: "system", content: instructions },
      { role: "user", content: `${commitMessage}\n\n${diff}` }
    ], {
      functions: [
        {
          name: "handleResponse",
          description: "Mark the change as significant or not. Provide a simplified reason for the decision.",
          parameters: {
            type: "object",
            properties: {
              significant: { type: "boolean", description: "true if the change is significant, false if not" },
              reason: { type: "string", description: "The reason for the decision" }
            },
            required: ["significant", "reason"]
          }
        }
      ],
      functionCall: { name: "handleResponse" },
      temperature: 0.2,
      maxTokens: 250,
      n: 1
    });

    updateUsage(result.usage);

    const parsedResponses = result.choices.map(choice => JSON.parse(choice.message.functionCall.arguments));
    const majorityResponse = getMajorityResponse(parsedResponses);

    if (majorityResponse) {
      return majorityResponse;
    } else {
      throw new Error("No majority result. Invalid JSON responses.");
    }
  } catch (error) {
    console.error(error);
    throw error;
  }
}

async function summarizeGPT(changeSummaries) {
  try {
    const result = await client.getChatCompletions(deploymentId35, [
      { role: "system", content: "You are a programmer developing on the Qt6 framework. Summarize the list of changes the user provides. Do not add commentary." },
      { role: "user", content: safeJsonStringify(changeSummaries) }
    ], {
      temperature: 0.2,
      maxTokens: 250
    });

    updateUsage(result.usage);

    return result.choices[0].message.content;
  } catch (error) {
    console.error(error);
    throw error;
  }
}

function updateUsage(newUsage) {
  usage.completionTokens += newUsage.completionTokens;
  usage.promptTokens += newUsage.promptTokens;
  usage.totalTokens += newUsage.totalTokens;
  console.log(`Current usage: ${usage.completionTokens} completion tokens, ${usage.promptTokens} prompt tokens, ${usage.totalTokens} total tokens.`);
}

function getMajorityResponse(responses) {
  const tally = responses.reduce((acc, res) => {
    acc[res.significant ? 'significant' : 'notSignificant']++;
    return acc;
  }, { significant: 0, notSignificant: 0 });

  const majorityFlag = tally.significant > tally.notSignificant;
  return responses.find(res => res.significant === majorityFlag);
}

function shouldExcludeFile(filePath, project) {
  const patterns = [
    /^(src\/)?tools\//i,
    /_p\//,
    /_(p|pch)\.h$/,
    /\/qt[a-z0-9]+-config\.h$/,
    /\/\./,
    /(^|\/)(private|doc|tests|examples|build|3rdparty)\//i,
    /\/ui_[^/]*\.h$/,
    /\/plugins\/platforms\//,
    /^src\/plugins\//
  ];

  if (!filePath.endsWith('.h')) {
    return true;
  }

  for (const pattern of patterns) {
    if (pattern.test(filePath)) {
      console.log(`Skipping file: ${filePath} due to pattern ${pattern}.`);
      return true;
    }
  }

  if (["qt/qttools", "qt/tqtc-qttools"].includes(project)) {
    const inclusions = ["src/assistant/help", "src/uiplugin", "src/uitools", "src/designer/src/lib/sdk"];
    return !inclusions.some(path => filePath.startsWith(path));
  }

  if (project.endsWith("qtwebengine")) {
    const inclusions = ["src/core/api", "src/webenginewidgets/api", "src/webenginequick/api",
    "src/pdfwidgets"];
    if (inclusions.some(path => filePath.startsWith(path))) {
      return false;
    }
    // Inclusions without subfolders
    const noSubdirsInclusions = ["src/pdf", "src/pdfquick"];
    const dirName = path.dirname(filePath);
    return !noSubdirsInclusions.some(inclusion => dirName === inclusion);
  }

  return false;
}

function splitDiff(diff, project) {
  const files = diff.split(/^diff --git/m).slice(1);
  return files.map(file => {
    // Matches a file path and captures the last directory or file name
    const pathMatch = file.match(/\sb(?:.+)?\/(.+)$/m);
    if (!pathMatch) return null;

    const path = pathMatch[0].trim().slice(2);
    if (shouldExcludeFile(path, project)) return null;
    // Regular expression to match .h files in a directory path, excluding _p.h files
    const fileNameMatch = file.match(/\sb(?:.+)?\/(.+\.h)(?<!_p\.h)$/m);
    if (!fileNameMatch) {
      console.log(`Skipping file: ${path} due to regex failure.`);
      return null;
    }

    const fileName = fileNameMatch[1];
    const diffLines = file.split('\n').slice(4).join('\n');

    return { fileName, diff: diffLines };
  }).filter(Boolean);
}

function checkApproval(change) {
  const approvals = change.approvals || [];
  return approvals.some(approval => approval.type === "Code-Review" && Number(approval.value) === 2 && 'oldValue' in approval);
}

async function processChange(change) {
  if (!change.change.project.startsWith("qt/") || change.change.project === "qt/qtwebengine-chromium") {
    return;
  }

  if (change.change.commitMessage.includes("(cherry picked from commit")) {
    console.log(`Skipping cherry-picked change ${change.change.number}`);
    return;
  }

  if (change.change.status === "MERGED") {
    console.log(`Skipping merged change ${change.change.number}`);
    return;
  }

  if (!checkApproval(change)) {
    return;
  }

  const number = change.change.number;
  const diffResponse = await fetch(`https://codereview.qt-project.org/changes/${number}/revisions/current/patch`);
  const diffText = await diffResponse.text();
  const decodedDiff = Buffer.from(diffText, 'base64').toString('utf-8');

  console.log(`\n New change: ${number}`);
  const files = splitDiff(decodedDiff, change.change.project);

  let summaries = [];
  let insignificants = [];
  for (const { fileName, diff } of files) {
    if (diff.length > 25000) {
      console.log(`${number}: Skipping ${fileName} due to length.`);
      summaries.push(`Skipping evaluation of ${fileName} due to diff size. (>25000 characters)`);
      continue;
    }

    console.log(`${number}: Evaluating ${fileName}.`);
    try {
      const result = await evaluateDiff(diff, change.change.commitMessage);
      if (result.significant) {
        summaries.push(`${fileName}: ${result.reason}`);
      } else {
        insignificants.push(`${fileName}: ${result.reason}`);
      }
    } catch (error) {
      console.log(`${number}: Error parsing JSON response from GPT: ${safeJsonStringify(error)}`);
    }
  }

  await finalizeSummaries(summaries, insignificants, change);
}

async function finalizeSummaries(summaries, insignificants, change) {
  if (summaries.length > 0) {
    const fixVersion = await getFixVersion(change.change.number);
    const hashtag = `Needs API-Review${fixVersion ? `_${fixVersion}` : ""}`;
    await addHashtags(change, hashtag);
    console.log("\nGenerating summary.\n");
    try {
      let finalSummary = await summarizeGPT(summaries);
      if (insignificants.length > 0) {
        finalSummary += `\n\nInsignificant changes:\n- ${insignificants.join('\n- ')}`;
      }

      const firstLine = summaries.length > 0
      ? "This change has been identified as having significant changes to the public API:\n"
      : "This change does not appear to have significant changes to the public API:\n";

      const comment = firstLine
      + `${finalSummary}\n\n`
      + `> This bot is powered by an LLM which evaluates the diffs of a change. It may make mistakes.\n`
      + `> The "${hashtag}" hashtag is non-blocking and does not currently prevent this change from being staged or merged.\n`
      + `> Current changes which require review can be viewed with the filter [hashtag:"${hashtag}"](https://codereview.qt-project.org/q/hashtag:%2522${encodeURIComponent(hashtag)}%2522)\n\n`
      + `Please comment on the jira issue QTQAINFRA-5781 or contact daniel.smith.at.qt.io directly,\n`
      + `especially if the bot produces a false positive or false negative.\n`

      await postGerritComment(change, comment, summaries.length > 0);
    } catch (error) {
      console.error(error);
    }
  }
}

async function getFixVersion(changeNumber) {
  try {
    const response = await fetch(`https://qt-cherry-pick-bot.herokuapp.com/jiracloser/fixversion?change=${changeNumber}`);
    const verData = await response.json();
    if (verData && verData.fixVersions.length > 0) {
      const { major, minor } = verData.fixVersions[0].parsedVersion;
      return `${major}.${minor}`;
    } else {
      if (verData && verData.errors) {
        console.log(verData.errors);
      }
      return "";
    }
  } catch (error) {
    console.error(error);
    return "";
  }
}

async function postGerritComment(change, comment) {
  const auth = Buffer.from(`${config.gerritUsername}:${config.gerritPassword}`).toString('base64');
  const options = {
    hostname: 'codereview.qt-project.org',
    path: `/a/changes/${change.change.number}/revisions/current/review`,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Basic ${auth}`
    }
  };

  const data = {
    message: comment,
    omit_duplicate_comments: true,
    ready: true
  };

  await sendHttpsRequest(options, data);
}



function sendHttpsRequest(options, data) {
  return new Promise((resolve, reject) => {
    const req = https.request(options, res => {
      console.log(`statusCode: ${res.statusCode}`);
      res.on('data', d => process.stdout.write(d));
      resolve();
    });

    req.on('error', reject);
    req.write(JSON.stringify(data));
    req.end();
  });
}

async function addHashtags(change, hashtag) {
  const auth = Buffer.from(`${config.gerritUsername}:${config.gerritPassword}`).toString('base64');
  const options = {
    hostname: 'codereview.qt-project.org',
    path: `/a/changes/${change.change.number}/hashtags`,
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Basic ${auth}`
    }
  };

  const hasPicks = change.change.commitMessage.includes("Pick-to: ");
  const hashtags = [hashtag, ...(hasPicks ? ["API change cherry-picked"] : [])];

  const data = { add: hashtags };
  await sendHttpsRequest(options, data);
}


function runWebserver() {
    const app = express();
    const port = config.webhook_port;

    app.use(bodyParser.json());

    app.get('/status', (req, res) => {
      res.json({
        completionTokens: usage.completionTokens,
        promptTokens: usage.promptTokens,
        totalTokens: usage.totalTokens
      });
    });
    app.post('/', async (req, res) => {
    const event = req.body;

    if (event.type === 'comment-added') {
        try {
            res.status(200);
            await processChange(event);
        } catch (error) {
            console.error(error);
            res.status(500).send('Error processing event');
        }
    } else {
        res.status(400).send('Event type not supported');
    }
    });

    app.listen(port, () => {
    console.log(`Listening for webhooks on port ${port}`);
    });
}

runWebserver();