Add a topic-level visualization that is auto-computed.

Author: Miltos Allamanis

.github/workflows/deploy.yml

@@ -26,6 +26,10 @@ jobs:
           python -m pip install transformers sklearn numpy
           python -m pip install torch==1.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
           python ${{ github.workspace }}/etc/compute_embeddings.py ${{ github.workspace }}/_site/paper-abstracts.json ${{ github.workspace }}/_site/tsne.json
+    - name: Compute topics
+      run: |
+          python -m pip install nltk gensim
+          python ${{ github.workspace }}/etc/compute_topics.py ${{ github.workspace }}/_site/paper-abstracts.json ${{ github.workspace }}/_site/topics.json
     - name: Deploy
       uses: peaceiris/actions-gh-pages@v3
       with:

_includes/sidebar.html

@@ -15,6 +15,7 @@ <h1>
    <a class="sidebar-nav-item{% if page.url == "/papers.html" %} active{% endif %}" href="{% link papers.html %}">List of Papers</a>
    <a class="sidebar-nav-item{% if page.url == "/tags.html" %} active{% endif %}" href="{% link tags.html %}">Papers by Tag</a>
    <a class="sidebar-nav-item{% if page.url == "/tsne-viz.html" %} active{% endif %}" href="{% link tsne-viz.html %}">2D Map of Papers</a>
+   <a class="sidebar-nav-item{% if page.url == "/topic-viz.html" %} active{% endif %}" href="{% link topic-viz.html %}">Topic-based Explorer</a>
 
    <a class="sidebar-nav-item{% if page.url == "/base-taxonomy/" %} active{% endif %}" href="{% link base-taxonomy/index.md %}">Core Taxonomy</a>
 

etc/compute_topics.py

@@ -0,0 +1,81 @@
+import argparse
+import json
+import nltk
+
+nltk.download('stopwords')
+nltk.download('wordnet')
+
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from gensim.corpora import Dictionary
+from gensim.models import LdaModel
+
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Topic Model of Papers in ML4Code")
+
+    parser.add_argument("json", default=False, help="the path the json containing all papers.")
+    parser.add_argument("outpath", default=False, help="the target path of the visualizations papers.")
+    parser.add_argument("--num-topics", default=20, help="The number of topics.", type=int)
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    with open(args.json) as f:
+        data = json.load(f)
+
+    print(f"Num papers: {len(data)}")
+
+    
+    lemmatizer = WordNetLemmatizer()
+    stopwords = set(stopwords.words('english'))
+    stopwords.update(["one", "two", "using"])
+    
+    tokens_per_paper = []
+    for paper_info in data:
+        text = paper_info["title"] + " " + paper_info["abstract"].replace("<p>", " ").replace("</p>", " ") + " ".join(paper_info["tags"])
+        lemmatized_tokens = [lemmatizer.lemmatize(w).lower() for w in nltk.word_tokenize(text) if w.lower() not in stopwords and w.isalpha()]
+        tokens_per_paper.append(lemmatized_tokens)
+
+    dictionary = Dictionary(tokens_per_paper)
+    dictionary.filter_extremes(no_below=20, no_above=0.5)
+
+    corpus = [dictionary.doc2bow(doc) for doc in tokens_per_paper]
+
+    passes = 100
+    iterations = 1000
+
+    temp = dictionary[0]  # This is needed to "load" the dictionary.
+
+    model = LdaModel(
+        corpus=corpus,
+        id2word=dictionary.id2token,
+        chunksize=1000,
+        alpha='asymmetric',
+        eta='auto',
+        iterations=iterations,
+        num_topics=args.num_topics,
+        passes=passes,
+        eval_every=None
+    )
+
+    topic_tokens = []
+    for topicid in range(args.num_topics):
+        topic_tokens.append([dictionary.id2token[k[0]] for i, k in enumerate(model.get_topic_terms(topicid, topn=4)) if i < 2 or k[1] > 0.025])
+
+    paper_topic_data = []
+    for paper, paper_bow in zip(data, corpus):
+        topic_distr = model.get_document_topics(paper_bow, minimum_probability=0)
+        paper_topic_data.append({
+            "key": paper["key"],
+            "year": paper["year"],
+            "title": paper["title"],
+            "topic_distr": {t: float(p) for t, p in topic_distr}
+        })
+
+    with open(args.outpath, 'w') as f:
+        json.dump({
+            "topics": topic_tokens,
+            "paper_data": paper_topic_data 
+        }, f)

topic-viz.html

@@ -0,0 +1,61 @@
+---
+layout: default
+title: Explore ML4Code papers with Topics
+description: A topic model for the papers in the ML4Code survey
+---
+<h2>Topic-based Explorer</h2>
+<p>Using topic-modelling the following topics have been extracted. The top stemmed words apprear below. 
+  Please change the slider to present the papers that mostly related to the appropria topics</p>
+<div id="topicslider">
+</div>
+<p>
+<ul id="toppapers">
+  <li>Please move the sliders to look at the papers.</li>
+</ul>
+</p>
+
+
+
+<script>
+  var all_papers = null;
+  var num_topics = -1;
+
+  $(document).ready(
+    function() {
+      $.getJSON('/topics.json', function(data) {
+        all_papers=data.paper_data;
+        num_topics = data.topics.length;
+        html = "";
+        for (let i=0; i < num_topics; i++) {
+            html += '<tag style="white-space: nowrap;">'+ data.topics[i].join(", ") +' <input type="range" min="0" max="10" value="0" style="width:50px" id="topicSlider'+i+'"></tag> '
+        }
+        $("#topicslider").append(html);
+        for (let i=0; i < num_topics; i++) {
+          $("#topicSlider"+i).on("change", renderPapers);
+        }
+      });
+    });
+
+  function scorePaper(paper_id) {
+    let score = 0;
+    topic_dist = all_papers[paper_id].topic_distr;
+    for (let i=0; i < num_topics; i++) {
+      score += $("#topicSlider"+i).val() * topic_dist[i];
+    }
+    return score;
+  }
+
+  function renderPapers(e, u) {
+    paper_idxs = [];
+    for (let i=0; i < all_papers.length; i++) {
+      paper_idxs.push([i, scorePaper(i)]);
+    }
+    paper_idxs = paper_idxs.sort(function(a,b){return b[1] - a[1]});
+    
+    $("#toppapers").text("");
+    for (let i=0; i < 20; i++) {
+      data = all_papers[paper_idxs[i][0]];
+      $("#toppapers").append("<li><a href='/publications/"+data.key+"'>"+ data.title +"</a>. " + data.year + "</li>");
+    }
+  }
+</script>