for release 2

Author: celine-lee

docs/README.md

@@ -2,7 +2,7 @@
 python update_submissions_dataset.py
 cd ../
 python docs/render_submissions.py --analyze_submissions --split SPLIT
-python docs/render_submissions.py --render_webpages --overwrite_previous_eval --split SPLIT
+python docs/render_submissions.py --render_webpages --overwrite_previous_eval
 ```
 
 ```

docs/javascripts/tablesort.js

@@ -1,6 +1,20 @@
 document$.subscribe(function() {
-    var tables = document.querySelectorAll("article table:not([class])")
-    tables.forEach(function(table) {
-      new Tablesort(table)
-    })
-  })
+  var tables = document.querySelectorAll("article table:not([class])")
+  tables.forEach(function(table) {
+    new Tablesort(table);
+    // Automatically sort the table by the specified column
+    var defaultSortColumn = 2; // Index of the column to sort (0-based)
+    var isAscending = False;   // Set to false for descending order
+
+    // Delay to ensure Tablesort is fully initialized
+    setTimeout(function () {
+      var header = table.querySelectorAll("thead th")[defaultSortColumn];
+      if (header) {
+        header.click(); // Simulate a click on the header
+        if (!isAscending) {
+          header.click(); // Click again for descending order
+        }
+      }
+    }, 100);
+  });
+});

docs/javascripts/tablesort.number.js

@@ -0,0 +1,26 @@
+(function(){
+    var cleanNumber = function(i) {
+      return i.replace(/[^\-?0-9.]/g, '');
+    },
+  
+    compareNumber = function(a, b) {
+      a = parseFloat(a);
+      b = parseFloat(b);
+  
+      a = isNaN(a) ? 0 : a;
+      b = isNaN(b) ? 0 : b;
+  
+      return a - b;
+    };
+  
+    Tablesort.extend('number', function(item) {
+      return item.match(/^[-+]?[£\x24Û¢´€]?\d+\s*([,\.]\d{0,2})/) || // Prefixed currency
+        item.match(/^[-+]?\d+\s*([,\.]\d{0,2})?[£\x24Û¢´€]/) || // Suffixed currency
+        item.match(/^[-+]?(\d)*-?([,\.]){0,1}-?(\d)+([E,e][\-+][\d]+)?%?$/); // Number
+    }, function(a, b) {
+      a = cleanNumber(a);
+      b = cleanNumber(b);
+  
+      return compareNumber(b, a);
+    });
+  }());

docs/render_submissions.py

@@ -185,28 +185,29 @@ def render_mds(overwrite_previous, subfolder="docs"):
         "lite": 3628,
         "all": 140926,
     }  # hard-coded to skip running it later
-    for split in tqdm.tqdm(["lite", "all"]):
+    for split in ["lite", "all"]:
         num_repos = len(SPLIT[split])
         # total_num_tests = 0
         # for repo_name in SPLIT[split]:
         #     repo_tests = subprocess.run(['commit0', 'get-tests', repo_name], capture_output=True, text=True).stdout.strip()
         #     total_num_tests += len(repo_tests.splitlines())
-        leaderboard[split] = leaderboard_header.format(
+        leaderboard[split] = []
+        leaderboard[split].append((split_to_total_tests[split]+1, leaderboard_header.format(
             split=split,
             num_repos=num_repos,
             total_num_tests=split_to_total_tests[split],
-        )
+        )))
 
     for org_path in tqdm.tqdm(glob.glob(os.path.join(analysis_files_path, "*"))):
         org_name = os.path.basename(org_path)
         if org_name in {"blank", "repos", "submission_repos"}:
             continue
         for branch_path in glob.glob(os.path.join(org_path, "*.json")):
             cum_tests_passed = 0
-            lite_cum_tests_passed = 0
             repos_resolved = 0
-            lite_repos_resolved = 0
             total_duration = 0.0
+            lite_cum_tests_passed = 0
+            lite_repos_resolved = 0
             lite_total_duration = 0.0
             branch_metrics = json.load(open(branch_path))
             submission_info = branch_metrics["submission_info"]
@@ -301,10 +302,11 @@ def render_mds(overwrite_previous, subfolder="docs"):
                         cum_tests_passed += pytest_info["summary"]["passed"]
                         total_duration += pytest_info["duration"]
                         repos_resolved += int(resolved)
-                        if repo_name in SPLIT["lite"]:
+                        if split == "all":
                             lite_cum_tests_passed += pytest_info["summary"]["passed"]
                             lite_total_duration += pytest_info["duration"]
                             lite_repos_resolved += int(resolved)
+
                         if write_submission:
                             pytest_details = f"{pytest_info['summary']['passed']} / {pytest_info['summary']['total']}"
                             duration = f"{pytest_info['duration']:.2f}"
@@ -329,29 +331,33 @@ def render_mds(overwrite_previous, subfolder="docs"):
                     wf.write(back_button + "\n" + submission_page)
             analysis_link = f"[Analysis](/{f'analysis_{org_name}_{branch_name}'})"
             github_link = f"[Github]({project_page_link})"
-            leaderboard[split] += (
+            leaderboard[split].append((cum_tests_passed,
                 f"\n|{display_name}|"
                 f"{repos_resolved}|"
                 f"{cum_tests_passed}|"
                 f"{total_duration:.2f}|"
                 f"{submission_date}|"
                 f"{analysis_link}|"
                 f"{github_link}|"
-            )
-            if split == "all":
-                leaderboard["lite"] += (
-                    f"\n|{display_name}|"
+            ))
+            if ((split == "all") and ("Reference (Gold)" not in display_name)):
+                leaderboard["lite"].append((lite_cum_tests_passed,
+                    f"\n|{display_name} (subset of `all`)|"
                     f"{lite_repos_resolved}|"
                     f"{lite_cum_tests_passed}|"
                     f"{lite_total_duration:.2f}|"
                     f"{submission_date}|"
                     f"{analysis_link}|"
                     f"{github_link}|"
-                )
+                ))
 
     leaderboard_filepath = os.path.join(subfolder, "analysis.md")
+    for split in ["lite", "all"]:
+        leaderboard[split] = sorted(leaderboard[split], key=lambda elt: -elt[0])
     with open(leaderboard_filepath, "w") as wf:
-        wf.write(leaderboard["lite"] + "\n\n" + leaderboard["all"])
+        lite_leaderboard_string = "".join(string for (_, string) in leaderboard["lite"])
+        all_leaderboard_string = "".join(string for (_, string) in leaderboard["all"])
+        wf.write(lite_leaderboard_string + "\n\n" + all_leaderboard_string)
 
 
 def get_args():

docs/update_submissions_dataset.py

@@ -2,10 +2,10 @@
 
 submissions = {
     "org_name": ["test-save-commit0", "commit0-lite-with-test", "commit0-lite-plain", "commit0-all-plain", "openhands-commit0", "sweagent-commit0"],
-    "branch": ["baseline", "fillin", "fillin", "fillin", "openhands", "sweagent"],
+    "branch": ["baseline", "fillin", "fillin", "fillin", "openhands", "sweagent", "openhands", "sweagent"],
     "display_name": ["Claude Sonnet 3.5 - Base", "Claude Sonnet 3.5 - Fill-in + Unit Test Feedback", "Claude Sonnet 3.5 - Fill-in", "Claude Sonnet 3.5 - Fill-in", "OpenHands", "SWE-Agent"], 
     "submission_date": ["09/25/2024", "09/25/2024", "09/25/2024", "09/25/2024", "11/25/2024", "11/26/2024"],
-    "split": ["lite", "lite", "lite", "all", "all", "all"],
+    "split": ["lite", "lite", "lite", "all", "all", "all", "lite", "lite"],
     "project_page": ["https://github.com/test-save-commit0", "https://github.com/commit0-lite-with-test", "https://github.com/commit0-lite-plain", "https://github.com/commit0-all-plain", "https://github.com/openhands-commit0", "https://github.com/sweagent-commit0"]
 }
 

mkdocs.yml

@@ -22,4 +22,5 @@ markdown_extensions:
 
 extra_javascript:
   - https://unpkg.com/[email protected]/dist/tablesort.min.js
-  - javascripts/tablesort.js
+  - javascripts/tablesort.js
+  - javascripts/tablesort.number.js