⚡️ Speed up function tokenize_code by 12%

[codeflash-ai]

📄 12% (0.12x) speedup for tokenize_code in evaluation/benchmarks/testgeneval/pygments_utils.py

⏱️ Runtime : 30.3 milliseconds → 27.1 milliseconds (best of 254 runs)

📝 Explanation and details

Key Optimizations.

1. Simplified Loop Logic: The code reduces complex looping and state-checking by maintaining a cleaner track of the '"STR"' sequence using a prev_token variable for efficient string matching.

2. Token Matching: Used sets for token type comparison to speed up checks and prevent extra string conversion.

3. Streamlined Control Flow: Removed nested conditions by using early continues and checks, which makes the code more efficient and easier to follow.

4. Final Check Handling: Managed trailing tokens efficiently without a second loop, precisely handling scenario for unmatched states.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 28 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	90.0%

🌀 Generated Regression Tests Details

import re

# imports
import pytest  # used for our unit tests
from evaluation.benchmarks.testgeneval.pygments_utils import tokenize_code
from pygments.lexers.python import PythonLexer

# unit tests

def test_simple_function():
    # Test basic function definition
    code = "def foo(): return 42"
    expected = ['def', 'foo', '(', ')', ':', 'return', '42']
    codeflash_output = tokenize_code(code)

def test_variable_assignment():
    # Test simple variable assignment
    code = "x = 1"
    expected = ['x', '=', '1']
    codeflash_output = tokenize_code(code)

def test_single_line_comment():
    # Test code with a single-line comment
    code = "# This is a comment"
    expected = []
    codeflash_output = tokenize_code(code)

def test_inline_comment():
    # Test code with an inline comment
    code = "x = 1  # Inline comment"
    expected = ['x', '=', '1']
    codeflash_output = tokenize_code(code)

def test_double_quoted_string():
    # Test code with a double-quoted string
    code = 's = "hello"'
    expected = ['s', '=', '"STR"']
    codeflash_output = tokenize_code(code)

def test_triple_quoted_string():
    # Test code with a triple-quoted string
    code = 's = """multi\nline"""'
    expected = ['s', '=', '"STR"']
    codeflash_output = tokenize_code(code)

def test_empty_input():
    # Test with empty input
    code = ""
    expected = []
    codeflash_output = tokenize_code(code)

def test_leading_trailing_spaces():
    # Test code with leading and trailing spaces
    code = "   x = 1   "
    expected = ['x', '=', '1']
    codeflash_output = tokenize_code(code)

def test_mixed_whitespace():
    # Test code with mixed tabs and spaces
    code = "\t x = 1"
    expected = ['x', '=', '1']
    codeflash_output = tokenize_code(code)

def test_unicode_characters():
    # Test code with Unicode characters
    code = "s = 'café'"
    expected = ['s', '=', '"STR"']
    codeflash_output = tokenize_code(code)

def test_large_input():
    # Test with a large block of code
    code = "def foo():\n" * 100 + "return 42\n"
    expected = ['def', 'foo', '(', ')', ':', 'return', '42'] * 100
    codeflash_output = tokenize_code(code)

def test_invalid_input():
    # Test with non-string input
    with pytest.raises(AttributeError):
        tokenize_code(123)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

import re

# imports
import pytest  # used for our unit tests
from evaluation.benchmarks.testgeneval.pygments_utils import tokenize_code
from pygments.lexers.python import PythonLexer

# unit tests

def test_simple_code():
    # Test a simple line of Python code
    code = 'print("Hello, World!")'
    expected_tokens = ['print', '(', '"STR"', ')']
    codeflash_output = tokenize_code(code)

def test_function_definition():
    # Test a simple function definition
    code = 'def greet(): return "Hello"'
    expected_tokens = ['def', 'greet', '(', ')', ':', 'return', '"STR"']
    codeflash_output = tokenize_code(code)

def test_leading_trailing_whitespace():
    # Test code with leading and trailing spaces
    code = '   x = 5   '
    expected_tokens = ['x', '=', '5']
    codeflash_output = tokenize_code(code)

def test_multiple_spaces_between_tokens():
    # Test code with multiple spaces between tokens
    code = 'x    =    5'
    expected_tokens = ['x', '=', '5']
    codeflash_output = tokenize_code(code)

def test_empty_input():
    # Test an empty string
    code = ''
    expected_tokens = []
    codeflash_output = tokenize_code(code)

def test_only_whitespace():
    # Test a string with only spaces
    code = '    '
    expected_tokens = []
    codeflash_output = tokenize_code(code)

def test_special_characters():
    # Test code with special characters
    code = 'print("Hello, World! #$%^&*()")'
    expected_tokens = ['print', '(', '"STR"', ')']
    codeflash_output = tokenize_code(code)

def test_embedded_quotes():
    # Test strings with embedded quotes
    code = 'print(\'He said, "Hello"\')'
    expected_tokens = ['print', '(', '"STR"', ')']
    codeflash_output = tokenize_code(code)

def test_escape_sequences():
    # Test strings with escape sequences
    code = 'print("Line1\\nLine2")'
    expected_tokens = ['print', '(', '"STR"', ')']
    codeflash_output = tokenize_code(code)

def test_multiline_code():
    # Test a function with multiple lines and indentation
    code = '''
def add(a, b):
    return a + b
'''
    expected_tokens = ['def', 'add', '(', 'a', ',', 'b', ')', ':', 'return', 'a', '+', 'b']
    codeflash_output = tokenize_code(code)

def test_nested_structures():
    # Test code with nested functions or loops
    code = '''
for i in range(5):
    def inner():
        pass
'''
    expected_tokens = ['for', 'i', 'in', 'range', '(', '5', ')', ':', 'def', 'inner', '(', ')', ':', 'pass']
    codeflash_output = tokenize_code(code)

def test_comments():
    # Test code with inline and block comments
    code = '''
# This is a comment
print("Hello")  # Inline comment
'''
    expected_tokens = ['print', '(', '"STR"', ')']
    codeflash_output = tokenize_code(code)

def test_large_code_base():
    # Test a large script with repetitive code
    code = 'x = 5\n' * 1000
    expected_tokens = ['x', '=', '5'] * 1000
    codeflash_output = tokenize_code(code)

def test_complex_expressions():
    # Test code with complex expressions and multiple operators
    code = 'result = (a * b) + (c / d) - (e ** f)'
    expected_tokens = ['result', '=', '(', 'a', '*', 'b', ')', '+', '(', 'c', '/', 'd', ')', '-', '(', 'e', '**', 'f', ')']
    codeflash_output = tokenize_code(code)

def test_syntax_errors():
    # Test code with syntax errors
    code = 'def func(: pass'
    expected_tokens = ['def', 'func', '(', ':', 'pass']
    codeflash_output = tokenize_code(code)

def test_unclosed_strings():
    # Test strings that are not properly closed
    code = 'print("Hello)'
    expected_tokens = ['print', '(', '"STR']
    codeflash_output = tokenize_code(code)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-tokenize_code-m8waewgc and push.