Merge pull request syntactic#7 from syntactic/2025_update

Add comprehensive Unicode support and publish to PyPI (v2.1.1)

- Add Unicode support for 10+ major language scripts (Chinese, Japanese,
  Korean, Arabic, Russian, Hebrew, Greek, Thai, Hindi, and more) in both
  parsers (legacy and modern)
- Add 11 comprehensive Unicode tests covering all supported scripts
- Fix argparse support in DeterministicGenerator CLI (--help now works)
- Publish to PyPI as 'jsgf-tools' package
- Add modern pyproject.toml for Python packaging standards
- Add MANIFEST.in for proper file inclusion in distribution
- Update README with PyPI installation instructions and Unicode examples
- Update setup.py with enhanced metadata and keywords

Author: syntactic

DeterministicGenerator.py

@@ -36,7 +36,7 @@
         a segmentation fault. 
 """
 
-import sys, itertools
+import sys, itertools, argparse
 import JSGFParser as parser
 import JSGFGrammar as gram
 
@@ -124,20 +124,25 @@ def main():
     """Main function for command line usage"""
     global grammar
 
-    if len(sys.argv) != 2:
-        print("Usage: python DeterministicGenerator.py <grammarFile>")
-        sys.exit(1)
+    arg_parser = argparse.ArgumentParser(
+        description='Generate all possible strings from a non-recursive JSGF grammar'
+    )
+    arg_parser.add_argument(
+        'grammarFile',
+        help='Path to the JSGF grammar file'
+    )
+    args = arg_parser.parse_args()
 
     try:
-        with open(sys.argv[1], 'r') as fileStream:
+        with open(args.grammarFile, 'r') as fileStream:
             grammar = parser.getGrammarObject(fileStream)
 
             for rule in grammar.publicRules:
                 expansions = processRHS(rule.rhs)
                 for expansion in expansions:
                     print(expansion)
     except FileNotFoundError:
-        print(f"Error: Grammar file '{sys.argv[1]}' not found")
+        print(f"Error: Grammar file '{args.grammarFile}' not found")
         sys.exit(1)
     except Exception as e:
         print(f"Error processing grammar: {e}")

JSGFParser.py

@@ -62,11 +62,30 @@
 import JSGFGrammar as gram
 from pyparsing import (Word, Literal, Group, Optional, ZeroOrMore, OneOrMore,
                       Forward, MatchFirst, Combine, alphas, alphanums, nums,
-                      stringEnd) 
+                      stringEnd, pyparsing_unicode) 
 
 sys.setrecursionlimit(100000)
 usePackrat = True
 
+# Unicode support: Tier 1 + Tier 2 scripts for comprehensive language coverage
+# Covers 5+ billion speakers: Latin, CJK, Arabic, Cyrillic, Devanagari, Hangul, Hebrew, Greek, Thai
+# Note: Using printables for scripts with combining characters (Thai, Devanagari)
+_unicode_letters = (
+    # Tier 1: Major scripts (Latin, CJK, Arabic, Cyrillic)
+    pyparsing_unicode.Latin1.alphas +
+    pyparsing_unicode.LatinA.alphas +
+    pyparsing_unicode.LatinB.alphas +
+    pyparsing_unicode.CJK.alphas +
+    pyparsing_unicode.Arabic.alphas +
+    pyparsing_unicode.Cyrillic.alphas +
+    # Tier 2: Common scripts (using printables for scripts with combining marks)
+    pyparsing_unicode.Devanagari.printables +
+    pyparsing_unicode.Hangul.alphas +
+    pyparsing_unicode.Hebrew.alphas +
+    pyparsing_unicode.Greek.alphas +
+    pyparsing_unicode.Thai.printables
+)
+
 def foundWeight(s, loc, toks):
     """
     PyParsing action to run when a weight is found.
@@ -165,11 +184,11 @@ def foundSeq(s, loc, toks):
 # PyParsing rule for a weight
 weight = (Literal('/').suppress() + (Word(nums + '.')).setResultsName('weightAmount') + Literal('/').suppress()).setParseAction(foundWeight).setResultsName("weight")
 
-# PyParsing rule for a token
-token = Word(alphanums+"'_-,.?@").setResultsName('token').setParseAction(foundToken)
+# PyParsing rule for a token (with Unicode support)
+token = Word(alphanums + _unicode_letters + "'_-,.?@").setResultsName('token').setParseAction(foundToken)
 
-# PyParsing rule for a nonterminal reference
-nonterminal = Combine(Literal('<') + Word(alphanums+'$_:;,=|/\\()[]@#%!^&~') + Literal('>')).setParseAction(foundNonterminal).setResultsName('NonTerminal')
+# PyParsing rule for a nonterminal reference (with Unicode support)
+nonterminal = Combine(Literal('<') + Word(alphanums + _unicode_letters + '$_:;,=|/\\()[]@#%!^&~') + Literal('>')).setParseAction(foundNonterminal).setResultsName('NonTerminal')
 
 Sequence = Forward()
 

MANIFEST.in

@@ -0,0 +1,9 @@
+include README.md
+include LICENSE
+include CLAUDE.md
+include requirements-dev.txt
+include pytest.ini
+include *.gram
+recursive-include jsgf *.py
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

README.md

@@ -15,6 +15,11 @@ A Python library for parsing and generating strings from JSGF (Java Speech Gramm
 
 ## Installation
 
+### From PyPI (Recommended)
+```bash
+pip install jsgf-tools
+```
+
 ### From Source
 ```bash
 git clone https://github.com/syntactic/JSGFTools.git
@@ -26,7 +31,7 @@ pip install -e .
 ```bash
 git clone https://github.com/syntactic/JSGFTools.git
 cd JSGFTools
-pip install -r requirements-dev.txt
+pip install -e ".[dev]"
 ```
 
 ## Quick Start
@@ -101,6 +106,26 @@ public <start> = <greeting> <target>;
 - Grouping with parentheses
 - Comments (// and /* */)
 - Public and private rules
+- **Unicode support** for 10+ major language scripts
+
+### Unicode Support
+
+JSGFTools fully supports Unicode characters in both tokens and rule names, covering:
+- **Latin scripts** (English, Spanish, French, etc.)
+- **CJK** (Chinese, Japanese Kanji, Korean Hanja)
+- **Arabic** (Arabic, Persian, Urdu)
+- **Cyrillic** (Russian, Ukrainian, Bulgarian)
+- **Devanagari** (Hindi, Sanskrit, Marathi)
+- **Hangul** (Korean)
+- **Hebrew**
+- **Greek**
+- **Thai**
+
+Example:
+```jsgf
+public <greeting> = hello | 你好 | こんにちは | مرحبا | привет | שלום;
+public <问候> = 您好 | 欢迎;
+```
 
 ### Not Yet Supported
 - Kleene operators (* and +)
@@ -162,5 +187,7 @@ MIT License. See [LICENSE](LICENSE) file for details.
 
 ## Version History
 
+- **2.1.1**: Fixed argparse support in DeterministicGenerator CLI (--help now works)
+- **2.1.0**: Added comprehensive Unicode support (10+ language scripts), published to PyPI
 - **2.0.0**: Complete Python 3 modernization, added test suite, improved packaging
 - **1.x**: Original Python 2.7 version

jsgf/__init__.py

@@ -0,0 +1,19 @@
+"""
+JSGF Tools - Modern Python library for parsing and generating from JSGF grammars.
+
+This package provides a clean, object-oriented API for working with JSGF grammars.
+"""
+
+from .grammar import Grammar
+from .generators import DeterministicGenerator, ProbabilisticGenerator
+from .exceptions import JSGFError, ParseError, GenerationError
+
+__version__ = "2.0.0"
+__all__ = [
+    "Grammar",
+    "DeterministicGenerator",
+    "ProbabilisticGenerator",
+    "JSGFError",
+    "ParseError",
+    "GenerationError"
+]

jsgf/ast_nodes.py

@@ -0,0 +1,152 @@
+"""
+Abstract Syntax Tree nodes for JSGF grammars.
+
+This module provides the core AST node classes that represent different
+parts of a JSGF grammar structure.
+"""
+
+from typing import List, Union, Any, Optional
+from abc import ABC, abstractmethod
+
+
+class JSGFNode(ABC):
+    """Base class for all JSGF AST nodes."""
+
+    @abstractmethod
+    def __str__(self) -> str:
+        """Return a string representation of this node."""
+        pass
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({str(self)})"
+
+
+class Terminal(JSGFNode):
+    """Represents a terminal symbol (token) in the grammar."""
+
+    def __init__(self, value: str):
+        self.value = value
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, Terminal) and self.value == other.value
+
+    def __hash__(self) -> int:
+        return hash(self.value)
+
+
+class NonTerminal(JSGFNode):
+    """Represents a non-terminal symbol in the grammar."""
+
+    def __init__(self, name: str):
+        self.name = name
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, NonTerminal) and self.name == other.name
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+
+class Sequence(JSGFNode):
+    """Represents a sequence of elements."""
+
+    def __init__(self, elements: List[JSGFNode]):
+        self.elements = elements
+
+    def __str__(self) -> str:
+        return " ".join(str(element) for element in self.elements)
+
+    def __iter__(self):
+        return iter(self.elements)
+
+    def __len__(self) -> int:
+        return len(self.elements)
+
+    def __getitem__(self, index: int) -> JSGFNode:
+        return self.elements[index]
+
+
+class Alternative(JSGFNode):
+    """Represents alternatives (choices) in the grammar."""
+
+    def __init__(self, choices: List[Union[JSGFNode, tuple]]):
+        """
+        Initialize alternatives.
+
+        Args:
+            choices: List of choices. Each choice can be:
+                    - A JSGFNode (unweighted)
+                    - A tuple of (JSGFNode, weight) (weighted)
+        """
+        self.choices = []
+        for choice in choices:
+            if isinstance(choice, tuple):
+                node, weight = choice
+                self.choices.append((node, float(weight)))
+            else:
+                self.choices.append((choice, 1.0))  # Default weight
+
+    def __str__(self) -> str:
+        choice_strs = []
+        for node, weight in self.choices:
+            if weight != 1.0:
+                choice_strs.append(f"/{weight}/ {node}")
+            else:
+                choice_strs.append(str(node))
+        return "( " + " | ".join(choice_strs) + " )"
+
+    def __iter__(self):
+        return iter(self.choices)
+
+    def __len__(self) -> int:
+        return len(self.choices)
+
+    def get_weights(self) -> List[float]:
+        """Return the weights of all choices."""
+        return [weight for _, weight in self.choices]
+
+    def get_nodes(self) -> List[JSGFNode]:
+        """Return the nodes of all choices."""
+        return [node for node, _ in self.choices]
+
+
+class Optional(JSGFNode):
+    """Represents an optional element in the grammar."""
+
+    def __init__(self, element: JSGFNode):
+        self.element = element
+
+    def __str__(self) -> str:
+        return f"[ {self.element} ]"
+
+
+class Group(JSGFNode):
+    """Represents a grouped element."""
+
+    def __init__(self, element: JSGFNode):
+        self.element = element
+
+    def __str__(self) -> str:
+        return f"( {self.element} )"
+
+
+class Rule:
+    """Represents a complete grammar rule."""
+
+    def __init__(self, name: str, expansion: JSGFNode, is_public: bool = False):
+        self.name = name
+        self.expansion = expansion
+        self.is_public = is_public
+
+    def __str__(self) -> str:
+        prefix = "public " if self.is_public else ""
+        return f"{prefix}<{self.name}> = {self.expansion};"
+
+    def __repr__(self) -> str:
+        return f"Rule(name='{self.name}', is_public={self.is_public})"