Merge pull request #20 from codellm-devkit/19-bug-report-numpy-build-failure-on-python-312-due-to-missing-distutils

Fix issue 19

Author: rahlk

CHANGELOG.md

@@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.11] - 2025-07-21
+
+### Fixed
+- **CRITICAL**: Fixed NumPy build failure on Python 3.12+ (addresses [#19](https://github.com/codellm-devkit/codeanalyzer-python/issues/19))
+  - Updated NumPy dependency constraints to handle Python 3.12+ compatibility
+  - Split NumPy version constraints into three tiers:
+    - `numpy>=1.21.0,<1.24.0` for Python < 3.11
+    - `numpy>=1.24.0,<2.0.0` for Python 3.11.x  
+    - `numpy>=1.26.0,<2.0.0` for Python 3.12+ (requires NumPy 1.26+ which supports Python 3.12)
+  - Resolves `ModuleNotFoundError: No module named 'distutils'` errors on Python 3.12+
+  - Ensures compatibility with Python 3.12 which removed `distutils` from the standard library
+- Fixed Pydantic v1/v2 compatibility issues in JSON serialization throughout codebase
+  - Added comprehensive Pydantic version detection and compatibility layer
+  - Introduced `model_dump_json()` and `model_validate_json()` helper functions for cross-version compatibility
+  - Fixed `PyApplication.parse_raw()` deprecated method usage (replaced with `model_validate_json()`)
+  - Updated CLI output methods to use compatible serialization functions
+  - Resolved forward reference updates only for Pydantic v1 (v2 handles these automatically)
+
+### Changed
+- Enhanced Pydantic compatibility infrastructure in schema module
+  - Added runtime Pydantic version detection using `importlib.metadata`
+  - Created compatibility abstraction layer for JSON serialization/deserialization
+  - Improved forward reference resolution logic to work with both Pydantic v1 and v2
+  - Updated all JSON serialization calls to use new compatibility functions
+  - Better error handling for missing Pydantic dependency
+
+### Technical Details
+- Added `packaging` dependency for robust version comparison
+- Enhanced schema module with runtime version detection and compatibility helpers
+- Updated core analysis caching system to use compatible Pydantic JSON methods
+- Improved CLI output formatting with cross-version Pydantic support
+
 ## [0.1.10] - 2025-07-20
 
 ### Added

codeanalyzer/__main__.py

@@ -6,6 +6,7 @@
 from codeanalyzer.core import Codeanalyzer
 from codeanalyzer.utils import _set_log_level, logger
 from codeanalyzer.config import OutputFormat
+from codeanalyzer.schema import model_dump_json
 
 def main(
     input: Annotated[
@@ -102,7 +103,7 @@ def main(
         # Handle output based on format
         if output is None:
             # Output to stdout (only for JSON)
-            print(artifacts.json(separators=(",", ":")))
+            print(model_dump_json(artifacts, separators=(",", ":")))
         else:
             # Output to file
             output.mkdir(parents=True, exist_ok=True)
@@ -113,8 +114,8 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
     """Write artifacts to file in the specified format."""
     if format == OutputFormat.JSON:
         output_file = output_dir / "analysis.json"
-        # Use Pydantic's json() with separators for compact output
-        json_str = artifacts.json(indent=None)
+        # Use Pydantic's model_dump_json() for compact output
+        json_str = model_dump_json(artifacts, indent=None)
         with output_file.open("w") as f:
             f.write(json_str)
         logger.info(f"Analysis saved to {output_file}")

codeanalyzer/core.py

@@ -8,7 +8,7 @@
 
 import ray
 from codeanalyzer.utils import logger
-from codeanalyzer.schema import PyApplication, PyModule
+from codeanalyzer.schema import PyApplication, PyModule, model_dump_json, model_validate_json
 from codeanalyzer.semantic_analysis.codeql import CodeQLLoader
 from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
 from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
@@ -408,7 +408,7 @@ def _load_pyapplication_from_cache(self, cache_file: Path) -> PyApplication:
         """
         with cache_file.open('r') as f:
             data = f.read()
-        return PyApplication.parse_raw(data)
+        return model_validate_json(PyApplication, data)
 
     def _save_analysis_cache(self, app: PyApplication, cache_file: Path) -> None:
         """Save analysis to cache file.
@@ -421,8 +421,8 @@ def _save_analysis_cache(self, app: PyApplication, cache_file: Path) -> None:
         cache_file.parent.mkdir(parents=True, exist_ok=True)
 
         with cache_file.open('w') as f:
-            f.write(app.json(indent=2))
-        
+            f.write(model_dump_json(app, indent=2))
+
         logger.info(f"Analysis cached to {cache_file}")
 
     def _file_unchanged(self, file_path: Path, cached_module: PyModule) -> bool:

codeanalyzer/schema/__init__.py

@@ -1,3 +1,6 @@
+from importlib.metadata import version, PackageNotFoundError
+from packaging.version import parse as parse_version
+
 from .py_schema import (
     PyApplication,
     PyCallable,
@@ -22,12 +25,48 @@
     "PyCallableParameter",
 ]
 
-# Resolve forward references
-PyCallable.update_forward_refs(PyClass=PyClass)
-PyClass.update_forward_refs(PyCallable=PyCallable)
-PyModule.update_forward_refs(PyCallable=PyCallable, PyClass=PyClass)
-PyApplication.update_forward_refs(
-    PyCallable=PyCallable,
-    PyClass=PyClass,
-    PyModule=PyModule
-)
+try:
+    pydantic_version = version("pydantic")
+except PackageNotFoundError:
+    pydantic_version = "0.0.0"  # fallback or raise if appropriate
+
+PYDANTIC_V2 = parse_version(pydantic_version) >= parse_version("2.0.0")
+
+if not PYDANTIC_V2:
+    # Safe to pass localns
+    PyCallable.update_forward_refs(PyClass=PyClass)
+    PyClass.update_forward_refs(PyCallable=PyCallable)
+    PyModule.update_forward_refs(PyCallable=PyCallable, PyClass=PyClass)
+    PyApplication.update_forward_refs(
+        PyCallable=PyCallable,
+        PyClass=PyClass,
+        PyModule=PyModule
+    )
+    
+# Compatibility helpers for Pydantic v1/v2
+def model_dump_json(model, **kwargs):
+    """Compatibility helper for JSON serialization."""
+    if PYDANTIC_V2:
+        return model.model_dump_json(**kwargs)
+    else:
+        # Map Pydantic v2 parameters to v1 equivalents
+        v1_kwargs = {}
+        if 'indent' in kwargs:
+            v1_kwargs['indent'] = kwargs['indent']
+        if 'separators' in kwargs:
+            # In v1, separators is passed to dumps_kwargs
+            v1_kwargs['separators'] = kwargs['separators']
+        return model.json(**v1_kwargs)
+
+def model_validate_json(model_class, json_data):
+    """Compatibility helper for JSON deserialization."""
+    if PYDANTIC_V2:
+        return model_class.model_validate_json(json_data)
+    else:
+        return model_class.parse_raw(json_data)
+
+__all__.extend([
+    "PYDANTIC_V2",
+    "model_dump_json",
+    "model_validate_json"
+])

pyproject.toml

@@ -1,37 +1,60 @@
 [project]
 name = "codeanalyzer-python"
-version = "0.1.10"
+version = "0.1.11"
 description = "Static Analysis on Python source code using Jedi, CodeQL and Treesitter."
 readme = "README.md"
 authors = [
-    { name = "Rahul Krishna", email = "i.m.ralk@gmail.com" }
+  { name = "Rahul Krishna", email = "i.m.ralk@gmail.com" }
 ]
 requires-python = ">=3.9"
 
 dependencies = [
-    "jedi>=0.18.0,<0.20.0",
-    "msgpack>=1.0.0,<1.0.7",
-    "networkx>=2.6.0,<3.2.0",
-    "pandas>=1.3.0,<2.0.0",
-    "numpy>=1.21.0,<1.24.0",
-    "pydantic>=1.8.0,<2.0.0",
-    "requests>=2.20.0,<3.0.0",
-    "rich>=12.6.0,<14.0.0",
-    "typer>=0.9.0,<1.0.0",
-    "ray>=2.0.0,<3.0.0",
-    "typing-extensions>=4.0.0"
+    # jedi
+    "jedi>=0.18.0,<0.20.0; python_version < '3.11'",
+    "jedi<=0.19.2; python_version >= '3.11'",
+    # msgpack
+    "msgpack>=1.0.0,<1.0.7; python_version < '3.11'",
+    "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'",
+    # networkx
+    "networkx>=2.6.0,<3.2.0; python_version < '3.11'",
+    "networkx>=3.0.0,<4.0.0; python_version >= '3.11'",
+    # pandas
+    "pandas>=1.3.0,<2.0.0; python_version < '3.11'",
+    "pandas>=2.0.0,<3.0.0; python_version >= '3.11'",
+    # numpy
+    "numpy>=1.21.0,<1.24.0; python_version < '3.11'",
+    "numpy>=1.24.0,<2.0.0; python_version >= '3.11' and python_version < '3.12'",
+    "numpy>=1.26.0,<2.0.0; python_version >= '3.12'",
+    # pydantic
+    "pydantic>=1.8.0,<2.0.0; python_version < '3.11'",
+    "pydantic>=2.0.0,<3.0.0; python_version >= '3.11'",
+    # requests
+    "requests>=2.20.0,<3.0.0; python_version >= '3.11'",
+    # rich
+    "rich>=12.6.0,<14.0.0; python_version < '3.11'",
+    "rich>=14.0.0,<15.0.0; python_version >= '3.11'",
+    # typer
+    "typer>=0.9.0,<1.0.0; python_version < '3.11'",
+    "typer>=0.9.0,<2.0.0; python_version >= '3.11'",
+    # typing-extensions
+    "typing-extensions>=4.0.0,<5.0.0; python_version < '3.11'",
+    "typing-extensions>=4.5.0,<6.0.0; python_version >= '3.11'",
+    # ray
+    "ray==2.0.0; python_version < '3.11'",
+    "ray>=2.10.0,<3.0.0; python_version >= '3.11'",
+    "packaging>=25.0",
 ]
 
 [dependency-groups]
 test = [
-    "pytest>=7.0.0,<8.0.0",
-    "pytest-asyncio>=0.14.0,<0.15.0",
-    "pytest-cov>=2.10.0,<3.0.0",
-    "pytest-pspec>=0.0.3"
+  "pytest>=7.0.0,<8.0.0",
+  "pytest-asyncio>=0.14.0,<0.15.0",
+  "pytest-cov>=2.10.0,<3.0.0",
+  "pytest-pspec>=0.0.3"
 ]
 dev = [
-    "ipdb>=0.13.0,<0.14.0",
-    "pre-commit>=2.9.0,<3.0.0"
+  "ipdb>=0.13.0,<0.14.0",
+  "pre-commit>=2.9.0,<3.0.0"
 ]
 
 [project.scripts]
@@ -47,45 +70,45 @@ include = ["codeanalyzer/py.typed"]
 
 [tool.hatch.build.targets.sdist]
 include = [
-    "codeanalyzer",
-    "codeanalyzer/py.typed",
-    "README.md",
-    "LICENSE",
-    "NOTICE"
+  "codeanalyzer",
+  "codeanalyzer/py.typed",
+  "README.md",
+  "LICENSE",
+  "NOTICE"
 ]
 
 [tool.pytest.ini_options]
 addopts = [
-    "-p", "coverage",
-    "--cov=codeanalyzer",
-    "--cov-report=html",
-    "--cov-report=term-missing",
-    "--cov-fail-under=40",
-    "--ignore=test/fixtures"
+  "-p", "coverage",
+  "--cov=codeanalyzer",
+  "--cov-report=html",
+  "--cov-report=term-missing",
+  "--cov-fail-under=40",
+  "--ignore=test/fixtures"
 ]
 testpaths = ["test"]
 
 [tool.coverage.run]
 source = ["codeanalyzer"]
 branch = true
 omit = [
-    "*/tests/*",
-    "*/test_*",
-    "*/__pycache__/*",
-    "*/venv/*",
-    "*/.venv/*",
-    "codeanalyzer/semantic_analysis/*"
+  "*/tests/*",
+  "*/test_*",
+  "*/__pycache__/*",
+  "*/venv/*",
+  "*/.venv/*",
+  "codeanalyzer/semantic_analysis/*"
 ]
 
 [tool.coverage.report]
 precision = 2
 show_missing = true
 exclude_lines = [
-    "pragma: no cover",
-    "def __repr__",
-    "raise AssertionError",
-    "raise NotImplementedError",
-    "if __name__ == .__main__.:"
+  "pragma: no cover",
+  "def __repr__",
+  "raise AssertionError",
+  "raise NotImplementedError",
+  "if __name__ == .__main__.:"
 ]
 
 [tool.coverage.html]

test_numpy_constraints.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Test script to verify NumPy dependency constraints work correctly."""
+
+import sys
+from packaging.version import parse as parse_version
+
+def test_numpy_constraints():
+    """Test that NumPy constraints are correct for different Python versions."""
+    python_version = parse_version(f"{sys.version_info.major}.{sys.version_info.minor}")
+    print(f"Testing on Python {python_version}")
+    
+    try:
+        import numpy
+        numpy_version = parse_version(numpy.__version__)
+        print(f"NumPy version: {numpy_version}")
+        
+        # Test constraints based on Python version
+        if python_version < parse_version("3.11"):
+            if not (parse_version("1.21.0") <= numpy_version < parse_version("1.24.0")):
+                print(f"ERROR: NumPy {numpy_version} not in expected range 1.21.0-1.24.0 for Python < 3.11")
+                return False
+        elif python_version >= parse_version("3.11") and python_version < parse_version("3.12"):
+            if not (parse_version("1.24.0") <= numpy_version < parse_version("2.0.0")):
+                print(f"ERROR: NumPy {numpy_version} not in expected range 1.24.0-2.0.0 for Python 3.11.x")
+                return False
+        elif python_version >= parse_version("3.12"):
+            if not (parse_version("1.26.0") <= numpy_version < parse_version("2.0.0")):
+                print(f"ERROR: NumPy {numpy_version} not in expected range 1.26.0-2.0.0 for Python 3.12+")
+                return False
+                
+        print("✅ NumPy constraints are satisfied")
+        return True
+        
+    except ImportError as e:
+        print(f"ERROR: Failed to import NumPy: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = test_numpy_constraints()
+    sys.exit(0 if success else 1)