Develop

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM ubuntu:24.04
 
 # Set environment variables
 ENV PYTHONUNBUFFERED=1 \
@@ -21,6 +21,7 @@ RUN apt-get update && apt-get install -y \
     jq \
     tree \
     htop \
+    python3 \
     ca-certificates \
     gnupg \
     lsb-release \

.github/workflows/build-and-test.yml

@@ -31,10 +31,11 @@ jobs:
         python -m pip install --upgrade pip
         pip install -e ./sai[test]
         pip install -e ./saigen[test]
+        pip install -e ./saitest[test]
 
     - name: Run tests
       run: |
-        pytest --cov=sai --cov=saigen --cov-report=xml --cov-report=term
+        pytest --cov=sai --cov=saigen --cov=saitest --cov-report=xml --cov-report=term
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
@@ -61,18 +62,19 @@ jobs:
         python -m pip install --upgrade pip
         pip install -e ./sai[dev]
         pip install -e ./saigen[dev]
+        pip install -e ./saitest[dev]
 
     - name: Run black
-      run: black --check sai saigen tests
+      run: black --check sai saigen saitest tests
 
     - name: Run isort
-      run: isort --check-only sai saigen tests
+      run: isort --check-only sai saigen saitest tests
 
     - name: Run flake8
-      run: flake8 sai saigen tests
+      run: flake8 sai saigen saitest tests
 
     - name: Run mypy
-      run: mypy sai saigen
+      run: mypy sai saigen saitest
 
   build:
     name: Build Packages
@@ -106,6 +108,12 @@ jobs:
         python -m build
         twine check dist/*
 
+    - name: Build SAITEST package
+      run: |
+        cd saitest
+        python -m build
+        twine check dist/*
+
     - name: Upload SAI artifacts
       uses: actions/upload-artifact@v4
       with:
@@ -117,3 +125,9 @@ jobs:
       with:
         name: saigen-dist
         path: saigen/dist/
+
+    - name: Upload SAITEST artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: saitest-dist
+        path: saitest/dist/

.github/workflows/ci.yml

@@ -45,17 +45,17 @@ jobs:
 
     - name: Run linting
       run: |
-        black --check --diff sai saigen tests
-        isort --check-only --diff sai saigen tests
-        flake8 sai saigen tests
+        black --check --diff sai saigen saitest tests
+        isort --check-only --diff sai saigen saitest tests
+        flake8 sai saigen saitest tests
 
     - name: Run type checking
       run: |
-        mypy sai saigen
+        mypy sai saigen saitest
 
     - name: Run tests
       run: |
-        pytest --cov=sai --cov=saigen --cov-report=xml --cov-report=term-missing
+        pytest --cov=sai --cov=saigen --cov=saitest --cov-report=xml --cov-report=term-missing
 
     - name: Upload coverage to Codecov
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
@@ -82,8 +82,8 @@ jobs:
 
     - name: Run security checks with bandit
       run: |
-        bandit -r sai saigen -f json -o bandit-report.json || true
-        bandit -r sai saigen
+        bandit -r sai saigen saitest -f json -o bandit-report.json || true
+        bandit -r sai saigen saitest
 
     - name: Check dependencies for known vulnerabilities
       run: |
@@ -153,6 +153,7 @@ jobs:
       run: |
         sai --help
         saigen --help
+        saitest --help
         sai providers list || true  # May fail if no providers available
         sai config show || true     # May fail if no config exists
 

.gitignore

@@ -163,6 +163,7 @@ cython_debug/
 .sai/
 *.cache
 cache/
+saidata/
 
 # Auto-generated version files (setuptools-scm)
 */_version.py
@@ -198,4 +199,5 @@ logs/
 .coverage
 htmlcov/
 .pytest_cache/
-test-results/
+test-results/
+llm-comparison*

.kiro/hooks/git-auto-commit.kiro.hook

@@ -0,0 +1,16 @@
+{
+  "enabled": true,
+  "name": "Git Auto Commit",
+  "description": "Automatically stages and commits all current changes in the git repository when files are modified",
+  "version": "1",
+  "when": {
+    "type": "userTriggered",
+    "patterns": [
+      "**/*"
+    ]
+  },
+  "then": {
+    "type": "askAgent",
+    "prompt": "Files have been modified in the workspace. Please:\n1. Update CHANGELOG. When runnging git diff use the --no-pager option\n2. Run `git add .` to stage all changes\n3. Run `git commit -m \"Auto-commit: [describe the changes]\"` with an appropriate commit message describing what was changed\n"
+  }
+}

.kiro/specs/llm-prompt-rag-refactoring/requirements.md

@@ -0,0 +1,138 @@
+# Requirements Document
+
+## Introduction
+
+This specification defines the requirements for refactoring saigen's LLM interaction system to support experimentation with different prompt strategies and RAG (Retrieval-Augmented Generation) approaches. The current implementation has a single, hardcoded prompt template and RAG strategy. This refactoring will enable flexible experimentation with various prompt engineering techniques and context-building strategies to optimize saidata generation quality.
+
+## Glossary
+
+- **Saigen**: The SAI data generation tool that creates software metadata YAML files
+- **LLM**: Large Language Model used for generating saidata content
+- **RAG**: Retrieval-Augmented Generation - technique for enhancing LLM responses with retrieved context
+- **Prompt Strategy**: A specific approach to constructing prompts for LLM queries
+- **Context Builder**: Component responsible for gathering and formatting context data for prompts
+- **Generation Engine**: Core orchestrator that coordinates LLM queries and saidata generation
+- **Provider Manager**: Component managing multiple LLM provider instances
+- **Saidata**: YAML-formatted software metadata following the saidata-0.3-schema.json specification
+
+## Requirements
+
+### Requirement 1: Pluggable Prompt Strategy System
+
+**User Story:** As a saigen developer, I want to experiment with different prompt engineering approaches, so that I can optimize saidata generation quality for different software types.
+
+#### Acceptance Criteria
+
+1. WHEN THE System initializes, THE Saigen SHALL load prompt strategies from a configurable registry
+2. WHEN a generation request is made, THE Generation Engine SHALL select the appropriate prompt strategy based on configuration or request parameters
+3. WHEN multiple prompt strategies are available, THE System SHALL support runtime selection without code changes
+4. WHERE a custom prompt strategy is defined, THE System SHALL validate the strategy interface before registration
+5. WHEN a prompt strategy is executed, THE System SHALL provide access to all available context data including repository information, similar saidata, and user hints
+
+### Requirement 2: Flexible RAG Context Building
+
+**User Story:** As a saigen developer, I want to experiment with different RAG approaches for gathering context, so that I can improve the relevance and accuracy of generated saidata.
+
+#### Acceptance Criteria
+
+1. WHEN THE System builds generation context, THE Context Builder SHALL support multiple RAG strategies for retrieving similar packages and saidata
+2. WHEN a RAG strategy is selected, THE System SHALL apply the strategy to gather repository data, similar saidata examples, and sample templates
+3. WHILE building context, THE Context Builder SHALL support configurable parameters for each RAG strategy including similarity thresholds, result limits, and filtering criteria
+4. WHERE multiple RAG strategies are available, THE System SHALL support combining strategies or selecting based on software characteristics
+5. WHEN context building completes, THE System SHALL provide metrics on context quality including relevance scores and data source coverage
+
+### Requirement 3: Prompt Template Management
+
+**User Story:** As a saigen developer, I want to manage prompt templates separately from code, so that I can iterate on prompts without modifying Python files.
+
+#### Acceptance Criteria
+
+1. WHEN THE System initializes, THE Saigen SHALL load prompt templates from external configuration files or a template directory
+2. WHEN a template is loaded, THE System SHALL validate template syntax and required variables
+3. WHILE rendering templates, THE Template Engine SHALL support variable substitution, conditional sections, and template inheritance
+4. WHERE templates reference context data, THE System SHALL provide clear error messages for missing or invalid data
+5. WHEN templates are updated, THE System SHALL support hot-reloading without restarting the application
+
+### Requirement 4: Strategy Configuration System
+
+**User Story:** As a saigen user, I want to configure which prompt and RAG strategies to use, so that I can optimize generation for my specific use cases.
+
+#### Acceptance Criteria
+
+1. WHEN THE System reads configuration, THE Saigen SHALL support strategy selection via configuration files, environment variables, or CLI parameters
+2. WHEN a strategy is configured, THE System SHALL validate strategy names against available implementations
+3. WHILE processing generation requests, THE System SHALL support per-request strategy overrides
+4. WHERE strategy parameters are provided, THE System SHALL validate parameter types and ranges
+5. WHEN configuration is invalid, THE System SHALL provide clear error messages indicating the configuration issue and available options
+
+### Requirement 5: Prompt Strategy Interface
+
+**User Story:** As a saigen developer, I want a clear interface for implementing new prompt strategies, so that I can easily add experimental approaches.
+
+#### Acceptance Criteria
+
+1. WHEN implementing a new strategy, THE Developer SHALL extend a base prompt strategy class with well-defined methods
+2. WHEN a strategy is registered, THE System SHALL verify the strategy implements all required interface methods
+3. WHILE executing a strategy, THE Strategy SHALL receive a complete generation context object with all available data
+4. WHERE a strategy needs custom configuration, THE System SHALL support strategy-specific configuration parameters
+5. WHEN a strategy completes, THE Strategy SHALL return a formatted prompt string ready for LLM submission
+
+### Requirement 6: RAG Strategy Interface
+
+**User Story:** As a saigen developer, I want a clear interface for implementing new RAG strategies, so that I can experiment with different context retrieval approaches.
+
+#### Acceptance Criteria
+
+1. WHEN implementing a new RAG strategy, THE Developer SHALL extend a base RAG strategy class with well-defined methods
+2. WHEN a RAG strategy is executed, THE Strategy SHALL receive software name, target providers, and search parameters
+3. WHILE retrieving context, THE RAG Strategy SHALL support asynchronous operations for efficient data gathering
+4. WHERE multiple data sources are available, THE RAG Strategy SHALL support prioritizing and filtering sources
+5. WHEN context retrieval completes, THE RAG Strategy SHALL return structured context data including packages, saidata examples, and metadata
+
+### Requirement 7: Built-in Strategy Implementations
+
+**User Story:** As a saigen user, I want several pre-built prompt and RAG strategies, so that I can choose effective approaches without custom development.
+
+#### Acceptance Criteria
+
+1. WHEN THE System is installed, THE Saigen SHALL include at least three prompt strategies: minimal, standard, and comprehensive
+2. WHEN THE System is installed, THE Saigen SHALL include at least three RAG strategies: similarity-based, category-based, and hybrid
+3. WHILE using built-in strategies, THE User SHALL access clear documentation describing each strategy's approach and use cases
+4. WHERE built-in strategies have parameters, THE System SHALL provide sensible defaults
+5. WHEN comparing strategies, THE System SHALL support running multiple strategies and comparing results
+
+### Requirement 8: Strategy Performance Metrics
+
+**User Story:** As a saigen developer, I want to track performance metrics for different strategies, so that I can evaluate and compare their effectiveness.
+
+#### Acceptance Criteria
+
+1. WHEN a strategy executes, THE System SHALL record execution time, token usage, and context size
+2. WHEN generation completes, THE System SHALL record validation results, quality scores, and error rates
+3. WHILE tracking metrics, THE System SHALL support aggregating metrics across multiple generations
+4. WHERE metrics are collected, THE System SHALL support exporting metrics for analysis
+5. WHEN comparing strategies, THE System SHALL provide summary statistics and performance comparisons
+
+### Requirement 9: Backward Compatibility
+
+**User Story:** As a saigen user, I want existing functionality to continue working, so that the refactoring doesn't break my current workflows.
+
+#### Acceptance Criteria
+
+1. WHEN THE System is upgraded, THE Saigen SHALL maintain compatibility with existing configuration files
+2. WHEN no strategy is specified, THE System SHALL use default strategies matching current behavior
+3. WHILE processing existing requests, THE System SHALL produce equivalent results to the pre-refactoring implementation
+4. WHERE new features are added, THE System SHALL not require changes to existing code or configurations
+5. WHEN running tests, THE Existing test suite SHALL pass without modifications
+
+### Requirement 10: Strategy Documentation and Examples
+
+**User Story:** As a saigen developer, I want comprehensive documentation and examples for creating strategies, so that I can quickly implement and test new approaches.
+
+#### Acceptance Criteria
+
+1. WHEN THE System is installed, THE Saigen SHALL include documentation describing the strategy architecture and interfaces
+2. WHEN implementing a strategy, THE Developer SHALL have access to example implementations with detailed comments
+3. WHILE developing strategies, THE Developer SHALL have access to testing utilities for validating strategy behavior
+4. WHERE strategies have configuration options, THE Documentation SHALL describe all available parameters and their effects
+5. WHEN troubleshooting strategies, THE System SHALL provide debug logging showing strategy selection and execution details