gmarth/LLMUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplified

Browse Source
This commit is contained in:
Alexander Thiess
2025-09-16 16:51:50 +02:00
parent 6772c1561c
commit 045a9f669a
9 changed files with 732 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
README.md
View File

@@ -1,6 +1,6 @@
# LLMUtils
A Python utility library for managing LLM prompts with Jinja2 template support and JSON schemas.
A Python utility library for managing LLM prompts with Jinja2 templating and automatic schema loading.
## Installation
@@ -19,12 +19,11 @@ pip install git+https://git.project-insanity.de/gmarth/LLMUtils.git
## Features
- **Jinja2 Template Engine**: Full support for loops, conditionals, filters, and complex data structures
- **Smart Prompt Management**: Load and manage prompt templates with variable substitution
- **Smart Prompt Management**: Load and manage prompt templates with automatic schema detection
- **On-demand Loading**: Prompts are loaded lazily at runtime for better performance
- **Caching Support**: Optional caching to avoid repeated disk reads
- **JSON Schema Support**: Associate structured output schemas with prompts
- **Variable Validation**: Automatic validation of required template variables
- **Flexible API**: Fill variables at retrieval or on-demand
- **Automatic Schema Loading**: JSON schemas are automatically loaded when found next to prompt files
- **Flexible Undefined Handling**: Configure how Jinja2 handles missing variables (strict, debug, silent)
## Quick Start
@@ -32,15 +31,18 @@ pip install git+https://git.project-insanity.de/gmarth/LLMUtils.git
```python
from llmutils.prompt_manager import PromptManager
from jinja2 import UndefinedError
# Get a prompt template
result = PromptManager.get_prompt('greeting')
print(result.variables) # See required variables: {'name', 'age'}
print(result.template) # View the template: "Hello {{name}}, you are {{age}} years old"
# Fill the template
filled = result.fill(name='Alice', age=30)
print(filled) # "Hello Alice, you are 30 years old"
# Fill the template - Jinja2 handles missing variables
try:
filled = result.fill(name='Alice', age=30)
print(filled) # "Hello Alice, you are 30 years old"
except UndefinedError as e:
print(f"Missing variable: {e}")
```
### Pre-filling Variables
@@ -51,18 +53,18 @@ result = PromptManager.get_prompt('greeting', name='Alice', age=30)
print(result.prompt) # Already filled: "Hello Alice, you are 30 years old"
```
### Validation
### Handling Optional Variables
Use Jinja2's built-in features for optional variables:
```python
result = PromptManager.get_prompt('greeting')
# Template with optional variables (greeting_flexible.md):
# Hello {{ name | default('Guest') }}!
# {% if age is defined %}You are {{ age }} years old.{% endif %}
# Check if variables are valid
if not result.validate(name='Alice'):
missing = result.get_missing_variables(name='Alice')
print(f"Missing variables: {missing}") # {'age'}
# Fill with all required variables
filled = result.fill(name='Alice', age=30)
result = PromptManager.get_prompt('greeting_flexible')
filled = result.fill(name='Alice') # Works! Age is optional
print(filled) # "Hello Alice!\n"
```
### Advanced Jinja2 Features
@@ -75,8 +77,9 @@ filled = result.fill(
priority='high'
)
# Using conditionals
# Using conditionals for optional variables
result = PromptManager.get_prompt('status_report')
# Template can handle optional variables with {% if variable %}
filled = result.fill(
error='Connection timeout', # Will show error message
items=[] # Will show "No items"
@@ -96,14 +99,35 @@ filled = result.fill(
)
```
### JSON Schema Support
### Configuring Undefined Behavior
```python
# Get prompt with associated schema
result = PromptManager.get_prompt('task_prompt')
from jinja2 import DebugUndefined, Undefined
# Default: StrictUndefined (raises errors on missing variables)
PromptManager.configure()
# Debug mode: shows undefined variable names in output
PromptManager.configure(undefined=DebugUndefined)
result = PromptManager.get_prompt('greeting')
filled = result.fill(name='Alice') # Output: "Hello Alice, you are {{ age }} years old"
# Silent mode: undefined variables become empty strings
PromptManager.configure(undefined=Undefined)
result = PromptManager.get_prompt('greeting')
filled = result.fill(name='Alice') # Output: "Hello Alice, you are years old"
```
### Automatic Schema Loading
Schemas are automatically loaded when a `.json` file exists next to the prompt:
```python
# If you have both greeting.md and greeting.json files:
result = PromptManager.get_prompt('greeting')
if result.schema:
print("This prompt has a structured output schema")
print("Schema automatically loaded!")
print(result.schema) # The JSON schema dictionary
```
@@ -111,11 +135,15 @@ if result.schema:
```python
from pathlib import Path
from jinja2 import DebugUndefined
from llmutils.prompt_manager import PromptManager
# Configure custom prompts directory (default: ./prompts)
PromptManager.configure(path=Path('/custom/prompts/location'))
# Configure undefined variable handling
PromptManager.configure(undefined=DebugUndefined)
# Disable caching for development
PromptManager.configure(caching=False)
@@ -200,25 +228,28 @@ No items to process.
The `ManagedPrompt` dataclass returned by `get_prompt()`:
- `template: str` - The original Jinja2 template string
- `template: str` - The Jinja2 template string (or filled result if pre-filled)
- `name: str` - The prompt name
- `variables: Set[str]` - Required template variables (auto-extracted from Jinja2)
- `schema: Optional[Dict]` - Associated JSON schema
- `prompt: str` - Property that returns filled prompt or template
- `schema: Optional[Dict]` - Associated JSON schema (automatically loaded)
- `prompt: str` - Property that returns the template (backward compatibility)
- `fill(**kwargs) -> str` - Fill template with variables using Jinja2
- `validate(**kwargs) -> bool` - Check if all variables provided
- `get_missing_variables(**kwargs) -> Set[str]` - Get missing variables
### PromptManager Methods
- `get_prompt(prompt_name, **kwargs) -> ManagedPrompt` - Get a prompt template
- `get_schema(prompt_name) -> Optional[Dict]` - Get just the schema
- `has_schema(prompt_name) -> bool` - Check if prompt has schema
- `list_prompts() -> Dict` - List all available prompts
- `get_prompt_info(prompt_name) -> Dict` - Get detailed prompt information
- `configure(path=None, caching=None)` - Configure settings
- `list_prompts() -> Dict[str, Dict[str, Any]]` - List all available prompts with schema info
- `configure(path=None, caching=None, undefined=None)` - Configure settings
- `reload_prompts()` - Clear the cache
### Configuration Options
- `path: Path` - Custom prompts directory (default: `./prompts`)
- `caching: bool` - Enable/disable prompt caching (default: `True`)
- `undefined: type` - Jinja2 undefined behavior:
- `StrictUndefined` (default): Raises error on undefined variables
- `DebugUndefined`: Shows `{{ variable }}` for undefined variables
- `Undefined`: Replaces undefined variables with empty string
### Jinja2 Template Features
The library supports all standard Jinja2 features:

8
character_test.py Normal file
View File

@@ -0,0 +1,8 @@
from src.llmutils import PromptManager
from pprint import pprint
char_prompt = PromptManager().get_prompt('character_sheet', char_name='Jessi', age=15, traits=['cautious', 'pervted', 'addicted'])
print(char_prompt.prompt)
char_prompt = PromptManager().get_prompt('character_sheet', char_name='Jessi', occupation='slut', age=15, traits=['cautious', 'pervted', 'addicted'])
# char_prompt.fill(char_name='Jessi', age='15', traits=['cautious, pervted, addicted'])
print(char_prompt.prompt)

6
prompts/character_sheet.md Normal file
View File

@@ -0,0 +1,6 @@
## {{ char_name }}
Age: {{ age }}{% if occupation is defined %}
Occupation: {{occupation}}{% endif %}
Traits:{% for trait in traits %}
- {{ trait }}{% endfor %}

292
src/llmutils/prompt_manager.py
View File

@@ -1,45 +1,21 @@
import os
import re
import json
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, Any, Set, Optional
import logging
from jinja2 import Template, Environment, meta, TemplateSyntaxError, UndefinedError
from typing import Dict, Any, Optional
from jinja2 import Template, Environment, DebugUndefined, StrictUndefined, TemplateSyntaxError
logger = logging.getLogger(__name__)
@dataclass
class ManagedPrompt:
"""Smart result object that holds template and can fill variables on demand."""
"""Smart prompt that can be filled with variables using Jinja2."""
template: str
name: str
variables: Set[str]
schema: Optional[Dict[str, Any]] = None
_filled_prompt: Optional[str] = field(default=None, init=False, repr=False)
_context: Optional[Dict[str, Any]] = field(default=None, init=False, repr=False)
_jinja_template: Optional[Template] = field(default=None, init=False, repr=False)
def validate(self, **kwargs) -> bool:
"""Validate that all required variables are provided.
Returns:
True if all required variables are present, False otherwise
"""
provided_vars = set(kwargs.keys())
missing_vars = self.variables - provided_vars
return len(missing_vars) == 0
def get_missing_variables(self, **kwargs) -> Set[str]:
"""Get the set of missing required variables.
Returns:
Set of variable names that are required but not provided
"""
provided_vars = set(kwargs.keys())
return self.variables - provided_vars
def fill(self, **kwargs) -> str:
"""Fill the template with provided variables using Jinja2.
@@ -50,60 +26,26 @@ class ManagedPrompt:
The filled prompt string
Raises:
ValueError: If required variables are missing or template syntax error
UndefinedError: If required variables are missing (Jinja2's default behavior)
"""
# Create Jinja2 template if not already created
if self._jinja_template is None:
try:
self._jinja_template = Template(self.template)
except TemplateSyntaxError as e:
raise ValueError(f"Invalid template syntax in prompt '{self.name}': {e}")
self._jinja_template = Template(self.template)
# If no variables required and none provided, return template as-is
if not self.variables and not kwargs:
self._filled_prompt = self.template
self._context = {}
return self.template
missing_vars = self.get_missing_variables(**kwargs)
if missing_vars:
raise ValueError(
f"Missing required variables for prompt '{self.name}': {missing_vars}. "
f"Required: {self.variables}, Provided: {set(kwargs.keys())}"
)
try:
# Render the template with Jinja2
result = self._jinja_template.render(**kwargs)
except UndefinedError as e:
raise ValueError(f"Error rendering template '{self.name}': {e}")
# Cache the filled result
self._filled_prompt = result
self._context = kwargs
return result
# Render the template - Jinja2 handles undefined variables
return self._jinja_template.render(**kwargs)
@property
def prompt(self) -> str:
"""Get the filled prompt if available, otherwise return the template.
This property provides backward compatibility.
"""
return self._filled_prompt if self._filled_prompt else self.template
def __str__(self) -> str:
"""String representation returns the filled prompt or template."""
return self.prompt
"""Get the template for backward compatibility."""
return self.template
class PromptManager:
"""Singleton class to manage prompt templates and JSON schemas"""
"""Singleton class to manage prompt templates with automatic schema loading"""
_instance: Optional['PromptManager'] = None
_initialized: bool = False
_prompt_path: Path
_caching: bool
def __new__(cls):
if cls._instance is None:
@@ -114,12 +56,13 @@ class PromptManager:
if not self._initialized:
self.prompts: Dict[str, str] = {}
self.schemas: Dict[str, Dict[str, Any]] = {}
self.prompt_variables: Dict[str, Set[str]] = {}
self._caching = True # Enable caching by default
self._caching = True
self._prompt_path = Path.cwd() / 'prompts'
self._undefined_mode = StrictUndefined # Default to strict
PromptManager._initialized = True
def _load_prompt(self, prompt_name: str) -> bool:
"""Load a specific prompt and its schema on-demand.
"""Load a specific prompt and its schema if it exists.
Args:
prompt_name: Name of the prompt to load
@@ -131,7 +74,7 @@ class PromptManager:
if self._caching and prompt_name in self.prompts:
return True
prompts_dir = self._get_path()
prompts_dir = self._prompt_path
if not prompts_dir.exists():
logger.warning(f"Prompts directory not found: {prompts_dir}")
@@ -148,25 +91,20 @@ class PromptManager:
with open(md_file, 'r', encoding='utf-8') as f:
content = f.read().strip()
# Extract variables from {{variable}} patterns
variables = self._extract_variables(content)
# Store in cache
self.prompts[prompt_name] = content
self.prompt_variables[prompt_name] = variables
# Look for corresponding JSON schema file
# Automatically look for JSON schema next to the prompt
schema_file = md_file.with_suffix('.json')
if schema_file.exists():
try:
with open(schema_file, 'r', encoding='utf-8') as f:
schema = json.load(f)
self.schemas[prompt_name] = schema
logger.debug(f"Loaded prompt '{prompt_name}' with schema and variables: {variables}")
logger.debug(f"Loaded prompt '{prompt_name}' with schema")
except json.JSONDecodeError as e:
logger.error(f"Invalid JSON schema in {schema_file}: {e}")
else:
logger.debug(f"Loaded prompt '{prompt_name}' (no schema) with variables: {variables}")
logger.debug(f"Loaded prompt '{prompt_name}' (no schema)")
return True
@@ -174,108 +112,56 @@ class PromptManager:
logger.error(f"Error loading prompt file {md_file}: {e}")
return False
def _get_path(self) -> Path:
"""Get the prompts directory path.
Returns the configured path if set via configure(),
otherwise defaults to 'prompts' in the current working directory.
"""
if hasattr(self, '_prompt_path') and self._prompt_path:
return self._prompt_path
# Default to 'prompts' directory in the current working directory
return Path.cwd() / 'prompts'
def _extract_variables(self, template: str) -> Set[str]:
"""Extract all variables from Jinja2 template"""
try:
# Create a Jinja2 environment and parse the template
env = Environment()
ast = env.parse(template)
# Get all undeclared variables from the template
variables = meta.find_undeclared_variables(ast)
return variables
except TemplateSyntaxError:
# Fallback to simple regex for backwards compatibility
pattern = r'\{\{\s*(\w+)\s*\}\}'
variables = set(re.findall(pattern, template))
return variables
def _validate_context(self, prompt_name: str, context: Dict[str, Any]) -> None:
"""Validate that all required variables are provided"""
if prompt_name not in self.prompt_variables:
raise ValueError(f"Unknown prompt: '{prompt_name}'")
required_vars = self.prompt_variables[prompt_name]
provided_vars = set(context.keys())
missing_vars = required_vars - provided_vars
if missing_vars:
raise ValueError(
f"Missing required variables for prompt '{prompt_name}': {missing_vars}. "
f"Required: {required_vars}, Provided: {provided_vars}"
)
# Warn about extra variables (not an error, but might indicate mistakes)
extra_vars = provided_vars - required_vars
if extra_vars:
logger.warning(f"Extra variables provided for prompt '{prompt_name}': {extra_vars}")
def _fill_template(self, template: str, context: Dict[str, Any]) -> str:
"""Fill template with context variables using Jinja2"""
try:
jinja_template = Template(template)
return jinja_template.render(**context)
except (TemplateSyntaxError, UndefinedError) as e:
raise ValueError(f"Error rendering template: {e}")
@classmethod
def configure(cls, path: Optional[Path] = None, caching: Optional[bool] = None):
def configure(cls, path: Optional[Path] = None, caching: Optional[bool] = None,
undefined: Optional[type] = None):
"""Configure the PromptManager settings.
Args:
path: Custom path to prompts directory
caching: Whether to cache loaded prompts (default: True)
undefined: Jinja2 undefined behavior (StrictUndefined, DebugUndefined, etc.)
"""
instance = cls()
if path is not None:
instance._prompt_path = path
# Clear cache when path changes
instance.prompts.clear()
instance.schemas.clear()
instance.prompt_variables.clear()
if caching is not None:
instance._caching = caching
# If disabling cache, clear existing cached prompts
if not caching:
instance.prompts.clear()
instance.schemas.clear()
instance.prompt_variables.clear()
if undefined is not None:
instance._undefined_mode = undefined
@classmethod
def get_prompt(cls, prompt_name: str, **kwargs) -> ManagedPrompt:
"""
Get a PromptResult that can be filled with variables.
"""Get a ManagedPrompt that can be filled with variables.
Args:
prompt_name: Name of the prompt template (filename without .md)
**kwargs: Optional variables to pre-fill the template
Returns:
PromptResult object with smart fill/validate methods
ManagedPrompt object
Raises:
ValueError: If prompt doesn't exist
UndefinedError: If pre-filling with missing required variables
Examples:
# Get unfilled template
result = PromptManager.get_prompt('greeting')
print(result.variables) # See required variables
filled = result.fill(name='Alice', age=30)
# Or pre-fill on retrieval
# Pre-fill on retrieval
result = PromptManager.get_prompt('greeting', name='Alice', age=30)
print(result.prompt) # Already filled
print(result.prompt) # Already filled if all vars provided
"""
instance = cls()
@@ -283,105 +169,55 @@ class PromptManager:
if not instance._load_prompt(prompt_name):
raise ValueError(f"Prompt '{prompt_name}' not found")
# Get template, variables and schema
# Get template and schema
template = instance.prompts[prompt_name]
variables = instance.prompt_variables.get(prompt_name, set())
schema = instance.schemas.get(prompt_name)
# Create the result object
# Create the result object with custom undefined mode
env = Environment(undefined=instance._undefined_mode)
result = ManagedPrompt(
template=template,
name=prompt_name,
variables=variables,
schema=schema
)
# If kwargs provided, pre-fill the template
# Create template with configured undefined behavior
result._jinja_template = env.from_string(template)
# If kwargs provided, try to pre-fill
if kwargs:
try:
result.fill(**kwargs)
except ValueError:
# If validation fails, return unfilled result and let user handle
pass
filled = result.fill(**kwargs)
# Store the filled result
result.template = filled
# If caching is disabled, clear the prompt after use
if not instance._caching:
del instance.prompts[prompt_name]
del instance.prompt_variables[prompt_name]
if prompt_name in instance.schemas:
del instance.schemas[prompt_name]
return result
@classmethod
def get_schema(cls, prompt_name: str) -> Optional[Dict[str, Any]]:
"""
Get the JSON schema for a prompt if it exists
Args:
prompt_name: Name of the prompt template
Returns:
JSON schema dictionary or None if no schema exists
"""
instance = cls()
# Try to load the prompt if not already loaded
if not instance._load_prompt(prompt_name):
raise ValueError(f"Prompt '{prompt_name}' not found")
return instance.schemas.get(prompt_name)
@classmethod
def has_schema(cls, prompt_name: str) -> bool:
"""Check if a prompt has a JSON schema"""
instance = cls()
# Try to load the prompt if not already loaded
instance._load_prompt(prompt_name)
return prompt_name in instance.schemas
@classmethod
def get_prompt_with_schema(cls, prompt_name: str, **kwargs) -> ManagedPrompt:
"""
Get both the processed prompt and its schema (if available)
This is now just an alias for get_prompt() since it returns PromptResult.
Kept for backward compatibility.
Args:
prompt_name: Name of the prompt template
**kwargs: Variables to fill in the template
Returns:
PromptResult object containing prompt, schema, variables, and name
"""
return cls.get_prompt(prompt_name, **kwargs)
@classmethod
def list_prompts(cls) -> Dict[str, Dict[str, Any]]:
"""
List all available prompts with their info
"""List all available prompts with their info.
Returns:
Dictionary mapping prompt names to their info (variables, has_schema)
Dictionary mapping prompt names to their info (has_schema)
"""
instance = cls()
prompts_dir = instance._get_path()
prompts_dir = instance._prompt_path
result = {}
if not prompts_dir.exists():
return result
# Scan for all .md files in the prompts directory
for md_file in prompts_dir.glob("*.md"):
prompt_name = md_file.stem
# Load prompt to get its details
if instance._load_prompt(prompt_name):
result[prompt_name] = {
'variables': instance.prompt_variables[prompt_name],
'has_schema': prompt_name in instance.schemas,
'variable_count': len(instance.prompt_variables[prompt_name])
}
schema_file = md_file.with_suffix('.json')
result[prompt_name] = {
'has_schema': schema_file.exists()
}
return result
@@ -391,32 +227,4 @@ class PromptManager:
if cls._instance:
cls._instance.prompts.clear()
cls._instance.schemas.clear()
cls._instance.prompt_variables.clear()
logger.info("Prompt cache cleared")
@classmethod
def get_prompt_info(cls, prompt_name: str) -> Dict[str, Any]:
"""
Get detailed information about a specific prompt
Returns:
Dictionary with prompt template, schema, and required variables
"""
instance = cls()
# Try to load the prompt if not already loaded
if not instance._load_prompt(prompt_name):
raise ValueError(f"Prompt '{prompt_name}' not found")
info = {
'name': prompt_name,
'template': instance.prompts[prompt_name],
'variables': instance.prompt_variables[prompt_name],
'variable_count': len(instance.prompt_variables[prompt_name]),
'has_schema': prompt_name in instance.schemas
}
if prompt_name in instance.schemas:
info['schema'] = instance.schemas[prompt_name]
return info
logger.info("Prompt cache cleared")

143
test_jinja2_analysis.py Normal file
View File

@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Explore Jinja2's AST to understand variable detection in conditionals"""
from jinja2 import Environment, meta, nodes
# Test different template patterns
templates = [
# Simple required variable
("simple", "Hello {{ name }}!"),
# Variable in conditional (optional)
("conditional", """{% if error %}
Error: {{ error }}
{% endif %}"""),
# Mixed required and optional
("mixed", """Status: {{ status }}
{% if error %}
Error: {{ error }}
{% endif %}"""),
# Nested conditionals
("nested", """{% if user %}
Name: {{ user.name }}
{% if user.email %}
Email: {{ user.email }}
{% endif %}
{% endif %}"""),
# Variable with default filter (makes it optional)
("default_filter", "Hello {{ name | default('Guest') }}!"),
# Variable in for loop
("for_loop", """{% for item in items %}
- {{ item }}
{% endfor %}"""),
# Complex with multiple contexts
("complex", """Required: {{ required }}
{% if optional1 %}
Optional1: {{ optional1 }}
{% endif %}
Default: {{ optional2 | default('N/A') }}
{% for item in items %}
- {{ item }}
{% endfor %}""")
]
env = Environment()
print("=" * 70)
print("Analyzing Jinja2 AST for Variable Detection")
print("=" * 70)
for name, template in templates:
print(f"\n{name.upper()} TEMPLATE:")
print("-" * 40)
print(template)
print("-" * 40)
# Parse the template
ast = env.parse(template)
# Get undeclared variables (what we currently use)
undeclared = meta.find_undeclared_variables(ast)
print(f"Undeclared variables: {undeclared}")
# Try to analyze the AST more deeply
print("\nAST Analysis:")
def analyze_node(node, indent=0, context="root"):
"""Recursively analyze AST nodes"""
prefix = " " * indent
if isinstance(node, nodes.Name):
print(f"{prefix}Variable '{node.name}' in context: {context}")
elif isinstance(node, nodes.If):
print(f"{prefix}If block:")
print(f"{prefix} Test expression:")
analyze_node(node.test, indent + 2, "if_test")
print(f"{prefix} Body:")
for child in node.body:
analyze_node(child, indent + 2, "if_body")
if node.elif_:
print(f"{prefix} Elif:")
for child in node.elif_:
analyze_node(child, indent + 2, "elif")
if node.else_:
print(f"{prefix} Else:")
for child in node.else_:
analyze_node(child, indent + 2, "else")
elif isinstance(node, nodes.For):
print(f"{prefix}For loop:")
print(f"{prefix} Target: {node.target}")
print(f"{prefix} Iter:")
analyze_node(node.iter, indent + 2, "for_iter")
print(f"{prefix} Body:")
for child in node.body:
analyze_node(child, indent + 2, "for_body")
elif isinstance(node, nodes.Filter):
print(f"{prefix}Filter: {node.name}")
if node.name == "default":
print(f"{prefix} (Makes variable optional!)")
analyze_node(node.node, indent + 1, f"filter_{node.name}")
elif isinstance(node, nodes.Getattr):
print(f"{prefix}Attribute access: .{node.attr}")
analyze_node(node.node, indent + 1, "getattr")
elif isinstance(node, nodes.Output):
print(f"{prefix}Output:")
for child in node.nodes:
analyze_node(child, indent + 1, "output")
elif isinstance(node, nodes.TemplateData):
# Skip raw text
pass
else:
# Recurse into child nodes
for child in node.iter_child_nodes():
analyze_node(child, indent, context)
analyze_node(ast)
print("\n" + "=" * 70)
print("INSIGHTS:")
print("-" * 70)
print("""
1. Jinja2's `find_undeclared_variables()` returns ALL variables, regardless of context
2. Variables in {% if %} conditions could be considered optional
3. Variables with | default() filter are definitely optional
4. We can walk the AST to identify context for each variable
Possible solution:
- Walk the AST to categorize variables by context
- Mark variables in if_test contexts as optional (unless also used elsewhere)
- Mark variables with default filter as optional
- Consider all others as required
""")

119
test_optional_vars.py Normal file
View File

@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""Test optional variables with Jinja2 conditionals"""
from pathlib import Path
from src.llmutils.prompt_manager import PromptManager
# Create test prompts directory
test_dir = Path("test_prompts")
test_dir.mkdir(exist_ok=True)
# Create a template with required and optional variables
report_template = test_dir / "report.md"
report_template.write_text("""# Status Report
Status: {{ status }}
{% if error %}
⚠️ ERROR: {{ error }}
{% endif %}
{% if warning %}
⚠️ WARNING: {{ warning }}
{% endif %}
{% if details %}
## Details
{{ details }}
{% endif %}
Generated at: {{ timestamp }}""")
# Configure PromptManager
PromptManager.configure(path=test_dir)
print("=" * 60)
print("Testing Optional Variables with Jinja2 Conditionals")
print("=" * 60)
# Test 1: All variables provided
print("\n1. All variables provided:")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
filled = prompt.fill(
status="Operational",
error="Database connection failed",
warning="High memory usage",
details="System recovered after 5 retries",
timestamp="2024-01-15 10:30:00"
)
print(filled)
# Test 2: Only required variables (status and timestamp)
print("\n2. Only truly required variables (will fail with strict=True):")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
try:
# This will fail because Jinja2 considers all variables as required
filled = prompt.fill(
status="Operational",
timestamp="2024-01-15 10:30:00"
)
print(filled)
except ValueError as e:
print(f"Error (expected): Missing variables {{'error', 'warning', 'details'}}")
print(f"Actual error: {e}")
# Test 3: Provide None/empty for optional variables
print("\n3. Provide None/empty values for optional variables:")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
filled = prompt.fill(
status="Operational",
error=None, # Will not show in output due to {% if error %}
warning="", # Empty string is falsy, won't show
details=None, # Will not show
timestamp="2024-01-15 10:30:00"
)
print(filled)
# Test 4: Mix of provided and None values
print("\n4. Mix of provided and None values:")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
filled = prompt.fill(
status="Degraded",
error="API timeout",
warning=None, # Won't show
details=None, # Won't show
timestamp="2024-01-15 10:35:00"
)
print(filled)
# Test 5: Using strict=False to handle missing variables
print("\n5. Using strict=False for partial filling:")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
filled = prompt.fill(
strict=False,
status="Unknown",
timestamp="2024-01-15 10:40:00"
)
print(filled)
print("Note: Unfilled variables remain as {{ variable }}")
# Test 6: Show extracted variables
print("\n6. Extracted variables from template:")
print("-" * 40)
prompt = PromptManager.get_prompt("report")
print(f"All variables detected: {prompt.variables}")
print("Note: Jinja2 extracts all variables, even those in conditionals")
print("Use None/empty values for optional ones, or use strict=False")
print("\n" + "=" * 60)
print("Tests completed!")
print("=" * 60)
# Cleanup
import shutil
shutil.rmtree(test_dir)

130
test_simplified.py Normal file
View File

@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Test the simplified prompt manager that relies on Jinja2's built-in handling"""
from pathlib import Path
from jinja2 import UndefinedError, DebugUndefined
from src.llmutils.prompt_manager_simple import PromptManager
# Create test prompts directory
test_dir = Path("test_prompts")
test_dir.mkdir(exist_ok=True)
# Create templates
greeting = test_dir / "greeting.md"
greeting.write_text("Hello {{ name }}, you are {{ age }} years old!")
# Template with optional variables using default filter
flexible = test_dir / "flexible.md"
flexible.write_text("""# Report for {{ title }}
Status: {{ status | default('Unknown') }}
{% if error is defined %}
Error: {{ error }}
{% endif %}
Author: {{ author | default('System') }}""")
# Create a schema file
schema_file = test_dir / "flexible.json"
schema_file.write_text("""{
"type": "object",
"properties": {
"title": {"type": "string"},
"status": {"type": "string"},
"error": {"type": "string"},
"author": {"type": "string"}
},
"required": ["title"]
}""")
# Configure PromptManager
PromptManager.configure(path=test_dir)
print("=" * 70)
print("Testing Simplified Prompt Manager")
print("=" * 70)
# Test 1: Normal usage with all variables
print("\n1. Normal usage with all variables:")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting")
filled = prompt.fill(name="Alice", age=30)
print(filled)
# Test 2: Missing required variable (will raise error)
print("\n2. Missing required variable (should raise error):")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting")
try:
filled = prompt.fill(name="Bob") # Missing 'age'
print(filled)
except UndefinedError as e:
print(f"✓ Error raised as expected: {e}")
# Test 3: Template with defaults and conditionals
print("\n3. Template with optional variables (using defaults and conditionals):")
print("-" * 40)
prompt = PromptManager.get_prompt("flexible")
filled = prompt.fill(title="Daily Report") # Only required variable
print(filled)
# Test 4: Same template with all variables
print("\n4. Same template with all variables provided:")
print("-" * 40)
prompt = PromptManager.get_prompt("flexible")
filled = prompt.fill(
title="Error Report",
status="Failed",
error="Database connection timeout",
author="Admin"
)
print(filled)
# Test 5: Pre-fill on retrieval
print("\n5. Pre-fill on retrieval:")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting", name="Charlie", age=25)
print(prompt.prompt) # Already filled
# Test 6: Schema is automatically loaded
print("\n6. Schema automatically loaded:")
print("-" * 40)
prompt = PromptManager.get_prompt("flexible")
if prompt.schema:
print(f"✓ Schema found: {prompt.schema}")
else:
print("✗ No schema")
# Test 7: Configure with DebugUndefined
print("\n7. Using DebugUndefined mode:")
print("-" * 40)
PromptManager.configure(undefined=DebugUndefined)
PromptManager.reload_prompts() # Clear cache
prompt = PromptManager.get_prompt("greeting")
filled = prompt.fill(name="Debug") # Missing 'age' - will show as undefined
print(filled)
# Test 8: List available prompts
print("\n8. List available prompts:")
print("-" * 40)
prompts = PromptManager.list_prompts()
for name, info in prompts.items():
print(f"- {name}: {'has schema' if info['has_schema'] else 'no schema'}")
print("\n" + "=" * 70)
print("Benefits of this simplified approach:")
print("-" * 70)
print("✓ Jinja2 handles undefined variables for us")
print("✓ No need for complex variable extraction and validation")
print("✓ Users can choose undefined behavior (strict, debug, silent)")
print("✓ Templates can use Jinja2's built-in features for optional vars:")
print(" - {{ var | default('value') }}")
print(" - {% if var is defined %}")
print("✓ Schemas are automatically loaded if present")
print("✓ Much simpler and cleaner codebase")
print("=" * 70)
# Cleanup
import shutil
shutil.rmtree(test_dir)

125
test_smart_variables.py Normal file
View File

@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Test the smart variable detection that distinguishes required from optional"""
from pathlib import Path
from src.llmutils.prompt_manager import PromptManager
# Create test prompts directory
test_dir = Path("test_prompts")
test_dir.mkdir(exist_ok=True)
# Test 1: Simple required variable
simple = test_dir / "simple.md"
simple.write_text("Hello {{ name }}!")
# Test 2: Optional variable in conditional
conditional = test_dir / "conditional.md"
conditional.write_text("""Status: OK
{% if error %}
Error: {{ error }}
{% endif %}""")
# Test 3: Variable with default filter
default_filter = test_dir / "default.md"
default_filter.write_text("""Hello {{ name | default('Guest') }}!
Age: {{ age }}""")
# Test 4: Complex mix
complex_template = test_dir / "complex.md"
complex_template.write_text("""# Report for {{ title }}
Status: {{ status }}
{% if error %}
⚠️ ERROR: {{ error }}
{% endif %}
{% if warning %}
⚠️ WARNING: {{ warning }}
{% endif %}
Debug Level: {{ debug_level | default(0) }}
{% for item in items %}
- {{ item }}
{% endfor %}
Generated by: {{ author | default('System') }}""")
# Configure PromptManager
PromptManager.configure(path=test_dir)
print("=" * 70)
print("Testing Smart Variable Detection (Required vs Optional)")
print("=" * 70)
# Test each template
test_cases = [
("simple", {"name": "Alice"}),
("conditional", {"error": "Timeout"}),
("default", {"age": 25}),
("complex", {"title": "Daily Report", "status": "OK", "items": ["task1", "task2"]})
]
for prompt_name, test_data in test_cases:
print(f"\n{'='*50}")
print(f"TESTING: {prompt_name}")
print("=" * 50)
prompt = PromptManager.get_prompt(prompt_name)
print(f"\nAll variables: {prompt.variables}")
print(f"Required variables: {prompt.required_variables}")
print(f"Optional variables: {prompt.optional_variables}")
print(f"\nProviding: {test_data}")
# Test validation
is_valid = prompt.validate(**test_data)
missing = prompt.get_missing_variables(**test_data)
print(f"Valid: {is_valid}")
if missing:
print(f"Missing required: {missing}")
# Try to fill
try:
filled = prompt.fill(**test_data)
print("\nFilled successfully (showing first 200 chars):")
print(filled[:200])
except ValueError as e:
print(f"\nError filling: {e}")
# Test 5: Demonstrate that optional variables don't need to be provided
print("\n" + "=" * 70)
print("DEMONSTRATING OPTIONAL VARIABLES")
print("=" * 70)
prompt = PromptManager.get_prompt("complex")
print(f"\nRequired: {prompt.required_variables}")
print(f"Optional: {prompt.optional_variables}")
# Provide only required variables
minimal_data = {
"title": "Test Report",
"status": "Running",
"items": ["item1", "item2", "item3"]
}
print(f"\nProviding only required variables: {minimal_data}")
try:
filled = prompt.fill(**minimal_data)
print("\n✓ Filled successfully with only required variables!")
print("\nOutput:")
print("-" * 40)
print(filled)
except ValueError as e:
print(f"\n✗ Error: {e}")
print("\n" + "=" * 70)
print("Tests completed!")
print("=" * 70)
# Cleanup
import shutil
shutil.rmtree(test_dir)

86
test_strict_mode.py Normal file
View File

@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Test the new strict mode for error handling"""
from pathlib import Path
from src.llmutils.prompt_manager import PromptManager
# Create test prompts directory
test_dir = Path("test_prompts")
test_dir.mkdir(exist_ok=True)
# Create a test prompt
prompt_file = test_dir / "greeting.md"
prompt_file.write_text("Hello {{name}}, you are {{age}} years old!")
# Configure PromptManager
PromptManager.configure(path=test_dir)
print("=" * 60)
print("Testing Strict Mode for Error Handling")
print("=" * 60)
# Test 1: Default behavior with fill() - should raise error
print("\n1. Direct fill() with missing variables (default strict=True):")
print("-" * 40)
try:
prompt = PromptManager.get_prompt("greeting")
filled = prompt.fill(name="Alice") # Missing 'age'
print(f"Result: {filled}")
except ValueError as e:
print(f"✓ Error raised as expected: {e}")
# Test 2: Non-strict mode with fill()
print("\n2. Direct fill() with strict=False:")
print("-" * 40)
try:
prompt = PromptManager.get_prompt("greeting")
filled = prompt.fill(strict=False, name="Alice") # Missing 'age'
print(f"✓ Result (unfilled template): {filled}")
except ValueError as e:
print(f"Unexpected error: {e}")
# Test 3: get_prompt with pre-fill (default non-strict)
print("\n3. get_prompt() with pre-fill (default strict=False):")
print("-" * 40)
try:
prompt = PromptManager.get_prompt("greeting", name="Alice") # Missing 'age'
print(f"✓ Result (unfilled template): {prompt.prompt}")
except ValueError as e:
print(f"Unexpected error: {e}")
# Test 4: get_prompt with pre-fill in strict mode
print("\n4. get_prompt() with pre-fill (strict=True):")
print("-" * 40)
try:
prompt = PromptManager.get_prompt("greeting", strict=True, name="Alice") # Missing 'age'
print(f"Result: {prompt.prompt}")
except ValueError as e:
print(f"✓ Error raised as expected: {e}")
# Test 5: Successful fill with all variables
print("\n5. Successful fill with all variables:")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting")
filled = prompt.fill(name="Bob", age=25)
print(f"✓ Result: {filled}")
# Test 6: Successful pre-fill with all variables
print("\n6. Successful pre-fill on retrieval:")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting", name="Charlie", age=30)
print(f"✓ Result: {prompt.prompt}")
# Test 7: Check that variables are correctly identified
print("\n7. Variable extraction:")
print("-" * 40)
prompt = PromptManager.get_prompt("greeting")
print(f"Required variables: {prompt.variables}")
print(f"Missing when only 'name' provided: {prompt.get_missing_variables(name='Test')}")
print("\n" + "=" * 60)
print("All tests completed!")
print("=" * 60)
# Cleanup
import shutil
shutil.rmtree(test_dir)