simplified

README.md

@@ -1,6 +1,6 @@
 # LLMUtils
 
-A Python utility library for managing LLM prompts with Jinja2 template support and JSON schemas.
+A Python utility library for managing LLM prompts with Jinja2 templating and automatic schema loading.
 
 ## Installation
 
@@ -19,12 +19,11 @@ pip install git+https://git.project-insanity.de/gmarth/LLMUtils.git
 ## Features
 
 - **Jinja2 Template Engine**: Full support for loops, conditionals, filters, and complex data structures
-- **Smart Prompt Management**: Load and manage prompt templates with variable substitution
+- **Smart Prompt Management**: Load and manage prompt templates with automatic schema detection
 - **On-demand Loading**: Prompts are loaded lazily at runtime for better performance
 - **Caching Support**: Optional caching to avoid repeated disk reads
-- **JSON Schema Support**: Associate structured output schemas with prompts
+- **Automatic Schema Loading**: JSON schemas are automatically loaded when found next to prompt files
-- **Variable Validation**: Automatic validation of required template variables
+- **Flexible Undefined Handling**: Configure how Jinja2 handles missing variables (strict, debug, silent)
-- **Flexible API**: Fill variables at retrieval or on-demand
 
 ## Quick Start
 
@@ -32,15 +31,18 @@ pip install git+https://git.project-insanity.de/gmarth/LLMUtils.git
 
 ```python
 from llmutils.prompt_manager import PromptManager
+from jinja2 import UndefinedError
 
 # Get a prompt template
 result = PromptManager.get_prompt('greeting')
-print(result.variables)  # See required variables: {'name', 'age'}
 print(result.template)   # View the template: "Hello {{name}}, you are {{age}} years old"
 
-# Fill the template
+# Fill the template - Jinja2 handles missing variables
+try:
     filled = result.fill(name='Alice', age=30)
     print(filled)  # "Hello Alice, you are 30 years old"
+except UndefinedError as e:
+    print(f"Missing variable: {e}")
 ```
 
 ### Pre-filling Variables
@@ -51,18 +53,18 @@ result = PromptManager.get_prompt('greeting', name='Alice', age=30)
 print(result.prompt)  # Already filled: "Hello Alice, you are 30 years old"
 ```
 
-### Validation
+### Handling Optional Variables
+
+Use Jinja2's built-in features for optional variables:
 
 ```python
-result = PromptManager.get_prompt('greeting')
+# Template with optional variables (greeting_flexible.md):
+# Hello {{ name | default('Guest') }}!
+# {% if age is defined %}You are {{ age }} years old.{% endif %}
 
-# Check if variables are valid
+result = PromptManager.get_prompt('greeting_flexible')
-if not result.validate(name='Alice'):
+filled = result.fill(name='Alice')  # Works! Age is optional
-    missing = result.get_missing_variables(name='Alice')
+print(filled)  # "Hello Alice!\n"
-    print(f"Missing variables: {missing}")  # {'age'}
-
-# Fill with all required variables
-filled = result.fill(name='Alice', age=30)
 ```
 
 ### Advanced Jinja2 Features
@@ -75,8 +77,9 @@ filled = result.fill(
     priority='high'
 )
 
-# Using conditionals
+# Using conditionals for optional variables
 result = PromptManager.get_prompt('status_report')
+# Template can handle optional variables with {% if variable %}
 filled = result.fill(
     error='Connection timeout',  # Will show error message
     items=[]  # Will show "No items"
@@ -96,14 +99,35 @@ filled = result.fill(
 )
 ```
 
-### JSON Schema Support
+### Configuring Undefined Behavior
 
 ```python
-# Get prompt with associated schema
+from jinja2 import DebugUndefined, Undefined
-result = PromptManager.get_prompt('task_prompt')
+
+# Default: StrictUndefined (raises errors on missing variables)
+PromptManager.configure()
+
+# Debug mode: shows undefined variable names in output
+PromptManager.configure(undefined=DebugUndefined)
+result = PromptManager.get_prompt('greeting')
+filled = result.fill(name='Alice')  # Output: "Hello Alice, you are {{ age }} years old"
+
+# Silent mode: undefined variables become empty strings
+PromptManager.configure(undefined=Undefined)
+result = PromptManager.get_prompt('greeting')
+filled = result.fill(name='Alice')  # Output: "Hello Alice, you are  years old"
+```
+
+### Automatic Schema Loading
+
+Schemas are automatically loaded when a `.json` file exists next to the prompt:
+
+```python
+# If you have both greeting.md and greeting.json files:
+result = PromptManager.get_prompt('greeting')
 
 if result.schema:
-    print("This prompt has a structured output schema")
+    print("Schema automatically loaded!")
     print(result.schema)  # The JSON schema dictionary
 ```
 
@@ -111,11 +135,15 @@ if result.schema:
 
 ```python
 from pathlib import Path
+from jinja2 import DebugUndefined
 from llmutils.prompt_manager import PromptManager
 
 # Configure custom prompts directory (default: ./prompts)
 PromptManager.configure(path=Path('/custom/prompts/location'))
 
+# Configure undefined variable handling
+PromptManager.configure(undefined=DebugUndefined)
+
 # Disable caching for development
 PromptManager.configure(caching=False)
 
@@ -200,25 +228,28 @@ No items to process.
 
 The `ManagedPrompt` dataclass returned by `get_prompt()`:
 
-- `template: str` - The original Jinja2 template string
+- `template: str` - The Jinja2 template string (or filled result if pre-filled)
 - `name: str` - The prompt name
-- `variables: Set[str]` - Required template variables (auto-extracted from Jinja2)
+- `schema: Optional[Dict]` - Associated JSON schema (automatically loaded)
-- `schema: Optional[Dict]` - Associated JSON schema
+- `prompt: str` - Property that returns the template (backward compatibility)
-- `prompt: str` - Property that returns filled prompt or template
 - `fill(**kwargs) -> str` - Fill template with variables using Jinja2
-- `validate(**kwargs) -> bool` - Check if all variables provided
-- `get_missing_variables(**kwargs) -> Set[str]` - Get missing variables
 
 ### PromptManager Methods
 
 - `get_prompt(prompt_name, **kwargs) -> ManagedPrompt` - Get a prompt template
-- `get_schema(prompt_name) -> Optional[Dict]` - Get just the schema
+- `list_prompts() -> Dict[str, Dict[str, Any]]` - List all available prompts with schema info
-- `has_schema(prompt_name) -> bool` - Check if prompt has schema
+- `configure(path=None, caching=None, undefined=None)` - Configure settings
-- `list_prompts() -> Dict` - List all available prompts
-- `get_prompt_info(prompt_name) -> Dict` - Get detailed prompt information
-- `configure(path=None, caching=None)` - Configure settings
 - `reload_prompts()` - Clear the cache
 
+### Configuration Options
+
+- `path: Path` - Custom prompts directory (default: `./prompts`)
+- `caching: bool` - Enable/disable prompt caching (default: `True`)
+- `undefined: type` - Jinja2 undefined behavior:
+  - `StrictUndefined` (default): Raises error on undefined variables
+  - `DebugUndefined`: Shows `{{ variable }}` for undefined variables
+  - `Undefined`: Replaces undefined variables with empty string
+
 ### Jinja2 Template Features
 
 The library supports all standard Jinja2 features:

character_test.py

@@ -0,0 +1,8 @@
+from src.llmutils import PromptManager
+from pprint import pprint
+
+char_prompt = PromptManager().get_prompt('character_sheet', char_name='Jessi', age=15, traits=['cautious', 'pervted', 'addicted'])
+print(char_prompt.prompt)
+char_prompt = PromptManager().get_prompt('character_sheet', char_name='Jessi', occupation='slut', age=15, traits=['cautious', 'pervted', 'addicted'])
+# char_prompt.fill(char_name='Jessi', age='15', traits=['cautious, pervted, addicted'])
+print(char_prompt.prompt)

prompts/character_sheet.md

@@ -0,0 +1,6 @@
+## {{ char_name }}
+
+Age: {{ age }}{% if occupation is defined %}
+Occupation: {{occupation}}{% endif %}
+Traits:{% for trait in traits %}
+- {{ trait }}{% endfor %}

src/llmutils/prompt_manager.py

@@ -1,45 +1,21 @@
-import os
-import re
 import json
+import logging
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, Any, Set, Optional
+from typing import Dict, Any, Optional
-import logging
+from jinja2 import Template, Environment, DebugUndefined, StrictUndefined, TemplateSyntaxError
-from jinja2 import Template, Environment, meta, TemplateSyntaxError, UndefinedError
 
 logger = logging.getLogger(__name__)
 
 
 @dataclass
 class ManagedPrompt:
-    """Smart result object that holds template and can fill variables on demand."""
+    """Smart prompt that can be filled with variables using Jinja2."""
     template: str
     name: str
-    variables: Set[str]
     schema: Optional[Dict[str, Any]] = None
-    _filled_prompt: Optional[str] = field(default=None, init=False, repr=False)
-    _context: Optional[Dict[str, Any]] = field(default=None, init=False, repr=False)
     _jinja_template: Optional[Template] = field(default=None, init=False, repr=False)
 
-    def validate(self, **kwargs) -> bool:
-        """Validate that all required variables are provided.
-
-        Returns:
-            True if all required variables are present, False otherwise
-        """
-        provided_vars = set(kwargs.keys())
-        missing_vars = self.variables - provided_vars
-        return len(missing_vars) == 0
-
-    def get_missing_variables(self, **kwargs) -> Set[str]:
-        """Get the set of missing required variables.
-
-        Returns:
-            Set of variable names that are required but not provided
-        """
-        provided_vars = set(kwargs.keys())
-        return self.variables - provided_vars
-
     def fill(self, **kwargs) -> str:
         """Fill the template with provided variables using Jinja2.
 
@@ -50,60 +26,26 @@ class ManagedPrompt:
             The filled prompt string
 
         Raises:
-            ValueError: If required variables are missing or template syntax error
+            UndefinedError: If required variables are missing (Jinja2's default behavior)
         """
         # Create Jinja2 template if not already created
         if self._jinja_template is None:
-            try:
             self._jinja_template = Template(self.template)
-            except TemplateSyntaxError as e:
-                raise ValueError(f"Invalid template syntax in prompt '{self.name}': {e}")
 
-        # If no variables required and none provided, return template as-is
+        # Render the template - Jinja2 handles undefined variables
-        if not self.variables and not kwargs:
+        return self._jinja_template.render(**kwargs)
-            self._filled_prompt = self.template
-            self._context = {}
-            return self.template
-
-        missing_vars = self.get_missing_variables(**kwargs)
-        if missing_vars:
-            raise ValueError(
-                f"Missing required variables for prompt '{self.name}': {missing_vars}. "
-                f"Required: {self.variables}, Provided: {set(kwargs.keys())}"
-            )
-
-        try:
-            # Render the template with Jinja2
-            result = self._jinja_template.render(**kwargs)
-        except UndefinedError as e:
-            raise ValueError(f"Error rendering template '{self.name}': {e}")
-
-        # Cache the filled result
-        self._filled_prompt = result
-        self._context = kwargs
-
-        return result
 
     @property
     def prompt(self) -> str:
-        """Get the filled prompt if available, otherwise return the template.
+        """Get the template for backward compatibility."""
-
+        return self.template
-        This property provides backward compatibility.
-        """
-        return self._filled_prompt if self._filled_prompt else self.template
-
-    def __str__(self) -> str:
-        """String representation returns the filled prompt or template."""
-        return self.prompt
 
 
 class PromptManager:
-    """Singleton class to manage prompt templates and JSON schemas"""
+    """Singleton class to manage prompt templates with automatic schema loading"""
 
     _instance: Optional['PromptManager'] = None
     _initialized: bool = False
-    _prompt_path: Path
-    _caching: bool
 
     def __new__(cls):
         if cls._instance is None:
@@ -114,12 +56,13 @@ class PromptManager:
         if not self._initialized:
             self.prompts: Dict[str, str] = {}
             self.schemas: Dict[str, Dict[str, Any]] = {}
-            self.prompt_variables: Dict[str, Set[str]] = {}
+            self._caching = True
-            self._caching = True  # Enable caching by default
+            self._prompt_path = Path.cwd() / 'prompts'
+            self._undefined_mode = StrictUndefined  # Default to strict
             PromptManager._initialized = True
 
     def _load_prompt(self, prompt_name: str) -> bool:
-        """Load a specific prompt and its schema on-demand.
+        """Load a specific prompt and its schema if it exists.
 
         Args:
             prompt_name: Name of the prompt to load
@@ -131,7 +74,7 @@ class PromptManager:
         if self._caching and prompt_name in self.prompts:
             return True
 
-        prompts_dir = self._get_path()
+        prompts_dir = self._prompt_path
 
         if not prompts_dir.exists():
             logger.warning(f"Prompts directory not found: {prompts_dir}")
@@ -148,25 +91,20 @@ class PromptManager:
             with open(md_file, 'r', encoding='utf-8') as f:
                 content = f.read().strip()
 
-            # Extract variables from {{variable}} patterns
-            variables = self._extract_variables(content)
-
-            # Store in cache
             self.prompts[prompt_name] = content
-            self.prompt_variables[prompt_name] = variables
 
-            # Look for corresponding JSON schema file
+            # Automatically look for JSON schema next to the prompt
             schema_file = md_file.with_suffix('.json')
             if schema_file.exists():
                 try:
                     with open(schema_file, 'r', encoding='utf-8') as f:
                         schema = json.load(f)
                     self.schemas[prompt_name] = schema
-                    logger.debug(f"Loaded prompt '{prompt_name}' with schema and variables: {variables}")
+                    logger.debug(f"Loaded prompt '{prompt_name}' with schema")
                 except json.JSONDecodeError as e:
                     logger.error(f"Invalid JSON schema in {schema_file}: {e}")
             else:
-                logger.debug(f"Loaded prompt '{prompt_name}' (no schema) with variables: {variables}")
+                logger.debug(f"Loaded prompt '{prompt_name}' (no schema)")
 
             return True
 
@@ -174,108 +112,56 @@ class PromptManager:
             logger.error(f"Error loading prompt file {md_file}: {e}")
             return False
 
-    def _get_path(self) -> Path:
-        """Get the prompts directory path.
-
-        Returns the configured path if set via configure(),
-        otherwise defaults to 'prompts' in the current working directory.
-        """
-        if hasattr(self, '_prompt_path') and self._prompt_path:
-            return self._prompt_path
-
-        # Default to 'prompts' directory in the current working directory
-        return Path.cwd() / 'prompts'
-
-    def _extract_variables(self, template: str) -> Set[str]:
-        """Extract all variables from Jinja2 template"""
-        try:
-            # Create a Jinja2 environment and parse the template
-            env = Environment()
-            ast = env.parse(template)
-            # Get all undeclared variables from the template
-            variables = meta.find_undeclared_variables(ast)
-            return variables
-        except TemplateSyntaxError:
-            # Fallback to simple regex for backwards compatibility
-            pattern = r'\{\{\s*(\w+)\s*\}\}'
-            variables = set(re.findall(pattern, template))
-            return variables
-
-    def _validate_context(self, prompt_name: str, context: Dict[str, Any]) -> None:
-        """Validate that all required variables are provided"""
-        if prompt_name not in self.prompt_variables:
-            raise ValueError(f"Unknown prompt: '{prompt_name}'")
-
-        required_vars = self.prompt_variables[prompt_name]
-        provided_vars = set(context.keys())
-
-        missing_vars = required_vars - provided_vars
-        if missing_vars:
-            raise ValueError(
-                f"Missing required variables for prompt '{prompt_name}': {missing_vars}. "
-                f"Required: {required_vars}, Provided: {provided_vars}"
-            )
-
-        # Warn about extra variables (not an error, but might indicate mistakes)
-        extra_vars = provided_vars - required_vars
-        if extra_vars:
-            logger.warning(f"Extra variables provided for prompt '{prompt_name}': {extra_vars}")
-
-    def _fill_template(self, template: str, context: Dict[str, Any]) -> str:
-        """Fill template with context variables using Jinja2"""
-        try:
-            jinja_template = Template(template)
-            return jinja_template.render(**context)
-        except (TemplateSyntaxError, UndefinedError) as e:
-            raise ValueError(f"Error rendering template: {e}")
-
     @classmethod
-    def configure(cls, path: Optional[Path] = None, caching: Optional[bool] = None):
+    def configure(cls, path: Optional[Path] = None, caching: Optional[bool] = None,
+                  undefined: Optional[type] = None):
         """Configure the PromptManager settings.
 
         Args:
             path: Custom path to prompts directory
             caching: Whether to cache loaded prompts (default: True)
+            undefined: Jinja2 undefined behavior (StrictUndefined, DebugUndefined, etc.)
         """
         instance = cls()
+
         if path is not None:
             instance._prompt_path = path
             # Clear cache when path changes
             instance.prompts.clear()
             instance.schemas.clear()
-            instance.prompt_variables.clear()
+
         if caching is not None:
             instance._caching = caching
-            # If disabling cache, clear existing cached prompts
             if not caching:
                 instance.prompts.clear()
                 instance.schemas.clear()
-                instance.prompt_variables.clear()
+
+        if undefined is not None:
+            instance._undefined_mode = undefined
 
     @classmethod
     def get_prompt(cls, prompt_name: str, **kwargs) -> ManagedPrompt:
-        """
+        """Get a ManagedPrompt that can be filled with variables.
-        Get a PromptResult that can be filled with variables.
 
         Args:
             prompt_name: Name of the prompt template (filename without .md)
             **kwargs: Optional variables to pre-fill the template
 
         Returns:
-            PromptResult object with smart fill/validate methods
+            ManagedPrompt object
 
         Raises:
             ValueError: If prompt doesn't exist
+            UndefinedError: If pre-filling with missing required variables
 
         Examples:
             # Get unfilled template
             result = PromptManager.get_prompt('greeting')
-            print(result.variables)  # See required variables
             filled = result.fill(name='Alice', age=30)
 
-            # Or pre-fill on retrieval
+            # Pre-fill on retrieval
             result = PromptManager.get_prompt('greeting', name='Alice', age=30)
-            print(result.prompt)  # Already filled
+            print(result.prompt)  # Already filled if all vars provided
         """
         instance = cls()
 
@@ -283,104 +169,54 @@ class PromptManager:
         if not instance._load_prompt(prompt_name):
             raise ValueError(f"Prompt '{prompt_name}' not found")
 
-        # Get template, variables and schema
+        # Get template and schema
         template = instance.prompts[prompt_name]
-        variables = instance.prompt_variables.get(prompt_name, set())
         schema = instance.schemas.get(prompt_name)
 
-        # Create the result object
+        # Create the result object with custom undefined mode
+        env = Environment(undefined=instance._undefined_mode)
         result = ManagedPrompt(
             template=template,
             name=prompt_name,
-            variables=variables,
             schema=schema
         )
 
-        # If kwargs provided, pre-fill the template
+        # Create template with configured undefined behavior
+        result._jinja_template = env.from_string(template)
+
+        # If kwargs provided, try to pre-fill
         if kwargs:
-            try:
+            filled = result.fill(**kwargs)
-                result.fill(**kwargs)
+            # Store the filled result
-            except ValueError:
+            result.template = filled
-                # If validation fails, return unfilled result and let user handle
-                pass
 
         # If caching is disabled, clear the prompt after use
         if not instance._caching:
             del instance.prompts[prompt_name]
-            del instance.prompt_variables[prompt_name]
             if prompt_name in instance.schemas:
                 del instance.schemas[prompt_name]
 
         return result
 
-    @classmethod
-    def get_schema(cls, prompt_name: str) -> Optional[Dict[str, Any]]:
-        """
-        Get the JSON schema for a prompt if it exists
-
-        Args:
-            prompt_name: Name of the prompt template
-
-        Returns:
-            JSON schema dictionary or None if no schema exists
-        """
-        instance = cls()
-
-        # Try to load the prompt if not already loaded
-        if not instance._load_prompt(prompt_name):
-            raise ValueError(f"Prompt '{prompt_name}' not found")
-
-        return instance.schemas.get(prompt_name)
-
-    @classmethod
-    def has_schema(cls, prompt_name: str) -> bool:
-        """Check if a prompt has a JSON schema"""
-        instance = cls()
-        # Try to load the prompt if not already loaded
-        instance._load_prompt(prompt_name)
-        return prompt_name in instance.schemas
-
-    @classmethod
-    def get_prompt_with_schema(cls, prompt_name: str, **kwargs) -> ManagedPrompt:
-        """
-        Get both the processed prompt and its schema (if available)
-
-        This is now just an alias for get_prompt() since it returns PromptResult.
-        Kept for backward compatibility.
-
-        Args:
-            prompt_name: Name of the prompt template
-            **kwargs: Variables to fill in the template
-
-        Returns:
-            PromptResult object containing prompt, schema, variables, and name
-        """
-        return cls.get_prompt(prompt_name, **kwargs)
-
     @classmethod
     def list_prompts(cls) -> Dict[str, Dict[str, Any]]:
-        """
+        """List all available prompts with their info.
-        List all available prompts with their info
 
         Returns:
-            Dictionary mapping prompt names to their info (variables, has_schema)
+            Dictionary mapping prompt names to their info (has_schema)
         """
         instance = cls()
-        prompts_dir = instance._get_path()
+        prompts_dir = instance._prompt_path
         result = {}
 
         if not prompts_dir.exists():
             return result
 
-        # Scan for all .md files in the prompts directory
         for md_file in prompts_dir.glob("*.md"):
             prompt_name = md_file.stem
-            # Load prompt to get its details
+            schema_file = md_file.with_suffix('.json')
-            if instance._load_prompt(prompt_name):
             result[prompt_name] = {
-                    'variables': instance.prompt_variables[prompt_name],
+                'has_schema': schema_file.exists()
-                    'has_schema': prompt_name in instance.schemas,
-                    'variable_count': len(instance.prompt_variables[prompt_name])
             }
 
         return result
@@ -391,32 +227,4 @@ class PromptManager:
         if cls._instance:
             cls._instance.prompts.clear()
             cls._instance.schemas.clear()
-            cls._instance.prompt_variables.clear()
             logger.info("Prompt cache cleared")
-
-    @classmethod
-    def get_prompt_info(cls, prompt_name: str) -> Dict[str, Any]:
-        """
-        Get detailed information about a specific prompt
-
-        Returns:
-            Dictionary with prompt template, schema, and required variables
-        """
-        instance = cls()
-
-        # Try to load the prompt if not already loaded
-        if not instance._load_prompt(prompt_name):
-            raise ValueError(f"Prompt '{prompt_name}' not found")
-
-        info = {
-            'name': prompt_name,
-            'template': instance.prompts[prompt_name],
-            'variables': instance.prompt_variables[prompt_name],
-            'variable_count': len(instance.prompt_variables[prompt_name]),
-            'has_schema': prompt_name in instance.schemas
-        }
-
-        if prompt_name in instance.schemas:
-            info['schema'] = instance.schemas[prompt_name]
-
-        return info

test_jinja2_analysis.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Explore Jinja2's AST to understand variable detection in conditionals"""
+
+from jinja2 import Environment, meta, nodes
+
+# Test different template patterns
+templates = [
+    # Simple required variable
+    ("simple", "Hello {{ name }}!"),
+
+    # Variable in conditional (optional)
+    ("conditional", """{% if error %}
+Error: {{ error }}
+{% endif %}"""),
+
+    # Mixed required and optional
+    ("mixed", """Status: {{ status }}
+{% if error %}
+Error: {{ error }}
+{% endif %}"""),
+
+    # Nested conditionals
+    ("nested", """{% if user %}
+Name: {{ user.name }}
+{% if user.email %}
+Email: {{ user.email }}
+{% endif %}
+{% endif %}"""),
+
+    # Variable with default filter (makes it optional)
+    ("default_filter", "Hello {{ name | default('Guest') }}!"),
+
+    # Variable in for loop
+    ("for_loop", """{% for item in items %}
+- {{ item }}
+{% endfor %}"""),
+
+    # Complex with multiple contexts
+    ("complex", """Required: {{ required }}
+{% if optional1 %}
+Optional1: {{ optional1 }}
+{% endif %}
+Default: {{ optional2 | default('N/A') }}
+{% for item in items %}
+- {{ item }}
+{% endfor %}""")
+]
+
+env = Environment()
+
+print("=" * 70)
+print("Analyzing Jinja2 AST for Variable Detection")
+print("=" * 70)
+
+for name, template in templates:
+    print(f"\n{name.upper()} TEMPLATE:")
+    print("-" * 40)
+    print(template)
+    print("-" * 40)
+
+    # Parse the template
+    ast = env.parse(template)
+
+    # Get undeclared variables (what we currently use)
+    undeclared = meta.find_undeclared_variables(ast)
+    print(f"Undeclared variables: {undeclared}")
+
+    # Try to analyze the AST more deeply
+    print("\nAST Analysis:")
+
+    def analyze_node(node, indent=0, context="root"):
+        """Recursively analyze AST nodes"""
+        prefix = "  " * indent
+
+        if isinstance(node, nodes.Name):
+            print(f"{prefix}Variable '{node.name}' in context: {context}")
+
+        elif isinstance(node, nodes.If):
+            print(f"{prefix}If block:")
+            print(f"{prefix}  Test expression:")
+            analyze_node(node.test, indent + 2, "if_test")
+            print(f"{prefix}  Body:")
+            for child in node.body:
+                analyze_node(child, indent + 2, "if_body")
+            if node.elif_:
+                print(f"{prefix}  Elif:")
+                for child in node.elif_:
+                    analyze_node(child, indent + 2, "elif")
+            if node.else_:
+                print(f"{prefix}  Else:")
+                for child in node.else_:
+                    analyze_node(child, indent + 2, "else")
+
+        elif isinstance(node, nodes.For):
+            print(f"{prefix}For loop:")
+            print(f"{prefix}  Target: {node.target}")
+            print(f"{prefix}  Iter:")
+            analyze_node(node.iter, indent + 2, "for_iter")
+            print(f"{prefix}  Body:")
+            for child in node.body:
+                analyze_node(child, indent + 2, "for_body")
+
+        elif isinstance(node, nodes.Filter):
+            print(f"{prefix}Filter: {node.name}")
+            if node.name == "default":
+                print(f"{prefix}  (Makes variable optional!)")
+            analyze_node(node.node, indent + 1, f"filter_{node.name}")
+
+        elif isinstance(node, nodes.Getattr):
+            print(f"{prefix}Attribute access: .{node.attr}")
+            analyze_node(node.node, indent + 1, "getattr")
+
+        elif isinstance(node, nodes.Output):
+            print(f"{prefix}Output:")
+            for child in node.nodes:
+                analyze_node(child, indent + 1, "output")
+
+        elif isinstance(node, nodes.TemplateData):
+            # Skip raw text
+            pass
+
+        else:
+            # Recurse into child nodes
+            for child in node.iter_child_nodes():
+                analyze_node(child, indent, context)
+
+    analyze_node(ast)
+
+print("\n" + "=" * 70)
+print("INSIGHTS:")
+print("-" * 70)
+print("""
+1. Jinja2's `find_undeclared_variables()` returns ALL variables, regardless of context
+2. Variables in {% if %} conditions could be considered optional
+3. Variables with | default() filter are definitely optional
+4. We can walk the AST to identify context for each variable
+
+Possible solution:
+- Walk the AST to categorize variables by context
+- Mark variables in if_test contexts as optional (unless also used elsewhere)
+- Mark variables with default filter as optional
+- Consider all others as required
+""")

test_optional_vars.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Test optional variables with Jinja2 conditionals"""
+
+from pathlib import Path
+from src.llmutils.prompt_manager import PromptManager
+
+# Create test prompts directory
+test_dir = Path("test_prompts")
+test_dir.mkdir(exist_ok=True)
+
+# Create a template with required and optional variables
+report_template = test_dir / "report.md"
+report_template.write_text("""# Status Report
+
+Status: {{ status }}
+
+{% if error %}
+⚠️ ERROR: {{ error }}
+{% endif %}
+
+{% if warning %}
+⚠️ WARNING: {{ warning }}
+{% endif %}
+
+{% if details %}
+## Details
+{{ details }}
+{% endif %}
+
+Generated at: {{ timestamp }}""")
+
+# Configure PromptManager
+PromptManager.configure(path=test_dir)
+
+print("=" * 60)
+print("Testing Optional Variables with Jinja2 Conditionals")
+print("=" * 60)
+
+# Test 1: All variables provided
+print("\n1. All variables provided:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+filled = prompt.fill(
+    status="Operational",
+    error="Database connection failed",
+    warning="High memory usage",
+    details="System recovered after 5 retries",
+    timestamp="2024-01-15 10:30:00"
+)
+print(filled)
+
+# Test 2: Only required variables (status and timestamp)
+print("\n2. Only truly required variables (will fail with strict=True):")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+try:
+    # This will fail because Jinja2 considers all variables as required
+    filled = prompt.fill(
+        status="Operational",
+        timestamp="2024-01-15 10:30:00"
+    )
+    print(filled)
+except ValueError as e:
+    print(f"Error (expected): Missing variables {{'error', 'warning', 'details'}}")
+    print(f"Actual error: {e}")
+
+# Test 3: Provide None/empty for optional variables
+print("\n3. Provide None/empty values for optional variables:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+filled = prompt.fill(
+    status="Operational",
+    error=None,  # Will not show in output due to {% if error %}
+    warning="",   # Empty string is falsy, won't show
+    details=None, # Will not show
+    timestamp="2024-01-15 10:30:00"
+)
+print(filled)
+
+# Test 4: Mix of provided and None values
+print("\n4. Mix of provided and None values:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+filled = prompt.fill(
+    status="Degraded",
+    error="API timeout",
+    warning=None,  # Won't show
+    details=None,  # Won't show
+    timestamp="2024-01-15 10:35:00"
+)
+print(filled)
+
+# Test 5: Using strict=False to handle missing variables
+print("\n5. Using strict=False for partial filling:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+filled = prompt.fill(
+    strict=False,
+    status="Unknown",
+    timestamp="2024-01-15 10:40:00"
+)
+print(filled)
+print("Note: Unfilled variables remain as {{ variable }}")
+
+# Test 6: Show extracted variables
+print("\n6. Extracted variables from template:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("report")
+print(f"All variables detected: {prompt.variables}")
+print("Note: Jinja2 extracts all variables, even those in conditionals")
+print("Use None/empty values for optional ones, or use strict=False")
+
+print("\n" + "=" * 60)
+print("Tests completed!")
+print("=" * 60)
+
+# Cleanup
+import shutil
+shutil.rmtree(test_dir)

test_simplified.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Test the simplified prompt manager that relies on Jinja2's built-in handling"""
+
+from pathlib import Path
+from jinja2 import UndefinedError, DebugUndefined
+from src.llmutils.prompt_manager_simple import PromptManager
+
+# Create test prompts directory
+test_dir = Path("test_prompts")
+test_dir.mkdir(exist_ok=True)
+
+# Create templates
+greeting = test_dir / "greeting.md"
+greeting.write_text("Hello {{ name }}, you are {{ age }} years old!")
+
+# Template with optional variables using default filter
+flexible = test_dir / "flexible.md"
+flexible.write_text("""# Report for {{ title }}
+
+Status: {{ status | default('Unknown') }}
+
+{% if error is defined %}
+Error: {{ error }}
+{% endif %}
+
+Author: {{ author | default('System') }}""")
+
+# Create a schema file
+schema_file = test_dir / "flexible.json"
+schema_file.write_text("""{
+    "type": "object",
+    "properties": {
+        "title": {"type": "string"},
+        "status": {"type": "string"},
+        "error": {"type": "string"},
+        "author": {"type": "string"}
+    },
+    "required": ["title"]
+}""")
+
+# Configure PromptManager
+PromptManager.configure(path=test_dir)
+
+print("=" * 70)
+print("Testing Simplified Prompt Manager")
+print("=" * 70)
+
+# Test 1: Normal usage with all variables
+print("\n1. Normal usage with all variables:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting")
+filled = prompt.fill(name="Alice", age=30)
+print(filled)
+
+# Test 2: Missing required variable (will raise error)
+print("\n2. Missing required variable (should raise error):")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting")
+try:
+    filled = prompt.fill(name="Bob")  # Missing 'age'
+    print(filled)
+except UndefinedError as e:
+    print(f"✓ Error raised as expected: {e}")
+
+# Test 3: Template with defaults and conditionals
+print("\n3. Template with optional variables (using defaults and conditionals):")
+print("-" * 40)
+prompt = PromptManager.get_prompt("flexible")
+filled = prompt.fill(title="Daily Report")  # Only required variable
+print(filled)
+
+# Test 4: Same template with all variables
+print("\n4. Same template with all variables provided:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("flexible")
+filled = prompt.fill(
+    title="Error Report",
+    status="Failed",
+    error="Database connection timeout",
+    author="Admin"
+)
+print(filled)
+
+# Test 5: Pre-fill on retrieval
+print("\n5. Pre-fill on retrieval:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting", name="Charlie", age=25)
+print(prompt.prompt)  # Already filled
+
+# Test 6: Schema is automatically loaded
+print("\n6. Schema automatically loaded:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("flexible")
+if prompt.schema:
+    print(f"✓ Schema found: {prompt.schema}")
+else:
+    print("✗ No schema")
+
+# Test 7: Configure with DebugUndefined
+print("\n7. Using DebugUndefined mode:")
+print("-" * 40)
+PromptManager.configure(undefined=DebugUndefined)
+PromptManager.reload_prompts()  # Clear cache
+prompt = PromptManager.get_prompt("greeting")
+filled = prompt.fill(name="Debug")  # Missing 'age' - will show as undefined
+print(filled)
+
+# Test 8: List available prompts
+print("\n8. List available prompts:")
+print("-" * 40)
+prompts = PromptManager.list_prompts()
+for name, info in prompts.items():
+    print(f"- {name}: {'has schema' if info['has_schema'] else 'no schema'}")
+
+print("\n" + "=" * 70)
+print("Benefits of this simplified approach:")
+print("-" * 70)
+print("✓ Jinja2 handles undefined variables for us")
+print("✓ No need for complex variable extraction and validation")
+print("✓ Users can choose undefined behavior (strict, debug, silent)")
+print("✓ Templates can use Jinja2's built-in features for optional vars:")
+print("  - {{ var | default('value') }}")
+print("  - {% if var is defined %}")
+print("✓ Schemas are automatically loaded if present")
+print("✓ Much simpler and cleaner codebase")
+print("=" * 70)
+
+# Cleanup
+import shutil
+shutil.rmtree(test_dir)

test_smart_variables.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Test the smart variable detection that distinguishes required from optional"""
+
+from pathlib import Path
+from src.llmutils.prompt_manager import PromptManager
+
+# Create test prompts directory
+test_dir = Path("test_prompts")
+test_dir.mkdir(exist_ok=True)
+
+# Test 1: Simple required variable
+simple = test_dir / "simple.md"
+simple.write_text("Hello {{ name }}!")
+
+# Test 2: Optional variable in conditional
+conditional = test_dir / "conditional.md"
+conditional.write_text("""Status: OK
+{% if error %}
+Error: {{ error }}
+{% endif %}""")
+
+# Test 3: Variable with default filter
+default_filter = test_dir / "default.md"
+default_filter.write_text("""Hello {{ name | default('Guest') }}!
+Age: {{ age }}""")
+
+# Test 4: Complex mix
+complex_template = test_dir / "complex.md"
+complex_template.write_text("""# Report for {{ title }}
+
+Status: {{ status }}
+
+{% if error %}
+⚠️ ERROR: {{ error }}
+{% endif %}
+
+{% if warning %}
+⚠️ WARNING: {{ warning }}
+{% endif %}
+
+Debug Level: {{ debug_level | default(0) }}
+
+{% for item in items %}
+- {{ item }}
+{% endfor %}
+
+Generated by: {{ author | default('System') }}""")
+
+# Configure PromptManager
+PromptManager.configure(path=test_dir)
+
+print("=" * 70)
+print("Testing Smart Variable Detection (Required vs Optional)")
+print("=" * 70)
+
+# Test each template
+test_cases = [
+    ("simple", {"name": "Alice"}),
+    ("conditional", {"error": "Timeout"}),
+    ("default", {"age": 25}),
+    ("complex", {"title": "Daily Report", "status": "OK", "items": ["task1", "task2"]})
+]
+
+for prompt_name, test_data in test_cases:
+    print(f"\n{'='*50}")
+    print(f"TESTING: {prompt_name}")
+    print("=" * 50)
+
+    prompt = PromptManager.get_prompt(prompt_name)
+
+    print(f"\nAll variables:      {prompt.variables}")
+    print(f"Required variables: {prompt.required_variables}")
+    print(f"Optional variables: {prompt.optional_variables}")
+
+    print(f"\nProviding: {test_data}")
+
+    # Test validation
+    is_valid = prompt.validate(**test_data)
+    missing = prompt.get_missing_variables(**test_data)
+
+    print(f"Valid: {is_valid}")
+    if missing:
+        print(f"Missing required: {missing}")
+
+    # Try to fill
+    try:
+        filled = prompt.fill(**test_data)
+        print("\nFilled successfully (showing first 200 chars):")
+        print(filled[:200])
+    except ValueError as e:
+        print(f"\nError filling: {e}")
+
+# Test 5: Demonstrate that optional variables don't need to be provided
+print("\n" + "=" * 70)
+print("DEMONSTRATING OPTIONAL VARIABLES")
+print("=" * 70)
+
+prompt = PromptManager.get_prompt("complex")
+print(f"\nRequired: {prompt.required_variables}")
+print(f"Optional: {prompt.optional_variables}")
+
+# Provide only required variables
+minimal_data = {
+    "title": "Test Report",
+    "status": "Running",
+    "items": ["item1", "item2", "item3"]
+}
+
+print(f"\nProviding only required variables: {minimal_data}")
+try:
+    filled = prompt.fill(**minimal_data)
+    print("\n✓ Filled successfully with only required variables!")
+    print("\nOutput:")
+    print("-" * 40)
+    print(filled)
+except ValueError as e:
+    print(f"\n✗ Error: {e}")
+
+print("\n" + "=" * 70)
+print("Tests completed!")
+print("=" * 70)
+
+# Cleanup
+import shutil
+shutil.rmtree(test_dir)

test_strict_mode.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Test the new strict mode for error handling"""
+
+from pathlib import Path
+from src.llmutils.prompt_manager import PromptManager
+
+# Create test prompts directory
+test_dir = Path("test_prompts")
+test_dir.mkdir(exist_ok=True)
+
+# Create a test prompt
+prompt_file = test_dir / "greeting.md"
+prompt_file.write_text("Hello {{name}}, you are {{age}} years old!")
+
+# Configure PromptManager
+PromptManager.configure(path=test_dir)
+
+print("=" * 60)
+print("Testing Strict Mode for Error Handling")
+print("=" * 60)
+
+# Test 1: Default behavior with fill() - should raise error
+print("\n1. Direct fill() with missing variables (default strict=True):")
+print("-" * 40)
+try:
+    prompt = PromptManager.get_prompt("greeting")
+    filled = prompt.fill(name="Alice")  # Missing 'age'
+    print(f"Result: {filled}")
+except ValueError as e:
+    print(f"✓ Error raised as expected: {e}")
+
+# Test 2: Non-strict mode with fill()
+print("\n2. Direct fill() with strict=False:")
+print("-" * 40)
+try:
+    prompt = PromptManager.get_prompt("greeting")
+    filled = prompt.fill(strict=False, name="Alice")  # Missing 'age'
+    print(f"✓ Result (unfilled template): {filled}")
+except ValueError as e:
+    print(f"Unexpected error: {e}")
+
+# Test 3: get_prompt with pre-fill (default non-strict)
+print("\n3. get_prompt() with pre-fill (default strict=False):")
+print("-" * 40)
+try:
+    prompt = PromptManager.get_prompt("greeting", name="Alice")  # Missing 'age'
+    print(f"✓ Result (unfilled template): {prompt.prompt}")
+except ValueError as e:
+    print(f"Unexpected error: {e}")
+
+# Test 4: get_prompt with pre-fill in strict mode
+print("\n4. get_prompt() with pre-fill (strict=True):")
+print("-" * 40)
+try:
+    prompt = PromptManager.get_prompt("greeting", strict=True, name="Alice")  # Missing 'age'
+    print(f"Result: {prompt.prompt}")
+except ValueError as e:
+    print(f"✓ Error raised as expected: {e}")
+
+# Test 5: Successful fill with all variables
+print("\n5. Successful fill with all variables:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting")
+filled = prompt.fill(name="Bob", age=25)
+print(f"✓ Result: {filled}")
+
+# Test 6: Successful pre-fill with all variables
+print("\n6. Successful pre-fill on retrieval:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting", name="Charlie", age=30)
+print(f"✓ Result: {prompt.prompt}")
+
+# Test 7: Check that variables are correctly identified
+print("\n7. Variable extraction:")
+print("-" * 40)
+prompt = PromptManager.get_prompt("greeting")
+print(f"Required variables: {prompt.variables}")
+print(f"Missing when only 'name' provided: {prompt.get_missing_variables(name='Test')}")
+
+print("\n" + "=" * 60)
+print("All tests completed!")
+print("=" * 60)
+
+# Cleanup
+import shutil
+shutil.rmtree(test_dir)