"""
Core Permission Management Services for Granular Page-Specific Permissions System
"""
from django.core.cache import cache
from django.db.models import Q
from django.utils import timezone
from typing import Dict, List, Optional, Set, Tuple
import logging

logger = logging.getLogger(__name__)


class PagePermissionManager:
    """
    Service class for managing page-specific permissions with enhanced Redis caching
    """
    
    def __init__(self):
        # Import here to avoid circular imports
        from .permission_cache_service import permission_cache
        self.cache_service = permission_cache
    
    def get_page_permissions(self, user, page_name: str) -> Dict[str, bool]:
        """
        Get all permissions for a specific page for the given user
        
        Args:
            user: CustomUser instance
            page_name: Name of the page (e.g., 'loans', 'clients', 'reports')
            
        Returns:
            Dict mapping action codes to boolean permission status
        """
        # Check cache first using enhanced cache service
        cached_permissions = self.cache_service.get_user_page_permissions(str(user.id), page_name)
        
        if cached_permissions is not None:
            logger.debug(f"Cache hit for user {user.id} page {page_name}")
            return cached_permissions
        
        try:
            from .enhanced_permissions_models import PagePermission
            
            # Get all permissions for this page
            page_permissions = PagePermission.objects.filter(
                page_name=page_name,
                is_active=True
            ).select_related().prefetch_related('required_permissions')
            
            permissions = {}
            
            for page_perm in page_permissions:
                # Check if user has this specific permission
                has_permission = self._check_user_permission(user, page_perm)
                permissions[page_perm.action_code] = has_permission
            
            # Cache the result using enhanced cache service
            self.cache_service.set_user_page_permissions(str(user.id), page_name, permissions)
            logger.debug(f"Cached permissions for user {user.id} page {page_name}")
            
            return permissions
            
        except Exception as e:
            logger.error(f"Error getting page permissions for user {user.id}, page {page_name}: {e}")
            return {}
    
    def check_action_permission(self, user, page_name: str, action_code: str) -> bool:
        """
        Check if user can perform a specific action on a page
        
        Args:
            user: CustomUser instance
            page_name: Name of the page
            action_code: Specific action code to check
            
        Returns:
            Boolean indicating if user has permission
        """
        # First try to get from page permissions cache
        cached_permissions = self.cache_service.get_user_page_permissions(str(user.id), page_name)
        
        if cached_permissions is not None and action_code in cached_permissions:
            logger.debug(f"Cache hit for user {user.id} action {page_name}:{action_code}")
            return cached_permissions[action_code]
        
        try:
            from .enhanced_permissions_models import PagePermission
            
            # Get the specific permission
            try:
                page_permission = PagePermission.objects.get(
                    page_name=page_name,
                    action_code=action_code,
                    is_active=True
                )
            except PagePermission.DoesNotExist:
                logger.warning(f"Permission not found: {page_name}:{action_code}")
                cache.set(cache_key, False, self.CACHE_TIMEOUT)
                return False
            
            # Check if user has this permission
            has_permission = self._check_user_permission(user, page_permission)
            
            # Update the page permissions cache with this result
            page_permissions = self.cache_service.get_user_page_permissions(str(user.id), page_name) or {}
            page_permissions[action_code] = has_permission
            self.cache_service.set_user_page_permissions(str(user.id), page_name, page_permissions)
            
            return has_permission
            
        except Exception as e:
            logger.error(f"Error checking action permission for user {user.id}, {page_name}:{action_code}: {e}")
            return False
    
    def get_available_actions(self, user, page_name: str) -> List[Dict[str, any]]:
        """
        Get list of actions user can perform on a page for UI rendering
        
        Args:
            user: CustomUser instance
            page_name: Name of the page
            
        Returns:
            List of dictionaries containing action details for allowed actions
        """
        # Check cache first using enhanced cache service
        cached_actions = self.cache_service.get_user_available_actions(str(user.id), page_name)
        
        if cached_actions is not None:
            logger.debug(f"Cache hit for available actions user {user.id} page {page_name}")
            return cached_actions
        
        try:
            from .enhanced_permissions_models import PagePermission
            
            # Get all permissions for this page
            page_permissions = PagePermission.objects.filter(
                page_name=page_name,
                is_active=True
            ).order_by('category', 'action_name')
            
            available_actions = []
            
            for page_perm in page_permissions:
                if self._check_user_permission(user, page_perm):
                    action_info = {
                        'action_code': page_perm.action_code,
                        'action_name': page_perm.action_name,
                        'description': page_perm.description,
                        'category': page_perm.category,
                        'is_critical': page_perm.is_critical,
                        'full_permission_code': page_perm.get_full_permission_code()
                    }
                    available_actions.append(action_info)
            
            # Cache the result using enhanced cache service
            self.cache_service.set_user_available_actions(str(user.id), page_name, available_actions)
            
            return available_actions
            
        except Exception as e:
            logger.error(f"Error getting available actions for user {user.id}, page {page_name}: {e}")
            return []
    
    def _check_user_permission(self, user, page_permission) -> bool:
        """
        Internal method to check if user has a specific page permission
        Checks custom user permissions first, then falls back to role defaults
        
        Args:
            user: CustomUser instance
            page_permission: PagePermission instance
            
        Returns:
            Boolean indicating if user has permission
        """
        try:
            from .enhanced_permissions_models import UserPagePermission, RolePermissionTemplate
            
            # Admin users have all permissions
            if user.role == 'admin':
                return True
            
            # Check for custom user permission override first
            try:
                user_permission = UserPagePermission.objects.get(
                    user=user,
                    page_permission=page_permission
                )
                
                # Check if permission has expired
                if not user_permission.is_expired():
                    logger.debug(f"Using custom permission for user {user.id}: {user_permission.is_allowed}")
                    return user_permission.is_allowed
                else:
                    logger.debug(f"Custom permission expired for user {user.id}")
                    # Permission expired, fall through to role default
                    
            except UserPagePermission.DoesNotExist:
                # No custom permission, fall through to role default
                pass
            
            # Check role permission template
            try:
                role_template = RolePermissionTemplate.objects.get(
                    role=user.role,
                    page_permission=page_permission
                )
                
                logger.debug(f"Using role permission for user {user.id} role {user.role}: {role_template.is_allowed}")
                return role_template.is_allowed
                
            except RolePermissionTemplate.DoesNotExist:
                # No role template defined, default to False
                logger.debug(f"No role template found for role {user.role}, permission {page_permission.action_code}")
                return False
                
        except Exception as e:
            logger.error(f"Error checking user permission: {e}")
            return False
    
    def invalidate_user_cache(self, user_id: str, page_name: Optional[str] = None):
        """
        Invalidate cached permissions for a user
        
        Args:
            user_id: User ID to invalidate cache for
            page_name: Optional specific page to invalidate, if None invalidates all
        """
        try:
            # Use enhanced cache service for invalidation
            invalidated_count = self.cache_service.invalidate_user_cache(user_id, page_name)
            
            if page_name:
                logger.info(f"Invalidated {invalidated_count} cache keys for user {user_id}, page {page_name}")
            else:
                logger.info(f"Invalidated {invalidated_count} cache keys for user {user_id} (all pages)")
                
        except Exception as e:
            logger.error(f"Error invalidating cache for user {user_id}: {e}")
    
    def get_permission_summary(self, user, page_name: str) -> Dict[str, any]:
        """
        Get a comprehensive permission summary for a user on a specific page
        
        Args:
            user: CustomUser instance
            page_name: Name of the page
            
        Returns:
            Dictionary containing permission summary and metadata
        """
        try:
            permissions = self.get_page_permissions(user, page_name)
            available_actions = self.get_available_actions(user, page_name)
            
            # Count permissions by category
            category_counts = {}
            for action in available_actions:
                category = action['category']
                category_counts[category] = category_counts.get(category, 0) + 1
            
            # Check if user has any critical permissions
            has_critical_permissions = any(action['is_critical'] for action in available_actions)
            
            summary = {
                'user_id': str(user.id),
                'user_name': user.get_full_name(),
                'user_role': user.role,
                'page_name': page_name,
                'total_permissions': len(permissions),
                'allowed_permissions': sum(1 for allowed in permissions.values() if allowed),
                'denied_permissions': sum(1 for allowed in permissions.values() if not allowed),
                'available_actions': len(available_actions),
                'category_breakdown': category_counts,
                'has_critical_permissions': has_critical_permissions,
                'permission_source': 'role_default',  # This could be enhanced to show mixed sources
                'last_updated': timezone.now().isoformat()
            }
            
            return summary
            
        except Exception as e:
            logger.error(f"Error getting permission summary for user {user.id}, page {page_name}: {e}")
            return {}
    
    def bulk_check_permissions(self, user, permission_checks: List[Tuple[str, str]]) -> Dict[str, bool]:
        """
        Efficiently check multiple permissions at once
        
        Args:
            user: CustomUser instance
            permission_checks: List of (page_name, action_code) tuples
            
        Returns:
            Dictionary mapping "page_name:action_code" to boolean permission status
        """
        results = {}
        
        try:
            for page_name, action_code in permission_checks:
                key = f"{page_name}:{action_code}"
                results[key] = self.check_action_permission(user, page_name, action_code)
            
            return results
            
        except Exception as e:
            logger.error(f"Error in bulk permission check for user {user.id}: {e}")
            return {f"{page}:{action}": False for page, action in permission_checks}

class RolePermissionTemplateManager:
    """
    Service class for managing role permission templates and applying defaults to users
    """
    
    def __init__(self):
        # Import here to avoid circular imports
        from .permission_cache_service import permission_cache
        self.cache_service = permission_cache
    
    def set_role_defaults(self, role: str, permissions_dict: Dict[str, Dict[str, bool]], 
                         created_by=None) -> Dict[str, any]:
        """
        Set default permissions for a role
        
        Args:
            role: Role name (e.g., 'loan_officer', 'team_leader')
            permissions_dict: Nested dict {page_name: {action_code: is_allowed}}
            created_by: User who is setting these defaults
            
        Returns:
            Dictionary with operation results and statistics
        """
        try:
            from .enhanced_permissions_models import RolePermissionTemplate, PagePermission
            from django.db import transaction
            
            results = {
                'role': role,
                'total_permissions': 0,
                'updated_permissions': 0,
                'created_permissions': 0,
                'errors': [],
                'success': True
            }
            
            with transaction.atomic():
                for page_name, actions in permissions_dict.items():
                    for action_code, is_allowed in actions.items():
                        try:
                            # Get the page permission
                            page_permission = PagePermission.objects.get(
                                page_name=page_name,
                                action_code=action_code,
                                is_active=True
                            )
                            
                            # Create or update role permission template
                            template, created = RolePermissionTemplate.objects.update_or_create(
                                role=role,
                                page_permission=page_permission,
                                defaults={
                                    'is_allowed': is_allowed,
                                    'can_override': True,  # Default to allowing overrides
                                    'created_by': created_by
                                }
                            )
                            
                            results['total_permissions'] += 1
                            if created:
                                results['created_permissions'] += 1
                            else:
                                results['updated_permissions'] += 1
                                
                        except PagePermission.DoesNotExist:
                            error_msg = f"Page permission not found: {page_name}:{action_code}"
                            results['errors'].append(error_msg)
                            logger.warning(error_msg)
                        except Exception as e:
                            error_msg = f"Error setting permission {page_name}:{action_code}: {str(e)}"
                            results['errors'].append(error_msg)
                            logger.error(error_msg)
            
            # Invalidate cache for this role using enhanced cache service
            self.cache_service.invalidate_role_cache(role)
            
            logger.info(f"Set role defaults for {role}: {results['created_permissions']} created, "
                       f"{results['updated_permissions']} updated")
            
            return results
            
        except Exception as e:
            logger.error(f"Error setting role defaults for {role}: {e}")
            return {
                'role': role,
                'success': False,
                'error': str(e),
                'total_permissions': 0,
                'updated_permissions': 0,
                'created_permissions': 0,
                'errors': [str(e)]
            }
    
    def apply_defaults_to_users(self, role: str, update_existing: bool = False, 
                               user_ids: Optional[List[str]] = None) -> Dict[str, any]:
        """
        Apply role defaults to users, optionally updating existing custom permissions
        
        Args:
            role: Role to apply defaults for
            update_existing: Whether to update users who already have custom permissions
            user_ids: Optional list of specific user IDs to update, if None updates all users with this role
            
        Returns:
            Dictionary with operation results and statistics
        """
        try:
            from .enhanced_permissions_models import RolePermissionTemplate, UserPagePermission
            from .models import CustomUser
            from django.db import transaction
            
            results = {
                'role': role,
                'users_processed': 0,
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'errors': [],
                'success': True
            }
            
            # Get users to update
            if user_ids:
                users = CustomUser.objects.filter(id__in=user_ids, role=role)
            else:
                users = CustomUser.objects.filter(role=role)
            
            # Get role permission templates
            role_templates = RolePermissionTemplate.objects.filter(role=role).select_related('page_permission')
            
            if not role_templates.exists():
                results['errors'].append(f"No role templates found for role {role}")
                results['success'] = False
                return results
            
            with transaction.atomic():
                for user in users:
                    user_permissions_applied = 0
                    
                    for template in role_templates:
                        try:
                            # Check if user already has custom permission
                            existing_permission = UserPagePermission.objects.filter(
                                user=user,
                                page_permission=template.page_permission
                            ).first()
                            
                            if existing_permission and not update_existing:
                                # Skip if user has custom permission and we're not updating existing
                                results['permissions_skipped'] += 1
                                continue
                            
                            # Apply the role default
                            if existing_permission:
                                # Update existing custom permission
                                existing_permission.is_allowed = template.is_allowed
                                existing_permission.reason = f"Updated from role template for {role}"
                                existing_permission.save()
                            else:
                                # Create new custom permission based on role template
                                # Only create if the role template grants permission or explicitly denies it
                                # This prevents creating unnecessary "False" permissions
                                if template.is_allowed:
                                    UserPagePermission.objects.create(
                                        user=user,
                                        page_permission=template.page_permission,
                                        is_allowed=template.is_allowed,
                                        reason=f"Applied from role template for {role}"
                                    )
                            
                            user_permissions_applied += 1
                            results['permissions_applied'] += 1
                            
                        except Exception as e:
                            error_msg = f"Error applying permission {template.page_permission.action_code} to user {user.id}: {str(e)}"
                            results['errors'].append(error_msg)
                            logger.error(error_msg)
                    
                    if user_permissions_applied > 0:
                        results['users_processed'] += 1
                        # Invalidate user's permission cache using enhanced cache service
                        self.cache_service.invalidate_user_cache(str(user.id))
            
            logger.info(f"Applied role defaults for {role}: {results['users_processed']} users, "
                       f"{results['permissions_applied']} permissions applied")
            
            return results
            
        except Exception as e:
            logger.error(f"Error applying role defaults for {role}: {e}")
            return {
                'role': role,
                'success': False,
                'error': str(e),
                'users_processed': 0,
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'errors': [str(e)]
            }
    
    def get_role_template(self, role: str) -> Dict[str, Dict[str, any]]:
        """
        Get role permission template for a specific role
        
        Args:
            role: Role name to get template for
            
        Returns:
            Nested dictionary {page_name: {action_code: permission_info}}
        """
        # Check cache first using enhanced cache service
        cached_template = self.cache_service.get_role_template(role)
        
        if cached_template is not None:
            logger.debug(f"Cache hit for role template {role}")
            return cached_template
        
        try:
            from .enhanced_permissions_models import RolePermissionTemplate
            
            templates = RolePermissionTemplate.objects.filter(
                role=role
            ).select_related('page_permission').order_by(
                'page_permission__page_name', 
                'page_permission__category',
                'page_permission__action_name'
            )
            
            role_template = {}
            
            for template in templates:
                page_name = template.page_permission.page_name
                action_code = template.page_permission.action_code
                
                if page_name not in role_template:
                    role_template[page_name] = {}
                
                role_template[page_name][action_code] = {
                    'is_allowed': template.is_allowed,
                    'can_override': template.can_override,
                    'priority': template.priority,
                    'action_name': template.page_permission.action_name,
                    'description': template.page_permission.description,
                    'category': template.page_permission.category,
                    'is_critical': template.page_permission.is_critical,
                    'created_at': template.created_at.isoformat(),
                    'updated_at': template.updated_at.isoformat()
                }
            
            # Cache the result using enhanced cache service
            self.cache_service.set_role_template(role, role_template)
            
            return role_template
            
        except Exception as e:
            logger.error(f"Error getting role template for {role}: {e}")
            return {}
    
    def validate_permission_combinations(self, role: str, permissions_dict: Dict[str, Dict[str, bool]]) -> Dict[str, any]:
        """
        Validate permission combinations for potential conflicts or issues
        
        Args:
            role: Role name
            permissions_dict: Permissions to validate {page_name: {action_code: is_allowed}}
            
        Returns:
            Dictionary with validation results, warnings, and errors
        """
        try:
            from .enhanced_permissions_models import PagePermission
            
            validation_results = {
                'valid': True,
                'warnings': [],
                'errors': [],
                'suggestions': [],
                'critical_permissions': [],
                'dependency_issues': []
            }
            
            # Check each permission
            for page_name, actions in permissions_dict.items():
                for action_code, is_allowed in actions.items():
                    try:
                        page_permission = PagePermission.objects.get(
                            page_name=page_name,
                            action_code=action_code,
                            is_active=True
                        )
                        
                        # Check for critical permissions
                        if page_permission.is_critical and is_allowed:
                            validation_results['critical_permissions'].append({
                                'page_name': page_name,
                                'action_code': action_code,
                                'action_name': page_permission.action_name,
                                'description': page_permission.description
                            })
                        
                        # Check permission dependencies
                        if is_allowed and page_permission.required_permissions.exists():
                            for required_perm in page_permission.required_permissions.all():
                                required_page = required_perm.page_name
                                required_action = required_perm.action_code
                                
                                # Check if required permission is granted
                                if (required_page not in permissions_dict or 
                                    required_action not in permissions_dict[required_page] or 
                                    not permissions_dict[required_page][required_action]):
                                    
                                    validation_results['dependency_issues'].append({
                                        'permission': f"{page_name}:{action_code}",
                                        'requires': f"{required_page}:{required_action}",
                                        'message': f"Permission {page_permission.action_name} requires {required_perm.action_name}"
                                    })
                        
                    except PagePermission.DoesNotExist:
                        validation_results['errors'].append(
                            f"Permission not found: {page_name}:{action_code}"
                        )
                        validation_results['valid'] = False
            
            # Role-specific validation rules
            if role == 'loan_officer':
                self._validate_loan_officer_permissions(permissions_dict, validation_results)
            elif role == 'team_leader':
                self._validate_team_leader_permissions(permissions_dict, validation_results)
            elif role == 'secretary':
                self._validate_secretary_permissions(permissions_dict, validation_results)
            elif role == 'auditor':
                self._validate_auditor_permissions(permissions_dict, validation_results)
            
            # Set overall validity
            if validation_results['errors'] or validation_results['dependency_issues']:
                validation_results['valid'] = False
            
            return validation_results
            
        except Exception as e:
            logger.error(f"Error validating permissions for role {role}: {e}")
            return {
                'valid': False,
                'error': str(e),
                'warnings': [],
                'errors': [str(e)],
                'suggestions': [],
                'critical_permissions': [],
                'dependency_issues': []
            }
    
    def _validate_loan_officer_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                         validation_results: Dict[str, any]):
        """Validate permissions specific to loan officer role"""
        # Loan officers should have client management permissions
        if 'clients' in permissions_dict:
            if not permissions_dict['clients'].get('clients_view_list', False):
                validation_results['warnings'].append(
                    "Loan officers typically need 'clients_view_list' permission"
                )
        
        # Check for loan processing permissions
        if 'loans' in permissions_dict:
            if permissions_dict['loans'].get('loans_approve_application', False):
                validation_results['suggestions'].append(
                    "Consider if loan officers should have approval permissions or if this should be team leaders only"
                )
    
    def _validate_team_leader_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                        validation_results: Dict[str, any]):
        """Validate permissions specific to team leader role"""
        # Team leaders should have broader access
        if 'reports_statements' in permissions_dict:
            if not permissions_dict['reports_statements'].get('reports_view_dashboard', False):
                validation_results['warnings'].append(
                    "Team leaders typically need access to reports dashboard"
                )
    
    def _validate_secretary_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                      validation_results: Dict[str, any]):
        """Validate permissions specific to secretary role"""
        # Secretaries should have data entry permissions but limited approval permissions
        critical_actions = ['loans_approve_application', 'clients_approve_client']
        for page_name, actions in permissions_dict.items():
            for action_code, is_allowed in actions.items():
                if action_code in critical_actions and is_allowed:
                    validation_results['warnings'].append(
                        f"Consider if secretaries should have {action_code} permission"
                    )
    
    def _validate_auditor_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                    validation_results: Dict[str, any]):
        """Validate permissions specific to auditor role"""
        # Auditors should have read access but limited write access
        write_actions = ['create', 'edit', 'delete', 'approve', 'process']
        for page_name, actions in permissions_dict.items():
            for action_code, is_allowed in actions.items():
                if any(write_action in action_code for write_action in write_actions) and is_allowed:
                    validation_results['warnings'].append(
                        f"Auditors typically should not have write permission: {action_code}"
                    )
    
    def _invalidate_role_cache(self, role: str):
        """Invalidate cached role template"""
        cache_key = f"{self.cache_prefix}:template:{role}"
        cache.delete(cache_key)
        logger.debug(f"Invalidated role template cache for {role}")
    
    def get_all_role_templates(self) -> Dict[str, Dict[str, Dict[str, any]]]:
        """
        Get all role permission templates
        
        Returns:
            Nested dictionary {role: {page_name: {action_code: permission_info}}}
        """
        try:
            from .enhanced_permissions_models import RolePermissionTemplate
            
            all_templates = {}
            
            # Get all available roles
            roles = [choice[0] for choice in RolePermissionTemplate.ROLE_CHOICES]
            
            for role in roles:
                all_templates[role] = self.get_role_template(role)
            
            return all_templates
            
        except Exception as e:
            logger.error(f"Error getting all role templates: {e}")
            return {}
    
    def compare_role_permissions(self, role1: str, role2: str) -> Dict[str, any]:
        """
        Compare permissions between two roles
        
        Args:
            role1: First role to compare
            role2: Second role to compare
            
        Returns:
            Dictionary with comparison results
        """
        try:
            template1 = self.get_role_template(role1)
            template2 = self.get_role_template(role2)
            
            comparison = {
                'role1': role1,
                'role2': role2,
                'role1_only': {},
                'role2_only': {},
                'different_permissions': {},
                'same_permissions': {},
                'summary': {
                    'role1_total': 0,
                    'role2_total': 0,
                    'role1_unique': 0,
                    'role2_unique': 0,
                    'different': 0,
                    'same': 0
                }
            }
            
            # Get all unique page/action combinations
            all_pages = set(template1.keys()) | set(template2.keys())
            
            for page_name in all_pages:
                page1_actions = template1.get(page_name, {})
                page2_actions = template2.get(page_name, {})
                
                all_actions = set(page1_actions.keys()) | set(page2_actions.keys())
                
                for action_code in all_actions:
                    perm1 = page1_actions.get(action_code)
                    perm2 = page2_actions.get(action_code)
                    
                    if perm1 and not perm2:
                        # Only in role1
                        if page_name not in comparison['role1_only']:
                            comparison['role1_only'][page_name] = {}
                        comparison['role1_only'][page_name][action_code] = perm1
                        comparison['summary']['role1_unique'] += 1
                        
                    elif perm2 and not perm1:
                        # Only in role2
                        if page_name not in comparison['role2_only']:
                            comparison['role2_only'][page_name] = {}
                        comparison['role2_only'][page_name][action_code] = perm2
                        comparison['summary']['role2_unique'] += 1
                        
                    elif perm1 and perm2:
                        if perm1['is_allowed'] != perm2['is_allowed']:
                            # Different permissions
                            if page_name not in comparison['different_permissions']:
                                comparison['different_permissions'][page_name] = {}
                            comparison['different_permissions'][page_name][action_code] = {
                                'role1': perm1,
                                'role2': perm2
                            }
                            comparison['summary']['different'] += 1
                        else:
                            # Same permissions
                            if page_name not in comparison['same_permissions']:
                                comparison['same_permissions'][page_name] = {}
                            comparison['same_permissions'][page_name][action_code] = perm1
                            comparison['summary']['same'] += 1
            
            # Calculate totals
            comparison['summary']['role1_total'] = sum(len(actions) for actions in template1.values())
            comparison['summary']['role2_total'] = sum(len(actions) for actions in template2.values())
            
            return comparison
            
        except Exception as e:
            logger.error(f"Error comparing roles {role1} and {role2}: {e}")
            return {
                'error': str(e),
                'role1': role1,
                'role2': role2
            }
    
    def apply_role_template_to_user(self, user, role_template: str = None, 
                                   update_existing: bool = False, 
                                   conflict_resolution: str = 'keep_custom',
                                   applied_by=None) -> Dict[str, any]:
        """
        Apply role template to a specific user with enhanced conflict resolution
        Create service to apply role templates to new users
        
        Args:
            user: CustomUser instance
            role_template: Role template to apply (defaults to user's current role)
            update_existing: Whether to update existing custom permissions
            conflict_resolution: How to handle conflicts ('keep_custom', 'use_role', 'merge')
            applied_by: User applying the template (for audit trail)
            
        Returns:
            Dictionary with operation results
        """
        try:
            from .enhanced_permissions_models import RolePermissionTemplate, UserPagePermission
            from django.db import transaction
            
            role = role_template or user.role
            
            results = {
                'success': True,
                'user_id': str(user.id),
                'role': role,
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'conflicts_resolved': 0,
                'errors': [],
                'conflicts': [],
                'approval_required': False
            }
            
            # Get role permission templates
            role_templates = RolePermissionTemplate.objects.filter(role=role).select_related('page_permission')
            
            if not role_templates.exists():
                results['errors'].append(f"No role templates found for role {role}")
                results['success'] = False
                return results
            
            # Check if any changes require approval
            critical_changes = []
            
            with transaction.atomic():
                for template in role_templates:
                    try:
                        # Get existing custom permission
                        existing_permission = UserPagePermission.objects.filter(
                            user=user,
                            page_permission=template.page_permission
                        ).first()
                        
                        # Determine if this is a critical permission change
                        if template.page_permission.is_critical and template.is_allowed:
                            critical_changes.append({
                                'permission': template.page_permission.action_code,
                                'page': template.page_permission.page_name,
                                'action': 'grant' if template.is_allowed else 'revoke'
                            })
                        
                        # Handle conflicts based on resolution strategy
                        conflict_result = self._resolve_template_conflict(
                            user=user,
                            template=template,
                            existing_permission=existing_permission,
                            resolution_strategy=conflict_resolution,
                            update_existing=update_existing
                        )
                        
                        if conflict_result['applied']:
                            results['permissions_applied'] += 1
                        else:
                            results['permissions_skipped'] += 1
                        
                        if conflict_result['conflict']:
                            results['conflicts_resolved'] += 1
                            results['conflicts'].append(conflict_result)
                        
                    except Exception as e:
                        error_msg = f"Error applying permission {template.page_permission.action_code}: {str(e)}"
                        results['errors'].append(error_msg)
                        logger.error(error_msg)
            
            # Check if approval is required for critical changes
            if critical_changes:
                results['approval_required'] = True
                results['critical_changes'] = critical_changes
                
                # Log for approval workflow
                try:
                    from .audit_service import AuditService
                    audit_service = AuditService()
                    audit_service.log_permission_change(
                        user=user,
                        changed_by=applied_by or user,
                        permission_name=f"role_template_{role}",
                        old_value=user.role,
                        new_value=role,
                        reason=f"Role template application with {len(critical_changes)} critical changes"
                    )
                except ImportError:
                    logger.info(f"Audit service not available, skipping audit log")
            
            # Invalidate user's permission cache
            if results['permissions_applied'] > 0:
                PagePermissionManager().invalidate_user_cache(str(user.id))
                
                logger.info(f"Applied role template {role} to user {user.id}: "
                           f"{results['permissions_applied']} applied, "
                           f"{results['conflicts_resolved']} conflicts resolved")
            
            return results
            
        except Exception as e:
            logger.error(f"Error applying role template to user {user.id}: {e}")
            return {
                'success': False,
                'error': str(e),
                'user_id': str(user.id),
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'conflicts_resolved': 0,
                'errors': [str(e)]
            }
    
    def bulk_update_users_with_template(self, user_ids: List[str], role: str, 
                                       update_existing: bool = False,
                                       conflict_resolution: str = 'keep_custom',
                                       applied_by=None,
                                       batch_size: int = 50) -> Dict[str, any]:
        """
        Implement bulk update functionality for existing users with enhanced progress tracking
        
        Args:
            user_ids: List of user IDs to update
            role: Role template to apply
            update_existing: Whether to update existing custom permissions
            conflict_resolution: How to handle conflicts ('keep_custom', 'use_role', 'merge')
            applied_by: User performing the bulk update
            batch_size: Number of users to process in each batch
            
        Returns:
            Dictionary with operation results and detailed progress
        """
        try:
            from .models import CustomUser
            from django.db import transaction
            import time
            
            results = {
                'success': True,
                'role': role,
                'total_users': len(user_ids),
                'users_processed': 0,
                'users_failed': 0,
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'conflicts_resolved': 0,
                'batches_processed': 0,
                'total_batches': (len(user_ids) + batch_size - 1) // batch_size,
                'errors': [],
                'user_results': {},
                'approval_required': False,
                'critical_changes_count': 0,
                'processing_time': 0
            }
            
            start_time = time.time()
            
            # Validate users exist and have correct role
            valid_users = CustomUser.objects.filter(
                id__in=user_ids,
                role=role,
                is_active=True
            ).values_list('id', flat=True)
            
            invalid_user_ids = set(user_ids) - set(str(uid) for uid in valid_users)
            if invalid_user_ids:
                results['errors'].append(f"Invalid or inactive user IDs: {list(invalid_user_ids)}")
            
            # Process users in batches
            for i in range(0, len(valid_users), batch_size):
                batch_user_ids = list(valid_users)[i:i + batch_size]
                batch_users = CustomUser.objects.filter(id__in=batch_user_ids)
                
                batch_results = {
                    'batch_number': results['batches_processed'] + 1,
                    'users_in_batch': len(batch_users),
                    'users_processed': 0,
                    'permissions_applied': 0,
                    'conflicts_resolved': 0,
                    'errors': []
                }
                
                try:
                    with transaction.atomic():
                        for user in batch_users:
                            try:
                                user_result = self.apply_role_template_to_user(
                                    user=user,
                                    role_template=role,
                                    update_existing=update_existing,
                                    conflict_resolution=conflict_resolution,
                                    applied_by=applied_by
                                )
                                
                                # Aggregate results
                                if user_result['success']:
                                    results['users_processed'] += 1
                                    batch_results['users_processed'] += 1
                                    results['permissions_applied'] += user_result['permissions_applied']
                                    batch_results['permissions_applied'] += user_result['permissions_applied']
                                    results['permissions_skipped'] += user_result['permissions_skipped']
                                    results['conflicts_resolved'] += user_result['conflicts_resolved']
                                    batch_results['conflicts_resolved'] += user_result['conflicts_resolved']
                                    
                                    if user_result.get('approval_required'):
                                        results['approval_required'] = True
                                        results['critical_changes_count'] += len(user_result.get('critical_changes', []))
                                else:
                                    results['users_failed'] += 1
                                    results['errors'].extend(user_result['errors'])
                                    batch_results['errors'].extend(user_result['errors'])
                                
                                # Store individual user result
                                results['user_results'][str(user.id)] = {
                                    'success': user_result['success'],
                                    'permissions_applied': user_result['permissions_applied'],
                                    'conflicts_resolved': user_result['conflicts_resolved'],
                                    'approval_required': user_result.get('approval_required', False)
                                }
                                
                            except Exception as e:
                                error_msg = f"Error processing user {user.id}: {str(e)}"
                                results['errors'].append(error_msg)
                                batch_results['errors'].append(error_msg)
                                results['users_failed'] += 1
                                logger.error(error_msg)
                    
                    results['batches_processed'] += 1
                    
                    # Log batch completion
                    logger.info(f"Completed batch {results['batches_processed']}/{results['total_batches']}: "
                               f"{batch_results['users_processed']} users, "
                               f"{batch_results['permissions_applied']} permissions applied")
                    
                except Exception as e:
                    error_msg = f"Error processing batch {results['batches_processed'] + 1}: {str(e)}"
                    results['errors'].append(error_msg)
                    logger.error(error_msg)
                    # Continue with next batch
                    continue
            
            results['processing_time'] = time.time() - start_time
            
            # Final validation
            if results['users_failed'] > 0:
                results['success'] = False
            
            # Log bulk update completion
            logger.info(f"Bulk update completed for role {role}: "
                       f"{results['users_processed']}/{results['total_users']} users processed, "
                       f"{results['permissions_applied']} permissions applied, "
                       f"{results['conflicts_resolved']} conflicts resolved in {results['processing_time']:.2f}s")
            
            # Create audit log for bulk operation
            if applied_by:
                try:
                    from .audit_service import AuditService
                    audit_service = AuditService()
                    audit_service.log_user_action(
                        user=applied_by,
                        action='bulk_role_template_update',
                        module='permissions',
                        target_count=results['users_processed'],
                        role=role,
                        permissions_applied=results['permissions_applied'],
                        conflicts_resolved=results['conflicts_resolved']
                    )
                except ImportError:
                    logger.info(f"Audit service not available, skipping audit log for bulk update")
            
            return results
            
        except Exception as e:
            logger.error(f"Error in bulk update with template {role}: {e}")
            return {
                'success': False,
                'error': str(e),
                'total_users': len(user_ids),
                'users_processed': 0,
                'users_failed': len(user_ids),
                'permissions_applied': 0,
                'permissions_skipped': 0,
                'conflicts_resolved': 0,
                'errors': [str(e)]
            }
    
    def _resolve_template_conflict(self, user, template, existing_permission, 
                                 resolution_strategy: str, update_existing: bool) -> Dict[str, any]:
        """
        Resolve conflicts between role template and existing custom permissions
        
        Args:
            user: CustomUser instance
            template: RolePermissionTemplate instance
            existing_permission: Existing UserPagePermission or None
            resolution_strategy: 'keep_custom', 'use_role', or 'merge'
            update_existing: Whether to update existing permissions
            
        Returns:
            Dictionary with conflict resolution result
        """
        try:
            from .enhanced_permissions_models import UserPagePermission
            
            result = {
                'permission_code': template.page_permission.action_code,
                'page_name': template.page_permission.page_name,
                'conflict': False,
                'applied': False,
                'action_taken': None,
                'final_permission': None,
                'reason': None
            }
            
            role_permission = template.is_allowed
            custom_permission = existing_permission.is_allowed if existing_permission else None
            
            # No existing custom permission - apply role template
            if not existing_permission:
                if role_permission:  # Only create if granting permission
                    UserPagePermission.objects.create(
                        user=user,
                        page_permission=template.page_permission,
                        is_allowed=role_permission,
                        reason=f"Applied from role template: {user.role}"
                    )
                    result['applied'] = True
                    result['final_permission'] = role_permission
                    result['action_taken'] = 'created_from_template'
                return result
            
            # Existing custom permission exists
            if custom_permission == role_permission:
                # No conflict - permissions match
                result['final_permission'] = custom_permission
                result['action_taken'] = 'no_change_needed'
                return result
            
            # Conflict detected
            result['conflict'] = True
            
            if not template.can_override:
                # Role template doesn't allow overrides - force role permission
                existing_permission.is_allowed = role_permission
                existing_permission.reason = f"Forced from role template (no override allowed): {user.role}"
                existing_permission.save()
                result['applied'] = True
                result['final_permission'] = role_permission
                result['action_taken'] = 'forced_role_permission'
                result['reason'] = 'Role template does not allow overrides'
                
            elif resolution_strategy == 'use_role' or (resolution_strategy == 'merge' and update_existing):
                # Use role permission
                existing_permission.is_allowed = role_permission
                existing_permission.reason = f"Updated from role template: {user.role}"
                existing_permission.save()
                result['applied'] = True
                result['final_permission'] = role_permission
                result['action_taken'] = 'updated_to_role_permission'
                result['reason'] = f'Applied {resolution_strategy} strategy'
                
            elif resolution_strategy == 'keep_custom':
                # Keep existing custom permission
                result['final_permission'] = custom_permission
                result['action_taken'] = 'kept_custom_permission'
                result['reason'] = 'Kept existing custom permission per strategy'
                
            else:
                # Default to keeping custom permission
                result['final_permission'] = custom_permission
                result['action_taken'] = 'kept_custom_permission'
                result['reason'] = 'Default conflict resolution'
            
            return result
            
        except Exception as e:
            logger.error(f"Error resolving template conflict: {e}")
            return {
                'permission_code': template.page_permission.action_code,
                'conflict': True,
                'applied': False,
                'error': str(e),
                'action_taken': 'error',
                'final_permission': False
            }

    def resolve_permission_conflicts(self, user, page_permission, 
                                   role_permission: bool, custom_permission: Optional[bool] = None) -> Dict[str, any]:
        """
        Add conflict resolution for custom vs role permissions
        
        Args:
            user: CustomUser instance
            page_permission: PagePermission instance
            role_permission: Permission from role template
            custom_permission: Existing custom permission (if any)
            
        Returns:
            Dictionary with conflict resolution result
        """
        try:
            from .enhanced_permissions_models import UserPagePermission, RolePermissionTemplate
            
            # Check if role template allows overrides
            try:
                role_template = RolePermissionTemplate.objects.get(
                    role=user.role,
                    page_permission=page_permission
                )
                can_override = role_template.can_override
            except RolePermissionTemplate.DoesNotExist:
                can_override = True  # Default to allowing overrides
            
            resolution = {
                'permission_id': str(page_permission.id),
                'action_code': page_permission.action_code,
                'role_permission': role_permission,
                'custom_permission': custom_permission,
                'can_override': can_override,
                'final_permission': None,
                'source': None,
                'conflict': False,
                'action_taken': None
            }
            
            # Determine final permission and source
            if custom_permission is not None and can_override:
                # Custom permission takes precedence if overrides are allowed
                resolution['final_permission'] = custom_permission
                resolution['source'] = 'custom'
                
                if custom_permission != role_permission:
                    resolution['conflict'] = True
                    resolution['action_taken'] = 'kept_custom_override'
            else:
                # Use role permission
                resolution['final_permission'] = role_permission
                resolution['source'] = 'role'
                
                if custom_permission is not None and not can_override:
                    resolution['conflict'] = True
                    resolution['action_taken'] = 'removed_custom_override'
                    
                    # Remove the custom permission since overrides are not allowed
                    UserPagePermission.objects.filter(
                        user=user,
                        page_permission=page_permission
                    ).delete()
            
            return resolution
            
        except Exception as e:
            logger.error(f"Error resolving permission conflict: {e}")
            return {
                'permission_id': str(page_permission.id),
                'error': str(e),
                'conflict': True,
                'final_permission': False,
                'source': 'error'
            }
    
    def create_template_rollback_point(self, role: str, created_by=None, description: str = "") -> str:
        """
        Create rollback functionality for template changes with database storage
        
        Args:
            role: Role to create rollback point for
            created_by: User creating the rollback point
            description: Optional description for the rollback point
            
        Returns:
            Rollback point identifier
        """
        try:
            from django.utils import timezone
            from .enhanced_permissions_models import RoleTemplateRollbackPoint
            import json
            import uuid
            
            # Get current role template
            current_template = self.get_role_template(role)
            
            if not current_template:
                logger.warning(f"No template found for role {role}, cannot create rollback point")
                return None
            
            # Create rollback point identifier
            rollback_id = f"{role}_{uuid.uuid4().hex[:8]}_{timezone.now().strftime('%Y%m%d_%H%M%S')}"
            
            # Store rollback data in database
            rollback_point = RoleTemplateRollbackPoint.objects.create(
                rollback_id=rollback_id,
                role=role,
                template_data=json.dumps(current_template),
                created_by=created_by,
                description=description or f"Rollback point for {role} role template",
                created_at=timezone.now()
            )
            
            # Also store in cache for quick access
            rollback_data = {
                'role': role,
                'template': current_template,
                'created_at': timezone.now().isoformat(),
                'rollback_id': rollback_id,
                'created_by': created_by.get_full_name() if created_by else 'System',
                'description': description
            }
            
            cache_key = f"rollback:{rollback_id}"
            cache.set(cache_key, rollback_data, timeout=86400)  # 24 hours
            
            logger.info(f"Created rollback point {rollback_id} for role {role}")
            return rollback_id
            
        except Exception as e:
            logger.error(f"Error creating rollback point for role {role}: {e}")
            return None
    
    def rollback_template_changes(self, rollback_id: str, applied_by=None) -> Dict[str, any]:
        """
        Rollback template changes to a previous state with enhanced tracking
        
        Args:
            rollback_id: Rollback point identifier
            applied_by: User applying the rollback
            
        Returns:
            Dictionary with rollback results
        """
        try:
            from .enhanced_permissions_models import RoleTemplateRollbackPoint
            import json
            
            # Get rollback point from database
            try:
                rollback_point = RoleTemplateRollbackPoint.objects.get(
                    rollback_id=rollback_id,
                    is_applied=False
                )
            except RoleTemplateRollbackPoint.DoesNotExist:
                return {
                    'success': False,
                    'error': 'Rollback point not found or already applied',
                    'rollback_id': rollback_id
                }
            
            role = rollback_point.role
            template = rollback_point.template_data
            
            # Convert template format for set_role_defaults
            permissions_dict = {}
            
            # Handle both string and dict template data
            if isinstance(template, str):
                import json
                template = json.loads(template)
            
            for page_name, actions in template.items():
                permissions_dict[page_name] = {}
                for action_code, perm_data in actions.items():
                    if isinstance(perm_data, dict):
                        permissions_dict[page_name][action_code] = perm_data['is_allowed']
                    else:
                        permissions_dict[page_name][action_code] = perm_data
            
            # Create a new rollback point before applying changes
            current_rollback_id = self.create_template_rollback_point(
                role=role,
                created_by=applied_by,
                description=f"Auto-created before rollback to {rollback_id}"
            )
            
            # Apply the rollback template
            result = self.set_role_defaults(
                role=role,
                permissions_dict=permissions_dict,
                created_by=applied_by
            )
            
            if result['success']:
                # Mark rollback point as applied
                rollback_point.mark_as_applied(applied_by=applied_by)
                
                logger.info(f"Successfully rolled back role {role} to {rollback_point.created_at}")
                
                # Clean up cache
                cache_key = f"rollback:{rollback_id}"
                cache.delete(cache_key)
                
                # Log the rollback operation
                try:
                    from .audit_service import AuditService
                    audit_service = AuditService()
                    audit_service.log_user_action(
                        user=applied_by or rollback_point.created_by,
                        action='role_template_rollback',
                        module='permissions',
                        rollback_id=rollback_id,
                        role=role,
                        rollback_date=rollback_point.created_at.isoformat(),
                        permissions_restored=result['total_permissions']
                    )
                except ImportError:
                    logger.info(f"Audit service not available, skipping audit log")
                
                return {
                    'success': True,
                    'role': role,
                    'rollback_id': rollback_id,
                    'rollback_date': rollback_point.created_at.isoformat(),
                    'permissions_restored': result['total_permissions'],
                    'description': rollback_point.description,
                    'created_by': rollback_point.created_by.get_full_name() if rollback_point.created_by else 'System',
                    'new_rollback_id': current_rollback_id
                }
            else:
                return {
                    'success': False,
                    'error': result.get('error', 'Failed to apply rollback'),
                    'rollback_id': rollback_id
                }
                
        except Exception as e:
            logger.error(f"Error rolling back template changes {rollback_id}: {e}")
            return {
                'success': False,
                'error': str(e),
                'rollback_id': rollback_id
            }
    
    def get_available_rollback_points(self, role: str, limit: int = 10) -> List[Dict[str, any]]:
        """
        Get available rollback points for a role from database
        
        Args:
            role: Role to get rollback points for
            limit: Maximum number of rollback points to return
            
        Returns:
            List of available rollback points
        """
        try:
            from .enhanced_permissions_models import RoleTemplateRollbackPoint
            
            rollback_points = RoleTemplateRollbackPoint.objects.filter(
                role=role,
                is_applied=False
            ).order_by('-created_at')[:limit]
            
            results = []
            for point in rollback_points:
                summary = point.get_template_summary()
                results.append({
                    'rollback_id': point.rollback_id,
                    'role': point.role,
                    'description': point.description,
                    'created_at': point.created_at.isoformat(),
                    'created_by': point.created_by.get_full_name() if point.created_by else 'System',
                    'template_summary': summary,
                    'is_applied': point.is_applied
                })
            
            return results
            
        except Exception as e:
            logger.error(f"Error getting rollback points for role {role}: {e}")
            return []
    
    def validate_role_permission_set(self, role: str, permissions_dict: Dict[str, Dict[str, bool]], 
                                   strict_mode: bool = False) -> Dict[str, any]:
        """
        Implement validation rules for permission combinations
        Add checks for critical permission dependencies
        Create warnings for potentially dangerous permission sets
        
        Args:
            role: Role to validate permissions for
            permissions_dict: Permissions to validate {page_name: {action_code: is_allowed}}
            
        Returns:
            Dictionary with comprehensive validation results
        """
        try:
            validation_results = {
                'valid': True,
                'role': role,
                'warnings': [],
                'errors': [],
                'suggestions': [],
                'critical_permissions': [],
                'dependency_issues': [],
                'security_concerns': [],
                'role_specific_issues': [],
                'permission_conflicts': [],
                'missing_required_permissions': []
            }
            
            # Get all page permissions for dependency checking
            from .enhanced_permissions_models import PagePermission
            all_permissions = PagePermission.objects.filter(is_active=True).prefetch_related('required_permissions')
            
            # Create lookup for permissions
            permission_lookup = {f"{p.page_name}_{p.action_code}": p for p in all_permissions}
            
            # Check each permission
            for page_name, actions in permissions_dict.items():
                for action_code, is_allowed in actions.items():
                    permission_key = f"{page_name}_{action_code}"
                    
                    if permission_key not in permission_lookup:
                        validation_results['errors'].append(
                            f"Unknown permission: {page_name}:{action_code}")
                        validation_results['valid'] = False
                        continue
                    
                    page_permission = permission_lookup[permission_key]
                    
                    # Check for critical permissions
                    if page_permission.is_critical and is_allowed:
                        validation_results['critical_permissions'].append({
                            'page_name': page_name,
                            'action_code': action_code,
                            'action_name': page_permission.action_name,
                            'description': page_permission.description
                        })
                    
                    # Check for security concerns
                    security_keywords = ['delete', 'admin', 'system', 'export_all', 'modify_permissions']
                    if any(keyword in action_code.lower() for keyword in security_keywords) and is_allowed:
                        validation_results['security_concerns'].append({
                            'page_name': page_name,
                            'action_code': action_code,
                            'concern': f"Granting {action_code} permission may pose security risks"
                        })
            
            # Check for dangerous permission combinations
            self._check_dangerous_combinations(permissions_dict, validation_results, role)
            
            # Check for missing essential permissions
            self._check_missing_essential_permissions(permissions_dict, validation_results, role)
            
            # Check for permission escalation risks
            self._check_permission_escalation_risks(permissions_dict, validation_results, role)
            
            # Role-specific validation
            if role == 'loan_officer':
                self._validate_loan_officer_permissions(permissions_dict, validation_results)
            elif role == 'team_leader':
                self._validate_team_leader_permissions(permissions_dict, validation_results)
            elif role == 'secretary':
                self._validate_secretary_permissions(permissions_dict, validation_results)
            elif role == 'auditor':
                self._validate_auditor_permissions(permissions_dict, validation_results)
            elif role == 'admin':
                self._validate_admin_permissions(permissions_dict, validation_results)
            
            # Apply strict mode validation if enabled
            if strict_mode:
                self._apply_strict_validation_rules(permissions_dict, validation_results, role)
            
            # Set overall validity
            if validation_results['errors'] or validation_results['dependency_issues']:
                validation_results['valid'] = False
            
            # Add severity assessment
            validation_results['severity'] = self._assess_validation_severity(validation_results)
            
            return validation_results
            
        except Exception as e:
            logger.error(f"Error validating permissions for role {role}: {e}")
            return {
                'valid': False,
                'error': str(e),
                'warnings': [],
                'errors': [str(e)],
                'suggestions': [],
                'critical_permissions': [],
                'dependency_issues': [],
                'security_concerns': [],
                'role_specific_issues': [],
                'permission_conflicts': [],
                'missing_required_permissions': []
            }
    
    def create_permission_change_approval_request(self, user, role: str, 
                                                 permissions_dict: Dict[str, Dict[str, bool]],
                                                 requested_by, reason: str = "") -> Dict[str, any]:
        """
        Implement approval workflow for sensitive permission changes
        
        Args:
            user: User whose permissions are being changed
            role: Role template being applied
            permissions_dict: Permissions being requested
            requested_by: User requesting the changes
            reason: Reason for the permission changes
            
        Returns:
            Dictionary with approval request results
        """
        try:
            from django.utils import timezone
            import uuid
            
            # Validate the permission changes
            validation_result = self.validate_role_permission_set(role, permissions_dict)
            
            # Check if approval is required
            requires_approval = (
                len(validation_result.get('critical_permissions', [])) > 0 or
                len(validation_result.get('security_concerns', [])) > 0 or
                not validation_result.get('valid', True)
            )
            
            if not requires_approval:
                # No approval needed, apply directly
                return self.apply_role_template_to_user(
                    user=user,
                    role_template=role,
                    update_existing=True,
                    applied_by=requested_by
                )
            
            # Create approval request ID
            request_id = f"perm_req_{uuid.uuid4().hex[:8]}"
            
            # Store approval request data in cache (in production, use database)
            approval_data = {
                'request_id': request_id,
                'user_id': str(user.id),
                'role': role,
                'permissions_data': permissions_dict,
                'requested_by': str(requested_by.id),
                'reason': reason,
                'validation_results': validation_result,
                'status': 'pending',
                'created_at': timezone.now().isoformat()
            }
            
            cache_key = f"approval_request:{request_id}"
            cache.set(cache_key, approval_data, timeout=604800)  # 7 days
            
            # Determine approvers based on criticality
            approvers_needed = self._determine_required_approvers(validation_result, role)
            
            # Store approver tasks
            for i, approver in enumerate(approvers_needed):
                task_data = {
                    'request_id': request_id,
                    'approver_id': str(approver.id),
                    'status': 'pending',
                    'created_at': timezone.now().isoformat()
                }
                task_key = f"approval_task:{request_id}:{i}"
                cache.set(task_key, task_data, timeout=604800)  # 7 days
            
            # Send notifications to approvers
            self._notify_permission_approvers(request_id, approvers_needed, user, role)
            
            logger.info(f"Created permission change approval request {request_id} for user {user.id}")
            
            return {
                'success': True,
                'approval_required': True,
                'request_id': request_id,
                'approvers_needed': [a.get_full_name() for a in approvers_needed],
                'critical_permissions': validation_result.get('critical_permissions', []),
                'security_concerns': validation_result.get('security_concerns', []),
                'estimated_approval_time': '1-2 business days'
            }
            
        except Exception as e:
            logger.error(f"Error creating permission change approval request: {e}")
            return {
                'success': False,
                'error': str(e),
                'approval_required': False
            }
    
    def _determine_required_approvers(self, validation_result: Dict[str, any], role: str) -> List:
        """
        Determine who needs to approve permission changes based on criticality
        
        Args:
            validation_result: Results from permission validation
            role: Role being modified
            
        Returns:
            List of users who need to approve the changes
        """
        try:
            from .models import CustomUser
            
            approvers = []
            critical_count = len(validation_result.get('critical_permissions', []))
            security_concerns = len(validation_result.get('security_concerns', []))
            
            # Always require admin approval for critical permissions
            if critical_count > 0 or security_concerns > 0:
                admins = CustomUser.objects.filter(
                    role='admin',
                    is_active=True,
                    is_superuser=True
                ).order_by('-last_login')[:2]  # Get 2 most recently active admins
                approvers.extend(admins)
            
            # For team leader role changes, require additional team leader approval
            if role == 'team_leader' and critical_count > 3:
                team_leaders = CustomUser.objects.filter(
                    role='team_leader',
                    is_active=True
                ).exclude(id__in=[a.id for a in approvers])[:1]
                approvers.extend(team_leaders)
            
            # For high-risk changes, require multiple approvals
            if critical_count > 5 or security_concerns > 2:
                additional_admins = CustomUser.objects.filter(
                    role='admin',
                    is_active=True
                ).exclude(id__in=[a.id for a in approvers])[:1]
                approvers.extend(additional_admins)
            
            return approvers
            
        except Exception as e:
            logger.error(f"Error determining required approvers: {e}")
            return []
    
    def _notify_permission_approvers(self, request_id: str, approvers, user, role: str):
        """
        Send notifications to approvers about pending permission changes
        
        Args:
            request_id: Approval request ID
            approvers: List of users who need to approve
            user: User whose permissions are being changed
            role: Role being applied
        """
        try:
            # Create simple notification (in production, use proper notification system)
            for approver in approvers:
                logger.info(f"APPROVAL NEEDED: {approver.get_full_name()} needs to approve "
                           f"permission changes for {user.get_full_name()} (role: {role}) "
                           f"Request ID: {request_id}")
            
            # In production, you would send emails, create database notifications, etc.
            
        except Exception as e:
            logger.error(f"Error sending approval notifications: {e}")
    
    def process_permission_approval(self, request_id: str, approver, 
                                   decision: str, comments: str = "") -> Dict[str, any]:
        """
        Process an approval decision for permission changes
        
        Args:
            request_id: Approval request ID
            approver: User making the approval decision
            decision: 'approve' or 'reject'
            comments: Optional comments from approver
            
        Returns:
            Dictionary with processing results
        """
        try:
            from django.utils import timezone
            
            # Get approval request
            cache_key = f"approval_request:{request_id}"
            approval_data = cache.get(cache_key)
            
            if not approval_data:
                return {
                    'success': False,
                    'error': 'Approval request not found or expired'
                }
            
            if approval_data['status'] != 'pending':
                return {
                    'success': False,
                    'error': 'Approval request has already been processed'
                }
            
            # Find approver's task
            approver_task = None
            task_keys = []
            
            # Look for approver tasks (simplified approach)
            for i in range(10):  # Assume max 10 approvers
                task_key = f"approval_task:{request_id}:{i}"
                task_data = cache.get(task_key)
                if task_data and task_data['approver_id'] == str(approver.id):
                    approver_task = task_data
                    approver_task['task_key'] = task_key
                    break
                elif task_data:
                    task_keys.append(task_key)
            
            if not approver_task:
                return {
                    'success': False,
                    'error': 'You are not authorized to approve this request'
                }
            
            if approver_task['status'] != 'pending':
                return {
                    'success': False,
                    'error': 'You have already processed this approval request'
                }
            
            # Update approver task
            approver_task['status'] = decision
            approver_task['comments'] = comments
            approver_task['decided_at'] = timezone.now().isoformat()
            cache.set(approver_task['task_key'], approver_task, timeout=604800)
            
            # Check if all approvals are complete
            pending_count = 0
            rejected_count = 0
            
            for task_key in task_keys + [approver_task['task_key']]:
                task_data = cache.get(task_key)
                if task_data:
                    if task_data['status'] == 'pending':
                        pending_count += 1
                    elif task_data['status'] == 'reject':
                        rejected_count += 1
            
            if rejected_count > 0:
                # Request rejected
                approval_data['status'] = 'rejected'
                approval_data['processed_at'] = timezone.now().isoformat()
                cache.set(cache_key, approval_data, timeout=604800)
                
                result = {
                    'success': True,
                    'decision': 'rejected',
                    'request_id': request_id,
                    'message': 'Permission change request has been rejected'
                }
                
            elif pending_count == 0:
                # All approvals complete - apply changes
                approval_data['status'] = 'approved'
                approval_data['processed_at'] = timezone.now().isoformat()
                cache.set(cache_key, approval_data, timeout=604800)
                
                # Get user and apply changes
                from .models import CustomUser
                user = CustomUser.objects.get(id=approval_data['user_id'])
                
                apply_result = self.apply_role_template_to_user(
                    user=user,
                    role_template=approval_data['role'],
                    update_existing=True,
                    applied_by=approver
                )
                
                result = {
                    'success': apply_result['success'],
                    'decision': 'approved_and_applied',
                    'request_id': request_id,
                    'permissions_applied': apply_result.get('permissions_applied', 0),
                    'message': 'Permission changes have been approved and applied'
                }
                
                if not apply_result['success']:
                    result['error'] = apply_result.get('error', 'Failed to apply permissions')
                
            else:
                # Still waiting for more approvals
                result = {
                    'success': True,
                    'decision': 'partial_approval',
                    'request_id': request_id,
                    'pending_approvals': pending_count,
                    'message': f'Your approval has been recorded. {pending_count} more approvals needed.'
                }
            
            # Log the approval decision
            try:
                from .audit_service import AuditService
                audit_service = AuditService()
                audit_service.log_user_action(
                    user=approver,
                    action='permission_approval_decision',
                    module='permissions',
                    request_id=request_id,
                    decision=decision,
                    comments=comments
                )
            except ImportError:
                logger.info(f"Audit service not available, skipping audit log")
            
            return result
            
        except Exception as e:
            logger.error(f"Error processing permission approval {request_id}: {e}")
            return {
                'success': False,
                'error': str(e)
            }  
  
    def _check_dangerous_combinations(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                    validation_results: Dict[str, any], role: str):
        """Check for dangerous permission combinations"""
        dangerous_combinations = [
            # Financial + Delete combinations
            (['loans_view_calculations', 'loans_delete_application'], 
             'Combination of financial access and delete permissions poses high risk'),
            
            # Export + Admin combinations  
            (['reports_export_excel', 'clients_delete_client'],
             'Export permissions with delete access can lead to data breaches'),
            
            # Approval + Processing combinations for non-admin roles
            (['loans_approve_application', 'loans_process_rollover'],
             'Approval and processing permissions should be separated for audit trail'),
        ]
        
        if role != 'admin':  # Admins can have any combination
            for combination, warning in dangerous_combinations:
                granted_perms = []
                for page_name, actions in permissions_dict.items():
                    for action_code, is_allowed in actions.items():
                        if is_allowed and action_code in combination:
                            granted_perms.append(action_code)
                
                if len(granted_perms) >= 2:
                    validation_results['security_concerns'].append({
                        'type': 'dangerous_combination',
                        'permissions': granted_perms,
                        'warning': warning,
                        'severity': 'high'
                    })
    
    def _check_missing_essential_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                           validation_results: Dict[str, any], role: str):
        """Check for missing essential permissions for each role"""
        essential_permissions = {
            'loan_officer': [
                'clients_view_list',
                'loans_view_applications',
                'clients_view_history'
            ],
            'team_leader': [
                'clients_view_list',
                'loans_view_applications', 
                'reports_view_dashboard',
                'clients_manage_documents'
            ],
            'secretary': [
                'clients_view_list',
                'clients_create_new',
                'clients_edit_info'
            ],
            'auditor': [
                'reports_view_dashboard',
                'clients_view_list',
                'loans_view_applications'
            ]
        }
        
        if role in essential_permissions:
            missing_permissions = []
            for essential_perm in essential_permissions[role]:
                found = False
                for page_name, actions in permissions_dict.items():
                    if essential_perm in actions and actions[essential_perm]:
                        found = True
                        break
                
                if not found:
                    missing_permissions.append(essential_perm)
            
            if missing_permissions:
                validation_results['missing_required_permissions'] = missing_permissions
                validation_results['warnings'].append(
                    f"Role {role} is missing essential permissions: {', '.join(missing_permissions)}"
                )
    
    def _check_permission_escalation_risks(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                         validation_results: Dict[str, any], role: str):
        """Check for permission escalation risks"""
        escalation_risks = {
            'secretary': [
                'loans_approve_application',
                'clients_approve_client', 
                'loans_delete_application',
                'clients_delete_client'
            ],
            'loan_officer': [
                'users_manage_permissions',
                'system_settings_modify',
                'reports_delete_reports'
            ],
            'auditor': [
                'loans_create_application',
                'clients_create_new',
                'loans_approve_application',
                'clients_approve_client'
            ]
        }
        
        if role in escalation_risks:
            risky_permissions = []
            for page_name, actions in permissions_dict.items():
                for action_code, is_allowed in actions.items():
                    if is_allowed and action_code in escalation_risks[role]:
                        risky_permissions.append(action_code)
            
            if risky_permissions:
                validation_results['security_concerns'].append({
                    'type': 'escalation_risk',
                    'role': role,
                    'risky_permissions': risky_permissions,
                    'warning': f'Role {role} has permissions that may allow privilege escalation',
                    'severity': 'medium'
                })
    
    def _validate_admin_permissions(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                  validation_results: Dict[str, any]):
        """Validate permissions specific to admin role"""
        # Admins should have comprehensive access but with warnings for critical actions
        critical_admin_actions = [
            'users_delete_user',
            'system_settings_modify',
            'database_direct_access'
        ]
        
        granted_critical = []
        for page_name, actions in permissions_dict.items():
            for action_code, is_allowed in actions.items():
                if is_allowed and action_code in critical_admin_actions:
                    granted_critical.append(action_code)
        
        if granted_critical:
            validation_results['warnings'].append(
                f"Admin role has critical system permissions: {', '.join(granted_critical)}. "
                "Ensure proper audit logging is enabled."
            )
    
    def _apply_strict_validation_rules(self, permissions_dict: Dict[str, Dict[str, bool]], 
                                     validation_results: Dict[str, any], role: str):
        """Apply strict validation rules for high-security environments"""
        # In strict mode, require explicit approval for any critical permissions
        critical_count = len(validation_results.get('critical_permissions', []))
        if critical_count > 0:
            validation_results['warnings'].append(
                f"Strict mode: {critical_count} critical permissions require explicit approval"
            )
        
        # In strict mode, limit export permissions
        export_permissions = []
        for page_name, actions in permissions_dict.items():
            for action_code, is_allowed in actions.items():
                if is_allowed and 'export' in action_code.lower():
                    export_permissions.append(action_code)
        
        if len(export_permissions) > 2 and role != 'admin':
            validation_results['security_concerns'].append({
                'type': 'excessive_export_permissions',
                'permissions': export_permissions,
                'warning': 'Strict mode: Excessive export permissions detected',
                'severity': 'medium'
            })
    
    def _assess_validation_severity(self, validation_results: Dict[str, any]) -> str:
        """Assess overall severity of validation issues"""
        error_count = len(validation_results.get('errors', []))
        critical_count = len(validation_results.get('critical_permissions', []))
        security_count = len(validation_results.get('security_concerns', []))
        dependency_count = len(validation_results.get('dependency_issues', []))
        
        if error_count > 0 or dependency_count > 0:
            return 'critical'
        elif critical_count > 3 or security_count > 2:
            return 'high'
        elif critical_count > 0 or security_count > 0:
            return 'medium'
        else:
            return 'low'
    
    def validate_permission_dependencies(self, permissions_dict: Dict[str, Dict[str, bool]]) -> Dict[str, any]:
        """
        Validate permission dependencies and hierarchies
        
        Args:
            permissions_dict: Permissions to validate {page_name: {action_code: is_allowed}}
            
        Returns:
            Dictionary with dependency validation results
        """
        try:
            dependency_results = {
                'valid': True,
                'missing_dependencies': [],
                'circular_dependencies': [],
                'hierarchy_violations': [],
                'suggestions': []
            }
            
            # Define permission hierarchies (higher level permissions require lower level ones)
            permission_hierarchies = {
                'loans': {
                    'loans_delete_application': ['loans_view_applications', 'loans_edit_application'],
                    'loans_approve_application': ['loans_view_applications'],
                    'loans_process_rollover': ['loans_view_applications', 'loans_edit_terms'],
                    'loans_modify_interest': ['loans_view_calculations', 'loans_edit_terms']
                },
                'clients': {
                    'clients_delete_client': ['clients_view_list', 'clients_edit_info'],
                    'clients_approve_client': ['clients_view_pending'],
                    'clients_assign_manager': ['clients_view_list', 'clients_edit_info']
                },
                'reports': {
                    'reports_export_excel': ['reports_view_dashboard'],
                    'reports_export_pdf': ['reports_view_dashboard'],
                    'reports_custom_reports': ['reports_view_dashboard'],
                    'reports_schedule_reports': ['reports_view_dashboard', 'reports_export_pdf']
                }
            }
            
            # Check hierarchies
            for page_name, actions in permissions_dict.items():
                if page_name in permission_hierarchies:
                    page_hierarchies = permission_hierarchies[page_name]
                    
                    for action_code, is_allowed in actions.items():
                        if is_allowed and action_code in page_hierarchies:
                            # Check if required dependencies are granted
                            required_permissions = page_hierarchies[action_code]
                            missing_deps = []
                            
                            for required_perm in required_permissions:
                                # Check if required permission is granted
                                if (page_name not in permissions_dict or 
                                    required_perm not in permissions_dict[page_name] or 
                                    not permissions_dict[page_name][required_perm]):
                                    missing_deps.append(required_perm)
                            
                            if missing_deps:
                                dependency_results['missing_dependencies'].append({
                                    'permission': action_code,
                                    'page': page_name,
                                    'missing': missing_deps,
                                    'message': f"{action_code} requires {', '.join(missing_deps)}"
                                })
                                dependency_results['valid'] = False
            
            # Add suggestions for common permission sets
            if dependency_results['missing_dependencies']:
                dependency_results['suggestions'].append(
                    "Consider granting the missing dependency permissions or removing the dependent permissions"
                )
            
            return dependency_results
            
        except Exception as e:
            logger.error(f"Error validating permission dependencies: {e}")
            return {
                'valid': False,
                'error': str(e),
                'missing_dependencies': [],
                'circular_dependencies': [],
                'hierarchy_violations': [],
                'suggestions': []
            }
    
    def create_permission_validation_report(self, role: str, permissions_dict: Dict[str, Dict[str, bool]], 
                                          include_recommendations: bool = True) -> Dict[str, any]:
        """
        Create a comprehensive permission validation report
        
        Args:
            role: Role to validate permissions for
            permissions_dict: Permissions to validate
            include_recommendations: Whether to include recommendations
            
        Returns:
            Comprehensive validation report
        """
        try:
            # Run all validations
            role_validation = self.validate_role_permission_set(role, permissions_dict, strict_mode=True)
            dependency_validation = self.validate_permission_dependencies(permissions_dict)
            
            # Count permissions
            total_permissions = 0
            granted_permissions = 0
            
            for page_name, actions in permissions_dict.items():
                for action_code, is_allowed in actions.items():
                    total_permissions += 1
                    if is_allowed:
                        granted_permissions += 1
            
            report = {
                'role': role,
                'validation_timestamp': timezone.now().isoformat(),
                'summary': {
                    'total_permissions': total_permissions,
                    'granted_permissions': granted_permissions,
                    'denied_permissions': total_permissions - granted_permissions,
                    'overall_valid': role_validation['valid'] and dependency_validation['valid'],
                    'severity': role_validation.get('severity', 'unknown')
                },
                'role_validation': role_validation,
                'dependency_validation': dependency_validation,
                'risk_assessment': {
                    'critical_permissions': len(role_validation.get('critical_permissions', [])),
                    'security_concerns': len(role_validation.get('security_concerns', [])),
                    'missing_dependencies': len(dependency_validation.get('missing_dependencies', [])),
                    'risk_level': self._calculate_risk_level(role_validation, dependency_validation)
                }
            }
            
            if include_recommendations:
                report['recommendations'] = self._generate_permission_recommendations(
                    role, role_validation, dependency_validation
                )
            
            return report
            
        except Exception as e:
            logger.error(f"Error creating permission validation report: {e}")
            return {
                'role': role,
                'error': str(e),
                'validation_timestamp': timezone.now().isoformat(),
                'summary': {'overall_valid': False}
            }
    
    def _calculate_risk_level(self, role_validation: Dict[str, any], 
                            dependency_validation: Dict[str, any]) -> str:
        """Calculate overall risk level based on validation results"""
        risk_score = 0
        
        # Add points for various risk factors
        risk_score += len(role_validation.get('critical_permissions', [])) * 3
        risk_score += len(role_validation.get('security_concerns', [])) * 2
        risk_score += len(dependency_validation.get('missing_dependencies', [])) * 1
        risk_score += len(role_validation.get('errors', [])) * 4
        
        if risk_score >= 15:
            return 'critical'
        elif risk_score >= 10:
            return 'high'
        elif risk_score >= 5:
            return 'medium'
        else:
            return 'low'
    
    def _generate_permission_recommendations(self, role: str, role_validation: Dict[str, any], 
                                           dependency_validation: Dict[str, any]) -> List[Dict[str, any]]:
        """Generate recommendations based on validation results"""
        recommendations = []
        
        # Recommendations for missing dependencies
        for missing_dep in dependency_validation.get('missing_dependencies', []):
            recommendations.append({
                'type': 'dependency',
                'priority': 'high',
                'title': f"Grant required dependency: {missing_dep['missing'][0]}",
                'description': f"Permission {missing_dep['permission']} requires {', '.join(missing_dep['missing'])}",
                'action': 'grant_permission',
                'permissions': missing_dep['missing']
            })
        
        # Recommendations for security concerns
        for concern in role_validation.get('security_concerns', []):
            if concern.get('type') == 'dangerous_combination':
                recommendations.append({
                    'type': 'security',
                    'priority': 'high',
                    'title': 'Review dangerous permission combination',
                    'description': concern['warning'],
                    'action': 'review_permissions',
                    'permissions': concern['permissions']
                })
        
        # Recommendations for missing essential permissions
        missing_essential = role_validation.get('missing_required_permissions', [])
        if missing_essential:
            recommendations.append({
                'type': 'essential',
                'priority': 'medium',
                'title': f'Grant essential permissions for {role} role',
                'description': f'Role {role} is missing essential permissions for normal operation',
                'action': 'grant_permission',
                'permissions': missing_essential
            })
        
        # Role-specific recommendations
        if role == 'auditor':
            # Auditors should have read-only access
            write_permissions = []
            for concern in role_validation.get('security_concerns', []):
                if concern.get('type') == 'escalation_risk':
                    write_permissions.extend(concern.get('risky_permissions', []))
            
            if write_permissions:
                recommendations.append({
                    'type': 'role_specific',
                    'priority': 'medium',
                    'title': 'Remove write permissions for auditor role',
                    'description': 'Auditors should typically have read-only access',
                    'action': 'revoke_permission',
                    'permissions': write_permissions
                })
        
        return recommendations

class PortfolioAnalyticsService:
    """
    Comprehensive portfolio analytics service for loan officers and team leaders
    Provides detailed portfolio performance metrics, trends, and risk analysis
    """
    
    def __init__(self):
        self.cache_prefix = "portfolio_analytics"
        self.cache_timeout = 1800  # 30 minutes
    
    def get_portfolio_overview(self, manager_id: str, date_range: Optional[Tuple[str, str]] = None) -> Dict[str, any]:
        """
        Get comprehensive portfolio overview metrics for a portfolio manager
        
        Args:
            manager_id: UUID of the portfolio manager (loan officer or team leader)
            date_range: Optional tuple of (start_date, end_date) strings in YYYY-MM-DD format
            
        Returns:
            Dictionary containing comprehensive portfolio metrics
        """
        try:
            from .models import CustomUser
            from loans.models import Loan, LoanApplication
            from payments.models import Payment
            from django.db.models import Sum, Count, Avg, Q, F
            from decimal import Decimal
            from datetime import datetime, timedelta
            
            # Get the manager
            try:
                manager = CustomUser.objects.get(id=manager_id)
                if manager.role not in ['loan_officer', 'team_leader']:
                    raise ValueError(f"User {manager_id} is not a portfolio manager")
            except CustomUser.DoesNotExist:
                raise ValueError(f"Manager {manager_id} not found")
            
            # Set date range
            if date_range:
                start_date = datetime.strptime(date_range[0], '%Y-%m-%d').date()
                end_date = datetime.strptime(date_range[1], '%Y-%m-%d').date()
            else:
                # Default to last 30 days
                end_date = timezone.now().date()
                start_date = end_date - timedelta(days=30)
            
            # Get clients in this manager's portfolio
            clients = manager.portfolio_clients.filter(status='active')
            client_ids = list(clients.values_list('id', flat=True))
            
            # Base querysets
            loans_qs = Loan.objects.filter(borrower_id__in=client_ids)
            applications_qs = LoanApplication.objects.filter(borrower_id__in=client_ids)
            
            # Apply date filters for period-specific metrics
            period_loans = loans_qs.filter(
                created_at__date__gte=start_date,
                created_at__date__lte=end_date
            )
            period_applications = applications_qs.filter(
                submitted_at__date__gte=start_date,
                submitted_at__date__lte=end_date
            )
            
            # Client metrics
            client_metrics = {
                'total_clients': clients.count(),
                'new_clients_period': clients.filter(
                    created_at__date__gte=start_date,
                    created_at__date__lte=end_date
                ).count(),
                'active_borrowers': clients.filter(loans__status='active').distinct().count(),
                'clients_with_multiple_loans': clients.annotate(
                    loan_count=Count('loans')
                ).filter(loan_count__gt=1).count()
            }
            
            # Loan metrics
            loan_aggregates = loans_qs.aggregate(
                total_loans=Count('id'),
                active_loans=Count('id', filter=Q(status='active')),
                paid_loans=Count('id', filter=Q(status='paid')),
                defaulted_loans=Count('id', filter=Q(status='defaulted')),
                total_disbursed=Sum('principal_amount'),
                total_outstanding=Sum('total_amount', filter=Q(status='active')) - Sum('_amount_paid_cache', filter=Q(status='active')),
                total_collected=Sum('_amount_paid_cache'),
                avg_loan_size=Avg('principal_amount')
            )
            
            # Period-specific loan metrics
            period_aggregates = period_loans.aggregate(
                new_loans=Count('id'),
                disbursed_amount=Sum('principal_amount'),
                collected_amount=Sum('_amount_paid_cache')
            )
            
            # Application metrics
            app_aggregates = applications_qs.aggregate(
                total_applications=Count('id'),
                pending_applications=Count('id', filter=Q(status='pending')),
                approved_applications=Count('id', filter=Q(status='approved')),
                rejected_applications=Count('id', filter=Q(status='rejected'))
            )
            
            # Calculate performance ratios
            total_disbursed = loan_aggregates['total_disbursed'] or Decimal('0')
            total_collected = loan_aggregates['total_collected'] or Decimal('0')
            total_outstanding = loan_aggregates['total_outstanding'] or Decimal('0')
            total_loans = loan_aggregates['total_loans'] or 0
            
            performance_metrics = {
                'collection_rate': float((total_collected / total_disbursed * 100) if total_disbursed > 0 else 0),
                'default_rate': float((loan_aggregates['defaulted_loans'] / total_loans * 100) if total_loans > 0 else 0),
                'approval_rate': float((app_aggregates['approved_applications'] / app_aggregates['total_applications'] * 100) if app_aggregates['total_applications'] > 0 else 0),
                'portfolio_at_risk': self._calculate_portfolio_at_risk(loans_qs),
                'average_loan_size': float(loan_aggregates['avg_loan_size'] or 0)
            }
            
            # Risk analysis
            risk_metrics = self._analyze_portfolio_risk(loans_qs)
            
            # Compile overview
            overview = {
                'manager_info': {
                    'id': str(manager.id),
                    'name': manager.get_full_name(),
                    'role': manager.role,
                    'branch': manager.branch.name if manager.branch else None
                },
                'period': {
                    'start_date': start_date.isoformat(),
                    'end_date': end_date.isoformat(),
                    'days': (end_date - start_date).days + 1
                },
                'client_metrics': client_metrics,
                'loan_metrics': {
                    **loan_aggregates,
                    'total_disbursed': float(total_disbursed),
                    'total_outstanding': float(total_outstanding),
                    'total_collected': float(total_collected),
                    'avg_loan_size': float(loan_aggregates['avg_loan_size'] or 0)
                },
                'period_metrics': {
                    **period_aggregates,
                    'disbursed_amount': float(period_aggregates['disbursed_amount'] or 0),
                    'collected_amount': float(period_aggregates['collected_amount'] or 0)
                },
                'application_metrics': app_aggregates,
                'performance_metrics': performance_metrics,
                'risk_metrics': risk_metrics,
                'generated_at': timezone.now().isoformat()
            }
            
            return overview
            
        except Exception as e:
            logger.error(f"Error getting portfolio overview for manager {manager_id}: {e}")
            return {
                'error': str(e),
                'manager_id': manager_id,
                'generated_at': timezone.now().isoformat()
            }
    
    def get_performance_trends(self, manager_id: str, period: str = 'monthly') -> Dict[str, any]:
        """
        Get performance trends over time for time-series analysis
        
        Args:
            manager_id: UUID of the portfolio manager
            period: Period for trend analysis ('daily', 'weekly', 'monthly', 'quarterly')
            
        Returns:
            Dictionary containing trend data for charts and analysis
        """
        try:
            from .enhanced_permissions_models import PortfolioSnapshot
            from .models import CustomUser
            from django.db.models import Q
            from datetime import datetime, timedelta
            import calendar
            
            # Get the manager
            manager = CustomUser.objects.get(id=manager_id)
            
            # Determine date range based on period
            end_date = timezone.now().date()
            if period == 'daily':
                start_date = end_date - timedelta(days=30)
                date_format = '%Y-%m-%d'
            elif period == 'weekly':
                start_date = end_date - timedelta(weeks=12)
                date_format = '%Y-W%U'
            elif period == 'monthly':
                start_date = end_date - timedelta(days=365)
                date_format = '%Y-%m'
            elif period == 'quarterly':
                start_date = end_date - timedelta(days=730)  # 2 years
                date_format = '%Y-Q'
            else:
                raise ValueError(f"Invalid period: {period}")
            
            # Get portfolio snapshots
            snapshots = PortfolioSnapshot.objects.filter(
                manager=manager,
                snapshot_date__gte=start_date,
                snapshot_date__lte=end_date
            ).order_by('snapshot_date')
            
            # If no snapshots exist, generate them from current data
            if not snapshots.exists():
                logger.info(f"No snapshots found for manager {manager_id}, generating from current data")
                trend_data = self._generate_trend_from_current_data(manager, start_date, end_date, period)
            else:
                # Process existing snapshots
                trend_data = self._process_snapshots_for_trends(snapshots, period)
            
            # Calculate trend indicators
            trend_analysis = self._calculate_trend_indicators(trend_data)
            
            return {
                'manager_id': str(manager.id),
                'manager_name': manager.get_full_name(),
                'period': period,
                'date_range': {
                    'start_date': start_date.isoformat(),
                    'end_date': end_date.isoformat()
                },
                'trend_data': trend_data,
                'trend_analysis': trend_analysis,
                'generated_at': timezone.now().isoformat()
            }
            
        except Exception as e:
            logger.error(f"Error getting performance trends for manager {manager_id}: {e}")
            return {
                'error': str(e),
                'manager_id': manager_id,
                'period': period,
                'generated_at': timezone.now().isoformat()
            }
    
    def get_risk_analysis(self, manager_id: str) -> Dict[str, any]:
        """
        Get comprehensive portfolio risk assessment
        
        Args:
            manager_id: UUID of the portfolio manager
            
        Returns:
            Dictionary containing detailed risk analysis and recommendations
        """
        try:
            from .models import CustomUser
            from loans.models import Loan
            from django.db.models import Q, Count, Sum, Avg
            from decimal import Decimal
            from datetime import datetime, timedelta
            
            manager = CustomUser.objects.get(id=manager_id)
            clients = manager.portfolio_clients.filter(status='active')
            client_ids = list(clients.values_list('id', flat=True))
            loans = Loan.objects.filter(borrower_id__in=client_ids)
            
            # Portfolio at Risk (PAR) analysis
            current_date = timezone.now().date()
            par_analysis = {
                'par_30': self._calculate_par(loans, 30),
                'par_60': self._calculate_par(loans, 60),
                'par_90': self._calculate_par(loans, 90),
                'par_120': self._calculate_par(loans, 120)
            }
            
            # Concentration risk analysis
            concentration_risk = self._analyze_concentration_risk(loans, clients)
            
            # Client risk distribution
            client_risk_distribution = self._analyze_client_risk_distribution(clients)
            
            # Loan size distribution
            loan_size_distribution = self._analyze_loan_size_distribution(loans)
            
            # Maturity profile
            maturity_profile = self._analyze_maturity_profile(loans)
            
            # Early warning indicators
            early_warnings = self._identify_early_warning_indicators(loans, clients)
            
            # Overall risk score calculation
            risk_score = self._calculate_portfolio_risk_score(
                par_analysis, concentration_risk, client_risk_distribution, early_warnings
            )
            
            # Risk recommendations
            recommendations = self._generate_risk_recommendations(
                risk_score, par_analysis, concentration_risk, early_warnings
            )
            
            return {
                'manager_info': {
                    'id': str(manager.id),
                    'name': manager.get_full_name(),
                    'role': manager.role
                },
                'risk_score': risk_score,
                'risk_level': self._get_risk_level(risk_score),
                'par_analysis': par_analysis,
                'concentration_risk': concentration_risk,
                'client_risk_distribution': client_risk_distribution,
                'loan_size_distribution': loan_size_distribution,
                'maturity_profile': maturity_profile,
                'early_warnings': early_warnings,
                'recommendations': recommendations,
                'generated_at': timezone.now().isoformat()
            }
            
        except Exception as e:
            logger.error(f"Error getting risk analysis for manager {manager_id}: {e}")
            return {
                'error': str(e),
                'manager_id': manager_id,
                'generated_at': timezone.now().isoformat()
            }
    
    def get_client_performance(self, manager_id: str) -> Dict[str, any]:
        """
        Get individual client performance metrics within the portfolio
        
        Args:
            manager_id: UUID of the portfolio manager
            
        Returns:
            Dictionary containing client-level performance analysis
        """
        try:
            from .models import CustomUser
            from loans.models import Loan
            from django.db.models import Sum, Count, Avg, Max, Min
            from decimal import Decimal
            
            manager = CustomUser.objects.get(id=manager_id)
            clients = manager.portfolio_clients.filter(status='active')
            
            client_performance = []
            
            for client in clients:
                # Get client's loan history
                client_loans = client.loans.all()
                active_loans = client_loans.filter(status='active')
                
                # Calculate client metrics
                loan_stats = client_loans.aggregate(
                    total_loans=Count('id'),
                    total_borrowed=Sum('principal_amount'),
                    total_paid=Sum('_amount_paid_cache'),
                    avg_loan_size=Avg('principal_amount'),
                    largest_loan=Max('principal_amount'),
                    latest_loan_date=Max('created_at')
                )
                
                # Calculate payment performance
                payment_performance = self._calculate_client_payment_performance(client)
                
                # Risk assessment
                risk_assessment = self._assess_client_risk(client, client_loans)
                
                # Profitability analysis
                profitability = self._calculate_client_profitability(client, client_loans)
                
                client_data = {
                    'client_info': {
                        'id': str(client.id),
                        'name': client.get_full_name(),
                        'phone': client.phone_number,
                        'registration_date': client.created_at.date().isoformat(),
                        'status': client.status
                    },
                    'loan_statistics': {
                        'total_loans': loan_stats['total_loans'] or 0,
                        'active_loans': active_loans.count(),
                        'total_borrowed': float(loan_stats['total_borrowed'] or 0),
                        'total_paid': float(loan_stats['total_paid'] or 0),
                        'avg_loan_size': float(loan_stats['avg_loan_size'] or 0),
                        'largest_loan': float(loan_stats['largest_loan'] or 0),
                        'latest_loan_date': loan_stats['latest_loan_date'].date().isoformat() if loan_stats['latest_loan_date'] else None
                    },
                    'payment_performance': payment_performance,
                    'risk_assessment': risk_assessment,
                    'profitability': profitability,
                    'relationship_duration_days': (timezone.now().date() - client.created_at.date()).days
                }
                
                client_performance.append(client_data)
            
            # Sort by various criteria for rankings
            performance_rankings = {
                'top_borrowers': sorted(client_performance, key=lambda x: x['loan_statistics']['total_borrowed'], reverse=True)[:10],
                'most_profitable': sorted(client_performance, key=lambda x: x['profitability']['total_profit'], reverse=True)[:10],
                'highest_risk': sorted(client_performance, key=lambda x: x['risk_assessment']['risk_score'], reverse=True)[:10],
                'best_payers': sorted(client_performance, key=lambda x: x['payment_performance']['payment_score'], reverse=True)[:10]
            }
            
            # Portfolio-level client statistics
            portfolio_stats = {
                'total_clients': len(client_performance),
                'avg_loans_per_client': sum(c['loan_statistics']['total_loans'] for c in client_performance) / len(client_performance) if client_performance else 0,
                'avg_relationship_duration': sum(c['relationship_duration_days'] for c in client_performance) / len(client_performance) if client_performance else 0,
                'high_risk_clients': len([c for c in client_performance if c['risk_assessment']['risk_level'] == 'high']),
                'profitable_clients': len([c for c in client_performance if c['profitability']['total_profit'] > 0])
            }
            
            return {
                'manager_info': {
                    'id': str(manager.id),
                    'name': manager.get_full_name(),
                    'role': manager.role
                },
                'client_performance': client_performance,
                'performance_rankings': performance_rankings,
                'portfolio_statistics': portfolio_stats,
                'generated_at': timezone.now().isoformat()
            }
            
        except Exception as e:
            logger.error(f"Error getting client performance for manager {manager_id}: {e}")
            return {
                'error': str(e),
                'manager_id': manager_id,
                'generated_at': timezone.now().isoformat()
            }
    
    # Helper methods for calculations
    
    def _calculate_portfolio_at_risk(self, loans_qs) -> Dict[str, float]:
        """Calculate Portfolio at Risk for different periods"""
        from django.db.models import Sum, Q
        from datetime import timedelta
        
        current_date = timezone.now().date()
        total_outstanding = loans_qs.filter(status='active').aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        if total_outstanding == 0:
            return {'par_30': 0, 'par_60': 0, 'par_90': 0}
        
        par_30_amount = loans_qs.filter(
            status='active',
            due_date__lt=current_date - timedelta(days=30)
        ).aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        par_60_amount = loans_qs.filter(
            status='active',
            due_date__lt=current_date - timedelta(days=60)
        ).aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        par_90_amount = loans_qs.filter(
            status='active',
            due_date__lt=current_date - timedelta(days=90)
        ).aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        return {
            'par_30': float(par_30_amount / total_outstanding * 100),
            'par_60': float(par_60_amount / total_outstanding * 100),
            'par_90': float(par_90_amount / total_outstanding * 100)
        }
    
    def _analyze_portfolio_risk(self, loans_qs) -> Dict[str, any]:
        """Analyze overall portfolio risk"""
        from django.db.models import Count, Avg
        
        total_loans = loans_qs.count()
        if total_loans == 0:
            return {'risk_level': 'low', 'risk_factors': []}
        
        # Calculate various risk metrics
        default_rate = loans_qs.filter(status='defaulted').count() / total_loans * 100
        overdue_rate = loans_qs.filter(
            status='active',
            due_date__lt=timezone.now().date()
        ).count() / total_loans * 100
        
        risk_factors = []
        risk_level = 'low'
        
        if default_rate > 10:
            risk_factors.append('High default rate')
            risk_level = 'high'
        elif default_rate > 5:
            risk_factors.append('Moderate default rate')
            risk_level = 'medium'
        
        if overdue_rate > 15:
            risk_factors.append('High overdue rate')
            risk_level = 'high'
        elif overdue_rate > 8:
            risk_factors.append('Moderate overdue rate')
            if risk_level == 'low':
                risk_level = 'medium'
        
        return {
            'risk_level': risk_level,
            'default_rate': default_rate,
            'overdue_rate': overdue_rate,
            'risk_factors': risk_factors
        }
    
    def _calculate_par(self, loans_qs, days: int) -> float:
        """Calculate Portfolio at Risk for specific number of days"""
        from django.db.models import Sum
        from datetime import timedelta
        
        current_date = timezone.now().date()
        cutoff_date = current_date - timedelta(days=days)
        
        total_outstanding = loans_qs.filter(status='active').aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        if total_outstanding == 0:
            return 0.0
        
        par_amount = loans_qs.filter(
            status='active',
            due_date__lt=cutoff_date
        ).aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        return float(par_amount / total_outstanding * 100)
    
    def _analyze_concentration_risk(self, loans_qs, clients_qs) -> Dict[str, any]:
        """Analyze concentration risk in the portfolio"""
        from django.db.models import Sum, Count
        
        # Client concentration (top 10 clients by outstanding amount)
        client_exposure = []
        total_outstanding = 0
        
        for client in clients_qs:
            client_outstanding = client.loans.filter(status='active').aggregate(
                total=Sum('total_amount') - Sum('_amount_paid_cache')
            )['total'] or 0
            
            if client_outstanding > 0:
                client_exposure.append({
                    'client_id': str(client.id),
                    'client_name': client.get_full_name(),
                    'outstanding_amount': float(client_outstanding)
                })
                total_outstanding += client_outstanding
        
        # Sort by exposure
        client_exposure.sort(key=lambda x: x['outstanding_amount'], reverse=True)
        
        # Calculate concentration percentages
        top_10_exposure = sum(c['outstanding_amount'] for c in client_exposure[:10])
        top_5_exposure = sum(c['outstanding_amount'] for c in client_exposure[:5])
        
        concentration_risk = 'low'
        if total_outstanding > 0:
            top_10_percentage = top_10_exposure / total_outstanding * 100
            top_5_percentage = top_5_exposure / total_outstanding * 100
            
            if top_5_percentage > 50:
                concentration_risk = 'high'
            elif top_10_percentage > 70:
                concentration_risk = 'medium'
        
        return {
            'risk_level': concentration_risk,
            'total_outstanding': float(total_outstanding),
            'top_10_clients': client_exposure[:10],
            'top_10_percentage': float(top_10_exposure / total_outstanding * 100) if total_outstanding > 0 else 0,
            'top_5_percentage': float(top_5_exposure / total_outstanding * 100) if total_outstanding > 0 else 0
        }
    
    def _analyze_client_risk_distribution(self, clients_qs) -> Dict[str, any]:
        """Analyze risk distribution across clients"""
        risk_distribution = {'low': 0, 'medium': 0, 'high': 0}
        
        for client in clients_qs:
            # Simple risk assessment based on loan history
            total_loans = client.loans.count()
            defaulted_loans = client.loans.filter(status='defaulted').count()
            
            if total_loans == 0:
                risk_level = 'medium'  # New clients are medium risk
            else:
                default_rate = defaulted_loans / total_loans
                if default_rate == 0:
                    risk_level = 'low'
                elif default_rate < 0.2:
                    risk_level = 'medium'
                else:
                    risk_level = 'high'
            
            risk_distribution[risk_level] += 1
        
        total_clients = sum(risk_distribution.values())
        
        return {
            'distribution': risk_distribution,
            'percentages': {
                level: (count / total_clients * 100) if total_clients > 0 else 0
                for level, count in risk_distribution.items()
            },
            'total_clients': total_clients
        }
    
    def _analyze_loan_size_distribution(self, loans_qs) -> Dict[str, any]:
        """Analyze loan size distribution"""
        from django.db.models import Count
        
        # Define size buckets
        size_buckets = {
            'small': (0, 10000),      # 0 - 10K
            'medium': (10000, 50000), # 10K - 50K
            'large': (50000, 100000), # 50K - 100K
            'xlarge': (100000, float('inf'))  # 100K+
        }
        
        distribution = {}
        for bucket_name, (min_amount, max_amount) in size_buckets.items():
            if max_amount == float('inf'):
                count = loans_qs.filter(principal_amount__gte=min_amount).count()
            else:
                count = loans_qs.filter(
                    principal_amount__gte=min_amount,
                    principal_amount__lt=max_amount
                ).count()
            distribution[bucket_name] = count
        
        total_loans = sum(distribution.values())
        
        return {
            'distribution': distribution,
            'percentages': {
                bucket: (count / total_loans * 100) if total_loans > 0 else 0
                for bucket, count in distribution.items()
            },
            'total_loans': total_loans
        }
    
    def _analyze_maturity_profile(self, loans_qs) -> Dict[str, any]:
        """Analyze loan maturity profile"""
        from datetime import timedelta
        
        current_date = timezone.now().date()
        
        # Define maturity buckets
        maturity_buckets = {
            'overdue': loans_qs.filter(status='active', due_date__lt=current_date).count(),
            'due_7_days': loans_qs.filter(
                status='active',
                due_date__gte=current_date,
                due_date__lte=current_date + timedelta(days=7)
            ).count(),
            'due_30_days': loans_qs.filter(
                status='active',
                due_date__gt=current_date + timedelta(days=7),
                due_date__lte=current_date + timedelta(days=30)
            ).count(),
            'due_90_days': loans_qs.filter(
                status='active',
                due_date__gt=current_date + timedelta(days=30),
                due_date__lte=current_date + timedelta(days=90)
            ).count(),
            'due_later': loans_qs.filter(
                status='active',
                due_date__gt=current_date + timedelta(days=90)
            ).count()
        }
        
        total_active = sum(maturity_buckets.values())
        
        return {
            'distribution': maturity_buckets,
            'percentages': {
                bucket: (count / total_active * 100) if total_active > 0 else 0
                for bucket, count in maturity_buckets.items()
            },
            'total_active_loans': total_active
        }
    
    def _identify_early_warning_indicators(self, loans_qs, clients_qs) -> List[Dict[str, any]]:
        """Identify early warning indicators for potential problems"""
        warnings = []
        
        # High concentration in single client
        from django.db.models import Sum
        total_outstanding = loans_qs.filter(status='active').aggregate(
            total=Sum('total_amount') - Sum('_amount_paid_cache')
        )['total'] or 0
        
        if total_outstanding > 0:
            for client in clients_qs:
                client_outstanding = client.loans.filter(status='active').aggregate(
                    total=Sum('total_amount') - Sum('_amount_paid_cache')
                )['total'] or 0
                
                if client_outstanding / total_outstanding > 0.2:  # More than 20% concentration
                    warnings.append({
                        'type': 'concentration_risk',
                        'severity': 'high',
                        'message': f'High concentration in client {client.get_full_name()} ({client_outstanding/total_outstanding*100:.1f}%)',
                        'client_id': str(client.id)
                    })
        
        # Multiple overdue loans
        overdue_count = loans_qs.filter(
            status='active',
            due_date__lt=timezone.now().date()
        ).count()
        
        total_active = loans_qs.filter(status='active').count()
        if total_active > 0 and overdue_count / total_active > 0.15:  # More than 15% overdue
            warnings.append({
                'type': 'overdue_loans',
                'severity': 'medium',
                'message': f'High percentage of overdue loans ({overdue_count/total_active*100:.1f}%)',
                'count': overdue_count
            })
        
        return warnings
    
    def _calculate_portfolio_risk_score(self, par_analysis: Dict, concentration_risk: Dict, 
                                      client_risk_distribution: Dict, early_warnings: List) -> float:
        """Calculate overall portfolio risk score (0-100, lower is better)"""
        score = 0
        
        # PAR contribution (40% of score)
        par_score = (par_analysis['par_30'] * 0.5 + par_analysis['par_60'] * 0.3 + par_analysis['par_90'] * 0.2)
        score += min(par_score, 40)
        
        # Concentration risk contribution (30% of score)
        if concentration_risk['risk_level'] == 'high':
            score += 30
        elif concentration_risk['risk_level'] == 'medium':
            score += 15
        
        # Client risk distribution contribution (20% of score)
        high_risk_percentage = client_risk_distribution['percentages']['high']
        score += min(high_risk_percentage * 0.5, 20)
        
        # Early warnings contribution (10% of score)
        warning_score = len([w for w in early_warnings if w['severity'] == 'high']) * 5
        warning_score += len([w for w in early_warnings if w['severity'] == 'medium']) * 2
        score += min(warning_score, 10)
        
        return min(score, 100)
    
    def _get_risk_level(self, risk_score: float) -> str:
        """Convert risk score to risk level"""
        if risk_score < 25:
            return 'low'
        elif risk_score < 50:
            return 'medium'
        else:
            return 'high'
    
    def _generate_risk_recommendations(self, risk_score: float, par_analysis: Dict, 
                                     concentration_risk: Dict, early_warnings: List) -> List[str]:
        """Generate risk management recommendations"""
        recommendations = []
        
        if risk_score > 50:
            recommendations.append("Immediate attention required - portfolio risk is high")
        
        if par_analysis['par_30'] > 10:
            recommendations.append("Focus on collection efforts for overdue loans")
        
        if concentration_risk['risk_level'] == 'high':
            recommendations.append("Diversify portfolio to reduce client concentration risk")
        
        if len(early_warnings) > 3:
            recommendations.append("Address early warning indicators to prevent further deterioration")
        
        if not recommendations:
            recommendations.append("Portfolio risk is well managed - maintain current practices")
        
        return recommendations
    
    def _generate_trend_from_current_data(self, manager, start_date, end_date, period) -> List[Dict]:
        """Generate trend data from current loan data when no snapshots exist"""
        # This is a simplified implementation - in production you'd want more sophisticated logic
        from datetime import timedelta
        
        trend_data = []
        current_date = start_date
        
        while current_date <= end_date:
            # Get data up to this date
            clients = manager.portfolio_clients.filter(
                status='active',
                created_at__date__lte=current_date
            )
            
            loans = manager.portfolio_clients.filter(
                status='active'
            ).values_list('loans', flat=True)
            
            # Create a basic data point
            data_point = {
                'date': current_date.isoformat(),
                'total_clients': clients.count(),
                'active_loans': 0,  # Would need more complex calculation
                'total_disbursed': 0,
                'total_collected': 0,
                'collection_rate': 0
            }
            
            trend_data.append(data_point)
            
            # Move to next period
            if period == 'daily':
                current_date += timedelta(days=1)
            elif period == 'weekly':
                current_date += timedelta(weeks=1)
            elif period == 'monthly':
                current_date += timedelta(days=30)
            else:
                current_date += timedelta(days=90)
        
        return trend_data
    
    def _process_snapshots_for_trends(self, snapshots, period) -> List[Dict]:
        """Process existing snapshots into trend data"""
        trend_data = []
        
        for snapshot in snapshots:
            data_point = {
                'date': snapshot.snapshot_date.isoformat(),
                'total_clients': snapshot.total_clients,
                'active_clients': snapshot.active_clients,
                'active_loans': snapshot.active_loans,
                'total_disbursed': float(snapshot.total_disbursed),
                'total_collected': float(snapshot.total_collected),
                'collection_rate': float(snapshot.collection_rate),
                'default_rate': float(snapshot.default_rate),
                'par_30': float(snapshot.par_30),
                'portfolio_yield': float(snapshot.portfolio_yield)
            }
            trend_data.append(data_point)
        
        return trend_data
    
    def _calculate_trend_indicators(self, trend_data: List[Dict]) -> Dict[str, any]:
        """Calculate trend indicators from trend data"""
        if len(trend_data) < 2:
            return {'trend': 'insufficient_data'}
        
        # Calculate growth rates
        first_point = trend_data[0]
        last_point = trend_data[-1]
        
        client_growth = ((last_point['total_clients'] - first_point['total_clients']) / 
                        first_point['total_clients'] * 100) if first_point['total_clients'] > 0 else 0
        
        disbursement_growth = ((last_point['total_disbursed'] - first_point['total_disbursed']) / 
                              first_point['total_disbursed'] * 100) if first_point['total_disbursed'] > 0 else 0
        
        return {
            'client_growth_rate': client_growth,
            'disbursement_growth_rate': disbursement_growth,
            'trend_direction': 'positive' if client_growth > 0 else 'negative' if client_growth < 0 else 'stable',
            'data_points': len(trend_data)
        }
    
    def _calculate_client_payment_performance(self, client) -> Dict[str, any]:
        """Calculate payment performance metrics for a client"""
        loans = client.loans.all()
        
        if not loans.exists():
            return {'payment_score': 0, 'performance_level': 'no_history'}
        
        total_loans = loans.count()
        paid_loans = loans.filter(status='paid').count()
        defaulted_loans = loans.filter(status='defaulted').count()
        
        # Calculate payment score (0-100)
        payment_score = 0
        if total_loans > 0:
            payment_score = (paid_loans / total_loans) * 100
            # Penalize for defaults
            payment_score -= (defaulted_loans / total_loans) * 50
            payment_score = max(0, payment_score)
        
        performance_level = 'excellent' if payment_score >= 90 else \
                           'good' if payment_score >= 70 else \
                           'fair' if payment_score >= 50 else 'poor'
        
        return {
            'payment_score': payment_score,
            'performance_level': performance_level,
            'total_loans': total_loans,
            'paid_loans': paid_loans,
            'defaulted_loans': defaulted_loans
        }
    
    def _assess_client_risk(self, client, loans) -> Dict[str, any]:
        """Assess individual client risk"""
        if not loans.exists():
            return {'risk_score': 50, 'risk_level': 'medium', 'factors': ['No loan history']}
        
        risk_factors = []
        risk_score = 0
        
        # Payment history (40% of score)
        total_loans = loans.count()
        defaulted_loans = loans.filter(status='defaulted').count()
        if defaulted_loans > 0:
            risk_score += (defaulted_loans / total_loans) * 40
            risk_factors.append(f'{defaulted_loans} defaulted loans')
        
        # Current overdue status (30% of score)
        overdue_loans = loans.filter(
            status='active',
            due_date__lt=timezone.now().date()
        ).count()
        if overdue_loans > 0:
            risk_score += min(overdue_loans * 15, 30)
            risk_factors.append(f'{overdue_loans} overdue loans')
        
        # Loan frequency (20% of score)
        if total_loans > 5:
            risk_score += 10
            risk_factors.append('High loan frequency')
        
        # Account age (10% of score)
        account_age_days = (timezone.now().date() - client.created_at.date()).days
        if account_age_days < 90:
            risk_score += 10
            risk_factors.append('New client')
        
        risk_level = 'high' if risk_score > 60 else 'medium' if risk_score > 30 else 'low'
        
        return {
            'risk_score': min(risk_score, 100),
            'risk_level': risk_level,
            'factors': risk_factors
        }
    
    def _calculate_client_profitability(self, client, loans) -> Dict[str, any]:
        """Calculate client profitability"""
        from django.db.models import Sum
        
        # Calculate total interest and fees collected
        total_interest = loans.aggregate(total=Sum('interest_amount'))['total'] or 0
        total_fees = loans.aggregate(total=Sum('processing_fee'))['total'] or 0
        total_revenue = total_interest + total_fees
        
        # Estimate costs (simplified)
        acquisition_cost = 500  # Estimated cost to acquire client
        servicing_cost = loans.count() * 100  # Estimated cost per loan
        total_costs = acquisition_cost + servicing_cost
        
        total_profit = total_revenue - total_costs
        
        return {
            'total_revenue': float(total_revenue),
            'total_costs': float(total_costs),
            'total_profit': float(total_profit),
            'profit_margin': float((total_profit / total_revenue * 100) if total_revenue > 0 else 0)
        }