from django.contrib.auth.models import AbstractUser
from django.db import models
from django.core.validators import RegexValidator
from django.utils import timezone
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from utils.datetime_utils import get_current_datetime, make_datetime_compatible
from typing import Dict, List, Optional, Any, Tuple
import uuid


class Branch(models.Model):
    """
    Branch model to represent different physical locations of the business
    """
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=100)
    code = models.CharField(max_length=20, unique=True)
    address = models.TextField(blank=True, null=True)
    phone_number = models.CharField(max_length=20, blank=True, null=True)
    email = models.EmailField(blank=True, null=True)
    is_main_branch = models.BooleanField(default=False)
    is_active = models.BooleanField(default=True)
    
    # M-Pesa Configuration
    mpesa_shortcode = models.CharField(
        max_length=10, 
        blank=True, 
        null=True,
        help_text="M-Pesa PayBill number for this branch. Default: 4159523"
    )
    mpesa_consumer_key = models.CharField(
        max_length=100, 
        blank=True, 
        null=True,
        help_text="M-Pesa Consumer Key for this branch"
    )
    mpesa_consumer_secret = models.CharField(
        max_length=100, 
        blank=True, 
        null=True,
        help_text="M-Pesa Consumer Secret for this branch"
    )
    mpesa_passkey = models.CharField(
        max_length=100, 
        blank=True, 
        null=True,
        help_text="M-Pesa Passkey for this branch"
    )
    
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        verbose_name = 'Branch'
        verbose_name_plural = 'Branches'
        ordering = ['name']
    
    def __str__(self):
        return self.name
    
    @property
    def is_main(self):
        """Alias for is_main_branch for compatibility"""
        return self.is_main_branch
    
    @is_main.setter
    def is_main(self, value):
        """Setter to allow setting is_main which updates is_main_branch"""
        self.is_main_branch = value
    
    def save(self, *args, **kwargs):
        # Ensure only one main branch exists
        if self.is_main_branch:
            Branch.objects.filter(is_main_branch=True).update(is_main_branch=False)
        super().save(*args, **kwargs)
    
    def get_mpesa_shortcode(self):
        """Get M-Pesa shortcode for this branch - must be configured by admin"""
        return self.mpesa_shortcode
    
    def get_mpesa_consumer_key(self):
        """Get M-Pesa consumer key for this branch - must be configured by admin"""
        return self.mpesa_consumer_key
    
    def get_mpesa_consumer_secret(self):
        """Get M-Pesa consumer secret for this branch - must be configured by admin"""
        return self.mpesa_consumer_secret
    
    def get_mpesa_passkey(self):
        """Get M-Pesa passkey for this branch"""
        return self.mpesa_passkey
    
    def has_mpesa_config(self):
        """Check if this branch has M-Pesa configuration"""
        return bool(self.mpesa_shortcode and self.mpesa_consumer_key and self.mpesa_consumer_secret)


class CustomUser(AbstractUser):
    """
    Custom user model with additional fields
    """
    # Override the id field to use UUID
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)

    ROLE_CHOICES = [
        ('admin', 'Admin'),
        ('team_leader', 'Team Leader'),
        ('loan_officer', 'Loan Officer'),
        ('secretary', 'Secretary'),
        ('auditor', 'Auditor'),
        ('borrower', 'Borrower'),
    ]
    
    # Branch fields
    branch = models.ForeignKey(Branch, on_delete=models.SET_NULL, null=True, blank=True, related_name='users')
    accessible_branches = models.ManyToManyField(Branch, blank=True, related_name='accessible_users')
    
    STATUS_CHOICES = [
        ('pending_approval', 'Pending Approval'),
        ('active', 'Active'),
        ('inactive', 'Inactive'),
        ('suspended', 'Suspended'),
        ('blacklisted', 'Blacklisted'),
    ]
    
    
    MARITAL_STATUS_CHOICES = [
        ('single', 'Single'),
        ('married', 'Married'),
        ('divorced', 'Divorced'),
        ('widowed', 'Widowed'),
    ]

    # Override email to make it nullable
    email = models.EmailField(unique=True, null=True, blank=True)

    # Basic Info
    role = models.CharField(max_length=20, choices=ROLE_CHOICES, default='borrower')
    phone_regex = RegexValidator(
        regex=r'^\+?254?\d{9,15}$',
        message="Phone number must be entered in the format: '+254XXXXXXXXX'"
    )
    phone_number = models.CharField(validators=[phone_regex], max_length=17, unique=True)
    id_number = models.CharField(max_length=20, unique=True, null=True, blank=True)  # Made nullable for existing records
    date_of_birth = models.DateField(null=True, blank=True)
    gender = models.CharField(max_length=10, choices=[('M', 'Male'), ('F', 'Female'), ('O', 'Other')], null=True, blank=True)
    marital_status = models.CharField(max_length=20, choices=MARITAL_STATUS_CHOICES, null=True, blank=True)
    nationality = models.CharField(max_length=50, null=True, blank=True)
    
    # Additional Personal Info
    nickname = models.CharField(max_length=100, blank=True, null=True)
    place_of_birth = models.CharField(max_length=200, blank=True, null=True)
    domicile = models.CharField(max_length=200, blank=True, null=True)
    cp_domicile = models.CharField(max_length=200, blank=True, null=True)
    start_time = models.TimeField(blank=True, null=True)
    registration_date = models.DateField(blank=True, null=True)
    
    # Address
    physical_address = models.TextField(blank=True, null=True)
    postal_address = models.CharField(max_length=100, blank=True, null=True)
    postal_code = models.CharField(max_length=20, blank=True, null=True)
    postal_code_business = models.CharField(max_length=20, blank=True, null=True)
    city = models.CharField(max_length=100, blank=True, null=True)
    county = models.CharField(max_length=100, blank=True, null=True)
    country = models.CharField(max_length=100, default='Kenya')
    physical_location = models.CharField(max_length=200, blank=True, null=True)
    
    # Business Info
    business_name = models.CharField(max_length=200, blank=True, null=True)
    business_type = models.CharField(max_length=100, blank=True, null=True)
    other_business_type = models.CharField(max_length=200, blank=True, null=True)
    business_address = models.TextField(blank=True, null=True)
    business_registration_number = models.CharField(max_length=50, blank=True, null=True)
    
    # Financial Information
    monthly_income = models.DecimalField(max_digits=12, decimal_places=2, null=True, blank=True, help_text="Monthly income in KES")
    employer = models.CharField(max_length=200, blank=True, null=True)
    capital_invested = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True, help_text="Capital invested in KES")
    source_of_funds = models.TextField(blank=True, null=True)
    expected_turnover = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True, help_text="Expected turnover in KES")
    
    # Profile Image
    profile_image = models.ImageField(upload_to='profile_images/', null=True, blank=True, help_text="Profile picture for system users")
    
    # Documents
    id_document = models.FileField(upload_to='kyc/id_documents/', null=True, blank=True)
    selfie = models.FileField(upload_to='kyc/selfies/', null=True, blank=True)
    utility_bill = models.FileField(upload_to='kyc/utility_bills/', null=True, blank=True)
    bank_statement = models.FileField(upload_to='kyc/bank_statements/', null=True, blank=True)
    business_license = models.FileField(upload_to='kyc/business_licenses/', null=True, blank=True)
    tax_certificate = models.FileField(upload_to='kyc/tax_certificates/', null=True, blank=True)
    logbook = models.FileField(upload_to='kyc/logbooks/', null=True, blank=True)
    title_deed = models.FileField(upload_to='kyc/title_deeds/', null=True, blank=True)
    signature = models.FileField(upload_to='kyc/signatures/', null=True, blank=True)
    other_documents = models.JSONField(default=list, blank=True, help_text="List of additional document URLs and descriptions")
    
    # Declaration
    declaration_name = models.CharField(max_length=200, blank=True, null=True)
    personal_pin = models.CharField(max_length=50, blank=True, null=True)
    
    # Recommenders
    recommender_name = models.CharField(max_length=200, blank=True, null=True)
    recommender_id = models.CharField(max_length=50, blank=True, null=True)
    recommender_tel = models.CharField(max_length=20, blank=True, null=True)
    recommender_mobile = models.CharField(max_length=20, blank=True, null=True)
    recommender_residence = models.CharField(max_length=200, blank=True, null=True)
    
    # Guarantors
    guarantor_name = models.CharField(max_length=200, blank=True, null=True)
    guarantor_id = models.CharField(max_length=50, blank=True, null=True)
    guarantor_tel = models.CharField(max_length=20, blank=True, null=True)
    guarantor_mobile = models.CharField(max_length=20, blank=True, null=True)
    guarantor_residence = models.CharField(max_length=200, blank=True, null=True)
    
    # Portfolio Management
    portfolio_manager = models.ForeignKey(
        'self', 
        on_delete=models.SET_NULL, 
        null=True, 
        blank=True, 
        related_name='portfolio_clients',
        limit_choices_to={'role__in': ['loan_officer', 'team_leader']},
        help_text="Loan officer or team leader managing this client's portfolio"
    )
    assigned_date = models.DateTimeField(null=True, blank=True, help_text="Date when client was assigned to portfolio manager")
    
    # Registration Fee
    registration_fee_amount = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True, help_text="Registration fee amount in KES")
    registration_fee_paid = models.BooleanField(default=False, help_text="Whether registration fee has been paid")
    registration_fee_payment_date = models.DateTimeField(null=True, blank=True, help_text="Date when registration fee was paid")
    registration_fee_payment_method = models.CharField(
        max_length=20, 
        choices=[
            ('mpesa', 'M-Pesa'),
            ('bank', 'Bank Transfer'),
            ('cash', 'Cash'),
            ('cheque', 'Cheque'),
            ('card', 'Card Payment'),
        ],
        null=True, blank=True,
        help_text="Method used to pay registration fee"
    )
    registration_fee_receipt_number = models.CharField(max_length=50, null=True, blank=True, help_text="Receipt number for registration fee payment")
    registration_fee_notes = models.TextField(null=True, blank=True, help_text="Additional notes about registration fee payment")
    
    # Status
    status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending_approval')
    is_verified = models.BooleanField(default=False)
    is_phone_verified = models.BooleanField(default=False)
    is_email_verified = models.BooleanField(default=False)
    verification_date = models.DateTimeField(null=True, blank=True)
    verified_by = models.ForeignKey('self', on_delete=models.SET_NULL, null=True, blank=True, related_name='verified_users')
    
    # Approval fields (for new client registration)
    approved_by = models.ForeignKey('self', on_delete=models.SET_NULL, null=True, blank=True, related_name='approved_clients')
    approved_at = models.DateTimeField(null=True, blank=True)
    approval_reason = models.TextField(blank=True, null=True, help_text="Reason for approval")
    rejected_by = models.ForeignKey('self', on_delete=models.SET_NULL, null=True, blank=True, related_name='rejected_clients')
    rejected_at = models.DateTimeField(null=True, blank=True)
    rejection_reason = models.TextField(blank=True, null=True, help_text="Reason for rejection")
    
    # Timestamps
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        db_table = 'users'
        ordering = ['-date_joined']
        
    def __str__(self):
        return f"{self.get_full_name()} ({self.email or self.phone_number})"
    
    def get_full_name(self):
        """Return full name or username if no name set"""
        full_name = super().get_full_name()
        return full_name if full_name else self.username
    
    def save(self, *args, **kwargs):
        """Override save to handle verification status"""
        if self.verified_by and not self.verification_date:
            self.verification_date = get_current_datetime()
            self.is_verified = True
        super().save(*args, **kwargs)
    
    def add_document(self, url, description, document_type):
        """Add a new document to other_documents"""
        if not self.other_documents:
            self.other_documents = []
        
        self.other_documents.append({
            'url': url,
            'description': description,
            'type': document_type,
            'uploaded_at': get_current_datetime().isoformat()
        })
        self.save()
    
    def get_short_name(self):
        return self.first_name or self.username
    
    def get_staff_status(self):
        """Get staff status based on is_staff and is_superuser flags"""
        if self.is_superuser:
            return 'superuser'
        elif self.is_staff:
            return 'admin'
        else:
            return 'regular'
    
    def set_staff_status(self, status):
        """Set staff status based on the status value"""
        if status == 'superuser':
            self.is_superuser = True
            self.is_staff = True
        elif status == 'admin':
            self.is_superuser = False
            self.is_staff = True
        else:  # regular
            self.is_superuser = False
            self.is_staff = False
    
    def get_profile_image_url(self):
        """Get profile image URL with fallback to selfie or default avatar"""
        if self.profile_image:
            return self.profile_image.url
        elif self.selfie:
            return self.selfie.url
        else:
            # Return a default avatar based on role
            role_avatars = {
                'admin': '/static/images/avatars/admin-avatar.png',
                'team_leader': '/static/images/avatars/team-leader-avatar.png',
                'loan_officer': '/static/images/avatars/loan-officer-avatar.png',
                'secretary': '/static/images/avatars/secretary-avatar.png',
                'auditor': '/static/images/avatars/auditor-avatar.png',
                'borrower': '/static/images/avatars/borrower-avatar.png',
            }
            return role_avatars.get(self.role, '/static/images/avatars/default-avatar.png')
    
    def get_initials(self):
        """Get user initials for avatar fallback"""
        if self.first_name and self.last_name:
            return f"{self.first_name[0]}{self.last_name[0]}".upper()
        elif self.first_name:
            return self.first_name[0].upper()
        elif self.username:
            return self.username[0].upper()
        else:
            return "U"
    
    def is_admin(self):
        return self.role == 'admin'
    
    def is_loan_officer(self):
        return self.role == 'loan_officer'
    
    def is_auditor(self):
        return self.role == 'auditor'
    
    def is_borrower(self):
        return self.role == 'borrower'
    
    def is_team_leader(self):
        return self.role == 'team_leader'
    
    def is_secretary(self):
        return self.role == 'secretary'
    
    def is_active_user(self):
        return self.status == 'active'
    
    def has_permission(self, module, action):
        """Check if user has permission for a specific module and action"""
        if self.role == 'admin':
            return True  # Admin has all permissions
        
        # Check for custom user permissions first
        try:
            from .models import UserPermission
            user_perm = UserPermission.objects.get(
                user=self,
                module=module,
                action=action
            )
            # Check if permission is expired
            if not user_perm.is_expired():
                return user_perm.is_allowed
        except (UserPermission.DoesNotExist, ImportError):
            pass
        
        # Fall back to role permissions
        try:
            from .models import RolePermission
            permission = RolePermission.objects.get(
                role=self.role,
                module=module,
                action=action
            )
            return permission.is_allowed
        except (RolePermission.DoesNotExist, ImportError):
            return False
    
    def get_permissions(self):
        """Get all permissions for the user's role including custom overrides"""
        if self.role == 'admin':
            # Return all possible permissions for admin
            permissions = {}
            try:
                from .models import RolePermission
                for module in RolePermission.MODULE_CHOICES:
                    permissions[module[0]] = {}
                    for action in RolePermission.ACTION_CHOICES:
                        permissions[module[0]][action[0]] = True
            except ImportError:
                # Fallback if RolePermission is not available
                pass
            return permissions
        
        try:
            from .models import RolePermission, UserPermission
            
            # Start with role permissions
            role_permissions = RolePermission.objects.filter(role=self.role)
            permissions = {}
            for perm in role_permissions:
                if perm.module not in permissions:
                    permissions[perm.module] = {}
                permissions[perm.module][perm.action] = perm.is_allowed
            
            # Override with custom user permissions
            user_permissions = UserPermission.objects.filter(user=self)
            for perm in user_permissions:
                if not perm.is_expired():
                    if perm.module not in permissions:
                        permissions[perm.module] = {}
                    permissions[perm.module][perm.action] = perm.is_allowed
            
            return permissions
        except ImportError:
            return {}
    
    def get_effective_permissions(self):
        """
        Get effective permissions with source information
        Enhanced to use new granular permission system with fallback to legacy system
        
        Returns:
            Dictionary with permission information including source tracking
        """
        try:
            # Try to use the enhanced granular permission system first
            enhanced_permissions = self.get_effective_permissions_enhanced()
            if enhanced_permissions and 'legacy' not in enhanced_permissions:
                return enhanced_permissions
        except:
            # Fall back to legacy system if enhanced system fails
            pass
        
        # Legacy permission system fallback
        if self.role == 'admin':
            return self.get_permissions()
        
        try:
            from .models import RolePermission, UserPermission
            
            permissions = {}
            
            # Get all possible modules and actions
            for module_choice in RolePermission.MODULE_CHOICES:
                module = module_choice[0]
                permissions[module] = {}
                
                for action_choice in RolePermission.ACTION_CHOICES:
                    action = action_choice[0]
                    
                    # Check custom user permission first
                    try:
                        user_perm = UserPermission.objects.get(user=self, module=module, action=action)
                        if not user_perm.is_expired():
                            permissions[module][action] = {
                                'allowed': user_perm.is_allowed,
                                'source': 'custom',
                                'granted_by': user_perm.granted_by.get_full_name() if user_perm.granted_by else None,
                                'reason': user_perm.reason,
                                'expires_at': user_perm.expires_at
                            }
                            continue
                    except UserPermission.DoesNotExist:
                        pass
                    
                    # Fall back to role permission
                    try:
                        role_perm = RolePermission.objects.get(role=self.role, module=module, action=action)
                        permissions[module][action] = {
                            'allowed': role_perm.is_allowed,
                            'source': 'role'
                        }
                    except RolePermission.DoesNotExist:
                        permissions[module][action] = {
                            'allowed': False,
                            'source': 'default'
                        }
            
            return permissions
        except ImportError:
            return {}
    
    def log_access(self, action, module, object_type=None, object_id=None, description="", request=None):
        """Log user access for audit trail"""
        try:
            from .models import UserAccessLog
            UserAccessLog.objects.create(
                user=self,
                action=action,
                module=module,
                object_type=object_type,
                object_id=object_id,
                description=description,
                ip_address=request.META.get('REMOTE_ADDR') if request else None,
                user_agent=request.META.get('HTTP_USER_AGENT') if request else None,
                session_id=request.session.session_key if request and request.session else None
            )
        except ImportError:
            # Fallback if UserAccessLog is not available
            pass
    
    def update_last_login(self):
        self.last_login_at = get_current_datetime()
        self.save(update_fields=['last_login_at'])
    
    def assign_to_portfolio_manager(self, manager):
        """Assign this client to a portfolio manager"""
        if manager.role not in ['loan_officer', 'team_leader']:
            raise ValueError("Portfolio manager must be a loan officer or team leader")
        
        self.portfolio_manager = manager
        self.assigned_date = get_current_datetime()
        self.save()
        
        # Log the assignment
        try:
            from utils.models import AuditLog
            AuditLog.objects.create(
                user=manager,
                action='assign_client',
                model_name='CustomUser',
                object_id=str(self.id),
                description=f'Assigned client {self.get_full_name()} to portfolio manager {manager.get_full_name()}'
            )
        except ImportError:
            pass
    
    def get_portfolio_stats(self):
        """Get portfolio statistics for this user if they are a portfolio manager"""
        if self.role not in ['loan_officer', 'team_leader']:
            return None
        
        from django.db.models import Sum, Count, Avg, Q
        from loans.models import Loan, LoanApplication
        from decimal import Decimal
        
        # Get all clients in this manager's portfolio
        clients = self.portfolio_clients.filter(status='active')
        client_ids = list(clients.values_list('id', flat=True))
        
        # Get loan statistics
        loans = Loan.objects.filter(borrower_id__in=client_ids)
        applications = LoanApplication.objects.filter(borrower_id__in=client_ids)
        
        # Calculate total collected from repayments
        from loans.models import Repayment
        total_collected = Repayment.objects.filter(
            loan__borrower_id__in=client_ids
        ).aggregate(total=Sum('amount'))['total'] or Decimal('0')
        
        # Calculate total outstanding for active loans
        active_loans = loans.filter(status='active')
        total_outstanding = Decimal('0')
        for loan in active_loans:
            amount_paid = getattr(loan, '_amount_paid_cache', 0) or 0
            total_outstanding += loan.total_amount - amount_paid
        
        stats = {
            'total_clients': clients.count(),
            'total_loans': loans.count(),
            'active_loans': active_loans.count(),
            'paid_loans': loans.filter(status='paid').count(),
            'defaulted_loans': loans.filter(status='defaulted').count(),
            'total_disbursed': loans.aggregate(total=Sum('principal_amount'))['total'] or Decimal('0'),
            'total_outstanding': total_outstanding,
            'total_collected': total_collected,
            'pending_applications': applications.filter(status='pending').count(),
            'approved_applications': applications.filter(status='approved').count(),
            'rejected_applications': applications.filter(status='rejected').count(),
        }
        
        # Calculate performance metrics
        if stats['total_loans'] > 0:
            stats['collection_rate'] = (stats['total_collected'] / stats['total_disbursed'] * Decimal('100')) if stats['total_disbursed'] > 0 else Decimal('0')
            stats['default_rate'] = (Decimal(str(stats['defaulted_loans'])) / Decimal(str(stats['total_loans'])) * Decimal('100'))
            stats['approval_rate'] = (Decimal(str(stats['approved_applications'])) / Decimal(str(stats['pending_applications'] + stats['approved_applications'] + stats['rejected_applications'])) * Decimal('100')) if (stats['pending_applications'] + stats['approved_applications'] + stats['rejected_applications']) > 0 else Decimal('0')
        else:
            stats['collection_rate'] = Decimal('0')
            stats['default_rate'] = Decimal('0')
            stats['approval_rate'] = Decimal('0')
        
        return stats
    
    def get_portfolio_performance_trend(self, months=6):
        """Get portfolio performance trend over the last N months"""
        if self.role not in ['loan_officer', 'team_leader']:
            return []
        
        from django.db.models import Sum, Count
        from loans.models import Loan
        from datetime import datetime, timedelta
        import calendar
        
        clients = self.portfolio_clients.filter(status='active')
        client_ids = list(clients.values_list('id', flat=True))
        
        trend_data = []
        for i in range(months):
            # Calculate date range for this month
            end_date = datetime.now().replace(day=1) - timedelta(days=i*30)
            start_date = end_date.replace(day=1)
            
            # Get loans created in this month
            month_loans = Loan.objects.filter(
                borrower_id__in=client_ids,
                created_at__gte=start_date,
                created_at__lt=end_date + timedelta(days=32)
            )
            
            # Calculate amount collected manually since amount_paid is not a direct field
            amount_collected = 0
            for loan in month_loans:
                amount_collected += getattr(loan, '_amount_paid_cache', 0) or 0
            
            month_stats = {
                'month': calendar.month_name[start_date.month],
                'year': start_date.year,
                'loans_disbursed': month_loans.count(),
                'amount_disbursed': month_loans.aggregate(total=Sum('principal_amount'))['total'] or 0,
                'amount_collected': amount_collected,
            }
            
            trend_data.append(month_stats)
        
        return list(reversed(trend_data))
    
    def mark_registration_fee_paid(self, amount, payment_method, receipt_number=None, notes=None):
        """Mark registration fee as paid"""
        self.registration_fee_amount = amount
        self.registration_fee_paid = True
        self.registration_fee_payment_date = get_current_datetime()
        self.registration_fee_payment_method = payment_method
        self.registration_fee_receipt_number = receipt_number
        self.registration_fee_notes = notes
        self.save()
    
    def get_registration_fee_status(self):
        """Get registration fee payment status"""
        if not self.registration_fee_amount:
            return {
                'status': 'not_set',
                'message': 'Registration fee not set',
                'amount': 0,
                'paid': False
            }
        
        if self.registration_fee_paid:
            return {
                'status': 'paid',
                'message': f'Paid KES {self.registration_fee_amount:,.2f} on {self.registration_fee_payment_date.strftime("%d %b %Y") if self.registration_fee_payment_date else "N/A"}',
                'amount': self.registration_fee_amount,
                'paid': True,
                'payment_date': self.registration_fee_payment_date,
                'payment_method': self.get_registration_fee_payment_method_display() if self.registration_fee_payment_method else None,
                'receipt_number': self.registration_fee_receipt_number
            }
        else:
            return {
                'status': 'pending',
                'message': f'Pending payment of KES {self.registration_fee_amount:,.2f}',
                'amount': self.registration_fee_amount,
                'paid': False
            }
    
    # Enhanced Permission Methods for Granular Page-Specific Permissions
    
    def has_page_permission(self, page_name: str, action_code: str) -> bool:
        """
        Check if user has a specific page permission using the new granular system
        
        Args:
            page_name: Name of the page (e.g., 'loans', 'clients', 'reports')
            action_code: Specific action code to check (e.g., 'create_application', 'view_list')
            
        Returns:
            Boolean indicating if user has permission for this page action
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            return permission_manager.check_action_permission(self, page_name, action_code)
        except ImportError:
            # Fallback to legacy permission system if services not available
            return self.has_permission(page_name, action_code)
    
    def get_page_permissions(self, page_name: str) -> Dict[str, bool]:
        """
        Get all permissions for a specific page for this user
        
        Args:
            page_name: Name of the page to get permissions for
            
        Returns:
            Dictionary mapping action codes to boolean permission status
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            return permission_manager.get_page_permissions(self, page_name)
        except ImportError:
            # Fallback to empty dict if services not available
            return {}
    
    def get_effective_permissions_enhanced(self) -> Dict[str, Dict[str, any]]:
        """
        Enhanced version of get_effective_permissions with source tracking and granular permissions
        
        Returns:
            Nested dictionary with page permissions and their sources
            Format: {page_name: {action_code: {'allowed': bool, 'source': str, 'metadata': dict}}}
        """
        try:
            from .enhanced_permissions_models import PagePermission, UserPagePermission, RolePermissionTemplate
            from django.utils import timezone
            
            # Admin users have all permissions
            if self.role == 'admin':
                permissions = {}
                all_page_permissions = PagePermission.objects.filter(is_active=True)
                
                for page_perm in all_page_permissions:
                    page_name = page_perm.page_name
                    if page_name not in permissions:
                        permissions[page_name] = {}
                    
                    permissions[page_name][page_perm.action_code] = {
                        'allowed': True,
                        'source': 'admin_role',
                        'metadata': {
                            'action_name': page_perm.action_name,
                            'description': page_perm.description,
                            'category': page_perm.category,
                            'is_critical': page_perm.is_critical
                        }
                    }
                
                return permissions
            
            permissions = {}
            
            # Get all page permissions
            all_page_permissions = PagePermission.objects.filter(is_active=True).order_by('page_name', 'category')
            
            for page_perm in all_page_permissions:
                page_name = page_perm.page_name
                action_code = page_perm.action_code
                
                if page_name not in permissions:
                    permissions[page_name] = {}
                
                # Check for custom user permission first
                try:
                    user_permission = UserPagePermission.objects.get(
                        user=self,
                        page_permission=page_perm
                    )
                    
                    if not user_permission.is_expired():
                        permissions[page_name][action_code] = {
                            'allowed': user_permission.is_allowed,
                            'source': 'custom_override',
                            'metadata': {
                                'action_name': page_perm.action_name,
                                'description': page_perm.description,
                                'category': page_perm.category,
                                'is_critical': page_perm.is_critical,
                                'granted_by': user_permission.granted_by.get_full_name() if user_permission.granted_by else None,
                                'reason': user_permission.reason,
                                'expires_at': user_permission.expires_at.isoformat() if user_permission.expires_at else None,
                                'created_at': user_permission.created_at.isoformat()
                            }
                        }
                        continue
                    else:
                        # Permission expired, fall through to role default
                        pass
                        
                except UserPagePermission.DoesNotExist:
                    # No custom permission, fall through to role default
                    pass
                
                # Check role permission template
                try:
                    role_template = RolePermissionTemplate.objects.get(
                        role=self.role,
                        page_permission=page_perm
                    )
                    
                    permissions[page_name][action_code] = {
                        'allowed': role_template.is_allowed,
                        'source': 'role_template',
                        'metadata': {
                            'action_name': page_perm.action_name,
                            'description': page_perm.description,
                            'category': page_perm.category,
                            'is_critical': page_perm.is_critical,
                            'can_override': role_template.can_override,
                            'priority': role_template.priority
                        }
                    }
                    
                except RolePermissionTemplate.DoesNotExist:
                    # No role template, default to denied
                    permissions[page_name][action_code] = {
                        'allowed': False,
                        'source': 'default_deny',
                        'metadata': {
                            'action_name': page_perm.action_name,
                            'description': page_perm.description,
                            'category': page_perm.category,
                            'is_critical': page_perm.is_critical,
                            'reason': 'No role template defined'
                        }
                    }
            
            return permissions
            
        except ImportError:
            # Fallback to legacy system
            return {'legacy': {'fallback': {'allowed': False, 'source': 'import_error', 'metadata': {}}}}
    
    def get_available_page_actions(self, page_name: str) -> List[Dict[str, any]]:
        """
        Get list of actions user can perform on a specific page for UI rendering
        
        Args:
            page_name: Name of the page to get available actions for
            
        Returns:
            List of dictionaries containing action details for allowed actions
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            return permission_manager.get_available_actions(self, page_name)
        except ImportError:
            # Fallback to empty list if services not available
            return []
    
    def inherit_role_permissions(self, apply_immediately: bool = True) -> Dict[str, any]:
        """
        Inherit permissions from role templates, optionally applying them immediately
        
        Args:
            apply_immediately: Whether to apply role permissions immediately or just return what would be applied
            
        Returns:
            Dictionary with inheritance results and statistics
        """
        try:
            from .services import RolePermissionTemplateManager
            template_manager = RolePermissionTemplateManager()
            
            if apply_immediately:
                return template_manager.apply_defaults_to_users(
                    role=self.role,
                    update_existing=False,
                    user_ids=[str(self.id)]
                )
            else:
                # Just return what would be inherited
                role_template = template_manager.get_role_template(self.role)
                
                inheritance_preview = {
                    'role': self.role,
                    'user_id': str(self.id),
                    'user_name': self.get_full_name(),
                    'permissions_to_inherit': 0,
                    'pages_affected': [],
                    'critical_permissions': [],
                    'template_data': role_template
                }
                
                for page_name, actions in role_template.items():
                    inheritance_preview['pages_affected'].append(page_name)
                    for action_code, perm_info in actions.items():
                        inheritance_preview['permissions_to_inherit'] += 1
                        if perm_info.get('is_critical') and perm_info.get('is_allowed'):
                            inheritance_preview['critical_permissions'].append({
                                'page_name': page_name,
                                'action_code': action_code,
                                'action_name': perm_info.get('action_name')
                            })
                
                return inheritance_preview
                
        except ImportError:
            return {
                'error': 'Enhanced permission services not available',
                'role': self.role,
                'user_id': str(self.id)
            }
    
    def get_permission_summary(self, page_name: Optional[str] = None) -> Dict[str, any]:
        """
        Get a comprehensive permission summary for this user
        
        Args:
            page_name: Optional specific page to get summary for, if None gets summary for all pages
            
        Returns:
            Dictionary containing comprehensive permission summary
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            
            if page_name:
                return permission_manager.get_permission_summary(self, page_name)
            else:
                # Get summary for all pages
                from .enhanced_permissions_models import PagePermission
                
                all_pages = PagePermission.objects.filter(is_active=True).values_list('page_name', flat=True).distinct()
                
                summary = {
                    'user_id': str(self.id),
                    'user_name': self.get_full_name(),
                    'user_role': self.role,
                    'total_pages': len(all_pages),
                    'pages': {},
                    'overall_stats': {
                        'total_permissions': 0,
                        'allowed_permissions': 0,
                        'denied_permissions': 0,
                        'critical_permissions': 0
                    }
                }
                
                for page in all_pages:
                    page_summary = permission_manager.get_permission_summary(self, page)
                    summary['pages'][page] = page_summary
                    
                    # Aggregate stats
                    summary['overall_stats']['total_permissions'] += page_summary.get('total_permissions', 0)
                    summary['overall_stats']['allowed_permissions'] += page_summary.get('allowed_permissions', 0)
                    summary['overall_stats']['denied_permissions'] += page_summary.get('denied_permissions', 0)
                    if page_summary.get('has_critical_permissions'):
                        summary['overall_stats']['critical_permissions'] += 1
                
                return summary
                
        except ImportError:
            return {
                'error': 'Enhanced permission services not available',
                'user_id': str(self.id),
                'user_role': self.role
            }
    
    def bulk_check_page_permissions(self, permission_checks: List[Tuple[str, str]]) -> Dict[str, bool]:
        """
        Efficiently check multiple page permissions at once
        
        Args:
            permission_checks: List of (page_name, action_code) tuples to check
            
        Returns:
            Dictionary mapping "page_name:action_code" to boolean permission status
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            return permission_manager.bulk_check_permissions(self, permission_checks)
        except ImportError:
            # Fallback to individual checks using legacy system
            results = {}
            for page_name, action_code in permission_checks:
                key = f"{page_name}:{action_code}"
                results[key] = self.has_permission(page_name, action_code)
            return results
    
    def invalidate_permission_cache(self, page_name: Optional[str] = None):
        """
        Invalidate cached permissions for this user
        
        Args:
            page_name: Optional specific page to invalidate cache for, if None invalidates all
        """
        try:
            from .services import PagePermissionManager
            permission_manager = PagePermissionManager()
            permission_manager.invalidate_user_cache(str(self.id), page_name)
        except ImportError:
            # No caching in legacy system, so nothing to invalidate
            pass
    
    def has_any_page_permission(self, page_name: str) -> bool:
        """
        Check if user has any permission on a specific page
        
        Args:
            page_name: Name of the page to check
            
        Returns:
            Boolean indicating if user has any permission on this page
        """
        page_permissions = self.get_page_permissions(page_name)
        return any(page_permissions.values())
    
    def get_accessible_pages(self) -> List[str]:
        """
        Get list of pages this user has any access to
        
        Returns:
            List of page names the user can access
        """
        try:
            from .enhanced_permissions_models import PagePermission
            
            all_pages = PagePermission.objects.filter(is_active=True).values_list('page_name', flat=True).distinct()
            accessible_pages = []
            
            for page_name in all_pages:
                if self.has_any_page_permission(page_name):
                    accessible_pages.append(page_name)
            
            return accessible_pages
            
        except ImportError:
            # Fallback to legacy system
            try:
                from .models import RolePermission
                
                role_permissions = RolePermission.objects.filter(
                    role=self.role,
                    is_allowed=True
                ).values_list('module', flat=True).distinct()
                
                return list(role_permissions)
            except:
                return []
    
    def can_override_permission(self, page_name: str, action_code: str) -> bool:
        """
        Check if this user's permission for a specific action can be overridden
        
        Args:
            page_name: Name of the page
            action_code: Specific action code
            
        Returns:
            Boolean indicating if permission can be overridden
        """
        try:
            from .enhanced_permissions_models import RolePermissionTemplate, PagePermission
            
            page_permission = PagePermission.objects.get(
                page_name=page_name,
                action_code=action_code,
                is_active=True
            )
            
            role_template = RolePermissionTemplate.objects.get(
                role=self.role,
                page_permission=page_permission
            )
            
            return role_template.can_override
            
        except:
            # Default to allowing overrides if we can't determine
            return True


class RolePermission(models.Model):
    """
    Role-based permissions for different user roles
    """
    ROLE_CHOICES = [
        ('admin', 'Admin'),
        ('team_leader', 'Team Leader'),
        ('loan_officer', 'Loan Officer'),
        ('secretary', 'Secretary'),
        ('borrower', 'Borrower'),
    ]
    
    MODULE_CHOICES = [
        # Main Navigation Pages - Core Access
        ('dashboard', 'Dashboard'),
        ('clients', 'Clients'),
        ('loans', 'Loans'),
        ('repayments', 'All Repayments'),
        ('portfolio', 'Portfolio'),
        ('reports_statements', 'Reports & Statements'),
        ('documents', 'Documents'),
        ('customer_documents', 'Customer Documents'),
        ('payment_receipts', 'Payment Receipts'),
        ('notifications', 'Notifications'),
        ('settings', 'Settings'),
        ('branch_settings', 'Branch Settings'),
        ('system_settings', 'System Settings'),
        
        # Dashboard Specific Features
        ('dashboard_overview', 'Dashboard Overview Cards'),
        ('dashboard_metrics', 'Key Performance Metrics'),
        ('dashboard_charts', 'Analytics Charts'),
        ('dashboard_recent_activities', 'Recent Activities'),
        ('dashboard_quick_actions', 'Quick Action Buttons'),
        ('dashboard_loan_summary', 'Loan Summary Widget'),
        ('dashboard_collection_summary', 'Collection Summary'),
        ('dashboard_branch_performance', 'Branch Performance'),
        ('dashboard_alerts', 'System Alerts & Warnings'),
        
        # Clients Page Features
        ('clients_view_list', 'View Clients List'),
        ('clients_search_filter', 'Search & Filter Clients'),
        ('clients_create_new', 'Create New Client'),
        ('clients_edit_profile', 'Edit Client Profile'),
        ('clients_delete_client', 'Delete Client'),
        ('clients_view_details', 'View Client Details'),
        ('clients_loan_history', 'View Loan History'),
        ('clients_payment_history', 'View Payment History'),
        ('clients_kyc_documents', 'Manage KYC Documents'),
        ('clients_assign_portfolio', 'Assign to Portfolio Manager'),
        ('clients_registration_fee', 'Manage Registration Fees'),
        ('clients_status_change', 'Change Client Status'),
        ('clients_export_data', 'Export Client Data'),
        ('clients_bulk_actions', 'Bulk Client Actions'),
        
        # Loans Page Features
        ('loans_view_list', 'View Loans List'),
        ('loans_search_filter', 'Search & Filter Loans'),
        ('loans_create_application', 'Create Loan Application'),
        ('loans_edit_application', 'Edit Loan Application'),
        ('loans_delete_loan', 'Delete Loan'),
        ('loans_approve_application', 'Approve Loan Applications'),
        ('loans_reject_application', 'Reject Loan Applications'),
        ('loans_disburse_funds', 'Disburse Loan Funds'),
        ('loans_view_details', 'View Loan Details'),
        ('loans_amortization_schedule', 'View Amortization Schedule'),
        ('loans_rollover_loan', 'Process Loan Rollover'),
        ('loans_calculate_interest', 'Calculate Interest & Fees'),
        ('loans_generate_receipt', 'Generate Loan Receipt'),
        ('loans_modify_terms', 'Modify Loan Terms'),
        ('loans_close_loan', 'Close/Complete Loan'),
        ('loans_export_data', 'Export Loan Data'),
        
        # All Repayments Page Features
        ('repayments_view_list', 'View All Repayments'),
        ('repayments_search_filter', 'Search & Filter Repayments'),
        ('repayments_record_payment', 'Record New Payment'),
        ('repayments_edit_payment', 'Edit Payment Record'),
        ('repayments_delete_payment', 'Delete Payment'),
        ('repayments_verify_payment', 'Verify Payment'),
        ('repayments_reconcile_mpesa', 'Reconcile M-Pesa Payments'),
        ('repayments_generate_receipt', 'Generate Payment Receipt'),
        ('repayments_bulk_import', 'Bulk Import Payments'),
        ('repayments_export_data', 'Export Repayment Data'),
        ('repayments_view_analytics', 'View Payment Analytics'),
        
        # Portfolio Page Features
        ('portfolio_view_overview', 'View Portfolio Overview'),
        ('portfolio_manager_stats', 'Portfolio Manager Statistics'),
        ('portfolio_performance_metrics', 'Performance Metrics'),
        ('portfolio_client_assignments', 'Manage Client Assignments'),
        ('portfolio_reassign_clients', 'Reassign Clients'),
        ('portfolio_view_analytics', 'Portfolio Analytics'),
        ('portfolio_generate_reports', 'Generate Portfolio Reports'),
        ('portfolio_target_tracking', 'Track Performance Targets'),
        
        # Reports & Statements Features
        ('reports_view_dashboard', 'View Reports Dashboard'),
        ('reports_loans_due', 'Loans Due Report'),
        ('reports_delinquent_loans', 'Delinquent Loans Report'),
        ('reports_arrears_analysis', 'Loans in Arrears Analysis'),
        ('reports_processing_fees', 'Processing Fees Report'),
        ('reports_interest_income', 'Interest Income Report'),
        ('reports_registration_fees', 'Registration Fees Report'),
        ('reports_customer_requests', 'Customer Requests Report'),
        ('reports_collection_summary', 'Collection Summary Report'),
        ('reports_branch_performance', 'Branch Performance Report'),
        ('reports_export_all', 'Export All Reports'),
        ('reports_schedule_automated', 'Schedule Automated Reports'),
        ('statements_generate_loan', 'Generate Loan Statements'),
        ('statements_generate_client', 'Generate Client Statements'),
        ('statements_download_pdf', 'Download PDF Statements'),
        ('statements_email_client', 'Email Statements to Clients'),
        
        # Documents Page Features
        ('documents_view_library', 'View Document Library'),
        ('documents_upload_files', 'Upload Documents'),
        ('documents_download_files', 'Download Documents'),
        ('documents_delete_files', 'Delete Documents'),
        ('documents_organize_folders', 'Organize Document Folders'),
        ('documents_search_content', 'Search Document Content'),
        ('documents_share_access', 'Share Document Access'),
        ('documents_version_control', 'Document Version Control'),
        
        # Customer Documents Features
        ('customer_docs_view_all', 'View All Customer Documents'),
        ('customer_docs_upload_kyc', 'Upload KYC Documents'),
        ('customer_docs_verify_identity', 'Verify Identity Documents'),
        ('customer_docs_approve_kyc', 'Approve KYC Documents'),
        ('customer_docs_reject_kyc', 'Reject KYC Documents'),
        ('customer_docs_request_additional', 'Request Additional Documents'),
        ('customer_docs_download_files', 'Download Customer Documents'),
        ('customer_docs_compliance_check', 'Compliance Verification'),
        
        # Payment Receipts Features
        ('receipts_view_all', 'View All Receipts'),
        ('receipts_generate_new', 'Generate New Receipt'),
        ('receipts_edit_existing', 'Edit Existing Receipt'),
        ('receipts_delete_receipt', 'Delete Receipt'),
        ('receipts_print_receipt', 'Print Receipt'),
        ('receipts_download_pdf', 'Download PDF Receipt'),
        ('receipts_email_client', 'Email Receipt to Client'),
        ('receipts_bulk_generate', 'Bulk Generate Receipts'),
        ('receipts_template_manage', 'Manage Receipt Templates'),
        
        # Notifications Features
        ('notifications_view_inbox', 'View Notification Inbox'),
        ('notifications_send_individual', 'Send Individual Notification'),
        ('notifications_send_bulk', 'Send Bulk Notifications'),
        ('notifications_manage_templates', 'Manage Notification Templates'),
        ('notifications_configure_settings', 'Configure Notification Settings'),
        ('notifications_view_history', 'View Notification History'),
        ('notifications_manage_channels', 'Manage Communication Channels'),
        
        # Settings Features
        ('settings_view_general', 'View General Settings'),
        ('settings_edit_system', 'Edit System Settings'),
        ('settings_manage_users', 'Manage User Accounts'),
        ('settings_configure_permissions', 'Configure Permissions'),
        ('settings_backup_restore', 'Backup & Restore'),
        ('settings_integration_config', 'Integration Configuration'),
        ('settings_audit_logs', 'View Audit Logs'),
        
        # Branch Settings Features
        ('branch_view_info', 'View Branch Information'),
        ('branch_edit_details', 'Edit Branch Details'),
        ('branch_manage_staff', 'Manage Branch Staff'),
        ('branch_configure_mpesa', 'Configure M-Pesa Settings'),
        ('branch_view_performance', 'View Branch Performance'),
        ('branch_manage_targets', 'Manage Branch Targets'),
        
        # System Settings Features
        ('system_database_management', 'Database Management'),
        ('system_server_monitoring', 'Server Monitoring'),
        ('system_security_settings', 'Security Settings'),
        ('system_maintenance_mode', 'Maintenance Mode'),
        ('system_api_management', 'API Management'),
        ('system_integration_logs', 'Integration Logs'),
    ]
    
    ACTION_CHOICES = [
        # Core Access Actions
        ('access', 'Access/View'),
        ('create', 'Create/Add'),
        ('edit', 'Edit/Modify'),
        ('delete', 'Delete/Remove'),
        
        # Approval & Workflow Actions
        ('approve', 'Approve'),
        ('reject', 'Reject'),
        ('verify', 'Verify'),
        ('validate', 'Validate'),
        
        # Data Management Actions
        ('export', 'Export'),
        ('import', 'Import'),
        ('download', 'Download'),
        ('upload', 'Upload'),
        ('print', 'Print'),
        
        # Processing Actions
        ('process', 'Process'),
        ('calculate', 'Calculate'),
        ('generate', 'Generate'),
        ('record', 'Record'),
        ('reconcile', 'Reconcile'),
        
        # Assignment & Management Actions
        ('assign', 'Assign'),
        ('reassign', 'Reassign'),
        ('manage', 'Manage'),
        ('configure', 'Configure'),
        
        # Communication Actions
        ('send', 'Send'),
        ('email', 'Email'),
        ('notify', 'Notify'),
        ('share', 'Share'),
        
        # Status & Control Actions
        ('activate', 'Activate'),
        ('deactivate', 'Deactivate'),
        ('suspend', 'Suspend'),
        ('close', 'Close'),
        
        # Advanced Actions
        ('monitor', 'Monitor'),
        ('audit', 'Audit'),
        ('backup', 'Backup'),
        ('restore', 'Restore'),
    ]
    
    role = models.CharField(max_length=20, choices=ROLE_CHOICES)
    module = models.CharField(max_length=50, choices=MODULE_CHOICES)
    action = models.CharField(max_length=30, choices=ACTION_CHOICES)
    is_allowed = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        unique_together = ['role', 'module', 'action']
        db_table = 'role_permissions'
        ordering = ['role', 'module', 'action']
        indexes = [
            # Remove the index that references the non-existent 'is_default' field
            models.Index(fields=['role']),
            models.Index(fields=['module', 'action']),
        ]
    
    def __str__(self):
        return f"{self.role} - {self.module} - {self.action} ({'Allowed' if self.is_allowed else 'Denied'})"


class DefaultRolePermission(models.Model):
    """
    Default permissions template for each role that can be customized by admin
    """
    # Add these class attributes to match RolePermission
    MODULE_CHOICES = RolePermission.MODULE_CHOICES
    ACTION_CHOICES = RolePermission.ACTION_CHOICES
    ROLE_CHOICES = RolePermission.ROLE_CHOICES
    
    role = models.CharField(max_length=20, choices=RolePermission.ROLE_CHOICES)
    module = models.CharField(max_length=50, choices=RolePermission.MODULE_CHOICES)
    action = models.CharField(max_length=30, choices=RolePermission.ACTION_CHOICES)
    is_allowed = models.BooleanField(default=False)
    description = models.TextField(blank=True, null=True, help_text="Description of what this permission allows")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        unique_together = ['role', 'module', 'action']
        db_table = 'default_role_permissions'
        ordering = ['role', 'module', 'action']
        indexes = [
            models.Index(fields=['role']),
            models.Index(fields=['module', 'action']),
        ]
    
    def __str__(self):
        return f"Default: {self.role} - {self.module} - {self.action} ({'Allowed' if self.is_allowed else 'Denied'})"


class UserPermission(models.Model):
    """
    Individual user permissions that override role defaults
    """
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='custom_permissions')
    module = models.CharField(max_length=50, choices=RolePermission.MODULE_CHOICES)
    action = models.CharField(max_length=30, choices=RolePermission.ACTION_CHOICES)
    is_allowed = models.BooleanField(default=False)
    granted_by = models.ForeignKey(CustomUser, on_delete=models.SET_NULL, null=True, related_name='permissions_granted')
    reason = models.TextField(blank=True, null=True, help_text="Reason for custom permission")
    expires_at = models.DateTimeField(null=True, blank=True, help_text="When this permission expires")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        unique_together = ['user', 'module', 'action']
        db_table = 'user_permissions'
        ordering = ['user', 'module', 'action']
        indexes = [
            models.Index(fields=['user', 'expires_at']),
            models.Index(fields=['module', 'action']),
        ]
    
    def __str__(self):
        return f"{self.user.get_full_name()} - {self.module} - {self.action} ({'Allowed' if self.is_allowed else 'Denied'})"
    
    def is_expired(self):
        """Check if this permission has expired"""
        if not self.expires_at:
            return False
        return get_current_datetime() > self.expires_at


class UserAccessLog(models.Model):
    """
    Detailed access logs for user activities
    """
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='access_logs')
    action = models.CharField(max_length=50)
    module = models.CharField(max_length=50)
    object_type = models.CharField(max_length=50, blank=True, null=True)
    object_id = models.CharField(max_length=50, blank=True, null=True)
    description = models.TextField()
    ip_address = models.GenericIPAddressField(null=True, blank=True)
    user_agent = models.CharField(max_length=500, blank=True, null=True)
    session_id = models.CharField(max_length=100, blank=True, null=True)
    accessed_at = models.DateTimeField(auto_now_add=True)
    response_time = models.FloatField(blank=True, null=True)  # in milliseconds
    status_code = models.IntegerField(blank=True, null=True)
    
    class Meta:
        db_table = 'user_access_logs'
        ordering = ['-accessed_at']
        indexes = [
            models.Index(fields=['user', 'accessed_at']),
            models.Index(fields=['action', 'accessed_at']),
            models.Index(fields=['module', 'accessed_at']),
        ]
    
    def __str__(self):
        return f"{self.user.get_full_name()} - {self.action} - {self.accessed_at}"


# Enhanced Audit Models

class AuditEventType(models.TextChoices):
    """Types of audit events"""
    LOGIN = 'login', 'Login'
    LOGOUT = 'logout', 'Logout'
    PERMISSION_CHANGE = 'permission_change', 'Permission Change'
    DATA_ACCESS = 'data_access', 'Data Access'
    DATA_MODIFICATION = 'data_modification', 'Data Modification'
    SECURITY_VIOLATION = 'security_violation', 'Security Violation'
    SYSTEM_CONFIGURATION = 'system_config', 'System Configuration'
    EXPORT_DATA = 'export_data', 'Data Export'
    REPORT_GENERATION = 'report_generation', 'Report Generation'
    FAILED_ACCESS = 'failed_access', 'Failed Access Attempt'
    BULK_OPERATION = 'bulk_operation', 'Bulk Operation'


class SecurityEventSeverity(models.TextChoices):
    """Security event severity levels"""
    LOW = 'low', 'Low'
    MEDIUM = 'medium', 'Medium'
    HIGH = 'high', 'High'
    CRITICAL = 'critical', 'Critical'


class EnhancedAuditLog(models.Model):
    """
    Enhanced audit log with detailed permission and security tracking
    """
    # Basic event information
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='enhanced_audit_logs', null=True, blank=True)
    event_type = models.CharField(max_length=50, choices=AuditEventType.choices)
    action = models.CharField(max_length=100)
    module = models.CharField(max_length=50)
    
    # Permission-specific fields
    permission_checked = models.CharField(max_length=100, blank=True, null=True)
    permission_granted = models.BooleanField(null=True, blank=True)
    role_at_time = models.CharField(max_length=50, blank=True, null=True)
    branch_context = models.CharField(max_length=100, blank=True, null=True)
    
    # Object tracking (generic foreign key for any model)
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True, blank=True)
    object_id = models.PositiveIntegerField(null=True, blank=True)
    content_object = GenericForeignKey('content_type', 'object_id')
    
    # Request context
    ip_address = models.GenericIPAddressField(null=True, blank=True)
    user_agent = models.CharField(max_length=500, blank=True, null=True)
    session_id = models.CharField(max_length=100, blank=True, null=True)
    request_method = models.CharField(max_length=10, blank=True, null=True)
    request_path = models.CharField(max_length=500, blank=True, null=True)
    request_params = models.JSONField(default=dict, blank=True)
    
    # Response information
    status_code = models.IntegerField(null=True, blank=True)
    response_time = models.FloatField(null=True, blank=True)  # in milliseconds
    
    # Event details
    description = models.TextField()
    additional_data = models.JSONField(default=dict, blank=True)
    
    # Security fields
    is_security_event = models.BooleanField(default=False)
    severity = models.CharField(max_length=20, choices=SecurityEventSeverity.choices, null=True, blank=True)
    requires_attention = models.BooleanField(default=False)
    
    # Timestamps
    timestamp = models.DateTimeField(auto_now_add=True)
    
    class Meta:
        db_table = 'enhanced_audit_logs'
        ordering = ['-timestamp']
        indexes = [
            models.Index(fields=['user', 'timestamp']),
            models.Index(fields=['event_type', 'timestamp']),
            models.Index(fields=['module', 'timestamp']),
            models.Index(fields=['permission_checked', 'timestamp']),
            models.Index(fields=['is_security_event', 'timestamp']),
            models.Index(fields=['severity', 'timestamp']),
            models.Index(fields=['ip_address', 'timestamp']),
        ]
    
    def __str__(self):
        user_name = self.user.get_full_name() if self.user else 'Anonymous'
        return f"{user_name} - {self.event_type} - {self.action} - {self.timestamp}"
    
    @classmethod
    def log_permission_check(cls, user, permission, granted, request=None, **kwargs):
        """Log a permission check event"""
        return cls.objects.create(
            user=user,
            event_type=AuditEventType.PERMISSION_CHANGE if granted else AuditEventType.FAILED_ACCESS,
            action=f"check_permission_{permission}",
            module=kwargs.get('module', 'permissions'),
            permission_checked=permission,
            permission_granted=granted,
            role_at_time=user.role if user else None,
            branch_context=getattr(user, 'branch', {}).get('name') if user and hasattr(user, 'branch') else None,
            ip_address=cls._get_client_ip(request) if request else None,
            user_agent=request.META.get('HTTP_USER_AGENT', '') if request else None,
            session_id=request.session.session_key if request and hasattr(request, 'session') else None,
            request_method=request.method if request else None,
            request_path=request.path if request else None,
            description=f"Permission check for '{permission}' - {'Granted' if granted else 'Denied'}",
            is_security_event=not granted,
            severity=SecurityEventSeverity.MEDIUM if not granted else None,
            requires_attention=not granted,
            additional_data=kwargs
        )
    
    @classmethod
    def log_data_access(cls, user, object_type, object_id, action, request=None, **kwargs):
        """Log data access event"""
        return cls.objects.create(
            user=user,
            event_type=AuditEventType.DATA_ACCESS,
            action=action,
            module=kwargs.get('module', object_type.lower()),
            role_at_time=user.role if user else None,
            content_type=ContentType.objects.get_for_model(object_type) if hasattr(object_type, '_meta') else None,
            object_id=object_id,
            ip_address=cls._get_client_ip(request) if request else None,
            user_agent=request.META.get('HTTP_USER_AGENT', '') if request else None,
            session_id=request.session.session_key if request and hasattr(request, 'session') else None,
            request_method=request.method if request else None,
            request_path=request.path if request else None,
            description=f"Accessed {object_type.__name__ if hasattr(object_type, '__name__') else object_type} ID: {object_id}",
            additional_data=kwargs
        )
    
    @classmethod
    def log_security_event(cls, user, action, severity, description, request=None, **kwargs):
        """Log security event"""
        return cls.objects.create(
            user=user,
            event_type=AuditEventType.SECURITY_VIOLATION,
            action=action,
            module=kwargs.get('module', 'security'),
            role_at_time=user.role if user else None,
            ip_address=cls._get_client_ip(request) if request else None,
            user_agent=request.META.get('HTTP_USER_AGENT', '') if request else None,
            session_id=request.session.session_key if request and hasattr(request, 'session') else None,
            request_method=request.method if request else None,
            request_path=request.path if request else None,
            description=description,
            is_security_event=True,
            severity=severity,
            requires_attention=severity in [SecurityEventSeverity.HIGH, SecurityEventSeverity.CRITICAL],
            additional_data=kwargs
        )
    
    @staticmethod
    def _get_client_ip(request):
        """Get client IP address from request"""
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for:
            ip = x_forwarded_for.split(',')[0]
        else:
            ip = request.META.get('REMOTE_ADDR')
        return ip


class PermissionChangeLog(models.Model):
    """
    Detailed logging for permission changes
    """
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='permission_changes')
    changed_by = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='permission_changes_made')
    
    # Permission details
    permission_type = models.CharField(max_length=50)  # 'role', 'custom', 'template'
    permission_name = models.CharField(max_length=100)
    old_value = models.BooleanField(null=True, blank=True)
    new_value = models.BooleanField(null=True, blank=True)
    
    # Context
    reason = models.TextField(blank=True)
    approval_required = models.BooleanField(default=False)
    approved_by = models.ForeignKey(CustomUser, on_delete=models.SET_NULL, null=True, blank=True, related_name='permission_approvals')
    approved_at = models.DateTimeField(null=True, blank=True)
    
    # Metadata
    ip_address = models.GenericIPAddressField(null=True, blank=True)
    session_id = models.CharField(max_length=100, blank=True, null=True)
    timestamp = models.DateTimeField(auto_now_add=True)
    
    class Meta:
        db_table = 'permission_change_logs'
        ordering = ['-timestamp']
        indexes = [
            models.Index(fields=['user', 'timestamp']),
            models.Index(fields=['changed_by', 'timestamp']),
            models.Index(fields=['permission_name', 'timestamp']),
        ]
    
    def __str__(self):
        return f"{self.user.get_full_name()} - {self.permission_name} - {self.timestamp}"


class SecurityAlert(models.Model):
    """
    Security alerts for suspicious activities
    """
    ALERT_TYPES = [
        ('multiple_failed_logins', 'Multiple Failed Logins'),
        ('permission_escalation', 'Permission Escalation Attempt'),
        ('unusual_access_pattern', 'Unusual Access Pattern'),
        ('bulk_data_access', 'Bulk Data Access'),
        ('after_hours_access', 'After Hours Access'),
        ('suspicious_ip', 'Suspicious IP Address'),
        ('role_change', 'Role Change'),
        ('critical_permission_grant', 'Critical Permission Granted'),
    ]
    
    alert_type = models.CharField(max_length=50, choices=ALERT_TYPES)
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='security_alerts', null=True, blank=True)
    severity = models.CharField(max_length=20, choices=SecurityEventSeverity.choices)
    
    title = models.CharField(max_length=200)
    description = models.TextField()
    
    # Context
    ip_address = models.GenericIPAddressField(null=True, blank=True)
    user_agent = models.CharField(max_length=500, blank=True, null=True)
    additional_context = models.JSONField(default=dict, blank=True)
    
    # Status
    is_resolved = models.BooleanField(default=False)
    resolved_by = models.ForeignKey(CustomUser, on_delete=models.SET_NULL, null=True, blank=True, related_name='resolved_alerts')
    resolved_at = models.DateTimeField(null=True, blank=True)
    resolution_notes = models.TextField(blank=True)
    
    # Timestamps
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        db_table = 'security_alerts'
        ordering = ['-created_at']
        indexes = [
            models.Index(fields=['alert_type', 'created_at']),
            models.Index(fields=['severity', 'created_at']),
            models.Index(fields=['user', 'created_at']),
            models.Index(fields=['is_resolved', 'created_at']),
        ]
    
    def __str__(self):
        return f"{self.alert_type} - {self.severity} - {self.created_at}"
    
    def resolve(self, resolved_by, notes=""):
        """Mark alert as resolved"""
        self.is_resolved = True
        self.resolved_by = resolved_by
        self.resolved_at = timezone.now()
        self.resolution_notes = notes
        self.save()


class DataAccessPattern(models.Model):
    """
    Track data access patterns for anomaly detection
    """
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='access_patterns')
    date = models.DateField()
    
    # Access metrics
    total_requests = models.IntegerField(default=0)
    unique_pages_accessed = models.IntegerField(default=0)
    data_exports = models.IntegerField(default=0)
    failed_access_attempts = models.IntegerField(default=0)
    
    # Time patterns
    first_access = models.TimeField(null=True, blank=True)
    last_access = models.TimeField(null=True, blank=True)
    after_hours_access = models.IntegerField(default=0)
    
    # Geographic patterns
    unique_ip_addresses = models.IntegerField(default=0)
    primary_ip = models.GenericIPAddressField(null=True, blank=True)
    
    # Behavioral flags
    is_anomalous = models.BooleanField(default=False)
    anomaly_score = models.FloatField(default=0.0)
    anomaly_reasons = models.JSONField(default=list, blank=True)
    
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        db_table = 'data_access_patterns'
        unique_together = ['user', 'date']
        ordering = ['-date']
        indexes = [
            models.Index(fields=['user', 'date']),
            models.Index(fields=['is_anomalous', 'date']),
            models.Index(fields=['anomaly_score', 'date']),
        ]
    
    def __str__(self):
        return f"{self.user.get_full_name()} - {self.date} - Score: {self.anomaly_score}"


class PortfolioAssignment(models.Model):
    """
    Track portfolio assignment history and changes
    """
    client = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='portfolio_history')
    portfolio_manager = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='managed_assignments')
    assigned_by = models.ForeignKey(CustomUser, on_delete=models.SET_NULL, null=True, related_name='assignments_made')
    assigned_date = models.DateTimeField(auto_now_add=True)
    unassigned_date = models.DateTimeField(null=True, blank=True)
    is_active = models.BooleanField(default=True)
    reason = models.TextField(blank=True, null=True, help_text="Reason for assignment/reassignment")
    
    class Meta:
        db_table = 'portfolio_assignments'
        ordering = ['-assigned_date']
        indexes = [
            models.Index(fields=['client', 'is_active']),
            models.Index(fields=['portfolio_manager', 'is_active']),
        ]
    
    def __str__(self):
        return f"{self.client.get_full_name()} -> {self.portfolio_manager.get_full_name()}"
    
    def deactivate(self, reason=""):
        """Deactivate this assignment"""
        self.is_active = False
        self.unassigned_date = get_current_datetime()
        if reason:
            self.reason = reason
        self.save()


class PortfolioPerformance(models.Model):
    """
    Monthly portfolio performance tracking
    """
    portfolio_manager = models.ForeignKey(CustomUser, on_delete=models.CASCADE, related_name='performance_records')
    month = models.DateField(help_text="First day of the month being tracked")
    
    # Client metrics
    total_clients = models.PositiveIntegerField(default=0)
    new_clients = models.PositiveIntegerField(default=0)
    active_clients = models.PositiveIntegerField(default=0)
    
    # Loan metrics
    loans_disbursed = models.PositiveIntegerField(default=0)
    total_disbursed_amount = models.DecimalField(max_digits=15, decimal_places=2, default=0)
    loans_repaid = models.PositiveIntegerField(default=0)
    total_repaid_amount = models.DecimalField(max_digits=15, decimal_places=2, default=0)
    loans_defaulted = models.PositiveIntegerField(default=0)
    total_defaulted_amount = models.DecimalField(max_digits=15, decimal_places=2, default=0)
    
    # Performance ratios
    collection_rate = models.DecimalField(max_digits=5, decimal_places=2, default=0, help_text="Percentage")
    default_rate = models.DecimalField(max_digits=5, decimal_places=2, default=0, help_text="Percentage")
    portfolio_at_risk = models.DecimalField(max_digits=15, decimal_places=2, default=0)
    
    # Rankings
    rank_by_disbursement = models.PositiveIntegerField(null=True, blank=True)
    rank_by_collection = models.PositiveIntegerField(null=True, blank=True)
    rank_by_clients = models.PositiveIntegerField(null=True, blank=True)
    
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    
    class Meta:
        db_table = 'portfolio_performance'
        unique_together = ['portfolio_manager', 'month']
        ordering = ['-month', 'rank_by_disbursement']
        indexes = [
            models.Index(fields=['month', 'portfolio_manager']),
            models.Index(fields=['month', 'rank_by_disbursement']),
        ]
    
    def __str__(self):
        return f"{self.portfolio_manager.get_full_name()} - {self.month.strftime('%B %Y')}"


class OTPVerification(models.Model):
    """
    OTP verification model for phone/email verification
    """
    user = models.ForeignKey(CustomUser, on_delete=models.CASCADE)
    otp_code = models.CharField(max_length=6)
    is_used = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)
    expires_at = models.DateTimeField()
    
    class Meta:
        db_table = 'otp_verifications'
    
    def __str__(self):
        return f"OTP for {self.user.phone_number} - {self.otp_code}"
    
    def is_expired(self):
        return get_current_datetime() > self.expires_at