""" Property-Based Tests for Demographic Categorization This module contains property-based tests using hypothesis to verify demographic categorization logic across a wide range of inputs. Validates: Requirements 2.2, 2.3, 2.4, 2.10, 2.11 """ from datetime import date, timedelta from decimal import Decimal from django.test import TestCase from django.utils import timezone from django.contrib.auth import get_user_model from hypothesis import given, strategies as st, settings from hypothesis.extra.django import TestCase as HypothesisTestCase from loans.models import Loan, LoanProduct, LoanApplication, Repayment from users.models import Branch from reports.demographic_service import DemographicAnalysisService User = get_user_model() # Custom strategies for generating test data @st.composite def birth_dates(draw): """Generate valid birth dates between 1920 and 2010""" year = draw(st.integers(min_value=1920, max_value=2010)) month = draw(st.integers(min_value=1, max_value=12)) # Handle different month lengths if month in [1, 3, 5, 7, 8, 10, 12]: max_day = 31 elif month in [4, 6, 9, 11]: max_day = 30 else: # February # Simple leap year check if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0): max_day = 29 else: max_day = 28 day = draw(st.integers(min_value=1, max_value=max_day)) return date(year, month, day) @st.composite def reference_dates(draw): """Generate reference dates between 2020 and 2030""" year = draw(st.integers(min_value=2020, max_value=2030)) month = draw(st.integers(min_value=1, max_value=12)) if month in [1, 3, 5, 7, 8, 10, 12]: max_day = 31 elif month in [4, 6, 9, 11]: max_day = 30 else: if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0): max_day = 29 else: max_day = 28 day = draw(st.integers(min_value=1, max_value=max_day)) return date(year, month, day) class DemographicCategorizationPropertyTests(HypothesisTestCase): """Property-based tests for demographic categorization""" def setUp(self): """Set up test data""" # Get or create a branch self.branch, _ = Branch.objects.get_or_create( code='TEST001', defaults={ 'name': 'Test Branch', 'is_active': True } ) # Get or create a loan product self.loan_product, _ = LoanProduct.objects.get_or_create( name='Test Product', defaults={ 'product_type': 'boost', 'description': 'Test product', 'min_amount': Decimal('1000.00'), 'max_amount': Decimal('100000.00'), 'interest_rate': Decimal('10.00'), 'processing_fee': Decimal('5.00'), 'min_duration': 7, 'max_duration': 90 } ) @settings(max_examples=100) @given( birth_date=birth_dates(), ref_date=reference_dates() ) def test_property_5_age_group_categorization(self, birth_date, ref_date): """ Feature: comprehensive-reports-and-fixes, Property 5: Age Group Categorization **Validates: Requirements 2.2, 2.11** For any loan application with a borrower birth date, the system should correctly categorize the application into exactly one age group (18-25, 26-35, 36-45, 46-55, 56-65, 66+, or Not Specified if birth date is missing). """ # Skip if birth date is after reference date (invalid scenario) if birth_date >= ref_date: return # Get age group age_group = DemographicAnalysisService.categorize_age_group(birth_date, ref_date) # Verify it's exactly one of the valid categories valid_categories = ['18-25', '26-35', '36-45', '46-55', '56-65', '66+', 'Not Specified'] self.assertIn( age_group, valid_categories, f"Age group '{age_group}' is not a valid category" ) # Calculate actual age age = DemographicAnalysisService.calculate_age(birth_date, ref_date) # Verify the categorization is correct based on age if age is None: self.assertEqual(age_group, 'Not Specified') elif age < 18: self.assertEqual(age_group, 'Not Specified') elif 18 <= age <= 25: self.assertEqual(age_group, '18-25') elif 26 <= age <= 35: self.assertEqual(age_group, '26-35') elif 36 <= age <= 45: self.assertEqual(age_group, '36-45') elif 46 <= age <= 55: self.assertEqual(age_group, '46-55') elif 56 <= age <= 65: self.assertEqual(age_group, '56-65') elif age >= 66: self.assertEqual(age_group, '66+') @settings(max_examples=100) @given(gender=st.one_of( st.just('M'), st.just('F'), st.just('O'), st.just(None), st.just(''), st.text(min_size=1, max_size=5) # Test invalid gender codes )) def test_property_6_gender_categorization(self, gender): """ Feature: comprehensive-reports-and-fixes, Property 6: Gender Categorization **Validates: Requirements 2.3, 2.11** For any loan application, the system should categorize it into exactly one gender category (Male, Female, Other, or Not Specified if gender is missing). """ # Get gender category gender_category = DemographicAnalysisService.categorize_gender(gender) # Verify it's exactly one of the valid categories valid_categories = ['Male', 'Female', 'Other', 'Not Specified'] self.assertIn( gender_category, valid_categories, f"Gender category '{gender_category}' is not a valid category" ) # Verify the categorization is correct if gender == 'M': self.assertEqual(gender_category, 'Male') elif gender == 'F': self.assertEqual(gender_category, 'Female') elif gender == 'O': self.assertEqual(gender_category, 'Other') elif gender is None or gender == '': self.assertEqual(gender_category, 'Not Specified') else: # Any other value should be 'Not Specified' self.assertEqual(gender_category, 'Not Specified') @settings(max_examples=100) @given( days_until_due=st.integers(min_value=1, max_value=90), payment_offset=st.integers(min_value=-30, max_value=30) ) def test_property_7_payment_pattern_classification(self, days_until_due, payment_offset): """ Feature: comprehensive-reports-and-fixes, Property 7: Payment Pattern Classification **Validates: Requirements 2.4, 2.5, 2.10** For any loan with a due_date and payment records, the system should correctly classify the payment as on-time if any payment was made on or before the due_date, otherwise as late. """ # Create a unique borrower for this test username = f'user_{timezone.now().timestamp()}_{days_until_due}_{payment_offset}' phone = f'+2547{str(abs(hash(username)))[:8]}' email = f'{username}@example.com' borrower = User.objects.create_user( username=username, phone_number=phone, email=email, date_of_birth=date(1990, 1, 1), gender='M', branch=self.branch ) # Create loan application app_number = f'APP-{timezone.now().timestamp()}-{days_until_due}-{payment_offset}' application = LoanApplication.objects.create( application_number=app_number, borrower=borrower, loan_product=self.loan_product, requested_amount=Decimal('10000.00'), requested_duration=30, purpose='Test loan', status='approved' ) # Create loan with due date in the past due_date = timezone.now() - timedelta(days=days_until_due) loan_number = f'LOAN-{timezone.now().timestamp()}-{days_until_due}-{payment_offset}' loan = Loan.objects.create( loan_number=loan_number, application=application, borrower=borrower, principal_amount=Decimal('10000.00'), interest_amount=Decimal('1000.00'), processing_fee=Decimal('500.00'), total_amount=Decimal('11500.00'), disbursement_date=due_date - timedelta(days=30), due_date=due_date, duration_days=30, status='active' ) # Create payment with offset from due date payment_date = due_date + timedelta(days=payment_offset) # Only create payment if payment_date is not in the future if payment_date <= timezone.now(): Repayment.objects.create( loan=loan, amount=Decimal('11500.00'), payment_date=payment_date, payment_method='mpesa' ) # Classify payment pattern pattern = DemographicAnalysisService.classify_payment_pattern(loan) # Verify classification if payment_date <= timezone.now(): # Payment was made if payment_offset <= 0: # Payment was on or before due date self.assertEqual( pattern, 'on-time', f"Payment made {abs(payment_offset)} days before due date should be on-time" ) else: # Payment was after due date self.assertEqual( pattern, 'late', f"Payment made {payment_offset} days after due date should be late" ) else: # No payment made yet if due_date.date() < date.today(): # Past due date with no payment self.assertEqual( pattern, 'late', "No payment past due date should be late" ) else: # Not yet due self.assertEqual( pattern, 'pending', "No payment before due date should be pending" ) @settings(max_examples=100) def test_property_5_none_birth_date_always_not_specified(self): """ Feature: comprehensive-reports-and-fixes, Property 5: Age Group Categorization **Validates: Requirements 2.11** For any loan application with missing birth date (None), the system should always categorize it as 'Not Specified'. """ age_group = DemographicAnalysisService.categorize_age_group(None) self.assertEqual(age_group, 'Not Specified') @settings(max_examples=100) @given( birth_date=birth_dates(), ref_date=reference_dates() ) def test_property_5_age_group_is_deterministic(self, birth_date, ref_date): """ Feature: comprehensive-reports-and-fixes, Property 5: Age Group Categorization **Validates: Requirements 2.2** For any given birth date and reference date, the age group categorization should always return the same result (deterministic). """ # Skip if birth date is after reference date if birth_date >= ref_date: return # Call categorization multiple times result1 = DemographicAnalysisService.categorize_age_group(birth_date, ref_date) result2 = DemographicAnalysisService.categorize_age_group(birth_date, ref_date) result3 = DemographicAnalysisService.categorize_age_group(birth_date, ref_date) # All results should be identical self.assertEqual(result1, result2) self.assertEqual(result2, result3) @settings(max_examples=100) @given(gender=st.one_of(st.just('M'), st.just('F'), st.just('O'), st.just(None), st.just(''))) def test_property_6_gender_is_deterministic(self, gender): """ Feature: comprehensive-reports-and-fixes, Property 6: Gender Categorization **Validates: Requirements 2.3** For any given gender value, the gender categorization should always return the same result (deterministic). """ # Call categorization multiple times result1 = DemographicAnalysisService.categorize_gender(gender) result2 = DemographicAnalysisService.categorize_gender(gender) result3 = DemographicAnalysisService.categorize_gender(gender) # All results should be identical self.assertEqual(result1, result2) self.assertEqual(result2, result3) @settings(max_examples=100) @given( num_loans=st.integers(min_value=1, max_value=20), seed=st.integers(min_value=0, max_value=1000000) ) def test_property_8_demographic_statistics_accuracy(self, num_loans, seed): """ Feature: comprehensive-reports-and-fixes, Property 8: Demographic Statistics Accuracy **Validates: Requirements 2.6** For any demographic group (age or gender), the calculated statistics (total applications, approved applications, approval rate, average loan amount, average repayment rate) should accurately reflect the underlying data for that group. """ import random random.seed(seed) # Create test loans with varied demographics loans = [] loan_ids = [] for i in range(num_loans): # Generate unique identifiers using seed and index unique_id = f'{seed}_{i}_{random.randint(100000, 999999)}' # Random demographics birth_year = random.randint(1950, 2000) gender = random.choice(['M', 'F', 'O', None]) # Create borrower username = f'user_{unique_id}'[:30] # Limit username length phone = f'+2547{str(abs(hash(username)))[:8]}' email = f'{username}@test.com'[:50] # Limit email length borrower = User.objects.create_user( username=username, phone_number=phone, email=email, date_of_birth=date(birth_year, 6, 15) if birth_year else None, gender=gender, branch=self.branch ) # Create loan application with shorter number app_number = f'APP{unique_id}'[:20] # Limit application number length application = LoanApplication.objects.create( application_number=app_number, borrower=borrower, loan_product=self.loan_product, requested_amount=Decimal('10000.00'), requested_duration=30, purpose='Test loan', status='approved' ) # Random loan status (some approved, some not) status = random.choice(['active', 'paid', 'pending', 'under_review']) principal = Decimal(str(random.randint(5000, 50000))) # Create loan with shorter number loan_number = f'LN{unique_id}'[:20] # Limit loan number length due_date = timezone.now() - timedelta(days=random.randint(1, 60)) loan = Loan.objects.create( loan_number=loan_number, application=application, borrower=borrower, principal_amount=principal, interest_amount=Decimal('1000.00'), processing_fee=Decimal('500.00'), total_amount=principal + Decimal('1500.00'), disbursement_date=due_date - timedelta(days=30), due_date=due_date, duration_days=30, status=status ) # Add payment for some loans if status in ['active', 'paid'] and random.random() > 0.3: payment_offset = random.randint(-10, 20) payment_date = due_date + timedelta(days=payment_offset) if payment_date <= timezone.now(): Repayment.objects.create( loan=loan, amount=loan.total_amount, payment_date=payment_date, payment_method='mpesa' ) loans.append(loan) loan_ids.append(loan.id) # Get all loans as queryset - use the exact IDs we just created from loans.models import Loan as LoanModel loans_qs = LoanModel.objects.filter(id__in=loan_ids) # Verify we got the exact number of loans we created self.assertEqual( loans_qs.count(), num_loans, f"Expected {num_loans} loans in queryset, got {loans_qs.count()}" ) # Analyze by age group age_results = DemographicAnalysisService.analyze_by_age_group(loans_qs) # Manually calculate expected statistics for each age group for age_group, stats in age_results.items(): # Filter loans for this age group - use queryset to ensure we're comparing same data group_loans_qs = loans_qs.filter( borrower__date_of_birth__isnull=False ) group_loans = [ loan for loan in group_loans_qs if DemographicAnalysisService.categorize_age_group( loan.borrower.date_of_birth ) == age_group ] # Also handle None birth dates if age_group == 'Not Specified': none_loans = loans_qs.filter(borrower__date_of_birth__isnull=True) group_loans.extend(list(none_loans)) # Verify total applications expected_total = len(group_loans) self.assertEqual( stats['total_applications'], expected_total, f"Total applications mismatch for {age_group}: expected {expected_total}, got {stats['total_applications']}" ) # Verify approved applications approved_loans = [ loan for loan in group_loans if loan.status in ['active', 'paid', 'defaulted', 'rolled_over', 'written_off'] ] expected_approved = len(approved_loans) self.assertEqual( stats['approved_applications'], expected_approved, f"Approved applications mismatch for {age_group}: expected {expected_approved}, got {stats['approved_applications']}" ) # Verify approval rate if expected_total > 0: expected_approval_rate = round(Decimal(expected_approved) / Decimal(expected_total) * Decimal('100'), 2) self.assertEqual( stats['approval_rate'], expected_approval_rate, f"Approval rate mismatch for {age_group}: expected {expected_approval_rate}, got {stats['approval_rate']}" ) else: self.assertEqual(stats['approval_rate'], Decimal('0.00')) # Verify average loan amount if expected_approved > 0: total_amount = sum(loan.principal_amount for loan in approved_loans) expected_avg = round(total_amount / Decimal(expected_approved), 2) self.assertEqual( stats['average_loan_amount'], expected_avg, f"Average loan amount mismatch for {age_group}: expected {expected_avg}, got {stats['average_loan_amount']}" ) else: self.assertEqual(stats['average_loan_amount'], Decimal('0.00')) # Verify payment pattern counts on_time_count = 0 late_count = 0 for loan in approved_loans: pattern = DemographicAnalysisService.classify_payment_pattern(loan) if pattern == 'on-time': on_time_count += 1 elif pattern == 'late': late_count += 1 self.assertEqual( stats['on_time_payments'], on_time_count, f"On-time payments mismatch for {age_group}: expected {on_time_count}, got {stats['on_time_payments']}" ) self.assertEqual( stats['late_payments'], late_count, f"Late payments mismatch for {age_group}: expected {late_count}, got {stats['late_payments']}" ) # Verify on-time rate total_payments = on_time_count + late_count if total_payments > 0: expected_on_time_rate = round(Decimal(on_time_count) / Decimal(total_payments) * Decimal('100'), 2) self.assertEqual( stats['on_time_rate'], expected_on_time_rate, f"On-time rate mismatch for {age_group}: expected {expected_on_time_rate}, got {stats['on_time_rate']}" ) else: self.assertEqual(stats['on_time_rate'], Decimal('0.00')) # Analyze by gender gender_results = DemographicAnalysisService.analyze_by_gender(loans_qs) # Manually calculate expected statistics for each gender for gender_category, stats in gender_results.items(): # Filter loans for this gender - use queryset to ensure we're comparing same data if gender_category == 'Male': group_loans = list(loans_qs.filter(borrower__gender='M')) elif gender_category == 'Female': group_loans = list(loans_qs.filter(borrower__gender='F')) elif gender_category == 'Other': group_loans = list(loans_qs.filter(borrower__gender='O')) else: # Not Specified group_loans = list(loans_qs.filter(borrower__gender__isnull=True) | loans_qs.filter(borrower__gender='')) # Verify total applications expected_total = len(group_loans) self.assertEqual( stats['total_applications'], expected_total, f"Total applications mismatch for {gender_category}: expected {expected_total}, got {stats['total_applications']}" ) # Verify approved applications approved_loans = [ loan for loan in group_loans if loan.status in ['active', 'paid', 'defaulted', 'rolled_over', 'written_off'] ] expected_approved = len(approved_loans) self.assertEqual( stats['approved_applications'], expected_approved, f"Approved applications mismatch for {gender_category}: expected {expected_approved}, got {stats['approved_applications']}" ) # Verify approval rate if expected_total > 0: expected_approval_rate = round(Decimal(expected_approved) / Decimal(expected_total) * Decimal('100'), 2) self.assertEqual( stats['approval_rate'], expected_approval_rate, f"Approval rate mismatch for {gender_category}: expected {expected_approval_rate}, got {stats['approval_rate']}" ) else: self.assertEqual(stats['approval_rate'], Decimal('0.00')) # Verify average loan amount if expected_approved > 0: total_amount = sum(loan.principal_amount for loan in approved_loans) expected_avg = round(total_amount / Decimal(expected_approved), 2) self.assertEqual( stats['average_loan_amount'], expected_avg, f"Average loan amount mismatch for {gender_category}: expected {expected_avg}, got {stats['average_loan_amount']}" ) else: self.assertEqual(stats['average_loan_amount'], Decimal('0.00')) # Verify payment pattern counts on_time_count = 0 late_count = 0 for loan in approved_loans: pattern = DemographicAnalysisService.classify_payment_pattern(loan) if pattern == 'on-time': on_time_count += 1 elif pattern == 'late': late_count += 1 self.assertEqual( stats['on_time_payments'], on_time_count, f"On-time payments mismatch for {gender_category}: expected {on_time_count}, got {stats['on_time_payments']}" ) self.assertEqual( stats['late_payments'], late_count, f"Late payments mismatch for {gender_category}: expected {late_count}, got {stats['late_payments']}" ) # Verify on-time rate total_payments = on_time_count + late_count if total_payments > 0: expected_on_time_rate = round(Decimal(on_time_count) / Decimal(total_payments) * Decimal('100'), 2) self.assertEqual( stats['on_time_rate'], expected_on_time_rate, f"On-time rate mismatch for {gender_category}: expected {expected_on_time_rate}, got {stats['on_time_rate']}" ) else: self.assertEqual(stats['on_time_rate'], Decimal('0.00'))