""" Performance Tests for Reports System Tests report generation performance with large datasets (10,000+ loans), export generation, and query optimization. """ from django.test import TestCase, Client from django.contrib.auth import get_user_model from django.utils import timezone from django.db import connection from django.test.utils import override_settings from datetime import timedelta, date from decimal import Decimal import time import os from loans.models import Loan, LoanApplication, Repayment, LoanProduct from users.models import CustomUser, Branch from reports.filter_service import ReportFilterService from reports.calculation_service import LoanCalculationService from reports.client_report_service import ClientReportService from reports.export_service import ReportExportService User = get_user_model() class PerformanceTestCase(TestCase): """Performance tests for reports system""" @classmethod def setUpClass(cls): """Set up test data once for all tests""" super().setUpClass() # Create test branch cls.branch = Branch.objects.create( name='Test Branch', code='TB001', is_active=True ) # Create test loan product cls.loan_product = LoanProduct.objects.create( name='Test Product', interest_rate=10.0, processing_fee_rate=2.0, is_active=True ) # Create admin user for testing cls.admin_user = User.objects.create_user( username='admin_test', email='admin@test.com', password='testpass123', role='admin', is_staff=True, is_superuser=True ) def setUp(self): """Set up for each test""" self.client = Client() self.client.force_login(self.admin_user) def _create_test_loans(self, count=100): """ Helper method to create test loans with borrowers and repayments. Args: count: Number of loans to create Returns: List of created loan objects """ loans = [] for i in range(count): # Create borrower borrower = User.objects.create_user( username=f'borrower_{i}', email=f'borrower_{i}@test.com', password='testpass123', role='borrower', first_name=f'Test', last_name=f'Borrower{i}', phone_number=f'07000000{i:03d}', branch=self.branch ) # Create loan application application = LoanApplication.objects.create( borrower=borrower, loan_product=self.loan_product, requested_amount=Decimal('10000.00'), status='approved' ) # Create loan loan = Loan.objects.create( borrower=borrower, application=application, loan_number=f'LN{i:06d}', principal_amount=Decimal('10000.00'), interest_amount=Decimal('1000.00'), processing_fee=Decimal('200.00'), total_amount=Decimal('11200.00'), disbursement_date=timezone.now().date() - timedelta(days=30), due_date=timezone.now().date() + timedelta(days=30), status='active', is_deleted=False, is_rolled_over=False ) # Create some repayments (50% of loans have repayments) if i % 2 == 0: Repayment.objects.create( loan=loan, amount=Decimal('5000.00'), payment_date=timezone.now().date() - timedelta(days=15), payment_method='mpesa' ) loans.append(loan) return loans def _count_queries(self, func): """ Count the number of database queries executed by a function. Args: func: Function to execute Returns: Tuple of (result, query_count) """ # Reset queries connection.queries_log.clear() # Enable query logging from django.conf import settings old_debug = settings.DEBUG settings.DEBUG = True try: # Execute function result = func() # Count queries query_count = len(connection.queries) return result, query_count finally: settings.DEBUG = old_debug def test_report_generation_with_1000_loans(self): """ Test report generation performance with 1,000 loans. Validates that report generation completes within acceptable time and uses optimized queries. """ # Create 1,000 test loans print("\nCreating 1,000 test loans...") start_time = time.time() loans = self._create_test_loans(count=1000) creation_time = time.time() - start_time print(f"Loan creation took {creation_time:.2f} seconds") # Test loans due report print("\nTesting loans due report...") start_time = time.time() response = self.client.get('/reports/loans-due/') generation_time = time.time() - start_time print(f"Report generation took {generation_time:.2f} seconds") # Assertions self.assertEqual(response.status_code, 200) self.assertLess(generation_time, 10.0, "Report generation should complete within 10 seconds") print(f"✓ Report generated successfully in {generation_time:.2f}s") def test_report_generation_with_10000_loans(self): """ Test report generation performance with 10,000+ loans. This is a stress test to ensure the system can handle large datasets. """ # Skip this test in CI/CD or if explicitly disabled if os.environ.get('SKIP_LARGE_TESTS', 'false').lower() == 'true': self.skipTest("Skipping large dataset test") # Create 10,000 test loans (in batches for memory efficiency) print("\nCreating 10,000 test loans...") start_time = time.time() batch_size = 1000 total_loans = 10000 for batch in range(total_loans // batch_size): print(f"Creating batch {batch + 1}/{total_loans // batch_size}...") self._create_test_loans(count=batch_size) creation_time = time.time() - start_time print(f"Loan creation took {creation_time:.2f} seconds") # Test loans due report print("\nTesting loans due report with 10,000 loans...") start_time = time.time() response = self.client.get('/reports/loans-due/') generation_time = time.time() - start_time print(f"Report generation took {generation_time:.2f} seconds") # Assertions self.assertEqual(response.status_code, 200) self.assertLess(generation_time, 30.0, "Report generation should complete within 30 seconds for 10k loans") print(f"✓ Report generated successfully in {generation_time:.2f}s") def test_export_generation_with_large_dataset(self): """ Test PDF and Excel export generation with large datasets. Validates that exports complete within acceptable time and handle large datasets appropriately. """ # Create 1,000 test loans print("\nCreating 1,000 test loans for export test...") loans = self._create_test_loans(count=1000) # Prepare report data report_data = { 'loans': [ { 'loan_number': loan.loan_number, 'borrower_name': loan.borrower.get_full_name(), 'principal_amount': loan.principal_amount, 'due_date': loan.due_date, 'outstanding_amount': loan.outstanding_amount } for loan in loans[:100] # Limit to 100 for export test ] } filters = { 'start_date': timezone.now().date() - timedelta(days=30), 'end_date': timezone.now().date(), 'period': 'monthly' } export_service = ReportExportService() # Test PDF export print("\nTesting PDF export...") start_time = time.time() pdf_response = export_service.export_to_pdf(report_data, 'loans_due', filters) pdf_time = time.time() - start_time print(f"PDF export took {pdf_time:.2f} seconds") self.assertEqual(pdf_response.status_code, 200) self.assertEqual(pdf_response['Content-Type'], 'application/pdf') self.assertLess(pdf_time, 5.0, "PDF export should complete within 5 seconds") # Test Excel export print("\nTesting Excel export...") start_time = time.time() excel_response = export_service.export_to_excel(report_data, 'loans_due', filters) excel_time = time.time() - start_time print(f"Excel export took {excel_time:.2f} seconds") self.assertEqual(excel_response.status_code, 200) self.assertIn('spreadsheet', excel_response['Content-Type']) self.assertLess(excel_time, 5.0, "Excel export should complete within 5 seconds") print(f"✓ Exports generated successfully (PDF: {pdf_time:.2f}s, Excel: {excel_time:.2f}s)") @override_settings(DEBUG=True) def test_query_count_optimization(self): """ Monitor query counts per request to ensure proper use of select_related and prefetch_related. Validates that N+1 query problems are avoided. """ # Create 100 test loans print("\nCreating 100 test loans for query optimization test...") loans = self._create_test_loans(count=100) # Enable query logging from django.conf import settings settings.DEBUG = True # Test loans due report query count print("\nTesting query count for loans due report...") # Reset queries from django.db import reset_queries reset_queries() response = self.client.get('/reports/loans-due/') query_count = len(connection.queries) print(f"Query count: {query_count}") # Assertions self.assertEqual(response.status_code, 200) # With proper optimization (select_related, prefetch_related), # query count should be reasonable (< 20 queries for 100 loans) self.assertLess( query_count, 50, f"Query count should be optimized. Found {query_count} queries. " "Consider using select_related() and prefetch_related()." ) print(f"✓ Query count is acceptable: {query_count} queries") # Print queries for debugging (first 5) if query_count > 20: print("\nFirst 5 queries:") for i, query in enumerate(connection.queries[:5], 1): print(f"{i}. {query['sql'][:100]}...") def test_filter_service_performance(self): """ Test filter service performance with various filter combinations. Validates that filtering operations are efficient. """ # Create 500 test loans print("\nCreating 500 test loans for filter test...") loans = self._create_test_loans(count=500) # Test date range filter print("\nTesting date range filter...") start_time = time.time() queryset = Loan.objects.all() filtered_qs = ReportFilterService.apply_date_range_filter( queryset, start_date=timezone.now().date() - timedelta(days=60), end_date=timezone.now().date(), date_field='disbursement_date' ) result_count = filtered_qs.count() filter_time = time.time() - start_time print(f"Date range filter took {filter_time:.4f} seconds ({result_count} results)") self.assertLess(filter_time, 1.0, "Date range filter should complete within 1 second") # Test loan status filter print("\nTesting loan status filter...") start_time = time.time() queryset = Loan.objects.all() filtered_qs = ReportFilterService.apply_loan_status_filter( queryset, exclude_rolled_over=True, exclude_deleted=True ) result_count = filtered_qs.count() filter_time = time.time() - start_time print(f"Loan status filter took {filter_time:.4f} seconds ({result_count} results)") self.assertLess(filter_time, 1.0, "Loan status filter should complete within 1 second") # Test combined filters print("\nTesting combined filters...") start_time = time.time() queryset = Loan.objects.all() filtered_qs = ReportFilterService.apply_date_range_filter( queryset, start_date=timezone.now().date() - timedelta(days=60), end_date=timezone.now().date(), date_field='disbursement_date' ) filtered_qs = ReportFilterService.apply_loan_status_filter( filtered_qs, exclude_rolled_over=True, exclude_deleted=True ) filtered_qs = ReportFilterService.apply_loan_product_filter( filtered_qs, product_id=str(self.loan_product.id) ) result_count = filtered_qs.count() filter_time = time.time() - start_time print(f"Combined filters took {filter_time:.4f} seconds ({result_count} results)") self.assertLess(filter_time, 1.0, "Combined filters should complete within 1 second") print(f"✓ All filter operations completed efficiently") def test_calculation_service_performance(self): """ Test calculation service performance with large datasets. Validates that financial calculations are efficient. """ # Create 1,000 test loans print("\nCreating 1,000 test loans for calculation test...") loans = self._create_test_loans(count=1000) # Test amount_paid calculation print("\nTesting amount_paid calculation...") start_time = time.time() for loan in loans[:100]: # Test on 100 loans amount_paid = LoanCalculationService.calculate_amount_paid(loan) calc_time = time.time() - start_time avg_time = calc_time / 100 print(f"Amount paid calculation took {calc_time:.4f} seconds (avg: {avg_time:.6f}s per loan)") self.assertLess(avg_time, 0.01, "Amount paid calculation should be < 10ms per loan") # Test outstanding_amount calculation print("\nTesting outstanding_amount calculation...") start_time = time.time() for loan in loans[:100]: # Test on 100 loans outstanding = LoanCalculationService.calculate_outstanding_amount(loan) calc_time = time.time() - start_time avg_time = calc_time / 100 print(f"Outstanding amount calculation took {calc_time:.4f} seconds (avg: {avg_time:.6f}s per loan)") self.assertLess(avg_time, 0.01, "Outstanding amount calculation should be < 10ms per loan") print(f"✓ All calculation operations completed efficiently") def test_client_report_service_performance(self): """ Test client report service performance with large client base. Validates that client metrics and scoring are efficient. """ # Create 500 test loans (which creates 500 borrowers) print("\nCreating 500 test loans for client report test...") loans = self._create_test_loans(count=500) # Test get_client_metrics print("\nTesting get_client_metrics...") start_time = time.time() metrics = ClientReportService.get_client_metrics(branch_id=str(self.branch.id)) metrics_time = time.time() - start_time print(f"Client metrics calculation took {metrics_time:.2f} seconds") print(f"Metrics: {metrics}") self.assertLess(metrics_time, 10.0, "Client metrics should complete within 10 seconds") # Test get_top_performers print("\nTesting get_top_performers...") start_time = time.time() top_performers = ClientReportService.get_top_performers(limit=10, branch_id=str(self.branch.id)) performers_time = time.time() - start_time print(f"Top performers calculation took {performers_time:.2f} seconds") print(f"Found {len(top_performers)} top performers") self.assertLess(performers_time, 10.0, "Top performers should complete within 10 seconds") print(f"✓ Client report operations completed efficiently") # Run tests with Django test runner if __name__ == '__main__': import django from django.core.management import call_command django.setup() call_command('test', 'reports.test_performance', verbosity=2)