""" Django views for batch processing and scheduling functionality """ from django.http import JsonResponse, HttpResponse, Http404 from django.views.decorators.http import require_http_methods from django.contrib.auth.decorators import login_required from django.views.decorators.csrf import csrf_exempt from django.utils.decorators import method_decorator from django.views import View from django.shortcuts import get_object_or_404, render from django.core.paginator import Paginator from django.utils import timezone import json import logging from datetime import datetime, time from typing import Dict, List, Any, Optional from .batch_processing_service import ( BatchProcessingService, BatchJob, ScheduledReport, process_batch_job, send_batch_job_email ) logger = logging.getLogger(__name__) class BatchProcessingView(View): """Base view for batch processing functionality""" def __init__(self): super().__init__() self.batch_service = BatchProcessingService() @method_decorator(login_required) def dispatch(self, request, *args, **kwargs): return super().dispatch(request, *args, **kwargs) class BatchJobView(BatchProcessingView): """API endpoint for managing batch jobs""" def get(self, request, job_id: str = None): """Get batch job(s)""" try: if job_id: # Get specific job status status_data = self.batch_service.get_job_status(job_id) return JsonResponse(status_data) else: # Get user's jobs status = request.GET.get('status') limit = int(request.GET.get('limit', 50)) jobs = self.batch_service.get_user_jobs( request.user, status=status, limit=limit ) jobs_data = [ { 'id': str(job.id), 'job_type': job.job_type, 'status': job.status, 'progress_percentage': job.progress_percentage, 'processed_items': job.processed_items, 'total_items': job.total_items, 'created_at': job.created_at.isoformat(), 'started_at': job.started_at.isoformat() if job.started_at else None, 'completed_at': job.completed_at.isoformat() if job.completed_at else None, 'duration': str(job.duration) if job.duration else None, 'report_types': job.report_types, 'export_formats': job.export_formats, 'output_files_count': len(job.output_files), 'error_count': len(job.error_messages), 'email_sent': job.email_sent } for job in jobs ] return JsonResponse(jobs_data, safe=False) except Exception as e: logger.error(f"Error getting batch jobs: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def post(self, request): """Create new batch job""" try: data = json.loads(request.body) job = self.batch_service.create_batch_job( user=request.user, job_type=data.get('job_type', 'batch_export'), report_types=data['report_types'], filters=data.get('filters', {}), export_formats=data.get('export_formats', ['excel']), email_recipients=data.get('email_recipients', []) ) response_data = { 'id': str(job.id), 'status': job.status, 'created_at': job.created_at.isoformat(), 'total_items': job.total_items } return JsonResponse(response_data, status=201) except Exception as e: logger.error(f"Error creating batch job: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def delete(self, request, job_id: str): """Cancel batch job""" try: success = self.batch_service.cancel_job(job_id, request.user) if success: return JsonResponse({'success': True}) else: return JsonResponse({'error': 'Job not found or cannot be cancelled'}, status=400) except Exception as e: logger.error(f"Error cancelling batch job: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class ScheduledReportView(BatchProcessingView): """API endpoint for managing scheduled reports""" def get(self, request, report_id: str = None): """Get scheduled report(s)""" try: if report_id: # Get specific scheduled report report = get_object_or_404(ScheduledReport, id=report_id, user=request.user) data = { 'id': str(report.id), 'name': report.name, 'description': report.description, 'frequency': report.frequency, 'time_of_day': report.time_of_day.strftime('%H:%M'), 'day_of_week': report.day_of_week, 'day_of_month': report.day_of_month, 'report_types': report.report_types, 'filters': report.filters, 'export_formats': report.export_formats, 'email_recipients': report.email_recipients, 'email_subject_template': report.email_subject_template, 'email_body_template': report.email_body_template, 'is_active': report.is_active, 'last_run': report.last_run.isoformat() if report.last_run else None, 'next_run': report.next_run.isoformat() if report.next_run else None, 'run_count': report.run_count, 'created_at': report.created_at.isoformat() } return JsonResponse(data) else: # Get user's scheduled reports active_only = request.GET.get('active_only', 'true').lower() == 'true' reports = self.batch_service.get_user_scheduled_reports( request.user, active_only=active_only ) reports_data = [ { 'id': str(report.id), 'name': report.name, 'description': report.description, 'frequency': report.frequency, 'time_of_day': report.time_of_day.strftime('%H:%M'), 'is_active': report.is_active, 'last_run': report.last_run.isoformat() if report.last_run else None, 'next_run': report.next_run.isoformat() if report.next_run else None, 'run_count': report.run_count, 'report_types': report.report_types, 'export_formats': report.export_formats } for report in reports ] return JsonResponse(reports_data, safe=False) except Exception as e: logger.error(f"Error getting scheduled reports: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def post(self, request): """Create new scheduled report""" try: data = json.loads(request.body) # Parse time time_str = data['time_of_day'] time_obj = datetime.strptime(time_str, '%H:%M').time() # Prepare schedule parameters schedule_params = {} if data['frequency'] == 'weekly': schedule_params['day_of_week'] = data.get('day_of_week') elif data['frequency'] == 'monthly': schedule_params['day_of_month'] = data.get('day_of_month', 1) report = self.batch_service.create_scheduled_report( user=request.user, name=data['name'], description=data.get('description', ''), frequency=data['frequency'], time_of_day=time_obj, report_types=data['report_types'], filters=data.get('filters', {}), export_formats=data.get('export_formats', ['excel']), email_recipients=data.get('email_recipients', []), **schedule_params ) response_data = { 'id': str(report.id), 'name': report.name, 'frequency': report.frequency, 'next_run': report.next_run.isoformat() if report.next_run else None, 'created_at': report.created_at.isoformat() } return JsonResponse(response_data, status=201) except Exception as e: logger.error(f"Error creating scheduled report: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def put(self, request, report_id: str): """Update scheduled report""" try: report = get_object_or_404(ScheduledReport, id=report_id, user=request.user) data = json.loads(request.body) # Update fields report.name = data.get('name', report.name) report.description = data.get('description', report.description) report.frequency = data.get('frequency', report.frequency) if 'time_of_day' in data: time_str = data['time_of_day'] report.time_of_day = datetime.strptime(time_str, '%H:%M').time() if 'day_of_week' in data: report.day_of_week = data['day_of_week'] if 'day_of_month' in data: report.day_of_month = data['day_of_month'] report.report_types = data.get('report_types', report.report_types) report.filters = data.get('filters', report.filters) report.export_formats = data.get('export_formats', report.export_formats) report.email_recipients = data.get('email_recipients', report.email_recipients) report.is_active = data.get('is_active', report.is_active) report.save() # Recalculate next run report.calculate_next_run() return JsonResponse({'success': True}) except Exception as e: logger.error(f"Error updating scheduled report: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def delete(self, request, report_id: str): """Delete scheduled report""" try: report = get_object_or_404(ScheduledReport, id=report_id, user=request.user) report.delete() return JsonResponse({'success': True}) except Exception as e: logger.error(f"Error deleting scheduled report: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchJobProgressView(BatchProcessingView): """API endpoint for real-time job progress""" def get(self, request, job_id: str): """Get real-time job progress""" try: job = get_object_or_404(BatchJob, id=job_id, user=request.user) # Get Celery task status if available celery_status = None if job.celery_task_id: from celery.result import AsyncResult task_result = AsyncResult(job.celery_task_id) celery_status = { 'state': task_result.state, 'info': task_result.info } progress_data = { 'id': str(job.id), 'status': job.status, 'progress_percentage': job.progress_percentage, 'processed_items': job.processed_items, 'total_items': job.total_items, 'estimated_time_remaining': str(job.estimated_time_remaining) if job.estimated_time_remaining else None, 'celery_status': celery_status, 'last_updated': timezone.now().isoformat() } return JsonResponse(progress_data) except Exception as e: logger.error(f"Error getting job progress: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchJobDownloadView(BatchProcessingView): """Download batch job results""" def get(self, request, job_id: str): """Download batch job results""" try: job = get_object_or_404(BatchJob, id=job_id, user=request.user) if job.status != 'completed' or not job.output_files: return JsonResponse({'error': 'Job not completed or no files available'}, status=400) # If single file, return it directly if len(job.output_files) == 1: file_path = job.output_files[0] if os.path.exists(file_path): with open(file_path, 'rb') as f: response = HttpResponse(f.read()) response['Content-Type'] = 'application/octet-stream' response['Content-Disposition'] = f'attachment; filename="{os.path.basename(file_path)}"' return response # Multiple files - create zip import zipfile import tempfile with tempfile.NamedTemporaryFile(delete=False, suffix='.zip') as temp_zip: with zipfile.ZipFile(temp_zip, 'w') as zip_file: for file_path in job.output_files: if os.path.exists(file_path): zip_file.write(file_path, os.path.basename(file_path)) temp_zip.seek(0) response = HttpResponse(temp_zip.read(), content_type='application/zip') response['Content-Disposition'] = f'attachment; filename="batch_reports_{job_id}.zip"' # Clean up temp file os.unlink(temp_zip.name) return response except Exception as e: logger.error(f"Error downloading batch job results: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchProcessingDashboardView(BatchProcessingView): """Dashboard view for batch processing""" def get(self, request): """Render batch processing dashboard""" try: # Get recent jobs recent_jobs = self.batch_service.get_user_jobs(request.user, limit=10) # Get scheduled reports scheduled_reports = self.batch_service.get_user_scheduled_reports(request.user) # Get job statistics job_stats = { 'total_jobs': BatchJob.objects.filter(user=request.user).count(), 'completed_jobs': BatchJob.objects.filter(user=request.user, status='completed').count(), 'failed_jobs': BatchJob.objects.filter(user=request.user, status='failed').count(), 'active_schedules': len([r for r in scheduled_reports if r.is_active]) } context = { 'recent_jobs': recent_jobs, 'scheduled_reports': scheduled_reports, 'job_stats': job_stats, 'available_report_types': [ 'loans_due', 'loans_in_arrears', 'delinquent_loans', 'processing_fees', 'interest_income' ], 'export_formats': ['pdf', 'excel'], 'frequency_choices': ScheduledReport.FREQUENCY_CHOICES } return render(request, 'reports/batch_processing_dashboard.html', context) except Exception as e: logger.error(f"Error rendering batch processing dashboard: {str(e)}") return render(request, 'reports/batch_processing_dashboard.html', {'error': str(e)}) # URL patterns would be added to urls.py: """ from django.urls import path from .batch_processing_views import ( BatchJobView, ScheduledReportView, BatchJobProgressView, BatchJobDownloadView, BatchProcessingDashboardView ) urlpatterns = [ path('batch-processing/', BatchProcessingDashboardView.as_view(), name='batch_processing_dashboard'), path('api/batch-jobs/', BatchJobView.as_view(), name='batch_jobs'), path('api/batch-jobs//', BatchJobView.as_view(), name='batch_job_detail'), path('api/batch-jobs//progress/', BatchJobProgressView.as_view(), name='batch_job_progress'), path('api/batch-jobs//download/', BatchJobDownloadView.as_view(), name='batch_job_download'), path('api/scheduled-reports/', ScheduledReportView.as_view(), name='scheduled_reports'), path('api/scheduled-reports//', ScheduledReportView.as_view(), name='scheduled_report_detail'), path('api/batch-templates/', BatchJobTemplateView.as_view(), name='batch_job_templates'), path('api/batch-templates//', BatchJobTemplateView.as_view(), name='batch_job_template_detail'), path('api/batch-queue/', BatchQueueView.as_view(), name='batch_queue'), path('api/batch-metrics/', BatchMetricsView.as_view(), name='batch_metrics'), path('api/batch-jobs//priority/', BatchJobPriorityView.as_view(), name='batch_job_priority'), ] """ class BatchJobTemplateView(BatchProcessingView): """API endpoint for managing batch job templates""" def get(self, request, template_id: int = None): """Get batch job template(s)""" try: if template_id: # Get specific template from .batch_processing_service import BatchJobTemplate template = get_object_or_404(BatchJobTemplate, id=template_id, user=request.user) data = { 'id': template.id, 'name': template.name, 'description': template.description, 'job_type': template.job_type, 'report_types': template.report_types, 'filters': template.filters, 'export_formats': template.export_formats, 'email_recipients': template.email_recipients, 'usage_count': template.usage_count, 'last_used': template.last_used.isoformat() if template.last_used else None, 'created_at': template.created_at.isoformat() } return JsonResponse(data) else: # Get user's templates templates = self.batch_service.get_user_templates(request.user) templates_data = [ { 'id': template.id, 'name': template.name, 'description': template.description, 'job_type': template.job_type, 'report_types': template.report_types, 'usage_count': template.usage_count, 'last_used': template.last_used.isoformat() if template.last_used else None, 'created_at': template.created_at.isoformat() } for template in templates ] return JsonResponse(templates_data, safe=False) except Exception as e: logger.error(f"Error getting batch job templates: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def post(self, request): """Create new batch job template""" try: data = json.loads(request.body) template = self.batch_service.create_job_template( user=request.user, name=data['name'], description=data.get('description', ''), job_type=data.get('job_type', 'batch_export'), report_types=data['report_types'], filters=data.get('filters', {}), export_formats=data.get('export_formats', ['excel']), email_recipients=data.get('email_recipients', []) ) response_data = { 'id': template.id, 'name': template.name, 'created_at': template.created_at.isoformat() } return JsonResponse(response_data, status=201) except Exception as e: logger.error(f"Error creating batch job template: {str(e)}") return JsonResponse({'error': str(e)}, status=500) def delete(self, request, template_id: int): """Delete batch job template""" try: from .batch_processing_service import BatchJobTemplate template = get_object_or_404(BatchJobTemplate, id=template_id, user=request.user) template.delete() return JsonResponse({'success': True}) except Exception as e: logger.error(f"Error deleting batch job template: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchQueueView(BatchProcessingView): """API endpoint for batch queue management""" def get(self, request): """Get batch queue status""" try: queue_status = self.batch_service.get_queue_status() return JsonResponse(queue_status) except Exception as e: logger.error(f"Error getting batch queue status: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchMetricsView(BatchProcessingView): """API endpoint for batch processing metrics""" def get(self, request): """Get batch processing metrics""" try: # Get date range from query parameters start_date = request.GET.get('start_date') end_date = request.GET.get('end_date') date_range = {} if start_date: date_range['start'] = start_date if end_date: date_range['end'] = end_date metrics = self.batch_service.get_processing_metrics(date_range) return JsonResponse(metrics) except Exception as e: logger.error(f"Error getting batch processing metrics: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BatchJobPriorityView(BatchProcessingView): """API endpoint for managing job priorities""" def post(self, request, job_id: str): """Set job priority""" try: data = json.loads(request.body) priority = data.get('priority', 'normal') success = self.batch_service.set_job_priority(job_id, priority, request.user) if success: return JsonResponse({'success': True}) else: return JsonResponse({'error': 'Job not found or cannot set priority'}, status=400) except Exception as e: logger.error(f"Error setting job priority: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class BulkJobView(BatchProcessingView): """API endpoint for creating bulk jobs with filter variations""" def post(self, request): """Create bulk job with multiple filter variations""" try: data = json.loads(request.body) bulk_job = self.batch_service.create_bulk_job_from_filters( user=request.user, base_filters=data.get('base_filters', {}), filter_variations=data['filter_variations'], report_types=data['report_types'], export_formats=data.get('export_formats', ['excel']) ) response_data = { 'id': str(bulk_job.id), 'status': bulk_job.status, 'total_items': bulk_job.total_items, 'created_at': bulk_job.created_at.isoformat() } return JsonResponse(response_data, status=201) except Exception as e: logger.error(f"Error creating bulk job: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class JobFromTemplateView(BatchProcessingView): """API endpoint for creating jobs from templates""" def post(self, request, template_id: int): """Create job from template""" try: from .batch_processing_service import BatchJobTemplate template = get_object_or_404(BatchJobTemplate, id=template_id, user=request.user) # Get override parameters data = json.loads(request.body) if request.body else {} override_params = data.get('overrides', {}) job = self.batch_service.create_job_from_template(template, override_params) response_data = { 'id': str(job.id), 'status': job.status, 'total_items': job.total_items, 'created_at': job.created_at.isoformat() } return JsonResponse(response_data, status=201) except Exception as e: logger.error(f"Error creating job from template: {str(e)}") return JsonResponse({'error': str(e)}, status=500) class JobControlView(BatchProcessingView): """API endpoint for job control operations (pause/resume)""" def post(self, request, job_id: str, action: str): """Control job execution""" try: if action == 'pause': success = self.batch_service.pause_job(job_id, request.user) elif action == 'resume': success = self.batch_service.resume_job(job_id, request.user) else: return JsonResponse({'error': 'Invalid action'}, status=400) if success: return JsonResponse({'success': True, 'action': action}) else: return JsonResponse({'error': f'Cannot {action} job'}, status=400) except Exception as e: logger.error(f"Error controlling job {job_id}: {str(e)}") return JsonResponse({'error': str(e)}, status=500)