"""Send task: real Gmail API/SMTP send with retry, backoff, DLQ, pause/cancel, rate limits. Status model (no fake success): queued -> processing -> sent (Gmail accepted, message ID recorded) temporary errors -> retry with exponential backoff + jitter, dead-letter after max attempts permanent errors -> perm_failed (no retry) suppressed / skipped -> dropped pre-send """ import logging import random import re as _re import time from datetime import date, datetime, timedelta, timezone import googleapiclient.errors import redis as redis_lib from celery import Task from sqlalchemy import select from app.core.config import settings from app.db.session import SyncSession from app.models import (Attachment, Campaign, CampaignRecipient, EmailEvent, EmailTemplate, GmailAccount, OAuthToken, Recipient, SendAttempt, SendJob, SubjectLine, SuppressionEntry) from app.services import attachments_service, message_builder from app.services.gmail_service import build_gmail_client_from_account from app.services.variable_engine import GeneratedValues, VariableEngine from app.worker.celery_app import celery_app log = logging.getLogger("send") try: rdb = redis_lib.from_url(settings.REDIS_URL, socket_timeout=2) except Exception: rdb = None TEMPORARY_PATTERNS = ["rate limit", "rate-limit", "429", "quota", "backend error", "transient", "temporarily", "503", "502", "500", "timeout", "connection", "try again", "resourceexhausted", "tempfail", "unavailable", "internal error", "deadline"] PERMANENT_PATTERNS = ["invalid grant", "401", "403", "unauthorized", "forbidden", "invalidrecipient", "failedprecondition", "invalid argument", "malformed", "does not exist", "no such user", "550", "553", "554", "rejected", "policy violation", "disabled", "authentication"] def classify_error(msg: str) -> str: m = (msg or "").lower() for p in TEMPORARY_PATTERNS: if p in m: return "temporary" for p in PERMANENT_PATTERNS: if p in m: return "permanent" return "unknown" def now(): return datetime.now(timezone.utc) class SendTask(Task): def on_failure(self, exc, task_id, args, kwargs, einfo): log.error("Task %s failed: %s", task_id, str(exc)[:200]) @celery_app.task(bind=True, base=SendTask, name="app.tasks.send_tasks.send_one", ignore_result=False) def send_one(self, campaign_id: int, campaign_recipient_id: int): """Process one recipient: build message, send through Gmail, record real outcome.""" with SyncSession() as db: campaign = db.get(Campaign, campaign_id) if campaign is None: return {"ok": False, "status": "cancelled", "reason": "campaign deleted"} if campaign.status in ("paused", "cancelled", "completed"): return {"ok": False, "status": "cancelled", "reason": f"campaign {campaign.status}"} cr = db.get(CampaignRecipient, campaign_recipient_id) if cr is None: return {"ok": False, "status": "cancelled", "reason": "campaign_recipient deleted"} if cr.status in ("sent", "suppressed", "perm_failed"): return {"ok": True, "status": cr.status, "reason": "already finalized"} job = (db.execute(select(SendJob) .where(SendJob.campaign_recipient_id == cr.id, SendJob.campaign_id == campaign.id))).scalar_one_or_none() # Re-check suppression at send time (recipient may have unsubscribed since enqueue) sup = (db.execute(select(SuppressionEntry) .where(SuppressionEntry.email == cr.email))).scalar_one_or_none() if sup: cr.status = "suppressed" campaign.suppressed_count += 1 cr.last_error = "Suppressed at send time" db.add(EmailEvent(campaign_id=campaign.id, campaign_recipient_id=cr.id, event_type="suppressed", detail="Suppressed at send time")) db.commit() _publish_progress(campaign.id) return {"ok": True, "status": "suppressed", "reason": "suppression hit"} account = db.get(GmailAccount, cr.assigned_account_id) if cr.assigned_account_id else None if account is None or not account.is_active or account.status != "authorized": cr.status = "skipped" campaign.skipped_count += 1 cr.last_error = "Assigned account unavailable/unauthorized" db.commit() _publish_progress(campaign.id) return {"ok": False, "status": "skipped", "reason": "account unavailable"} # Per-account daily cap (never exceed provider limits) today = now().date()