From ece073239b779643cff63f8d49095136af81bb54 Mon Sep 17 00:00:00 2001 From: Antoine LAURENT Date: Wed, 18 Sep 2024 06:07:46 +0200 Subject: [PATCH] users: Send emails to Brevo instead of Mailjet --- clevercloud/cron.json | 2 +- config/settings/base.py | 4 + .../management/commands/new_users_to_brevo.py | 203 ++++ .../commands/new_users_to_mailjet.py | 163 --- tests/users/test_management_commands.py | 1003 +++++++++++------ 5 files changed, 841 insertions(+), 534 deletions(-) create mode 100644 itou/users/management/commands/new_users_to_brevo.py delete mode 100644 itou/users/management/commands/new_users_to_mailjet.py diff --git a/clevercloud/cron.json b/clevercloud/cron.json index b5f05ce22ce..158bd92815a 100644 --- a/clevercloud/cron.json +++ b/clevercloud/cron.json @@ -19,7 +19,7 @@ "1 0 * * * $ROOT/clevercloud/run_management_command.sh update_prescriber_organization_with_api_entreprise --verbosity 2", "30 0 * * * $ROOT/clevercloud/run_management_command.sh collect_analytics_data --save", - "30 1 * * * $ROOT/clevercloud/run_management_command.sh new_users_to_mailjet --wet-run", + "30 1 * * * $ROOT/clevercloud/run_management_command.sh new_users_to_brevo --wet-run", "0 3 * * * $ROOT/clevercloud/run_management_command.sh clearsessions", "15 5 * * * $ROOT/clevercloud/run_management_command.sh prolongation_requests_chores email_reminder --wet-run", "0 9 * * * $ROOT/clevercloud/run_management_command.sh send_check_authorized_members_email", diff --git a/config/settings/base.py b/config/settings/base.py index 2718e98b8f8..f18ecf816c5 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -692,3 +692,7 @@ RDV_INSERTION_API_BASE_URL = os.getenv("RDV_INSERTION_API_BASE_URL") RDV_INSERTION_INVITE_HOLD_DURATION = datetime.timedelta(days=int(os.getenv("RDV_INSERTION_INVITE_HOLD_DAYS", 10))) RDV_INSERTION_WEBHOOK_SECRET = os.getenv("RDV_INSERTION_WEBHOOK_SECRET") + +# Brevo +# ------------------------------------------------------------------------------ +BREVO_API_KEY = os.getenv("BREVO_API_KEY") diff --git a/itou/users/management/commands/new_users_to_brevo.py b/itou/users/management/commands/new_users_to_brevo.py new file mode 100644 index 00000000000..972868a95f1 --- /dev/null +++ b/itou/users/management/commands/new_users_to_brevo.py @@ -0,0 +1,203 @@ +import enum +import logging +import time + +import httpx +from allauth.account.models import EmailAddress +from django.conf import settings +from django.db.models import Exists, OuterRef, Q +from sentry_sdk.crons import monitor + +from itou.companies.enums import SIAE_WITH_CONVENTION_KINDS +from itou.companies.models import CompanyMembership +from itou.prescribers.enums import PrescriberOrganizationKind +from itou.prescribers.models import PrescriberMembership +from itou.users.enums import IdentityProvider, UserKind +from itou.users.models import User +from itou.utils.command import BaseCommand +from itou.utils.iterators import chunks + + +logger = logging.getLogger(__name__) + +# https://app.brevo.com/contact/list-listing +BREVO_LIST_ID = 31 +BREVO_API_URL = "https://api.brevo.com/v3" + + +class UserCategory(enum.Enum): + PRECSRIBER_FT = "prescripteur FT" + PRESCRIBER = "prescripteur habilité" + ORIENTEUR = "orienteur" + EMPLOYEUR = "employeur" + + +class BrevoClient: + IMPORT_BATCH_SIZE = 1000 + DELETE_BATCH_SIZE = 150 # see https://developers.brevo.com/reference/removecontactfromlist + EXPORT_BATCH_SIZE = 500 # see https://developers.brevo.com/reference/getcontactsfromlist + + def __init__(self): + self.client = httpx.Client( + headers={ + "accept": "application/json", + "api-key": settings.BREVO_API_KEY, + } + ) + + def _import_contacts(self, users_data, category): + data = [ + { + "email": user["email"], + "attributes": { + "prenom": user["first_name"].title(), + "nom": user["last_name"].upper(), + "date_inscription": user["date_joined"].strftime("%Y-%m-%d"), + "type": category.value, + }, + } + for user in users_data + ] + + response = self.client.post( + f"{BREVO_API_URL}/contacts/import", + headers={"Content-Type": "application/json"}, + json={ + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": data, + }, + ) + if response.status_code != 202: + logger.error("Brevo API: Some emails were not imported", exc_info=True) + + def import_users(self, users, category): + for chunk in chunks(users, self.IMPORT_BATCH_SIZE): + if chunk: + self._import_contacts(chunk, category) + time.sleep(1) + + def _get_contacts_from_list(self, limit, offset): + response = self.client.get( + f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts?limit={limit}&offset={offset}" + ) + if response.status_code != 200: + return [], 0 + data = response.json() + return [user_data["email"] for user_data in data["contacts"]], data["count"] + + def all_emails(self): + emails = [] + limit = self.EXPORT_BATCH_SIZE + count = 0 + offset = 0 + while True: + batch, new_count = self._get_contacts_from_list(limit, offset) + emails += batch + count = count or new_count # set count + offset += limit + if offset >= count: + break + return emails + + def _remove_contacts_from_list(self, emails): + response = self.client.post( + f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove", + headers={"Content-Type": "application/json"}, + json={"emails": emails}, + ) + if response.status_code != 201 or response.json()["contacts"]["failure"]: + logger.error("Brevo API: Some emails were not deleted", exc_info=True) + + def delete_users(self, emails): + for chunk in chunks(emails, self.DELETE_BATCH_SIZE): + if chunk: + self._remove_contacts_from_list(chunk) + + +class Command(BaseCommand): + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + "--wet-run", + action="store_true", + help="Enroll new users to a mailing list in Brevo", + ) + + @monitor(monitor_slug="new-users-to-mailjet") + def handle(self, *args, wet_run, **options): + client = BrevoClient() + + users = ( + User.objects.filter(kind__in=[UserKind.PRESCRIBER, UserKind.EMPLOYER]) + .filter( + # Someday only filter on identity_provider ? + Exists( + EmailAddress.objects.filter( + user_id=OuterRef("pk"), + email=OuterRef("email"), + primary=True, + verified=True, + ) + ) + | Q(identity_provider=IdentityProvider.INCLUSION_CONNECT), # IC verifies emails on its own + is_active=True, + ) + .order_by("email") + ) + employers = list( + users.filter(kind=UserKind.EMPLOYER) + .filter( + Exists( + CompanyMembership.objects.filter( + user_id=OuterRef("pk"), + is_active=True, + company__kind__in=SIAE_WITH_CONVENTION_KINDS, + ) + ) + ) + .values("email", "first_name", "last_name", "date_joined") + ) + + all_prescribers = users.filter(kind=UserKind.PRESCRIBER) + prescriber_membership_qs = PrescriberMembership.objects.filter(user_id=OuterRef("pk"), is_active=True) + prescriber_ft_membership_qs = prescriber_membership_qs.filter(organization__kind=PrescriberOrganizationKind.PE) + prescribers_ft = list( + users.filter(Exists(prescriber_ft_membership_qs)).values("email", "first_name", "last_name", "date_joined") + ) + prescribers = list( + all_prescribers.filter(Exists(prescriber_membership_qs.filter(organization__is_authorized=True))) + .exclude(Exists(prescriber_ft_membership_qs)) + .values("email", "first_name", "last_name", "date_joined") + ) + orienteurs = list( + all_prescribers.exclude(Exists(prescriber_membership_qs.filter(organization__is_authorized=True))).values( + "email", "first_name", "last_name", "date_joined" + ) + ) + + logger.info("SIAE users count: %d", len(employers)) + logger.info("FT prescribers count: %d", len(prescribers_ft)) + logger.info("Prescribers count: %d", len(prescribers)) + logger.info("Orienteurs count: %d", len(orienteurs)) + + all_emails = [user["email"] for user in employers + prescribers_ft + prescribers + orienteurs] + + brevo_emails = client.all_emails() + emails_to_delete = sorted(set(brevo_emails) - set(all_emails)) + logger.info("Brevo emails count: %d", len(brevo_emails)) + logger.info("Brevo emails to delete count: %d", len(emails_to_delete)) + + if wet_run: + client.delete_users(emails_to_delete) + + for category, users in [ + (UserCategory.EMPLOYEUR, employers), + (UserCategory.PRECSRIBER_FT, prescribers_ft), + (UserCategory.PRESCRIBER, prescribers), + (UserCategory.ORIENTEUR, orienteurs), + ]: + client.import_users(users, category) diff --git a/itou/users/management/commands/new_users_to_mailjet.py b/itou/users/management/commands/new_users_to_mailjet.py deleted file mode 100644 index ae35914be3d..00000000000 --- a/itou/users/management/commands/new_users_to_mailjet.py +++ /dev/null @@ -1,163 +0,0 @@ -import datetime -import logging -import time - -import httpx -from allauth.account.models import EmailAddress -from django.conf import settings -from django.db.models import Exists, OuterRef, Q -from django.utils import timezone -from sentry_sdk.crons import monitor - -from itou.companies.enums import SIAE_WITH_CONVENTION_KINDS -from itou.companies.models import CompanyMembership -from itou.prescribers.enums import PrescriberOrganizationKind -from itou.prescribers.models import PrescriberMembership -from itou.users.enums import IdentityProvider, UserKind -from itou.users.models import User -from itou.utils.command import BaseCommand -from itou.utils.iterators import chunks - - -logger = logging.getLogger(__name__) - -MAILJET_API_URL = "https://api.mailjet.com/v3/" -# https://app.mailjet.com/contacts/lists/show/aG3w -NEW_SIAE_LISTID = 2544946 -# https://app.mailjet.com/contacts/lists/show/aG3x -NEW_PE_LISTID = 2544947 -# https://app.mailjet.com/contacts/lists/show/aG3z -NEW_PRESCRIBERS_LISTID = 2544949 -# https://app.mailjet.com/contacts/lists/show/aG3y -NEW_ORIENTEURS_LISTID = 2544948 - - -class Command(BaseCommand): - BATCH_SIZE = 1_000 - - def add_arguments(self, parser): - super().add_arguments(parser) - parser.add_argument( - "--wet-run", - action="store_true", - help="Enroll new users to a mailing list in MailJet", - ) - - @staticmethod - def campaign_start(): - # Users who joined before this date are not part of the mailing campaign. - return max( - timezone.make_aware(datetime.datetime(2023, 4, 23)), - timezone.now() - datetime.timedelta(days=365), - ) - - @staticmethod - def manage_url(list_id): - # https://dev.mailjet.com/email/guides/contact-management/#manage-multiple-contacts-in-a-list - # https://dev.mailjet.com/email/reference/contacts/bulk-contact-management/#v3_post_contactslist_list_ID_managemanycontacts - return f"{MAILJET_API_URL}REST/contactslist/{list_id}/managemanycontacts" - - @staticmethod - def monitor_url(list_id, job_id): - # https://dev.mailjet.com/email/reference/contacts/bulk-contact-management/#v3_get_contactslist_list_ID_managemanycontacts_job_ID - return f"{MAILJET_API_URL}REST/contactslist/{list_id}/managemanycontacts/{job_id}" - - def send_to_mailjet(self, client, list_id, users): - response = client.post( - self.manage_url(list_id), - json={ - "Action": "addnoforce", - "Contacts": [ - { - "Email": user.email, - "Name": user.get_full_name(), - } - for user in users - ], - }, - ) - response.raise_for_status() - response = response.json() - [data] = response["Data"] - return data["JobID"] - - def poll_completion(self, client, list_id, job_id): - end = timezone.now() + datetime.timedelta(minutes=5) - while timezone.now() < end: - response = client.get(self.monitor_url(list_id, job_id)) - response.raise_for_status() - response = response.json() - [data] = response["Data"] - if data["Status"] in ["Completed", "Error"]: - if error := data["Error"]: - logger.error("MailJet errors for list ID %s: %s", list_id, error) - if errorfile := data["ErrorFile"]: - logger.error("MailJet errors file for list ID %s: %s", list_id, errorfile) - job_end = datetime.datetime.fromisoformat(data["JobEnd"]) - job_start = datetime.datetime.fromisoformat(data["JobStart"]) - duration = (job_end - job_start).total_seconds() - logger.info("MailJet processed batch for list ID %s in %d seconds.", list_id, duration) - return - else: - time.sleep(2) - - @monitor(monitor_slug="new-users-to-mailjet") - def handle(self, *args, wet_run, **options): - users = ( - User.objects.exclude(kind=UserKind.JOB_SEEKER) - .filter( - Exists( - EmailAddress.objects.filter( - user_id=OuterRef("pk"), - email=OuterRef("email"), - primary=True, - verified=True, - ) - ) - | Q(identity_provider=IdentityProvider.INCLUSION_CONNECT), # IC verifies emails on its own - date_joined__gte=self.campaign_start(), - is_active=True, - ) - .order_by("email") - ) - employers = users.filter(kind=UserKind.EMPLOYER).filter( - Exists( - CompanyMembership.objects.filter( - user_id=OuterRef("pk"), - is_active=True, - company__kind__in=SIAE_WITH_CONVENTION_KINDS, - ) - ) - ) - all_prescribers = users.filter(kind=UserKind.PRESCRIBER) - prescriber_membership_qs = PrescriberMembership.objects.filter(user_id=OuterRef("pk"), is_active=True) - pe_prescribers = users.filter( - Exists(prescriber_membership_qs.filter(organization__kind=PrescriberOrganizationKind.PE)) - ) - prescribers = all_prescribers.filter(Exists(prescriber_membership_qs.filter(organization__is_authorized=True))) - orienteurs = all_prescribers.exclude( - pk__in=PrescriberMembership.objects.filter(is_active=True, organization__is_authorized=True).values_list( - "user_id", flat=True - ) - ) - - logger.info("SIAE users count: %d", len(employers)) - logger.info("PE prescribers count: %d", len(pe_prescribers)) - logger.info("Prescribers count: %d", len(prescribers)) - logger.info("Orienteurs count: %d", len(orienteurs)) - - if wet_run: - with httpx.Client( - auth=(settings.MAILJET_API_KEY_PRINCIPAL, settings.MAILJET_SECRET_KEY_PRINCIPAL), - headers={"Content-Type": "application/json"}, - ) as client: - for list_id, users in [ - (NEW_SIAE_LISTID, employers), - (NEW_PE_LISTID, pe_prescribers), - (NEW_PRESCRIBERS_LISTID, prescribers), - (NEW_ORIENTEURS_LISTID, orienteurs), - ]: - for chunk in chunks(users, self.BATCH_SIZE): - if chunk: - job_id = self.send_to_mailjet(client, list_id, chunk) - self.poll_completion(client, list_id, job_id) diff --git a/tests/users/test_management_commands.py b/tests/users/test_management_commands.py index 387af997a5a..a04374b37fd 100644 --- a/tests/users/test_management_commands.py +++ b/tests/users/test_management_commands.py @@ -21,13 +21,7 @@ from itou.prescribers.enums import PrescriberOrganizationKind from itou.users.enums import IdentityProvider from itou.users.management.commands import send_check_authorized_members_email -from itou.users.management.commands.new_users_to_mailjet import ( - MAILJET_API_URL, - NEW_ORIENTEURS_LISTID, - NEW_PE_LISTID, - NEW_PRESCRIBERS_LISTID, - NEW_SIAE_LISTID, -) +from itou.users.management.commands.new_users_to_brevo import BREVO_API_URL, BREVO_LIST_ID from itou.users.models import User from itou.utils.apis.pole_emploi import PoleEmploiAPIBadResponse from itou.utils.mocks.pole_emploi import API_RECHERCHE_ERROR, API_RECHERCHE_RESULT_KNOWN @@ -370,12 +364,9 @@ def test_shorten_active_sessions(): ] -class TestCommandNewUsersToMailJet: +class TestCommandNewUsersToBrevo: @freeze_time("2023-05-02") - def test_wet_run_siae(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - + def test_wet_run_siae(self, caplog, respx_mock, mocker): # Job seekers are ignored. JobSeekerFactory(with_verified_email=True) for kind in set(CompanyKind) - set(SIAE_WITH_CONVENTION_KINDS): @@ -386,12 +377,6 @@ def test_wet_run_siae(self, caplog, respx_mock, settings): company__kind=CompanyKind.EI, user__identity_provider=IdentityProvider.DJANGO ).user EmailAddress.objects.create(user=not_primary, email=not_primary.email, primary=False, verified=True) - # Past users are ignored. - CompanyMembershipFactory( - user__date_joined=datetime.datetime(2023, 1, 12, tzinfo=datetime.UTC), - user__with_verified_email=True, - company__kind=CompanyKind.EI, - ) # Inactive memberships are ignored. CompanyMembershipFactory(user__with_verified_email=True, company__kind=CompanyKind.EI, is_active=False) # Inactive users are ignored. @@ -440,101 +425,116 @@ def test_wet_run_siae(self, caplog, respx_mock, settings): CompanyMembershipFactory(user=cindy, company__kind=CompanyKind.ACI) CompanyMembershipFactory(user=dave, company__kind=CompanyKind.ETTI) CompanyMembershipFactory(user=eve, company__kind=CompanyKind.EITI) - post_mock = respx_mock.post(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts").mock( - return_value=httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 123456789}], "Total": 1}) - ) - respx_mock.get(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/123456789").mock( + + respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( side_effect=[ httpx.Response( 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_SIAE_LISTID, "Action": "addnoforce"}], - "Count": 2, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "", - "Status": "In Progress", - } - ], - "Total": 1, - }, - ), - httpx.Response( - 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_SIAE_LISTID, "Action": "addnoforce"}], - "Count": 5, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T12:34:56", - "Status": "Completed", - } - ], - "Total": 1, - }, - ), + json={"contacts": [], "count": 0}, + ) ] ) - with mock.patch("itou.users.management.commands.new_users_to_mailjet.time.sleep") as time_mock: - call_command("new_users_to_mailjet", wet_run=True) - time_mock.assert_called_once_with(2) - [postcall] = post_mock.calls - - assert json.loads(postcall.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "annie.amma@mailinator.com", "Name": "Annie AMMA"}, - {"Email": "bob.bailey@mailinator.com", "Name": "Bob BAILEY"}, - {"Email": "cindy.cinnamon@mailinator.com", "Name": "Cindy CINNAMON"}, - {"Email": "dave.doll@mailinator.com", "Name": "Dave DOLL"}, - {"Email": "eve.ebi@mailinator.com", "Name": "Eve EBI"}, - ], - } + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( + return_value=httpx.Response( + 201, + json={ + "contacts": { + "success": [], # it should contain the sent emails + "failure": [], + } + }, + ) + ) + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock( + return_value=httpx.Response(202, json={"processId": 106}) + ) + time_mock = mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + assert time_mock.call_args_list == [mocker.call(1)] + + assert [json.loads(call.request.content) for call in delete_mock.calls] == [] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + { + "email": "bob.bailey@mailinator.com", + "attributes": { + "prenom": "Bob", + "nom": "BAILEY", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + { + "email": "cindy.cinnamon@mailinator.com", + "attributes": { + "prenom": "Cindy", + "nom": "CINNAMON", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + { + "email": "dave.doll@mailinator.com", + "attributes": { + "prenom": "Dave", + "nom": "DOLL", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + { + "email": "eve.ebi@mailinator.com", + "attributes": { + "prenom": "Eve", + "nom": "EBI", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 5"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 5"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/123456789 "HTTP/1.1 200 OK"', # noqa: E501 + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( - "httpx", - logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/123456789 "HTTP/1.1 200 OK"', # noqa: E501 - ), - ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - f"MailJet processed batch for list ID {NEW_SIAE_LISTID} in 5025 seconds.", - ), - ( - "itou.users.management.commands.new_users_to_mailjet", - logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ] @freeze_time("2023-05-02") - def test_wet_run_prescribers(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - + def test_wet_run_prescribers(self, caplog, respx_mock, mocker): pe = PrescriberPoleEmploiFactory() other_org = PrescriberOrganizationFactory(kind=PrescriberOrganizationKind.ML, authorized=True) alice = PrescriberFactory( @@ -556,12 +556,6 @@ def test_wet_run_prescribers(self, caplog, respx_mock, settings): organization=organization, user__identity_provider=IdentityProvider.DJANGO ).user EmailAddress.objects.create(user=not_primary, email=not_primary.email, primary=False, verified=True) - # Past users are ignored. - PrescriberMembershipFactory( - user__date_joined=datetime.datetime(2023, 1, 12, tzinfo=datetime.UTC), - user__with_verified_email=True, - organization=organization, - ) # Inactive users are ignored. PrescriberMembershipFactory( user__is_active=False, user__with_verified_email=True, organization=organization @@ -575,115 +569,92 @@ def test_wet_run_prescribers(self, caplog, respx_mock, settings): changed_email.email = f"changed+{organization}@mailinator.com" changed_email.save(update_fields=["email"]) - pe_post_mock = respx_mock.post(f"{MAILJET_API_URL}REST/contactslist/{NEW_PE_LISTID}/managemanycontacts").mock( - return_value=httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 123456789}], "Total": 1}) - ) - respx_mock.get(f"{MAILJET_API_URL}REST/contactslist/{NEW_PE_LISTID}/managemanycontacts/123456789").mock( - return_value=httpx.Response( - 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_PE_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T11:11:56", - "Status": "Completed", - } - ], - "Total": 1, - }, - ), + respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + side_effect=[ + httpx.Response( + 200, + json={"contacts": [], "count": 0}, + ) + ] ) - other_org_post_mock = respx_mock.post( - f"{MAILJET_API_URL}REST/contactslist/{NEW_PRESCRIBERS_LISTID}/managemanycontacts" - ).mock(return_value=httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 123456789}], "Total": 1})) - respx_mock.get( - f"{MAILJET_API_URL}REST/contactslist/{NEW_PRESCRIBERS_LISTID}/managemanycontacts/123456789" - ).mock( - return_value=httpx.Response( - 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_PRESCRIBERS_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T11:11:56", - "Status": "Completed", - } - ], - "Total": 1, - }, - ), + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock() + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock( + return_value=httpx.Response(202, json={"processId": 106}) ) - call_command("new_users_to_mailjet", wet_run=True) - [pe_postcall] = pe_post_mock.calls - assert json.loads(pe_postcall.request.content) == { - "Action": "addnoforce", - "Contacts": [{"Email": "alice.aamar@mailinator.com", "Name": "Alice AAMAR"}], - } - [other_org_postcall] = other_org_post_mock.calls - assert json.loads(other_org_postcall.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "alice.aamar@mailinator.com", "Name": "Alice AAMAR"}, - {"Email": "justin.wood@mailinator.com", "Name": "Justin WOOD"}, - ], - } + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert [json.loads(call.request.content) for call in delete_mock.calls] == [] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "alice.aamar@mailinator.com", + "attributes": { + "prenom": "Alice", + "nom": "AAMAR", + "date_inscription": "2023-05-02", + "type": "prescripteur FT", + }, + }, + ], + }, + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "justin.wood@mailinator.com", + "attributes": { + "prenom": "Justin", + "nom": "WOOD", + "date_inscription": "2023-05-02", + "type": "prescripteur habilité", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 1"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 2"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_PE_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_PE_LISTID}/managemanycontacts/123456789 "HTTP/1.1 200 OK"', # noqa: E501 - ), - ( - "itou.users.management.commands.new_users_to_mailjet", - logging.INFO, - f"MailJet processed batch for list ID {NEW_PE_LISTID} in 45 seconds.", - ), - ( - "httpx", - logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_PRESCRIBERS_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_PRESCRIBERS_LISTID}/managemanycontacts/123456789 "HTTP/1.1 200 OK"', # noqa: E501 - ), - ( - "itou.users.management.commands.new_users_to_mailjet", - logging.INFO, - f"MailJet processed batch for list ID {NEW_PRESCRIBERS_LISTID} in 45 seconds.", + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ] @freeze_time("2023-05-02") - def test_wet_run_orienteurs(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - + def test_wet_run_orienteurs(self, caplog, respx_mock, mocker): PrescriberFactory( first_name="Billy", last_name="Boo", @@ -703,8 +674,6 @@ def test_wet_run_orienteurs(self, caplog, respx_mock, settings): email="timmy.timber@mailinator.com", ) PrescriberMembershipFactory(user=timmy, organization__kind=PrescriberOrganizationKind.OTHER) - # Past users are ignored. - PrescriberFactory(with_verified_email=True, date_joined=datetime.datetime(2023, 1, 12, tzinfo=datetime.UTC)) # Inactive users are ignored. PrescriberFactory( with_verified_email=True, @@ -719,73 +688,87 @@ def test_wet_run_orienteurs(self, caplog, respx_mock, settings): changed_email.email = "changed@mailinator.com" changed_email.save(update_fields=["email"]) - post_mock = respx_mock.post( - f"{MAILJET_API_URL}REST/contactslist/{NEW_ORIENTEURS_LISTID}/managemanycontacts" - ).mock(return_value=httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 123456789}], "Total": 1})) - respx_mock.get( - f"{MAILJET_API_URL}REST/contactslist/{NEW_ORIENTEURS_LISTID}/managemanycontacts/123456789" - ).mock( - return_value=httpx.Response( - 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_ORIENTEURS_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T11:11:56", - "Status": "Completed", - } - ], - "Total": 1, - }, - ), + respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + side_effect=[ + httpx.Response( + 200, + json={"contacts": [], "count": 0}, + ) + ] ) - call_command("new_users_to_mailjet", wet_run=True) - [postcall] = post_mock.calls - assert json.loads(postcall.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "billy.boo@mailinator.com", "Name": "Billy BOO"}, - {"Email": "sonny.sunder@mailinator.com", "Name": "Sonny SUNDER"}, - {"Email": "timmy.timber@mailinator.com", "Name": "Timmy TIMBER"}, - ], - } + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock() + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock( + return_value=httpx.Response(202, json={"processId": 106}) + ) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert [json.loads(call.request.content) for call in delete_mock.calls] == [] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "billy.boo@mailinator.com", + "attributes": { + "prenom": "Billy", + "nom": "BOO", + "date_inscription": "2023-05-02", + "type": "orienteur", + }, + }, + { + "email": "sonny.sunder@mailinator.com", + "attributes": { + "prenom": "Sonny", + "nom": "SUNDER", + "date_inscription": "2023-05-02", + "type": "orienteur", + }, + }, + { + "email": "timmy.timber@mailinator.com", + "attributes": { + "prenom": "Timmy", + "nom": "TIMBER", + "date_inscription": "2023-05-02", + "type": "orienteur", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 3"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 3"), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_ORIENTEURS_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_ORIENTEURS_LISTID}/managemanycontacts/123456789 "HTTP/1.1 200 OK"', # noqa: E501 - ), - ( - "itou.users.management.commands.new_users_to_mailjet", - logging.INFO, - f"MailJet processed batch for list ID {NEW_ORIENTEURS_LISTID} in 45 seconds.", + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ] @freeze_time("2023-05-02") - def test_wet_run_batch(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - + def test_wet_run_batch(self, caplog, respx_mock, mocker): annie = EmployerFactory( first_name="Annie", last_name="Amma", @@ -798,212 +781,492 @@ def test_wet_run_batch(self, caplog, respx_mock, settings): ) CompanyMembershipFactory(user=annie, company__kind=CompanyKind.EI) CompanyMembershipFactory(user=bob, company__kind=CompanyKind.AI) - post_mock = respx_mock.post(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts").mock( + + export_mock = respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( side_effect=[ - httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 1}], "Total": 1}), - httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 2}], "Total": 1}), + httpx.Response( + 200, + json={ + "contacts": [ + {"email": "old_1@mailinator.com"} # there are other attributes but we don't read them + ], + "count": 2, + }, + ), + httpx.Response( + 200, + json={ + "contacts": [ + {"email": "old_2@mailinator.com"} # there are other attributes but we don't read them + ], + "count": 2, + }, + ), ] ) - respx_mock.get(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/1").mock( + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( return_value=httpx.Response( - 200, + 201, json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_SIAE_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T11:12:00", - "Status": "Completed", - } - ], - "Total": 1, + "contacts": { + "success": [], # it should contain the sent emails + "failure": [], + } }, - ), + ) ) - respx_mock.get(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/2").mock( - return_value=httpx.Response( - 200, - json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_SIAE_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "", - "ErrorFile": "", - "JobStart": "2023-05-02T11:23:00", - "JobEnd": "2023-05-02T11:24:00", - "Status": "Completed", - } - ], - "Total": 1, - }, - ), + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock( + return_value=httpx.Response(202, json={"processId": 106}) ) - with mock.patch("itou.users.management.commands.new_users_to_mailjet.Command.BATCH_SIZE", 1): - call_command("new_users_to_mailjet", wet_run=True) - [postcall1, postcall2] = post_mock.calls - - assert json.loads(postcall1.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "annie.amma@mailinator.com", "Name": "Annie AMMA"}, - ], - } - assert json.loads(postcall2.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "bob.bailey@mailinator.com", "Name": "Bob BAILEY"}, - ], - } + mocker.patch("itou.users.management.commands.new_users_to_brevo.BrevoClient.EXPORT_BATCH_SIZE", 1) + mocker.patch("itou.users.management.commands.new_users_to_brevo.BrevoClient.DELETE_BATCH_SIZE", 1) + mocker.patch("itou.users.management.commands.new_users_to_brevo.BrevoClient.IMPORT_BATCH_SIZE", 1) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert len(export_mock.calls) == 2 + assert [json.loads(call.request.content) for call in delete_mock.calls] == [ + {"emails": ["old_1@mailinator.com"]}, + {"emails": ["old_2@mailinator.com"]}, + ] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "bob.bailey@mailinator.com", + "attributes": { + "prenom": "Bob", + "nom": "BAILEY", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 2"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 2"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=1&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 ), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/1 "HTTP/1.1 200 OK"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=1&offset=1 "HTTP/1.1 200 OK"', # noqa: E501 ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 2"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 2"), ( - "itou.users.management.commands.new_users_to_mailjet", + "httpx", logging.INFO, - f"MailJet processed batch for list ID {NEW_SIAE_LISTID} in 49 seconds.", + f'HTTP Request: POST https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts/remove "HTTP/1.1 201 Created"', # noqa: E501 ), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: POST https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts/remove "HTTP/1.1 201 Created"', # noqa: E501 ), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/2 "HTTP/1.1 200 OK"', # noqa: E501 + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( - "itou.users.management.commands.new_users_to_mailjet", + "httpx", logging.INFO, - f"MailJet processed batch for list ID {NEW_SIAE_LISTID} in 60 seconds.", + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ] @freeze_time("2023-05-02") - def test_wet_run_errors(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - + def test_wet_run_delete(self, caplog, respx_mock, mocker): annie = EmployerFactory( first_name="Annie", last_name="Amma", email="annie.amma@mailinator.com", ) CompanyMembershipFactory(user=annie, company__kind=CompanyKind.EI) - post_mock = respx_mock.post(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts").mock( - return_value=httpx.Response(201, json={"Count": 1, "Data": [{"JobID": 1}], "Total": 1}), + + export_mock = respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + side_effect=[ + httpx.Response( + 200, + json={ + "contacts": [ + {"email": "old@mailinator.com"}, # there are other attributes but we don't read them + { + "email": "annie.amma@mailinator.com" + }, # there are other attributes but we don't read them + ], + "count": 2, + }, + ), + ] ) - respx_mock.get(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/1").mock( + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( return_value=httpx.Response( - 200, + 201, json={ - "Count": 1, - "Data": [ - { - "ContactsLists": [{"ListID": NEW_SIAE_LISTID, "Action": "addnoforce"}], - "Count": 1, - "Error": "The blips failed to blap.", - "ErrorFile": "https://mailjet.com/my-errors.html", - "JobStart": "2023-05-02T11:11:11", - "JobEnd": "2023-05-02T11:12:00", - "Status": "Error", - } - ], - "Total": 1, + "contacts": { + "success": [], # it should contain the sent emails + "failure": [], + } }, + ) + ) + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock( + return_value=httpx.Response(202, json={"processId": 106}) + ) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert len(export_mock.calls) == 1 + assert [json.loads(call.request.content) for call in delete_mock.calls] == [ + {"emails": ["old@mailinator.com"]}, + ] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] + assert caplog.record_tuples == [ + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), + ( + "httpx", + logging.INFO, + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 + ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 2"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 1"), + ( + "httpx", + logging.INFO, + f'HTTP Request: POST https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts/remove "HTTP/1.1 201 Created"', # noqa: E501 + ), + ( + "httpx", + logging.INFO, + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 202 Accepted"', + ), + ( + "itou.users.management.commands.new_users_to_brevo", + logging.INFO, + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), + ] + + @freeze_time("2023-05-02") + def test_wet_run_errors_all_400(self, caplog, respx_mock, mocker): + annie = EmployerFactory( + first_name="Annie", + last_name="Amma", + email="annie.amma@mailinator.com", ) - call_command("new_users_to_mailjet", wet_run=True) - [postcall] = post_mock.calls - - assert json.loads(postcall.request.content) == { - "Action": "addnoforce", - "Contacts": [ - {"Email": "annie.amma@mailinator.com", "Name": "Annie AMMA"}, - ], - } + CompanyMembershipFactory(user=annie, company__kind=CompanyKind.EI) + export_mock = respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + httpx.Response(400) + ) + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( + return_value=httpx.Response(400) + ) + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock(return_value=httpx.Response(400)) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert len(export_mock.calls) == 1 + assert [json.loads(call.request.content) for call in delete_mock.calls] == [] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 1"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: POST https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts "HTTP/1.1 201 Created"', # noqa: E501 + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 400 Bad Request"', # noqa: E501 ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 0"), ( "httpx", logging.INFO, - f'HTTP Request: GET https://api.mailjet.com/v3/REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts/1 "HTTP/1.1 200 OK"', # noqa: E501 + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 400 Bad Request"', ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.ERROR, - f"MailJet errors for list ID {NEW_SIAE_LISTID}: The blips failed to blap.", + "BrevoAPI: Some emails were not imported", ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", + logging.INFO, + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", + ), + ] + + @freeze_time("2023-05-02") + def test_wet_run_errors_export_200_other_400(self, caplog, respx_mock, mocker): + annie = EmployerFactory( + first_name="Annie", + last_name="Amma", + email="annie.amma@mailinator.com", + ) + CompanyMembershipFactory(user=annie, company__kind=CompanyKind.EI) + export_mock = respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + side_effect=[ + httpx.Response( + 200, + json={ + "contacts": [ + {"email": "old@mailinator.com"}, # there are other attributes but we don't read them + ], + "count": 1, + }, + ), + ] + ) + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( + return_value=httpx.Response(400) + ) + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock(return_value=httpx.Response(400)) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert len(export_mock.calls) == 1 + assert [json.loads(call.request.content) for call in delete_mock.calls] == [ + {"emails": ["old@mailinator.com"]}, + ] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] + assert caplog.record_tuples == [ + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), + ( + "httpx", + logging.INFO, + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 + ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 1"), + ( + "httpx", + logging.INFO, + f'HTTP Request: POST https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts/remove "HTTP/1.1 400 Bad Request"', # noqa: E501 + ), + ( + "itou.users.management.commands.new_users_to_brevo", logging.ERROR, - f"MailJet errors file for list ID {NEW_SIAE_LISTID}: https://mailjet.com/my-errors.html", + "BrevoAPI: Some emails were not deleted", ), ( - "itou.users.management.commands.new_users_to_mailjet", + "httpx", logging.INFO, - f"MailJet processed batch for list ID {NEW_SIAE_LISTID} in 49 seconds.", + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 400 Bad Request"', + ), + ( + "itou.users.management.commands.new_users_to_brevo", + logging.ERROR, + "BrevoAPI: Some emails were not imported", ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ] - @freeze_time("2026-05-02") - def test_wet_run_limits_history_to_a_year(self, caplog, respx_mock, settings): - settings.MAILJET_API_KEY = "MAILJET_KEY" - settings.MAILJET_SECRET_KEY = "MAILJET_SECRET_KEY" - - # Past users are ignored. - CompanyMembershipFactory( - user__date_joined=datetime.datetime(2025, 5, 1, tzinfo=datetime.UTC), - company__kind=CompanyKind.EI, + @freeze_time("2023-05-02") + def test_wet_run_errors_export_200_remove_201_but_failed_other_400(self, caplog, respx_mock, mocker): + annie = EmployerFactory( + first_name="Annie", + last_name="Amma", + email="annie.amma@mailinator.com", ) - post_mock = respx_mock.post(f"{MAILJET_API_URL}REST/contactslist/{NEW_SIAE_LISTID}/managemanycontacts") - call_command("new_users_to_mailjet", wet_run=True) - assert post_mock.called is False + CompanyMembershipFactory(user=annie, company__kind=CompanyKind.EI) + export_mock = respx_mock.get(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts").mock( + side_effect=[ + httpx.Response( + 200, + json={ + "contacts": [ + {"email": "old@mailinator.com"}, # there are other attributes but we don't read them + ], + "count": 1, + }, + ), + ] + ) + delete_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/lists/{BREVO_LIST_ID}/contacts/remove").mock( + return_value=httpx.Response( + 201, + json={ + "contacts": { + "success": [], # it should contain the sent emails + "failure": ["old@mailinator.com"], + } + }, + ) + ) + import_mock = respx_mock.post(f"{BREVO_API_URL}/contacts/import").mock(return_value=httpx.Response(400)) + + mocker.patch("itou.users.management.commands.new_users_to_brevo.time.sleep") + call_command("new_users_to_brevo", wet_run=True) + + assert len(export_mock.calls) == 1 + assert [json.loads(call.request.content) for call in delete_mock.calls] == [ + {"emails": ["old@mailinator.com"]}, + ] + assert [json.loads(call.request.content) for call in import_mock.calls] == [ + { + "listIds": [BREVO_LIST_ID], + "emailBlacklist": False, + "smsBlacklist": False, + "updateExistingContacts": True, + "emptyContactsAttributes": False, + "jsonBody": [ + { + "email": "annie.amma@mailinator.com", + "attributes": { + "prenom": "Annie", + "nom": "AMMA", + "date_inscription": "2023-05-02", + "type": "employeur", + }, + }, + ], + }, + ] assert caplog.record_tuples == [ - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "SIAE users count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "PE prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Prescribers count: 0"), - ("itou.users.management.commands.new_users_to_mailjet", logging.INFO, "Orienteurs count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "SIAE users count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "FT prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Prescribers count: 0"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Orienteurs count: 0"), + ( + "httpx", + logging.INFO, + f'HTTP Request: GET https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts?limit=500&offset=0 "HTTP/1.1 200 OK"', # noqa: E501 + ), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails count: 1"), + ("itou.users.management.commands.new_users_to_brevo", logging.INFO, "Brevo emails to delete count: 1"), + ( + "httpx", + logging.INFO, + f'HTTP Request: POST https://api.brevo.com/v3/contacts/lists/{BREVO_LIST_ID}/contacts/remove "HTTP/1.1 201 Created"', # noqa: E501 + ), + ( + "itou.users.management.commands.new_users_to_brevo", + logging.ERROR, + "BrevoAPI: Some emails were not deleted", + ), + ( + "httpx", + logging.INFO, + 'HTTP Request: POST https://api.brevo.com/v3/contacts/import "HTTP/1.1 400 Bad Request"', + ), + ( + "itou.users.management.commands.new_users_to_brevo", + logging.ERROR, + "BrevoAPI: Some emails were not imported", + ), ( - "itou.users.management.commands.new_users_to_mailjet", + "itou.users.management.commands.new_users_to_brevo", logging.INFO, - "Management command itou.users.management.commands.new_users_to_mailjet succeeded in 0.00 seconds", + "Management command itou.users.management.commands.new_users_to_brevo succeeded in 0.00 seconds", ), ]