Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pc 32007 retrieve special event #15606

Merged
merged 5 commits into from
Jan 15, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/.env.development
Original file line number Diff line number Diff line change
@@ -120,6 +120,7 @@ SUPPORT_PRO_EMAIL_ADDRESS=support-pro@example.com
TITELIVE_EPAGINE_API_AUTH_URL=https://login.epagine.fr/v1
TITELIVE_EPAGINE_API_URL=https://catsearch.epagine.fr/v1
TITELIVE_GENERATE_FROM_FILE_IN_DEV=1
TYPEFORM_IMPORT_CHUNK_SIZE=100
TZ=UTC
UBBLE_MOCK_API_URL=https://mock-ubble.ehp.passculture.team/api
UBBLE_MOCK_CONFIG_URL=https://mock-ubble.ehp.passculture.team
1 change: 1 addition & 0 deletions api/.env.production
Original file line number Diff line number Diff line change
@@ -117,6 +117,7 @@ SUBCATEGORY_SUGGESTION_BACKEND=pcapi.core.external.subcategory_suggestion_backen
TITELIVE_EPAGINE_API_AUTH_URL=https://login.epagine.fr/v1
TITELIVE_EPAGINE_API_URL=https://catsearch.epagine.fr/v1
TYPEFORM_BACKEND=pcapi.connectors.typeform.TypeformBackend
TYPEFORM_IMPORT_CHUNK_SIZE=500
VIRUSTOTAL_BACKEND=pcapi.connectors.virustotal.VirusTotalBackend
WEBAPP_V2_REDIRECT_URL=https://redirect.passculture.app
WEBAPP_V2_URL=https://passculture.app
2 changes: 1 addition & 1 deletion api/alembic_version_conflict_detection.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
ab6fe51d82bb (pre) (head)
3a2ce7aea173 (post) (head)
138018be9e5a (post) (head)
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""drop externalId column in special_operation_answer table"""

from alembic import op
import sqlalchemy as sa


# pre/post deployment: post
# revision identifiers, used by Alembic.
revision = "8a3126f15848"
down_revision = "3a2ce7aea173"
branch_labels: tuple[str] | None = None
depends_on: list[str] | None = None


def upgrade() -> None:
op.drop_column("special_event_answer", "externalId")


def downgrade() -> None:
op.add_column("special_event_answer", sa.Column("externalId", sa.TEXT(), autoincrement=False, nullable=False))
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Set responseId not nullable for table special_event_answer"""

from alembic import op
import sqlalchemy as sa


# pre/post deployment: post
# revision identifiers, used by Alembic.
revision = "7f870f74ea1e"
down_revision = "8a3126f15848"
branch_labels: tuple[str] | None = None
depends_on: list[str] | None = None


def upgrade() -> None:
op.alter_column("special_event_answer", "responseId", existing_type=sa.BIGINT(), nullable=False)


def downgrade() -> None:
op.alter_column("special_event_answer", "responseId", existing_type=sa.BIGINT(), nullable=True)
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Set questionId not nullable for table special_event_answer"""

from alembic import op
import sqlalchemy as sa


# pre/post deployment: post
# revision identifiers, used by Alembic.
revision = "723ca088adc4"
down_revision = "7f870f74ea1e"
branch_labels: tuple[str] | None = None
depends_on: list[str] | None = None


def upgrade() -> None:
op.alter_column("special_event_answer", "questionId", existing_type=sa.BIGINT(), nullable=False)


def downgrade() -> None:
op.alter_column("special_event_answer", "questionId", existing_type=sa.BIGINT(), nullable=True)
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Set eventId not nullable for table special_event_response"""

from alembic import op
import sqlalchemy as sa


# pre/post deployment: post
# revision identifiers, used by Alembic.
revision = "30a53e3cc7b2"
down_revision = "723ca088adc4"
branch_labels: tuple[str] | None = None
depends_on: list[str] | None = None


def upgrade() -> None:
op.alter_column("special_event_response", "eventId", existing_type=sa.BIGINT(), nullable=False)


def downgrade() -> None:
op.alter_column("special_event_response", "eventId", existing_type=sa.BIGINT(), nullable=True)
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
"""Set eventId not nullable for table special_event_question"""

from alembic import op
import sqlalchemy as sa


# pre/post deployment: post
# revision identifiers, used by Alembic.
revision = "138018be9e5a"
down_revision = "30a53e3cc7b2"
branch_labels: tuple[str] | None = None
depends_on: list[str] | None = None


def upgrade() -> None:
op.alter_column("special_event_question", "eventId", existing_type=sa.BIGINT(), nullable=False)


def downgrade() -> None:
op.alter_column("special_event_question", "eventId", existing_type=sa.BIGINT(), nullable=True)
48 changes: 43 additions & 5 deletions api/src/pcapi/connectors/typeform.py
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
from datetime import datetime
import json
import logging
import typing

import pydantic.v1 as pydantic_v1

@@ -79,6 +80,32 @@ def get_responses(
)


def get_responses_generator(
last_date_retriever: typing.Callable[[], datetime | None], form_id: str
) -> typing.Iterator[TypeformResponse]:
previous_date = object()
while True:

last_date = last_date_retriever()
if last_date == previous_date:
logger.error(
"typeform import error: infinite loop detected",
extra={"last_chronicle_id": str(last_date), "form_id": form_id},
)
break
previous_date = last_date
forms = get_responses(
form_id=form_id,
num_results=settings.TYPEFORM_IMPORT_CHUNK_SIZE,
sort="submitted_at,asc",
since=last_date,
)
yield from forms
prouzet-pass marked this conversation as resolved.
Show resolved Hide resolved

if len(forms) < settings.TYPEFORM_IMPORT_CHUNK_SIZE:
break


class BaseBackend:
def search_forms(self, search_query: str) -> list[TypeformForm]:
raise NotImplementedError()
@@ -159,7 +186,7 @@ def _extract_questions(self, fields: dict) -> list[TypeformQuestion]:
for field in fields:
if "fields" in field["properties"]:
questions += self._extract_questions(field["properties"]["fields"])
elif field["type"] in ("multiple_choice", "short_text", "long_text"):
elif field["type"] not in ("phone_number", "email"):
questions.append(TypeformQuestion(field_id=field["id"], title=field["title"].strip()))
return questions

@@ -221,6 +248,10 @@ def get_responses(
phone_number = _strip(answer["phone_number"])
case "email":
email = _strip(answer["email"])
case "boolean":
answers.append(
TypeformAnswer(field_id=answer["field"]["id"], text="Oui" if answer["boolean"] else "Non")
)
case "choice":
answers.append(
TypeformAnswer(
@@ -229,10 +260,17 @@ def get_responses(
choice_id=answer["choice"]["id"],
)
)
case "number":
answers.append(TypeformAnswer(field_id=answer["field"]["id"], text=str(answer["number"])))
case "text":
answers.append(TypeformAnswer(field_id=answer["field"]["id"], text=_strip(answer["text"])))
case "date":
answers.append(
TypeformAnswer(
field_id=answer["field"]["id"],
text=datetime.fromisoformat(answer["date"]).strftime("%d/%m/%Y"),
)
)
case "number" | "text" | "url" | "file_url":
answers.append(
TypeformAnswer(field_id=answer["field"]["id"], text=_strip(str(answer[answer["type"]])))
)
case _:
raise ValueError("Unexpected answer type from Typeform API: %s" % (answer["type"],))

33 changes: 6 additions & 27 deletions api/src/pcapi/core/chronicles/api.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from datetime import datetime
import logging
from re import search
import typing

from dateutil.relativedelta import relativedelta
from sqlalchemy import orm as sa_orm
@@ -30,35 +29,15 @@ def anonymize_unlinked_chronicles() -> None:


def import_book_club_chronicles() -> None:
for form in _book_club_forms_generator():
form_id = constants.BOOK_CLUB_FORM_ID
for form in typeform.get_responses_generator(_get_last_chronicle_date, form_id):
save_book_club_chronicle(form)


def _book_club_forms_generator() -> typing.Iterator[typeform.TypeformResponse]:
previous_last_chronicle = object()
while True:
last_chronicle = models.Chronicle.query.order_by(models.Chronicle.dateCreated.desc()).first()

if last_chronicle == previous_last_chronicle:
logger.error(
"Import chronicle for book club: error: infinite loop detected",
extra={
"last_chronicle_id": last_chronicle.id if last_chronicle else None,
},
)
break

forms = typeform.get_responses(
form_id=constants.BOOK_CLUB_FORM_ID,
num_results=constants.IMPORT_CHUNK_SIZE,
sort="submitted_at,asc",
since=last_chronicle.dateCreated if last_chronicle else None,
)
yield from forms

if len(forms) < constants.IMPORT_CHUNK_SIZE:
break
previous_last_chronicle = last_chronicle
def _get_last_chronicle_date() -> datetime | None:
return (
db.session.query(models.Chronicle.dateCreated).order_by(models.Chronicle.dateCreated.desc()).limit(1).scalar()
)


def _extract_book_club_ean(answer: typeform.TypeformAnswer) -> str | None:
4 changes: 3 additions & 1 deletion api/src/pcapi/core/chronicles/commands.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import logging

from pcapi.models.feature import FeatureToggle
from pcapi.scheduled_tasks import decorators as cron_decorators
from pcapi.utils.blueprint import Blueprint

@@ -13,4 +14,5 @@
@blueprint.cli.command("import_book_club_chronicle")
@cron_decorators.log_cron
def import_book_club_chronicle() -> None:
api.import_book_club_chronicles()
if FeatureToggle.ENABLE_CHRONICLES_SYNC.is_active():
api.import_book_club_chronicles()
1 change: 0 additions & 1 deletion api/src/pcapi/core/chronicles/constants.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@


ANONYMIZED_EMAIL = "anonymized_email@anonymized.passculture"
IMPORT_CHUNK_SIZE = 500
BOOK_CLUB_FORM_ID = "YeTcrzM0"


Loading