Merge pull request #1493 from estuary/luis/source-hubspot-native

Modified OAuth2 and Access Token specs in estuary_cdk & source-hubspot-native fixes

estuary-cdk/estuary_cdk/flow.py

@@ -1,6 +1,6 @@
 import abc
 from dataclasses import dataclass
-from pydantic import BaseModel, NonNegativeInt, PositiveInt
+from pydantic import BaseModel, NonNegativeInt, PositiveInt, Field
 from typing import Any, Literal, TypeVar, Generic, Literal
 
 from .pydantic_polyfill import GenericModel
@@ -90,8 +90,14 @@ class ConnectorStateUpdate(GenericModel, Generic[ConnectorState]):
 
 
 class AccessToken(BaseModel):
-    credentials_title: Literal["Private App Credentials"] = "Private App Credentials"
-    access_token: str
+    credentials_title: Literal["Private App Credentials"] = Field(
+        default="Private App Credentials",
+        json_schema_extra={"type": "string"}
+    )
+    access_token: str = Field(
+        title="Access Token",
+        json_schema_extra={"secret": True},
+    )
 
 
 class BasicAuth(BaseModel):
@@ -110,10 +116,22 @@ class ValidationError(Exception):
 
 
 class BaseOAuth2Credentials(abc.ABC, BaseModel):
-    credentials_title: Literal["OAuth Credentials"] = "OAuth Credentials"
-    client_id: str
-    client_secret: str
-    refresh_token: str
+    credentials_title: Literal["OAuth Credentials"] = Field(
+        default="OAuth Credentials",
+        json_schema_extra={"type": "string"}
+    )
+    client_id: str = Field(
+        title="Client Id",
+        json_schema_extra={"secret": True},
+    )
+    client_secret: str = Field(
+        title="Client Secret",
+        json_schema_extra={"secret": True},
+    )
+    refresh_token: str = Field(
+        title="Refresh Token",
+        json_schema_extra={"secret": True},
+    )
 
     @abc.abstractmethod
     def _you_must_build_oauth2_credentials_for_a_provider(self): ...

source-google-sheets-native/tests/snapshots/source_google_sheets_native_tests_test_snapshots__spec__stdout.json

@@ -8,9 +8,11 @@
             "credentials_title": {
               "const": "Private App Credentials",
               "default": "Private App Credentials",
-              "title": "Credentials Title"
+              "title": "Credentials Title",
+              "type": "string"
             },
             "access_token": {
+              "secret": true,
               "title": "Access Token",
               "type": "string"
             }
@@ -26,17 +28,21 @@
             "credentials_title": {
               "const": "OAuth Credentials",
               "default": "OAuth Credentials",
-              "title": "Credentials Title"
+              "title": "Credentials Title",
+              "type": "string"
             },
             "client_id": {
+              "secret": true,
               "title": "Client Id",
               "type": "string"
             },
             "client_secret": {
+              "secret": true,
               "title": "Client Secret",
               "type": "string"
             },
             "refresh_token": {
+              "secret": true,
               "title": "Refresh Token",
               "type": "string"
             }

source-hubspot-native/acmeCo/companies.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/contacts.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/deals.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements_calls.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements_emails.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements_meetings.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements_notes.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/engagements_tasks.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/feedback_submissions.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/goal_targets.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/line_items.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/products.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/ticket_pipelines.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/acmeCo/tickets.schema.yaml

@@ -1,7 +1,7 @@
 ---
 $defs:
   History:
-    additionalProperties: false
+    additionalProperties: true
     properties:
       _meta:
         allOf:

source-hubspot-native/source_hubspot_native/__init__.py

@@ -30,7 +30,7 @@ def request_class(self):
 
     async def spec(self, log: Logger, _: request.Spec) -> ConnectorSpec:
         return ConnectorSpec(
-            documentationUrl="https://go.estuary.dev/source-hubspot-native",
+            documentationUrl="https://go.estuary.dev/hubspot-real-time",
             configSchema=EndpointConfig.model_json_schema(),
             oauth2=OAUTH2_SPEC,
             resourceConfigSchema=ResourceConfig.model_json_schema(),

source-hubspot-native/source_hubspot_native/api.py

@@ -349,7 +349,7 @@ async def fetch_changes(
 
     recent.sort()  # Oldest updates first.
 
-    for batch_it in itertools.batched(recent, 100):
+    for batch_it in itertools.batched(recent, 50):
         batch = list(batch_it)
 
         documents: BatchResult[CRMObject] = await fetch_batch_with_associations(
@@ -397,7 +397,9 @@ async def fetch_changes_no_batch(
 
         for ts, id, result in iter:
             recent.append((ts, id)) # Used later to update log_cursor value
-            yield V1CRMObject.model_validate_json(json.dumps(result)) 
+            doc = V1CRMObject.model_validate_json(json.dumps(result))
+            doc.meta_= V1CRMObject.Meta(op="u") 
+            yield  doc
 
 
         if not next_page:
@@ -447,11 +449,18 @@ async def fetch_recent_deals(
 ) -> tuple[Iterable[tuple[datetime, str]], PageCursor]:
 
     url = f"{HUB}/deals/v1/deal/recent/modified"
-    params = {"count": 100, "offset": page} if page else {"count": 1}
+    params = {"count": 100, "offset": page, "since": int(since.timestamp())} if page else {"count": 100}
 
     result = OldRecentDeals.model_validate_json(
         await http.request(log, url, params=params)
     )
+    if page:
+        if page == 9800:
+            return (
+        (_ms_to_dt(r.properties.hs_lastmodifieddate.timestamp), str(r.dealId))
+        for r in result.results
+    ), None
+
     return (
         (_ms_to_dt(r.properties.hs_lastmodifieddate.timestamp), str(r.dealId))
         for r in result.results
@@ -463,11 +472,18 @@ async def fetch_recent_engagements(
 ) -> tuple[Iterable[tuple[datetime, str]], PageCursor]:
 
     url = f"{HUB}/engagements/v1/engagements/recent/modified"
-    params = {"count": 100, "offset": page} if page else {"count": 1}
+    params = {"count": 100, "offset": page, "since": int(since.timestamp())} if page else {"count": 100}
 
     result = OldRecentEngagements.model_validate_json(
         await http.request(log, url, params=params)
     )
+    if page:
+        if page == 9800:
+            return(
+        (_ms_to_dt(r.engagement.lastUpdated), str(r.engagement.id))
+        for r in result.results
+    ), None
+
     return (
         (_ms_to_dt(r.engagement.lastUpdated), str(r.engagement.id))
         for r in result.results
@@ -660,6 +676,8 @@ async def fetch_campaigns(
     result = V1CRMObject.model_validate_json(
         data
     )
+    for item in result.campaigns:
+        item["id"] = str(item["id"])
 
     if result.hasMore is False:
         return (

source-hubspot-native/source_hubspot_native/models.py

@@ -80,9 +80,10 @@ class EndpointConfig(BaseModel):
     credentials: OAuth2Credentials | AccessToken = Field(
         discriminator="credentials_title",
         title="Authentication",
+        json_schema_extra={"type": "object"}
     )
     start_date: AwareDatetime = Field(
-        title="start_date",
+        title="Start Date",
     )
 
 
@@ -172,7 +173,7 @@ class BaseCRMObject(BaseDocument, extra="forbid"):
     IGNORE_PROPERTY_SEARCH: ClassVar[bool] = False
     ENFORCE_URL: ClassVar[str]
 
-    class History(BaseDocument, extra="forbid"):
+    class History(BaseDocument, extra="allow"):
         timestamp: datetime
         value: str
         sourceType: str