Update cursor logic in POST calls (#4)

immuta · Aug 5, 2021 · 094a339 · 094a339
1 parent 6f73006
commit 094a339
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 23 deletions.
diff --git a/tap_gong/client.py b/tap_gong/client.py
@@ -1,6 +1,5 @@
 """REST client handling, including GongStream base class."""
 
-import requests
 from pathlib import Path
 from typing import Any, Dict, Optional, Union, List, Iterable
 

diff --git a/tap_gong/streams/call_transcripts.py b/tap_gong/streams/call_transcripts.py
@@ -1,12 +1,12 @@
 import time
-from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable
 
+from typing import Any, Dict, Optional, Union, List, Iterable
 from singer_sdk import typing as th  # JSON Schema typing helpers
 
 from tap_gong.client import GongStream
 from tap_gong.streams.calls import CallsStream
 
+
 class CallTranscriptsStream(GongStream):
     """Define custom stream."""
     name = "call_transcripts"
@@ -38,13 +38,10 @@ def prepare_request_payload(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Optional[dict]:
         """Prepare the data payload for the REST API request."""
-        start_time = self.get_starting_timestamp(context).strftime('%Y-%m-%dT%H:%M:%SZ')
         request_body = {
+            "cursor": next_page_token,
             "filter": {
-                "callIds": [context["call_id"]],
-                "fromDateTime": start_time,
-                "toDateTime": None,
-                "cursor": next_page_token,
+                "callIds": [context["call_id"]]
             }
         }
         time.sleep(self.request_delay_seconds)

diff --git a/tap_gong/streams/calls.py b/tap_gong/streams/calls.py
@@ -1,20 +1,17 @@
-"""Stream type classes for tap-gong."""
-
-import time
-from pathlib import Path
 from typing import Any, Dict, Optional, Union, List, Iterable
-
-from singer_sdk import typing as th  # JSON Schema typing helpers
+from singer_sdk import typing as th
 
 from tap_gong.client import GongStream
 
+
 class CallsStream(GongStream):
-    """Define custom stream."""
+    "Stream for all call data"
     name = "calls"
     path = "/v2/calls/extensive"
     primary_keys = ["id"]
     replication_key = "started"
     records_jsonpath = "$.calls[*]"
+    next_page_token_jsonpath = "$.records.cursor"
     rest_method = "POST"
 
     schema = th.PropertiesList(
@@ -122,12 +119,13 @@ def prepare_request_payload(
         self, context: Optional[dict], next_page_token: Optional[Any]
     ) -> Optional[dict]:
         """Prepare the data payload for the REST API request."""
-        start_time = self.get_starting_timestamp(context).strftime('%Y-%m-%dT%H:%M:%SZ')
+        start_time = self.get_starting_timestamp(context)
+        start_time_fmt = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') if start_time else None
         request_body = {
+            "cursor": next_page_token,
             "filter": {
-                "fromDateTime": start_time,
-                "toDateTime": None,
-                "cursor": next_page_token
+                "fromDateTime": start_time_fmt,
+                "toDateTime": None
             },
             "contentSelector": {
                 "context": "Extended",

diff --git a/tap_gong/streams/users.py b/tap_gong/streams/users.py
@@ -1,7 +1,4 @@
-import time
-from pathlib import Path
 from typing import Any, Dict, Optional, Union, List, Iterable
-
 from singer_sdk import typing as th  # JSON Schema typing helpers
 
 from tap_gong.client import GongStream

diff --git a/tap_gong/tap.py b/tap_gong/tap.py
@@ -25,7 +25,7 @@ class TapGong(Tap):
     config_jsonschema = th.PropertiesList(
         th.Property("access_key", th.StringType, required=True),
         th.Property("access_key_secret", th.StringType, required=True),
-        th.Property("start_date", th.DateTimeType),
+        th.Property("start_date", th.DateTimeType, default=None),
     ).to_dict()
 
     def discover_streams(self) -> List[Stream]: