From f2a9e1f6a81e2ead4ad3ef04989325acb800a7ae Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Fri, 23 Jan 2015 19:56:41 -0500 Subject: [PATCH 01/78] Bump precision of format_seconds_minutes --- tapiriik/web/templatetags/displayutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/web/templatetags/displayutils.py b/tapiriik/web/templatetags/displayutils.py index f5bf2fb92..feff1eca7 100644 --- a/tapiriik/web/templatetags/displayutils.py +++ b/tapiriik/web/templatetags/displayutils.py @@ -34,7 +34,7 @@ def meters_per_day_to_km_per_hour(value): @register.filter(name="format_seconds_minutes") def meters_to_kms(value): try: - return round(value / 60, 1) + return round(value / 60, 3) except: return "NaN" From 7a4f34792aca7927a330f3e76ea454d9e1da5b5a Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Fri, 23 Jan 2015 20:13:01 -0500 Subject: [PATCH 02/78] More logging to profile sync_worker perf --- sync_worker.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sync_worker.py b/sync_worker.py index e42cd73bf..cf21b3a6b 100644 --- a/sync_worker.py +++ b/sync_worker.py @@ -1,12 +1,15 @@ from datetime import datetime, timedelta import os -print("Sync worker %s booting at %s" % (os.getpid(), datetime.now())) +# I'm trying to track down where some missing seconds are going in the sync process +# Will grep these out of the log at some later date +def worker_message(state): + print("Sync worker %d %s at %s" % (os.getpid(), state, datetime.now())) + +worker_message("booting") from tapiriik.requests_lib import patch_requests_with_default_timeout, patch_requests_source_address from tapiriik import settings from tapiriik.database import db, close_connections -import time -import signal import sys import subprocess import socket @@ -20,7 +23,7 @@ def sync_heartbeat(state, user=None): db.sync_workers.update({"Process": os.getpid(), "Host": socket.gethostname()}, {"$set": {"Heartbeat": datetime.utcnow(), "State": state, "User": user}}) -print("Sync worker " + str(os.getpid()) + " initialized at " + str(datetime.now())) +worker_message("initialized") db.sync_workers.update({"Process": os.getpid(), "Host": socket.gethostname()}, {"Process": os.getpid(), "Heartbeat": datetime.utcnow(), "Startup": datetime.utcnow(), "Version": WorkerVersion, "Host": socket.gethostname(), "Index": settings.WORKER_INDEX, "State": "startup"}, upsert=True) sys.stdout.flush() @@ -30,7 +33,7 @@ def sync_heartbeat(state, user=None): settings.HTTP_SOURCE_ADDR = settings.HTTP_SOURCE_ADDR[settings.WORKER_INDEX % len(settings.HTTP_SOURCE_ADDR)] patch_requests_source_address((settings.HTTP_SOURCE_ADDR, 0)) -print(" -> Index %s\n -> Interface %s" % (settings.WORKER_INDEX, settings.HTTP_SOURCE_ADDR)) +print(" %d -> Index %s\n -> Interface %s" % (os.getpid(), settings.WORKER_INDEX, settings.HTTP_SOURCE_ADDR)) # We defer including the main body of the application till here so the settings aren't captured before we've set them up. # The better way would be to defer initializing services until they're requested, but it's 10:30 and this will work just as well. @@ -40,10 +43,12 @@ def sync_heartbeat(state, user=None): sync_heartbeat("ready") +worker_message("ready") + Sync.PerformGlobalSync(heartbeat_callback=sync_heartbeat, version=WorkerVersion, max_users=RecycleInterval) -print("Sync worker shutting down cleanly") +worker_message("shutting down cleanly") db.sync_workers.remove({"Process": os.getpid(), "Host": socket.gethostname()}) -print("Closing database connections") close_connections() +worker_message("shut down") sys.stdout.flush() From 455f96522dd26f6cdf2ded22f25d6f31473c72e1 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Fri, 23 Jan 2015 20:26:36 -0500 Subject: [PATCH 03/78] Log the specific TS sanity mishap in the warning --- tapiriik/sync/sync.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index 82334b0b0..1a0f60a86 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -622,13 +622,13 @@ def _downloadActivityList(self, conn, exhaustive, no_add=False): logger.info("\tRetrieving list from " + svc.ID) svcActivities, svcExclusions = svc.DownloadActivityList(conn, exhaustive) except (ServiceException, ServiceWarning) as e: - # Historical note: there used to be a special case here where rate-limiting errors were fiddled with to trigger a full sync - # Wouldn't have been a problem except, elsewhere, exceptions are set as 'blocking' if they cause the retry counter to overfill - # at which point those users would be indefinitely stuck doing full synchronizations, since those blocking exceptions aren't automatically cleared - # so, now we don't set the special force-exhaustive flag that would stick around forever - # The original special case was there to ensure that, if there was a rate-limiting error while listing a service that previously had some other error, - # ...the user would continue to receive full synchronizations until the rate limit error cleared (even after the old error was forgotten) - # ...this being because of the fiddling mentioned above when the retry count was depleted - except that doesn't happen in the listing phase + # Historical note: there used to be a special case here where rate-limiting errors were fiddled with to trigger a full sync + # Wouldn't have been a problem except, elsewhere, exceptions are set as 'blocking' if they cause the retry counter to overfill + # at which point those users would be indefinitely stuck doing full synchronizations, since those blocking exceptions aren't automatically cleared + # so, now we don't set the special force-exhaustive flag that would stick around forever + # The original special case was there to ensure that, if there was a rate-limiting error while listing a service that previously had some other error, + # ...the user would continue to receive full synchronizations until the rate limit error cleared (even after the old error was forgotten) + # ...this being because of the fiddling mentioned above when the retry count was depleted - except that doesn't happen in the listing phase # so, it was never needed in the first place. I hope. self._syncErrors[conn._id].append(_packServiceException(SyncStep.List, e)) self._excludeService(conn, e.UserException) @@ -1012,8 +1012,8 @@ def Run(self, exhaustive=False, null_next_sync_on_unlock=False, heartbeat_callba try: full_activity.CheckTimestampSanity() - except ValueError: - logger.warning("\t\t...failed timestamp sanity check") + except ValueError as e: + logger.warning("\t\t...failed timestamp sanity check - %s" % e) # self._accumulateExclusions(full_activity.SourceConnection, APIExcludeActivity("Timestamp sanity check failed", activity=full_activity, permanent=True)) # activity.Record.MarkAsNotPresentOtherwise(UserException(UserExceptionType.SanityError)) # raise ActivityShouldNotSynchronizeException() From e70cfd47b6a186e3e037f9b1439876e7f2e98d4e Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Tue, 27 Jan 2015 11:14:28 -0500 Subject: [PATCH 04/78] Fix seemingly undocumented MIN interval type in N+ --- tapiriik/services/NikePlus/nikeplus.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tapiriik/services/NikePlus/nikeplus.py b/tapiriik/services/NikePlus/nikeplus.py index 0acc6a8dc..5d1387841 100644 --- a/tapiriik/services/NikePlus/nikeplus.py +++ b/tapiriik/services/NikePlus/nikeplus.py @@ -190,11 +190,15 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): return activities, exclusions def _nikeStream(self, stream, values_collection="values"): - if stream["intervalUnit"] != "SEC": + interval_secs = { + "SEC": 1, + "MIN": 60 + } + if stream["intervalUnit"] not in interval_secs: # Who knows if they ever return it in a different unit? Their docs don't give a list raise Exception("Unknown stream interval unit %s" % stream["intervalUnit"]) - interval = timedelta(seconds=stream["intervalMetric"]).total_seconds() + interval = stream["intervalMetric"] * interval_secs[stream["intervalUnit"]] for x in range(len(stream[values_collection])): yield (interval * x, stream[values_collection][x]) From df44ca3a9223e20935127d246451e0e4b8bdb861 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Thu, 5 Feb 2015 10:54:49 -0500 Subject: [PATCH 05/78] Stop using questionable $eq operator in PaidUserMongoQuery --- tapiriik/auth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/auth/__init__.py b/tapiriik/auth/__init__.py index 6be1fb093..edff2290e 100644 --- a/tapiriik/auth/__init__.py +++ b/tapiriik/auth/__init__.py @@ -93,7 +93,7 @@ def PaidUserMongoQuery(): "$or": [ {"Payments.Expiry": {"$gt": datetime.utcnow()}}, {"Promos.Expiry": {"$gt": datetime.utcnow()}}, - {"Promos.Expiry": {"$eq": None, "$exists": True}} + {"Promos.Expiry": {"$type": 10, "$exists": True}} # === null ] } From 37c85a0f5f425c010e02bcde71adee11cebae061 Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Mon, 9 Feb 2015 15:44:50 -0800 Subject: [PATCH 06/78] proposed workaround to retain both Name and Notes fields in TCX files http://github.com/cpfair/tapiriik/issues/99 --- tapiriik/services/tcx.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tapiriik/services/tcx.py b/tapiriik/services/tcx.py index 859efc148..7eb73afa2 100644 --- a/tapiriik/services/tcx.py +++ b/tapiriik/services/tcx.py @@ -55,7 +55,10 @@ def Parse(tcxData, act=None): xnotes = xact.find("tcx:Notes", namespaces=ns) if xnotes is not None: - act.Notes = xnotes.text + xnotes_lines = xnotes.splitlines() + act.Name = xnotes_lines[0] + if len(xnotes_lines)>1: + act.Notes = '\n'.join(xnotes.text[1:]) xcreator = xact.find("tcx:Creator", namespaces=ns) if xcreator is not None and xcreator.attrib["{" + TCXIO.Namespaces["xsi"] + "}type"] == "Device_t": @@ -257,8 +260,12 @@ def Dump(activity): dateFormat = "%Y-%m-%dT%H:%M:%S.000Z" - if activity.Name is not None: + if activity.Name is not None and activity.Notes is not None: + etree.SubElement(act, "Notes").text = '\n'.join(activity.Name, activity.Notes) + elif activity.Name is not None: etree.SubElement(act, "Notes").text = activity.Name + elif activity.Notes is not None: + etree.SubElement(act, "Notes").text = '\n' + activity.Notes if activity.Type == ActivityType.Cycling: act.attrib["Sport"] = "Biking" From e2afd4dca5bc789b9e995e6b9113aa64e766bd12 Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Mon, 9 Feb 2015 16:13:38 -0800 Subject: [PATCH 07/78] add TomTom GPS Sport Watch device IDs as seen in TCX and FIT files produced by TomTom MySports Connect --- tapiriik/services/devices.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tapiriik/services/devices.py b/tapiriik/services/devices.py index c273a14d9..82ef99167 100644 --- a/tapiriik/services/devices.py +++ b/tapiriik/services/devices.py @@ -119,3 +119,6 @@ def _garminIdentifier(name, *fitIds): DeviceIdentifier.AddIdentifierGroup(*_garminIdentifier("Forerunner 620", 1623)) +# TomTom MySports Connect appears to produce these IDs for all of their +# models of GPS watches (Runner, MultiSport, and Cardio versions of the same). +DeviceIdentifier.AddIdentifierGroup(TCXDeviceIdentifier("TomTom GPS Sport Watch", 0), FITDeviceIdentifier(71, 0)) From c2b3dd5aebba29b1992256815536c6563c9d0b05 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Thu, 12 Feb 2015 18:31:37 -0500 Subject: [PATCH 08/78] Update GC name retrieval - un-breaks first signin --- tapiriik/services/GarminConnect/garminconnect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index b730ea0b4..7b0e2e9f6 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -256,7 +256,8 @@ def Authorize(self, email, password): session = self._get_session(email=email, password=password) # TODO: http://connect.garmin.com/proxy/userprofile-service/socialProfile/ has the proper immutable user ID, not that anyone ever changes this one... self._rate_limit() - username = session.get("http://connect.garmin.com/user/username").json()["username"] + username_res = session.get("https://connect.garmin.com/modern/") + username = re.search(r'\\"fullName\\":\\"([^\\]+)', username_res.text).group(1) if not len(username): raise APIException("Unable to retrieve username", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) return (username, {}, {"Email": CredentialStore.Encrypt(email), "Password": CredentialStore.Encrypt(password)}) From 6869516bb4a3d1025e886cec8499f7cdd585375d Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Thu, 12 Feb 2015 20:16:11 -0500 Subject: [PATCH 09/78] Fix GC authorization --- .../services/GarminConnect/garminconnect.py | 182 ++++++++---------- 1 file changed, 80 insertions(+), 102 deletions(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index 7b0e2e9f6..8fea2af50 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -112,7 +112,7 @@ class GarminConnectService(ServiceBase): def __init__(self): cachedHierarchy = cachedb.gc_type_hierarchy.find_one() if not cachedHierarchy: - rawHierarchy = requests.get("http://connect.garmin.com/proxy/activity-service-1.2/json/activity_types", headers=self._obligatory_headers).text + rawHierarchy = requests.get("https://connect.garmin.com/modern/proxy/activity-service-1.2/json/activity_types", headers=self._obligatory_headers).text self._activityHierarchy = json.loads(rawHierarchy)["dictionary"] cachedb.gc_type_hierarchy.insert({"Hierarchy": rawHierarchy}) else: @@ -154,93 +154,73 @@ def _get_session(self, record=None, email=None, password=None, skip_cache=False) email = CredentialStore.Decrypt(record.ExtendedAuthorization["Email"]) session = requests.Session() - self._rate_limit() - gcPreResp = session.get("http://connect.garmin.com/", allow_redirects=False) - # New site gets this redirect, old one does not - if gcPreResp.status_code == 200: - self._rate_limit() - gcPreResp = session.get("https://connect.garmin.com/signin", allow_redirects=False) - req_count = int(re.search("j_id(\d+)", gcPreResp.text).groups(1)[0]) - params = {"login": "login", "login:loginUsernameField": email, "login:password": password, "login:signInButton": "Sign In"} - auth_retries = 3 # Did I mention Garmin Connect is silly? - for retries in range(auth_retries): - params["javax.faces.ViewState"] = "j_id%d" % req_count - req_count += 1 - self._rate_limit() - resp = session.post("https://connect.garmin.com/signin", data=params, allow_redirects=False) - if resp.status_code >= 500 and resp.status_code < 600: - raise APIException("Remote API failure") - if resp.status_code != 302: # yep - if "errorMessage" in resp.text: - if retries < auth_retries - 1: - time.sleep(1) - continue - else: - raise APIException("Invalid login", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) - else: - raise APIException("Mystery login error %s" % resp.text) - break - elif gcPreResp.status_code == 302: - # JSIG CAS, cool I guess. - # Not quite OAuth though, so I'll continue to collect raw credentials. - # Commented stuff left in case this ever breaks because of missing parameters... - data = { - "username": email, - "password": password, - "_eventId": "submit", - "embed": "true", - # "displayNameRequired": "false" - } - params = { - "service": "http://connect.garmin.com/post-auth/login", - # "redirectAfterAccountLoginUrl": "http://connect.garmin.com/post-auth/login", - # "redirectAfterAccountCreationUrl": "http://connect.garmin.com/post-auth/login", - # "webhost": "olaxpw-connect00.garmin.com", - "clientId": "GarminConnect", - # "gauthHost": "https://sso.garmin.com/sso", - # "rememberMeShown": "true", - # "rememberMeChecked": "false", - "consumeServiceTicket": "false", - # "id": "gauth-widget", - # "embedWidget": "false", - # "cssUrl": "https://static.garmincdn.com/com.garmin.connect/ui/src-css/gauth-custom.css", - # "source": "http://connect.garmin.com/en-US/signin", - # "createAccountShown": "true", - # "openCreateAccount": "false", - # "usernameShown": "true", - # "displayNameShown": "false", - # "initialFocus": "true", - # "locale": "en" - } - # I may never understand what motivates people to mangle a perfectly good protocol like HTTP in the ways they do... - preResp = session.get("https://sso.garmin.com/sso/login", params=params) - if preResp.status_code != 200: - raise APIException("SSO prestart error %s %s" % (preResp.status_code, preResp.text)) - data["lt"] = re.search("name=\"lt\"\s+value=\"([^\"]+)\"", preResp.text).groups(1)[0] - - ssoResp = session.post("https://sso.garmin.com/sso/login", params=params, data=data, allow_redirects=False) - if ssoResp.status_code != 200: - raise APIException("SSO error %s %s" % (ssoResp.status_code, ssoResp.text)) - - ticket_match = re.search("ticket=([^']+)'", ssoResp.text) - if not ticket_match: - raise APIException("Invalid login", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) - ticket = ticket_match.groups(1)[0] - # ...AND WE'RE NOT DONE YET! - - self._rate_limit() - gcRedeemResp1 = session.get("http://connect.garmin.com/post-auth/login", params={"ticket": ticket}, allow_redirects=False) - if gcRedeemResp1.status_code != 302: - raise APIException("GC redeem 1 error %s %s" % (gcRedeemResp1.status_code, gcRedeemResp1.text)) + # JSIG CAS, cool I guess. + # Not quite OAuth though, so I'll continue to collect raw credentials. + # Commented stuff left in case this ever breaks because of missing parameters... + data = { + "username": email, + "password": password, + "_eventId": "submit", + "embed": "true", + # "displayNameRequired": "false" + } + params = { + "service": "https://connect.garmin.com/post-auth/login", + # "redirectAfterAccountLoginUrl": "http://connect.garmin.com/post-auth/login", + # "redirectAfterAccountCreationUrl": "http://connect.garmin.com/post-auth/login", + # "webhost": "olaxpw-connect00.garmin.com", + "clientId": "GarminConnect", + # "gauthHost": "https://sso.garmin.com/sso", + # "rememberMeShown": "true", + # "rememberMeChecked": "false", + "consumeServiceTicket": "false", + # "id": "gauth-widget", + # "embedWidget": "false", + # "cssUrl": "https://static.garmincdn.com/com.garmin.connect/ui/src-css/gauth-custom.css", + # "source": "http://connect.garmin.com/en-US/signin", + # "createAccountShown": "true", + # "openCreateAccount": "false", + # "usernameShown": "true", + # "displayNameShown": "false", + # "initialFocus": "true", + # "locale": "en" + } + # I may never understand what motivates people to mangle a perfectly good protocol like HTTP in the ways they do... + preResp = session.get("https://sso.garmin.com/sso/login", params=params) + if preResp.status_code != 200: + raise APIException("SSO prestart error %s %s" % (preResp.status_code, preResp.text)) + data["lt"] = re.search("name=\"lt\"\s+value=\"([^\"]+)\"", preResp.text).groups(1)[0] + + ssoResp = session.post("https://sso.garmin.com/sso/login", params=params, data=data, allow_redirects=False) + if ssoResp.status_code != 200: + raise APIException("SSO error %s %s" % (ssoResp.status_code, ssoResp.text)) + + ticket_match = re.search("ticket=([^']+)'", ssoResp.text) + if not ticket_match: + raise APIException("Invalid login", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) + ticket = ticket_match.groups(1)[0] + + # ...AND WE'RE NOT DONE YET! + self._rate_limit() + gcRedeemResp = session.get("https://connect.garmin.com/post-auth/login", params={"ticket": ticket}, allow_redirects=False) + if gcRedeemResp.status_code != 302: + raise APIException("GC redeem-start error %s %s" % (gcRedeemResp.status_code, gcRedeemResp.text)) + + # There are 6 redirects that need to be followed to get the correct cookie + # ... :( + expected_redirect_count = 6 + current_redirect_count = 1 + while True: self._rate_limit() - gcRedeemResp2 = session.get(gcRedeemResp1.headers["location"], allow_redirects=False) - if gcRedeemResp2.status_code != 302: - raise APIException("GC redeem 2 error %s %s" % (gcRedeemResp2.status_code, gcRedeemResp2.text)) + gcRedeemResp = session.get(gcRedeemResp.headers["location"], allow_redirects=False) - else: - raise APIException("Unknown GC prestart response %s %s" % (gcPreResp.status_code, gcPreResp.text)) + if (current_redirect_count < expected_redirect_count and gcRedeemResp.status_code != 302) or (current_redirect_count >= expected_redirect_count and gcRedeemResp.status_code != 200): + raise APIException("GC redeem %d/%d error %s %s" % (current_redirect_count, expected_redirect_count, gcRedeemResp.status_code, gcRedeemResp.text)) + current_redirect_count += 1 + if current_redirect_count > expected_redirect_count: + break self._sessionCache.Set(record.ExternalID if record else email, session) @@ -286,7 +266,7 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): retried_auth = False while True: - res = session.get("http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities", params={"start": (page - 1) * pageSz, "limit": pageSz}) + res = session.get("https://connect.garmin.com/modern/proxy/activity-search-service-1.0/json/activities", params={"start": (page - 1) * pageSz, "limit": pageSz}) # It's 10 PM and I have no clue why it's throwing these errors, maybe we just need to log in again? if res.status_code == 403 and not retried_auth: retried_auth = True @@ -297,7 +277,7 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): res = res.json()["results"] except ValueError: res_txt = res.text # So it can capture in the log message - raise APIException("Parse failure in GC list resp: %s" % res.status_code) + raise APIException("Parse failure in GC list resp: %s - %s" % (res.status_code, res.text)) if "activities" not in res: break # No activities on this page - empty account. for act in res["activities"]: @@ -365,9 +345,7 @@ def _downloadActivitySummary(self, serviceRecord, activity): activityID = activity.ServiceData["ActivityID"] session = self._get_session(record=serviceRecord) self._rate_limit() - res = session.get("http://connect.garmin.com/proxy/activity-service-1.3/json/activity/" + str(activityID)) - - + res = session.get("https://connect.garmin.com/modern/proxy/activity-service-1.3/json/activity/" + str(activityID)) try: raw_data = res.json() @@ -461,7 +439,7 @@ def DownloadActivity(self, serviceRecord, activity): activityID = activity.ServiceData["ActivityID"] session = self._get_session(record=serviceRecord) self._rate_limit() - res = session.get("http://connect.garmin.com/proxy/activity-service-1.3/json/activityDetails/" + str(activityID) + "?maxSize=999999999") + res = session.get("https://connect.garmin.com/modern/proxy/activity-service-1.3/json/activityDetails/" + str(activityID) + "?maxSize=999999999") try: raw_data = res.json()["com.garmin.activity.details.json.ActivityDetails"] except ValueError: @@ -548,7 +526,7 @@ def UploadActivity(self, serviceRecord, activity): files = {"data": ("tap-sync-" + str(os.getpid()) + "-" + activity.UID + ".fit", fit_file)} session = self._get_session(record=serviceRecord) self._rate_limit() - res = session.post("http://connect.garmin.com/proxy/upload-service-1.1/json/upload/.fit", files=files) + res = session.post("https://connect.garmin.com/proxy/upload-service-1.1/json/upload/.fit", files=files) res = res.json()["detailedImportResult"] if len(res["successes"]) == 0: @@ -564,7 +542,7 @@ def UploadActivity(self, serviceRecord, activity): try: if activity.Name and activity.Name.strip(): self._rate_limit() - res = session.post("http://connect.garmin.com/proxy/activity-service-1.2/json/name/" + str(actid), data=urlencode({"value": activity.Name}).encode("UTF-8"), headers=encoding_headers) + res = session.post("https://connect.garmin.com/proxy/activity-service-1.2/json/name/" + str(actid), data=urlencode({"value": activity.Name}).encode("UTF-8"), headers=encoding_headers) try: res = res.json() except: @@ -577,7 +555,7 @@ def UploadActivity(self, serviceRecord, activity): try: if activity.Notes and activity.Notes.strip(): self._rate_limit() - res = session.post("http://connect.garmin.com/proxy/activity-service-1.2/json/description/" + str(actid), data=urlencode({"value": activity.Notes}).encode("UTF-8"), headers=encoding_headers) + res = session.post("https://connect.garmin.com/proxy/activity-service-1.2/json/description/" + str(actid), data=urlencode({"value": activity.Notes}).encode("UTF-8"), headers=encoding_headers) try: res = res.json() except: @@ -596,7 +574,7 @@ def UploadActivity(self, serviceRecord, activity): else: acttype = acttype[0] self._rate_limit() - res = session.post("http://connect.garmin.com/proxy/activity-service-1.2/json/type/" + str(actid), data={"value": acttype}) + res = session.post("https://connect.garmin.com/proxy/activity-service-1.2/json/type/" + str(actid), data={"value": acttype}) res = res.json() if "activityType" not in res or res["activityType"]["key"] != acttype: raise APIWarning("Unable to set activity type") @@ -606,7 +584,7 @@ def UploadActivity(self, serviceRecord, activity): try: if activity.Private: self._rate_limit() - res = session.post("http://connect.garmin.com/proxy/activity-service-1.2/json/privacy/" + str(actid), data={"value": "private"}) + res = session.post("https://connect.garmin.com/proxy/activity-service-1.2/json/privacy/" + str(actid), data={"value": "private"}) res = res.json() if "definition" not in res or res["definition"]["key"] != "private": raise APIWarning("Unable to set activity privacy") @@ -632,7 +610,7 @@ def SubscribeToPartialSyncTrigger(self, serviceRecord): user_name = self._user_watch_user(serviceRecord)["Name"] logger.info("Requesting connection to %s from %s" % (user_name, serviceRecord.ExternalID)) self._rate_limit() - resp = self._get_session(record=serviceRecord, skip_cache=True).put("http://connect.garmin.com/proxy/userprofile-service/connection/request/%s" % user_name) + resp = self._get_session(record=serviceRecord, skip_cache=True).put("https://connect.garmin.com/proxy/userprofile-service/connection/request/%s" % user_name) try: assert resp.status_code == 200 assert resp.json()["requestStatus"] == "Created" @@ -652,7 +630,7 @@ def UnsubscribeFromPartialSyncTrigger(self, serviceRecord): session = self._get_session(email=active_watch_user["Username"], password=active_watch_user["Password"], skip_cache=True) if "WatchConnectionID" in serviceRecord.GetConfiguration(): self._rate_limit() - dc_resp = session.put("http://connect.garmin.com/proxy/userprofile-service/connection/end/%s" % serviceRecord.GetConfiguration()["WatchConnectionID"]) + dc_resp = session.put("https://connect.garmin.com/modern/proxy/userprofile-service/connection/end/%s" % serviceRecord.GetConfiguration()["WatchConnectionID"]) if dc_resp.status_code != 200: raise APIException("Error disconnecting user watch accunt %s from %s: %s %s" % (active_watch_user, serviceRecord.ExternalID, dc_resp.status_code, dc_resp.text)) @@ -670,7 +648,7 @@ def ShouldForcePartialSyncTrigger(self, serviceRecord): def PollPartialSyncTrigger(self, multiple_index): # TODO: ensure the appropriate users are connected - # GET http://connect.garmin.com/proxy/userprofile-service/connection/pending to get ID + # GET http://connect.garmin.com/modern/proxy/userprofile-service/connection/pending to get ID # [{"userId":6244126,"displayName":"tapiriik-sync-ulukhaktok","fullName":"tapiriik sync ulukhaktok","profileImageUrlSmall":null,"connectionRequestId":1904086,"requestViewed":true,"userRoles":["ROLE_CONNECTUSER"],"userPro":false}] # PUT http://connect.garmin.com/proxy/userprofile-service/connection/accept/1904086 # ...later... @@ -683,7 +661,7 @@ def PollPartialSyncTrigger(self, multiple_index): # Then, check for users with new activities self._rate_limit() - watch_activities_resp = session.get("http://connect.garmin.com/proxy/activitylist-service/activities/subscriptionFeed?limit=1000") + watch_activities_resp = session.get("https://connect.garmin.com/modern/proxy/activitylist-service/activities/subscriptionFeed?limit=1000") try: watch_activities = watch_activities_resp.json() except ValueError: @@ -707,7 +685,7 @@ def PollPartialSyncTrigger(self, multiple_index): active_user_rec.SetConfiguration({"WatchUserLastID": this_active_id, "WatchUserKey": watch_user_key}) self._rate_limit() - pending_connections_resp = session.get("http://connect.garmin.com/proxy/userprofile-service/connection/pending") + pending_connections_resp = session.get("https://connect.garmin.com/modern/proxy/userprofile-service/connection/pending") try: pending_connections = pending_connections_resp.json() except ValueError: @@ -718,12 +696,12 @@ def PollPartialSyncTrigger(self, multiple_index): for pending_connect in pending_connections: if pending_connect["displayName"] in valid_pending_connections_external_ids: self._rate_limit() - connect_resp = session.put("http://connect.garmin.com/proxy/userprofile-service/connection/accept/%s" % pending_connect["connectionRequestId"]) + connect_resp = session.put("https://connect.garmin.com/modern/proxy/userprofile-service/connection/accept/%s" % pending_connect["connectionRequestId"]) if connect_resp.status_code != 200: logger.error("Error accepting request on watch account %s: %s %s" % (watch_user["Name"], connect_resp.status_code, connect_resp.text)) else: self._rate_limit() - ignore_resp = session.put("http://connect.garmin.com/proxy/userprofile-service/connection/decline/%s" % pending_connect["connectionRequestId"]) + ignore_resp = session.put("https://connect.garmin.com/modern/proxy/userprofile-service/connection/decline/%s" % pending_connect["connectionRequestId"]) return to_sync_ids From b055a424db8c381db339953137298bac726da96b Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Thu, 12 Feb 2015 20:23:27 -0500 Subject: [PATCH 10/78] Revert "Update GC name retrieval - un-breaks first signin" This reverts commit c2b3dd5aebba29b1992256815536c6563c9d0b05. --- tapiriik/services/GarminConnect/garminconnect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index 8fea2af50..e7b5d3add 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -236,8 +236,7 @@ def Authorize(self, email, password): session = self._get_session(email=email, password=password) # TODO: http://connect.garmin.com/proxy/userprofile-service/socialProfile/ has the proper immutable user ID, not that anyone ever changes this one... self._rate_limit() - username_res = session.get("https://connect.garmin.com/modern/") - username = re.search(r'\\"fullName\\":\\"([^\\]+)', username_res.text).group(1) + username = session.get("http://connect.garmin.com/user/username").json()["username"] if not len(username): raise APIException("Unable to retrieve username", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) return (username, {}, {"Email": CredentialStore.Encrypt(email), "Password": CredentialStore.Encrypt(password)}) From 88df03e2ff32f8312e8637e83f8b16a003b6afe7 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Fri, 13 Feb 2015 17:13:17 -0500 Subject: [PATCH 11/78] Enable logo cycling, new Arabic logo variant. --- tapiriik/web/static/img/tapiriik-arabic.png | Bin 2859 -> 4715 bytes tapiriik/web/static/js/tapiriik.js | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tapiriik/web/static/img/tapiriik-arabic.png b/tapiriik/web/static/img/tapiriik-arabic.png index b32312b124bc223163367bd52d9f9f03b6474a3e..6568b808369816dfdceb54fbfc0c39e3b887a64f 100644 GIT binary patch literal 4715 zcmaJ_c|4Ts+b3I*lq``jBPGj>F&HLt?1t=1mWgJVu{2|b8Ovm;WE~ZwoQ_9#W2H8%=AynRRey8($|9H>)d_K=}U)OzK-|ux@_x=6j{yZrTc9x>Tvcfz( zJfa9IGbipi%6)Nr1-W0|^wBl$AVW5HCp#0cNjPoTs7vD=4ef9l-MrRZaBH>&DJcf)?jKt%DN$^NR;9tCOuDv@A z1}gp)LcVAS{7+HtwhoG>giws44oFkOS5sG0Q5OP&K(%2on7X2tCPWjgsRf2;X+U6b zs1{sPQ}OQy$juh&hlM+tS^S-g>lp$A$YdfM433D107XDSgiwDlL|M4<^`WJ4gg(*Lx8C;melO!|A9xEltJL=nLd zkmhbnzXEM-|Nl@t{-0E7pI*PU^-@xEq|F3vkTR0+^L`DVsVi0DAKrRC)FwhTfqHC%L z)zyX>n^-^~5OZx4W0;-^6l!K-YOD)|m|6V7nh|`%@R(roFRb5xv3mcC-IW5K$jxkq z2@MR#_*sM!@QQ!63=jPGT6F%E?{BQ%zt^JsuUIg*81U}F{;z}nHN{oWuKAB_xz0br z#{_fL9?F$9Ij|VW!*gH)VP@Ozh6e4sUjwfUQZd#%(jEQcC$j^u5wJ zmF3dB9FYumoXx#Y;>r0<*%Uhuc79hT`beHyVw_0z49G#9I(a~)d*|{Cdrs`urasA^ z-B+}U{Yfb%-QnATZS?gPQHr09L`{r1)w&_R?z2OW_aRHt%E+v=XNCGY%v6^({mXGr za{Zy_=I*K-6NC^@bs6gkE=2jh3*Oy}2UTBRWAsMlj^)<45a&OW*=C!q^oMUw(wp)w zV#63z>J5-nlNuf>j`mV}lbYi}tgL2PYtVZ%H0h==PB2Z zJPC$53o>3XC?JL0-M+J5}^^)@^{z0JgX!lHQoIDfYD034MSQB#B&a z^uRx?Z+l#mt=fo#I4{-JM`r7L3Eb9@f*QGus3|{MW7~2u0VmI|xMAySCLf!p4i}mT zR60~mN(d(i+m$*nBn4%kn3cx8IoO^%_UBWz-TtF`7uK@5)==_Ngv1$*wJIhUdy)#(LFRMDf_i}+LYr{$kh4=knj-i`-ru3+Umkk)kkpTmDHe-wUYG)GruoLQ z(r4|kX^G^+F?}&IPbviVg~s!hV;96}SX1a~b?#umkDc3zb3zafpkw^VyRiuBb)j5S z(Jo7?i(T7+L#u!3Id4+AG?t1BTv z4Hd7DYyz-6>SlD#71z+?bT1%Pv_N&aJ^I}zc$Yi1)|+fu%{+y3JXOM4&$({^x&-MD zC$`R}!}tqNc;*+j%BYy;dF5*~{_xdS%1jo$(qQc)w9$Nsa6)Wn{$p!c-5N9C8qU^hzxf=!-eQm zmW|caImK@ZC1`T7zbS+vU(x@tVfXkc#76)*(d^fSR8eo$0OMHaIzu9F|oX=Ik(TdAM+Tm#<*eqHEsZx|Zxh_cyOLcrQ>`fqcKrF>t(`E( z3C=Lddq`HKf7OqzeD$;-`+7&X*IAmO_a2&Us3JX8Zq%~>d8~usb1Ufkrz|_G^qs;z z$;`%ZD4ua689;C7@sjkCq7~&R?gZZK5G?eo>Oro5t|KbaIaGCey?_vHOx_e%H6*Wg zl?Ad^p@#t7rN2@4g7)(DFxP}S!qIWx{b!N5jJx9H%8gTyf{dtaGu&-!=eJV+h}M2d zhl{!QU{9a2qfAt>7McRE4N^m^ski(?0sw;MV-BQ zjd?2XGiPgUo+=>Srj^~URNgLr*sGH7Y>7g~q&&>;r)C%0)M&#}f~fiKMORDC> z3)2FxC?l($8;JKG=o2Z!lCe3h%=y#i>W9^_v3k^{%HNY}ZoTAJ_dff$6C&4AsoX3y zaKs1$N=si)?L8*$z2aFtg}?e@2h09eg#JEtuS(|@(5sEWvESDpV)OEQRjnqfr9j*D zuC`j{1sP5s|K?P3%*DQm8eveE6?|~+wTOSBHn=_K>HZL6?Y#ael4o1XwUw&L)fDr6 z9B>Lt?)vb(AhTq!{cH*k1^nfl)S%1hb z9ax>!M|)akY2BZs?i>7inr#-qUcQ%Rs6UKunRe3xk0rgPX_uasJQ?eox`sNV@=U{^ zdvuK0Oh;ON4#ghVdtl%GU?<>ONLgNa(+#P^Vo7pIgWYbuvwu1cDFtPUGxwdPqdzJ@ zHnzXqT_Crqj5MdGaX?$S@1*5A*8Qy%DW=1RdP6I!SRr#!yBRZeAiBYjT*}b!U0Xe zPRpw31q4-#cl*j1x{XZe-=#!qUJ8%FZ&5=Y>Z$A~0VTImH^eQf)|aB;Cyc0$Wv+ zoosj}LpkcfZ@A*7by~Up7Nu@)VnN63SAbU1YCGBn*E(QmkvAn9o(HhS?<5^~<1cRh zi1nZxHbj`gXlS1r#7V3^ynnTq2BBP~#O*nXf_8$P|2Sy4ry>87Iq2l}QZWy1_WBq*3~YrW3x2FWN)^YwnPS%XqblZ3MKsvKm7#Me`RgJ2{XJg~oYWrhTd;Bt zE>3R-?OD!ss~F+ue@UHt#5l4DjAz2Xn0^5qSKL#(e8)J?*|K0b_WhNs_oo1Xk9vS{ zPtPbbt9d0fPR#fTZAcH?-V&n}xDE4c{JE5_{kcaZce{Qs;Yw+sS6Y-XB*>{Y$fYR@ z-XrlY#S)+3Dt#2*qT8}IQ8F!J4k3P$HkcD+Z-~wWLy%E1)kA|3NNl|5ckH44rUhBc zLZ)>};7^(#Zv*t~SFY6Wb=^pnSVqP1(HIEI`zyY&#sQEDGA~JrBI5Ep|DyEUvoC-N zT06~}QgO0&^YLB~X&0k3>Aa)-)76@lZevrIEHN??#Gip@0;9APIEd{tB>&>I^SNUB z>YNYej}O;Qjq+Y~-TwByMOlE5mYP3jRedP)Q>-<8$J6kD$hRLR#S@QWaV7mLhI@jD z*N@8^_W&V-nh6x^jO%6svw{l=YXHrcQrffYaFV^X)SN^xh2`?eR0zY6n+PbLI8+vM zzkj8D|Ln^aey`2%Hv!)b{VsG!u}bHE6Ba|)#4W_nsqQy{Y1t2>$M=iG(xy78M+Rf; zgK~fLoFAYe);7(%prSe{f#=F}g~xUL>!Qm+si$PuV!CcMd=8i@eBeq9=3z7k=>0_U z8<&K??1PU`550u86SAJ17d-JQa~2e)|4ih3(jytAqp?wlpj#!s(e8N)mtA5Kqx1I4 zWE4HTwHAxXHh3^ErZZU}VpRxM_UE>o@3* zx*xpYgV*Z14m*LIwZV(&TPmBUThnfwy7{5JU&QBq*tDsW4}1NkSISKgd_+lzTfnq)7-izZO~K@ z7p(g|Am-#*wNaJ0Cpy$RU~~C38I5=4NM%0LB0+e-Cn*+6Fp3Dyx|-GVt}7*Osi)xF zCEH{`Wb0Dy#z99#X3aFduhB7{(`qyR_RN+`^2tl;l#Kd@jH}7M-K31ihmO{XiDhj% ztqJdo(>1i_L6!8#l99wvI!!P_>c~tgL zC1Pe-X^>!+G)gEWE%kh7%_2Fu1!Td~zsvS5sZ7Y8X>D)XDNJuZ^2z7T?1sFSi_e16 z`QLulbU^OI>JT&XCkHn!N_~oLGBdE(crhU?rG2%3;6z1P;y5p}Rd(`)g2YXp_IW;0 zN`wTUy=pr_yEC8neb~0+TGi+g;!qz^?rNQl8dO{?T2%gJU znOy>CLlAU7)&SMLT|V6@(JqomexL6W{yr?E)ls?zt;(neM#C`*-z|k~Lwgy$GiQ$d zRPM!_uCDvHEM#u*vF*aUymsx-eMqZ|l+Pf|j;%A<@-2VX4XNk`M delta 2850 zcmV+-3*GeVB&!x7iBL{Q4GJ0x0000DNk~Le0002|0001N2nGNE08Y){B#|K}e+tb> zL_t(|+U#BJaoac$r9An6yb6w~z)S^UI*=(5*Q!~3V|#YjWz{DYaPxai3`M|4ND3G^OjNn~Yhq4G z>*lX84V@FDJs`8H0Z9P_e_ax;b{WjSmAkK>+geSZ#6PY)6>cb9@}a_gf~o=|VIA11 z+eqri&$wQaym?VYH1PLP+$bbT0IEK%4P*FbGVXZlhr&lehKd_$$`O8VP7r7URR#=o zH8YIiL;2)A;{V7u0VXZ0UpiBg1%wBB#qG-R(v{G1Sk2Q0byg~9vc z3pc;;NhoWRm}i8~1K|N1Zhq908>KgiI>M!rqt_A>5+pfP6adC2d)K9s`BbYKf$*rO z2b_URm3FE&PY?C;e}1)Ey_OD6fJdka;0xY4V%^0>6G`Jog4}}S`x!p|8EOIm^O1e6 zaNoljKVQ2whgkxEY2x;cg)zh+JOIo?wt_=89E1menO%9yW2tar4)6h-v-I0b*V^J( zo;3dt!UG`u0E?O_NL0#%UCfa1T;!JH_R>Xr#XBc!=m{=#eBd#zJJ-)eb{-D#_b%@j05j;v*)CimarF}S)S}f zbxR6puo$whsV7FSy2u{9^hm#}pLE9Y|1sdbQ^ofO=5u~`@ozDA8&F{}xNLbAwORSD zq+K4^X@~rcf15N3-DIzATKC}Bvg0%U0uL5p-375Df)Oxs5tEs}M9#^jSHwtcPRwb> zm4Xtp%t0*Aq30GpyBd7`nnH;}8!MOLmA*%d#M)U>G+l0?WJr$0r-kUE!Q;*XM}}Qd ztx%%UvOr^bZ&zhAai&QomX>5gxxF-IMv$FP*C$_;e}7k&7MlK5po*(=w~U0_)?;h( zthBD1tR)YKf}GL(r9NF8qS~&V@w2JBSy|UFi8vOOD8^aNJw~1TY(WMa4x!G#cr$gF z>`DvEFwa$)t{NdTLFYC+5Zy6ZE(*uE5-_vjnOU%~;f(W7Z9f%LSKu9^)l`+lD`y@D zt613;f45$voP~pA@A^IoL$`iQwP2ZL!LFr6R|&VA+d7Q01!5<0ceFf@Tkamnh(t>m z>14ZMY>5dI3oB>ZL2_$kCy>h);gdmb-Sp#&s0|Z6u;RdB2er4jpi>Ysod5e;Zg(1^^HCQEqN9pgyqsAc zf15UzvotPhy?_(0y`lz$g&q*(TxwqoiYpT~Y+Fw0VmXWBGCd5nW~>7RQ^ITa<(n~l z*b##PGvzc!zyl$swt>%v;m~Bg0!P3%#=7~Sd!6nQ%PnaB27AV1`m0K z{Dz_K9UPqh=cYna%QN%Sf^I{VL?>G$CSwkbk4ery#@OJb7sqq`1!D`JXvd}5e{NFM zJ#Ty_zG11a^?G`u%eiAF<4q6w?wEun8Rz>4NDC&Rpg=|Z*}A{XU~y7O^5dyLxDbBl zNG!bTA>-rRy%Jgs0^7VpSAD>$@gfl?4RM>E8?lXu6~8m)__d71#ba7>D!yBrvfVTJ z-hV2_cq>XUxrI#iALW3&$U)A>f82ZHqIW_{7~8&n{<9!j)fLs^49dPqcp>!4fuxj8 zh(+y_R8rUQL2iw4v-08;AL>1($M2#Z;x$VzizB{3E}i(oku(%=9d|vhuL193CLi#0 zt`fb1@X6Rq`2-Hii2Hb9Xr{&MVz1hVzX=@ahd#3_f8)c11B=9CE8(T-f4>Rnb8a4Gt`OOvs@jS|9$fxSDKaKxqd!N;_nwk zcA;qRJ2KlYS$OXACbs2Ne_O%gO0(O*>{#v8xM_rTt}ObGWWqOHg8KO+iGS~De1DMM zj3D}L*JT%uZOyTb-=%d|T?+YpmcJ|(`oKW%zBFB9sRIW`^!lgn4w$SYc&SnolO`&U zA{X4-;MYpHgH*14kMHlo$vJ^rt}eYS_f%Ddoehj?fjH~kZ^3*Qe{&wBr>h#zUBLT- z&>IJCY65(RrdMAE2?2m#gfCs`X%7IvFtTrZwKoO;z&Wy~9Itr>0AQN9U3cF=1W5tF zGRU4oQUKsS*+b%uN=ViLz(UB5UWNd$5V9k72Y>~U9kDwAEQIXGCG|Q07EB6|{RcMC z2Ecc+r_!>eJ>Hvee_bqkbd)I>euoGi2Qan;JYIBDi4EU-T(0PP4<|Oz)Cw@`L3R8F|!5CHCH~=cqloD z511+ywU^o(0!wSorn|37Xrt>rCJAwGX7;)~HI9vT_z<0<7w$d&+X1z7J z5vSUO1H6=~9AKr#L_u|{rA6P2Foq*nD>z_YUgZf52fE_;qajhwq&xO>H@kj!A%Ayr zE;x~Y-wbqff7vJL8gr5l{C;rf0j<@fy7<|}`%IKGYb2y{i{V7h) Date: Fri, 13 Feb 2015 17:15:47 -0500 Subject: [PATCH 12/78] Clarify the logo situation. --- tapiriik/web/templates/static/faq.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tapiriik/web/templates/static/faq.html b/tapiriik/web/templates/static/faq.html index 6ccfc7eb5..9fc1296da 100644 --- a/tapiriik/web/templates/static/faq.html +++ b/tapiriik/web/templates/static/faq.html @@ -99,7 +99,10 @@

Can tapiriik and CopyMySports be used together?

I wouldn't recommend it - CopyMySports will often create duplicates of activities tapiriik has already synchronized. - +
+

The logo's another language - what's up?

+ The best journeys answer questions that, in the beginning, you didn't even think to ask - Jeff Johnson +
From 82e8afe203c2f24070198ec49692fa0345633b74 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sat, 14 Feb 2015 15:11:54 -0500 Subject: [PATCH 13/78] First pass at automated activity rollback --- rollback_worker.py | 32 +++++++ tapiriik/services/Endomondo/endomondo.py | 5 + .../services/GarminConnect/garminconnect.py | 10 ++ tapiriik/services/NikePlus/nikeplus.py | 5 + tapiriik/services/RunKeeper/runkeeper.py | 6 ++ tapiriik/services/Strava/strava.py | 10 ++ tapiriik/services/rollback.py | 93 +++++++++++++++++++ tapiriik/services/service_base.py | 12 +++ tapiriik/urls.py | 5 + tapiriik/web/static/css/style.css | 2 +- tapiriik/web/static/js/tapiriik-ng.js | 33 +++++++ tapiriik/web/templates/rollback.html | 63 +++++++++++++ tapiriik/web/views/__init__.py | 1 + tapiriik/web/views/rollback.py | 29 ++++++ 14 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 rollback_worker.py create mode 100644 tapiriik/services/rollback.py create mode 100644 tapiriik/web/templates/rollback.html create mode 100644 tapiriik/web/views/rollback.py diff --git a/rollback_worker.py b/rollback_worker.py new file mode 100644 index 000000000..2fb274a99 --- /dev/null +++ b/rollback_worker.py @@ -0,0 +1,32 @@ +from tapiriik.database import db, close_connections +from tapiriik.settings import RABBITMQ_BROKER_URL, MONGO_HOST, MONGO_FULL_WRITE_CONCERN +from tapiriik import settings +from datetime import datetime + +from celery import Celery +from celery.signals import worker_shutdown +from datetime import datetime + +class _celeryConfig: + CELERY_ROUTES = { + "rollback_worker.rollback_task": {"queue": "tapiriik-rollback"} + } + CELERYD_CONCURRENCY = 1 + CELERYD_PREFETCH_MULTIPLIER = 1 + +celery_app = Celery('rollback_worker', broker=RABBITMQ_BROKER_URL) +celery_app.config_from_object(_celeryConfig()) + +@worker_shutdown.connect +def celery_shutdown(): + close_connections() + +@celery_app.task() +def rollback_task(task_id): + from tapiriik.services.rollback import RollbackTask + print("Starting rollback task %s" % task_id) + task = RollbackTask.Get(task_id) + task.Execute() + +def schedule_rollback_task(task_id): + rollback_task.apply_async(args=[task_id]) diff --git a/tapiriik/services/Endomondo/endomondo.py b/tapiriik/services/Endomondo/endomondo.py index 36348b150..f8bb93e1b 100644 --- a/tapiriik/services/Endomondo/endomondo.py +++ b/tapiriik/services/Endomondo/endomondo.py @@ -386,3 +386,8 @@ def UploadActivity(self, serviceRecord, activity): def DeleteCachedData(self, serviceRecord): pass + + def DeleteActivity(self, serviceRecord, uploadId): + session = self._oauthSession(serviceRecord) + del_res = session.delete("https://api.endomondo.com/api/1/workouts/%s" % uploadId) + del_res.raise_for_status() diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index e7b5d3add..580f47a53 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -82,6 +82,8 @@ class GarminConnectService(ServiceBase): SupportsHR = SupportsCadence = True + SupportsActivityDeletion = True + _sessionCache = SessionCache(lifetime=timedelta(minutes=30), freshen_on_get=True) _unitMap = { @@ -241,6 +243,8 @@ def Authorize(self, email, password): raise APIException("Unable to retrieve username", block=True, user_exception=UserException(UserExceptionType.Authorization, intervention_required=True)) return (username, {}, {"Email": CredentialStore.Encrypt(email), "Password": CredentialStore.Encrypt(password)}) + def UserUploadedActivityURL(self, uploadId): + return "https://connect.garmin.com/modern/activity/%d" % uploadId def _resolveActivityType(self, act_type): # Mostly there are two levels of a hierarchy, so we don't really need this as the parent is included in the listing. @@ -712,3 +716,9 @@ def RevokeAuthorization(self, serviceRecord): def DeleteCachedData(self, serviceRecord): # nothing cached... pass + + def DeleteActivity(self, serviceRecord, uploadId): + session = self._get_session(record=serviceRecord) + self._rate_limit() + del_res = session.delete("https://connect.garmin.com/modern/proxy/activity-service/activity/%d" % uploadId) + del_res.raise_for_status() diff --git a/tapiriik/services/NikePlus/nikeplus.py b/tapiriik/services/NikePlus/nikeplus.py index 5d1387841..6cdf50c79 100644 --- a/tapiriik/services/NikePlus/nikeplus.py +++ b/tapiriik/services/NikePlus/nikeplus.py @@ -338,3 +338,8 @@ def RevokeAuthorization(self, serviceRecord): def DeleteCachedData(self, serviceRecord): # nothing cached... pass + + def DeleteActivity(self, serviceRecord, uploadId): + session = self._get_session(serviceRecord) + del_res = session.delete("https://api.nike.com/v1/me/sport/activities/%d" % uploadId) + del_res.raise_for_status() diff --git a/tapiriik/services/RunKeeper/runkeeper.py b/tapiriik/services/RunKeeper/runkeeper.py index 59c5c021c..ca37b4f81 100644 --- a/tapiriik/services/RunKeeper/runkeeper.py +++ b/tapiriik/services/RunKeeper/runkeeper.py @@ -305,3 +305,9 @@ def _createUploadData(self, activity): def DeleteCachedData(self, serviceRecord): cachedb.rk_activity_cache.remove({"Owner": serviceRecord.ExternalID}) + + def DeleteActivity(self, serviceRecord, uri): + headers = self._apiHeaders(serviceRecord) + del_res = requests.delete("https://api.runkeeper.com/%s" % uri, headers=headers) + del_res.raise_for_status() + diff --git a/tapiriik/services/Strava/strava.py b/tapiriik/services/Strava/strava.py index 7588754fe..28978473a 100644 --- a/tapiriik/services/Strava/strava.py +++ b/tapiriik/services/Strava/strava.py @@ -31,6 +31,8 @@ class StravaService(ServiceBase): SupportsHR = SupportsCadence = SupportsTemp = SupportsPower = True + SupportsActivityDeletion = True + # For mapping common->Strava; no ambiguity in Strava activity type _activityTypeMappings = { ActivityType.Cycling: "Ride", @@ -70,6 +72,9 @@ class StravaService(ServiceBase): GlobalRateLimits = STRAVA_RATE_LIMITS + def UserUploadedActivityURL(self, uploadId): + return "https://www.strava.com/activities/%d" % uploadId + def WebInit(self): params = {'scope':'write view_private', 'client_id':STRAVA_CLIENT_ID, @@ -341,3 +346,8 @@ def UploadActivity(self, serviceRecord, activity): def DeleteCachedData(self, serviceRecord): cachedb.strava_cache.remove({"Owner": serviceRecord.ExternalID}) cachedb.strava_activity_cache.remove({"Owner": serviceRecord.ExternalID}) + + def DeleteActivity(self, serviceRecord, uploadId): + headers = self._apiHeaders(serviceRecord) + del_res = requests.delete("https://www.strava.com/api/v3/activities/%d" % uploadId, headers=headers) + del_res.raise_for_status() diff --git a/tapiriik/services/rollback.py b/tapiriik/services/rollback.py new file mode 100644 index 000000000..251f5f2ca --- /dev/null +++ b/tapiriik/services/rollback.py @@ -0,0 +1,93 @@ +from tapiriik.database import db +from tapiriik.auth import User +from tapiriik.services import Service +import datetime +import logging +import json +from bson.objectid import ObjectId +logger = logging.getLogger(__name__) + +class RollbackTask: + def __new__(cls, dbRec): + if not dbRec: + return None + return super(RollbackTask, cls).__new__(cls) + + def __init__(self, dbRec): + self.__dict__.update(dbRec) + + def _create(user): + # Pull all the records that need to be rolled back + logger.info("Finding activities for %s" % user["_id"]) + conns = User.GetConnectionRecordsByUser(user) + my_services = [conn.Service.ID for conn in conns] + my_ext_ids = [conn.ExternalID for conn in conns] + logger.info("Scanning uploads table for %s accounts with %s extids" % (my_services, my_ext_ids)) + uploads = db.uploaded_activities.find({"Service": {"$in": my_services}, "UserExternalID": {"$in": my_ext_ids}}) + pending_deletions = {} + for upload in uploads: + svc = upload["Service"] + upload_id = upload["ExternalID"] + svc_ext_id = upload["UserExternalID"] + # Filter back down to the pairing we actually need + if my_services.index(svc) != my_ext_ids.index(svc_ext_id): + continue + if svc not in pending_deletions: + pending_deletions[svc] = [] + pending_deletions[svc].append(upload_id) + + # Another case of "I should have an ORM" + return RollbackTask({"PendingDeletions": pending_deletions}) + + def Create(user): + task = RollbackTask._create(user) + uid = db.rollback_tasks.insert({"PendingDeletions": task.PendingDeletions, "Created": datetime.datetime.utcnow(), "UserID": user["_id"]}) + logger.info("Created rollback task %s" % uid) + task._id = uid + return task + + def Get(id): + dbRec = db.rollback_tasks.find_one({"_id": ObjectId(id)}) + if not dbRec: + return + return RollbackTask(dbRec) + + def json(self): + # Augment with the requisite URLs + self.ActivityURLs = {svc: {} for svc in self.PendingDeletions.keys()} + for svc_id, urls in self.ActivityURLs.items(): + svc = Service.FromID(svc_id) + for upload in self.PendingDeletions[svc_id]: + try: + urls[upload] = svc.UserUploadedActivityURL(upload) + except NotImplementedError: + pass + self.PendingDeletionCount = sum([len(v) for k, v in self.PendingDeletions.items()]) + dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date) else str(obj) + return json.dumps(self.__dict__, default=dthandler) + + def Schedule(self): + db.rollback_tasks.update({"_id": self._id}, {"$set": {"Scheduled": datetime.datetime.utcnow()}}) + from rollback_worker import schedule_rollback_task + schedule_rollback_task(str(self._id)) + + def Execute(self): + logger.info("Starting rollback %s" % self._id) + deletion_status = {} + user = User.Get(self.UserID) + for svc_id, upload_ids in self.PendingDeletions.items(): + svcrec = User.GetConnectionRecord(user, svc_id) + deletion_status[svc_id] = {} + if not svcrec.Service.SupportsActivityDeletion: + continue + for upload_id in upload_ids: + logger.info("Deleting activity %s on %s" % (upload_id, svc_id)) + try: + svcrec.Service.DeleteActivity(svcrec, upload_id) + except Exception as e: + deletion_status[svc_id][str(upload_id)] = False + logger.exception("Deletion failed - %s" % e) + else: + deletion_status[svc_id][str(upload_id)] = True + db.rollback_tasks.update({"_id": self._id}, {"$set": {"DeletionStatus": deletion_status}}) + logger.info("Finished rollback %s" % self._id) diff --git a/tapiriik/services/service_base.py b/tapiriik/services/service_base.py index b68a65266..991a5c044 100644 --- a/tapiriik/services/service_base.py +++ b/tapiriik/services/service_base.py @@ -41,6 +41,10 @@ class ServiceBase: ReceivesStationaryActivities = True # Manually-entered? ReceivesNonGPSActivitiesWithOtherSensorData = True # Trainer-ish? + + SupportsActivityDeletion = False + + # Causes synchronizations to be skipped until... # - One is triggered (via IDs returned by ExternalIDsForPartialSyncTrigger or PollPartialSyncTrigger) # - One is necessitated (non-partial sync, possibility of uploading new activities, etc) @@ -77,6 +81,10 @@ def RequiresConfiguration(self, serviceRecord): # Should convert this into a re def WebInit(self): pass + # Return an URL pointing directly to the specified activity on the remote site + def UserUploadedActivityURL(self, uploadId): + raise NotImplementedError + def GenerateUserAuthorizationURL(self, level=None): raise NotImplementedError @@ -92,9 +100,13 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): def DownloadActivity(self, serviceRecord, activity): raise NotImplementedError + # Should return an uploadId for storage and potential use in DeleteActivity def UploadActivity(self, serviceRecord, activity): raise NotImplementedError + def DeleteActivity(self, serviceRecord, uploadId): + raise NotImplementedError + def DeleteCachedData(self, serviceRecord): raise NotImplementedError diff --git a/tapiriik/urls.py b/tapiriik/urls.py index ecc6dc001..07bc45384 100644 --- a/tapiriik/urls.py +++ b/tapiriik/urls.py @@ -25,6 +25,11 @@ url(r'^account/settz$', 'tapiriik.web.views.account_settimezone', {}, name='account_set_timezone', ), url(r'^account/configure$', 'tapiriik.web.views.account_setconfig', {}, name='account_set_config', ), + url(r'^account/rollback/?$', 'tapiriik.web.views.account_rollback_initiate', {}, name='account_rollback_initiate', ), + url(r'^account/rollback/(?P.+)$', 'tapiriik.web.views.account_rollback_status', {}, name='account_rollback_status', ), + + url(r'^rollback$', 'tapiriik.web.views.rollback_dashboard', {}, name='rollback_dashboard', ), + url(r'^configure/save/(?P.+)?$', 'tapiriik.web.views.config.config_save', {}, name='config_save', ), url(r'^configure/dropbox$', 'tapiriik.web.views.config.dropbox', {}, name='dropbox_config', ), url(r'^configure/flow/save/(?P.+)?$', 'tapiriik.web.views.config.config_flow_save', {}, name='config_flow_save', ), diff --git a/tapiriik/web/static/css/style.css b/tapiriik/web/static/css/style.css index d78f36325..71c62bf76 100644 --- a/tapiriik/web/static/css/style.css +++ b/tapiriik/web/static/css/style.css @@ -933,7 +933,7 @@ form.dropboxConfig #filename { color: #aaa; } -.contactBlock { +.contactBlock, .splitBlock { display: inline-block; width:49%; vertical-align: top; diff --git a/tapiriik/web/static/js/tapiriik-ng.js b/tapiriik/web/static/js/tapiriik-ng.js index a1e3ca50a..16c7df149 100644 --- a/tapiriik/web/static/js/tapiriik-ng.js +++ b/tapiriik/web/static/js/tapiriik-ng.js @@ -125,6 +125,39 @@ function RecentSyncActivityController($scope, $http) { update_recent_activity(); }; +function RollbackDashboardController($scope, $http) { + $scope.step = 'pre'; + $scope.executing = false; + + $scope.DisplayNameByService = function(svcId){ return tapiriik.ServiceInfo[svcId].DisplayName; }; + + $scope.fetchList = function(){ + $scope.step = 'fetch-list'; + $http.get("/account/rollback/").success(function(task){ + $scope.task = task; + $scope.step = 'list'; + }); + }; + + $scope.execute = function() { + var confirm_coefficient = Math.floor(Math.random() * 11); + var confirm_base = $scope.task.PendingDeletionCount; + var confirm_res = prompt("Just to confirm, what's the value of " + confirm_base + " (the number of activities about to be deleted) multiplied by " + confirm_coefficient + "? LAST CHANCE - ONCE THE PROCESS BEGINS IT CANNOT BE CANCELLED."); + if (parseInt(confirm_res) === confirm_base * confirm_coefficient) { + $http.post("/account/rollback/" + $scope.task._id).success(function(task) { + $scope.executing = true; + setInterval(function(){ + $http.get("/account/rollback/" + $scope.task._id).success(function(task){ + $scope.task = task; + }); + }, 5000); + }); + } else { + alert("Rollback aborted - no activities will be deleted."); + } + }; +}; + angular.module('tapiriik', []).config(function($interpolateProvider) { $interpolateProvider.startSymbol('{[').endSymbol(']}'); }).run(function($rootScope, $http) { diff --git a/tapiriik/web/templates/rollback.html b/tapiriik/web/templates/rollback.html new file mode 100644 index 000000000..afefaee59 --- /dev/null +++ b/tapiriik/web/templates/rollback.html @@ -0,0 +1,63 @@ +{% extends "site.html" %} +{% load services %} +{% load users %} +{% load displayutils %} +{% block title %}roll back activities{% endblock %} + +{% block content %} +

Roll back all activities uploaded by tapiriik

+

READ ALL OF THIS.

+
+

This page lets you roll back all activities tapiriik has uploaded to most of your accounts

+

To emphasize, performing a roll-back will delete each and every activity that tapiriik has ever uploaded, where possible. Gone with those activities will be any associated comments, pictures, kudos, KOMs, etc. - all deleted permanently.

+ +

If you deleted any of the original copies after they were synced, those activities will be lost forever.

+ +

No other activities will be deleted. tapiriik tracks the unique identifiers assigned to each of its uploads by the remote service - it's these unique identifiers that are used to perform the rollback.

+ +

If the above sounds too scary, you can use the dry-run option to retrieve a list of links to the activities which would be deleted. You can then manually delete the undesired activities.

+
+ +
+

These services support rollback

+ {% for provider in service_providers %} + {% if provider.SupportsActivityDeletion %} + {{ provider.DisplayName }} + {% endif %} + {% endfor %} +
+
+

These services don't support rollback

+ {% for provider in service_providers %} + {% if not provider.SupportsActivityDeletion %} + {{ provider.DisplayName }} + {% endif %} + {% endfor %} +

Why not? Not all services offer a method to automatically delete activities. You can still manually delete activities on these services using the activity list below.

+
+ +
+ +

Fetching list (it'll take a bit)

+
+

tapiriik-originating activities:

+

(if some activities aren't listed, make sure you're connected to all the accounts you wish to roll back. Roll-back is not available for activities uploaded prior to January 18th 2014)

+
+

{[ DisplayNameByService(svc) ]}

+
+ {[ svc ]}-{[ upload ]} See on site » + + {[ task.DeletionStatus[svc][upload] ? "deleted" : "delete failed" ]} + +
+
+ + +
+

Rollback scheduled - check list above for status

+

Closing your browser will not cancel the process - however you must remain on this page to receive status updates.

+

Deletions may fail if the activity was already deleted, or if tapiriik no longer has access to your account. You can use this tool as many times as required.

+
+
+
+{% endblock %} \ No newline at end of file diff --git a/tapiriik/web/views/__init__.py b/tapiriik/web/views/__init__.py index 29c021273..3de421e46 100644 --- a/tapiriik/web/views/__init__.py +++ b/tapiriik/web/views/__init__.py @@ -9,4 +9,5 @@ from .settings import * from .ab import * from .activities_dashboard import * +from .rollback import * # why did I do it this way? should make it less bad diff --git a/tapiriik/web/views/rollback.py b/tapiriik/web/views/rollback.py new file mode 100644 index 000000000..54957de0a --- /dev/null +++ b/tapiriik/web/views/rollback.py @@ -0,0 +1,29 @@ +from tapiriik.services.rollback import RollbackTask +from django.http import HttpResponse +from django.views.decorators.http import require_GET +from django.shortcuts import redirect, render + +def account_rollback_initiate(req): + if not req.user: + return HttpResponse(status=403) + + task = RollbackTask.Create(req.user) + + return HttpResponse(task.json()) + +def account_rollback_status(req, task_id): + if not req.user: + return HttpResponse(status=403) + task = RollbackTask.Get(task_id) + + if not task: + return HttpResponse(status=404) + + if req.method == 'POST': + task.Schedule() + return HttpResponse(task.json()) + +def rollback_dashboard(req): + if not req.user: + return redirect('/') + return render(req, "rollback.html") \ No newline at end of file From b8460bc17686ce85675af721060bb59c662e06dc Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Fri, 20 Feb 2015 09:51:24 -0500 Subject: [PATCH 14/78] Less strict GC sign-in redirect following --- tapiriik/services/GarminConnect/garminconnect.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index 580f47a53..32ade06b7 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -218,8 +218,10 @@ def _get_session(self, record=None, email=None, password=None, skip_cache=False) self._rate_limit() gcRedeemResp = session.get(gcRedeemResp.headers["location"], allow_redirects=False) - if (current_redirect_count < expected_redirect_count and gcRedeemResp.status_code != 302) or (current_redirect_count >= expected_redirect_count and gcRedeemResp.status_code != 200): + if current_redirect_count >= expected_redirect_count and gcRedeemResp.status_code != 200: raise APIException("GC redeem %d/%d error %s %s" % (current_redirect_count, expected_redirect_count, gcRedeemResp.status_code, gcRedeemResp.text)) + if gcRedeemResp.status_code == 200: + break current_redirect_count += 1 if current_redirect_count > expected_redirect_count: break From a199e603d9e8f05044b4266bc5c4791923a96a90 Mon Sep 17 00:00:00 2001 From: Walter Huf Date: Sun, 22 Feb 2015 11:57:12 -0600 Subject: [PATCH 15/78] Adds some more useful local settings --- tapiriik/local_settings.py.example | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tapiriik/local_settings.py.example b/tapiriik/local_settings.py.example index 81a91bf13..d3876e100 100644 --- a/tapiriik/local_settings.py.example +++ b/tapiriik/local_settings.py.example @@ -1,3 +1,17 @@ +# Look in settings.py for more settings to override +# including mongodb, rabbitmq, and redis connection settings + +# This is the url that is used for redirects after logging in to each service +# It only needs to be accessible to the client browser +WEB_ROOT = "http://localhost:8000" + +# This is where sync logs show up +# It is the only directory that needs to be writable by the webapp user +USER_SYNC_LOGS = "./" + +# These settings are used to communicate with each respective service +# Register your installation with each service to get these values + RUNKEEPER_CLIENT_ID="####" RUNKEEPER_CLIENT_SECRET="####" From e5967fa90be929fb724f48a713e37797922b929b Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 23 Feb 2015 19:55:42 -0500 Subject: [PATCH 16/78] Revert mistaken reversion of rate-limiting special case during list, add more special casing around rate limits elsewhere to address the issue prompting said reversion --- tapiriik/sync/sync.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index 1a0f60a86..b38abc6b4 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -622,14 +622,18 @@ def _downloadActivityList(self, conn, exhaustive, no_add=False): logger.info("\tRetrieving list from " + svc.ID) svcActivities, svcExclusions = svc.DownloadActivityList(conn, exhaustive) except (ServiceException, ServiceWarning) as e: - # Historical note: there used to be a special case here where rate-limiting errors were fiddled with to trigger a full sync - # Wouldn't have been a problem except, elsewhere, exceptions are set as 'blocking' if they cause the retry counter to overfill - # at which point those users would be indefinitely stuck doing full synchronizations, since those blocking exceptions aren't automatically cleared - # so, now we don't set the special force-exhaustive flag that would stick around forever - # The original special case was there to ensure that, if there was a rate-limiting error while listing a service that previously had some other error, - # ...the user would continue to receive full synchronizations until the rate limit error cleared (even after the old error was forgotten) - # ...this being because of the fiddling mentioned above when the retry count was depleted - except that doesn't happen in the listing phase - # so, it was never needed in the first place. I hope. + # Special-case rate limiting errors thrown during listing + # Otherwise, things will melt down when the limit is reached + # (lots of users will hit this error, then be marked for full synchronization later) + # (but that's not really required) + # Though we don't want to play with things if this exception needs to take the place of an earlier, more significant one + # + # I had previously removed this because I forgot that TriggerExhaustive defaults to true - this exception was *un*setting it + # The issue prompting that change stemmed more from the fact that the rate-limiting errors were being marked as blocking, + # ...not that they were getting marked as *not* triggering exhaustive synchronization + + if e.UserException and e.UserException.Type == UserExceptionType.RateLimited: + e.TriggerExhaustive = conn._id in self._hasTransientSyncErrors and self._hasTransientSyncErrors[conn._id] self._syncErrors[conn._id].append(_packServiceException(SyncStep.List, e)) self._excludeService(conn, e.UserException) if not _isWarning(e): @@ -735,7 +739,7 @@ def _downloadActivity(self, activity): # Persist the exception if we just exceeded the failure count # (but not if a more useful blocking exception was provided) activity.Record.IncrementFailureCount(dlSvcRecord) - if activity.Record.GetFailureCount(dlSvcRecord) >= dlSvc.DownloadRetryCount and not e.Block: + if activity.Record.GetFailureCount(dlSvcRecord) >= dlSvc.DownloadRetryCount and not e.Block and (not e.UserException or e.UserException.Type != UserExceptionType.RateLimited): e.Block = True e.Scope = ServiceExceptionScope.Activity @@ -793,7 +797,8 @@ def _uploadActivity(self, activity, destinationServiceRec): except (ServiceException, ServiceWarning) as e: if not _isWarning(e): activity.Record.IncrementFailureCount(destinationServiceRec) - if activity.Record.GetFailureCount(destinationServiceRec) >= destSvc.UploadRetryCount and not e.Block: + # The rate-limiting special case here is so that users don't get stranded due to rate limiting issues outside of their control + if activity.Record.GetFailureCount(destinationServiceRec) >= destSvc.UploadRetryCount and not e.Block and (not e.UserException or e.UserException.Type != UserExceptionType.RateLimited): e.Block = True e.Scope = ServiceExceptionScope.Activity From d025c8b496751eebdcd5ce958456fa3d3ca91aa2 Mon Sep 17 00:00:00 2001 From: Matthew Duggan Date: Mon, 2 Mar 2015 17:45:20 +0900 Subject: [PATCH 17/78] Small fixes for tcx notes field --- tapiriik/services/tcx.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tapiriik/services/tcx.py b/tapiriik/services/tcx.py index 7eb73afa2..84b0b2b77 100644 --- a/tapiriik/services/tcx.py +++ b/tapiriik/services/tcx.py @@ -55,10 +55,10 @@ def Parse(tcxData, act=None): xnotes = xact.find("tcx:Notes", namespaces=ns) if xnotes is not None: - xnotes_lines = xnotes.splitlines() + xnotes_lines = xnotes.text.splitlines() act.Name = xnotes_lines[0] - if len(xnotes_lines)>1: - act.Notes = '\n'.join(xnotes.text[1:]) + if len(xnotes_lines) > 1: + act.Notes = '\n'.join(xnotes_lines[1:]) xcreator = xact.find("tcx:Creator", namespaces=ns) if xcreator is not None and xcreator.attrib["{" + TCXIO.Namespaces["xsi"] + "}type"] == "Device_t": @@ -261,7 +261,7 @@ def Dump(activity): dateFormat = "%Y-%m-%dT%H:%M:%S.000Z" if activity.Name is not None and activity.Notes is not None: - etree.SubElement(act, "Notes").text = '\n'.join(activity.Name, activity.Notes) + etree.SubElement(act, "Notes").text = '\n'.join((activity.Name, activity.Notes)) elif activity.Name is not None: etree.SubElement(act, "Notes").text = activity.Name elif activity.Notes is not None: From eb41af760273b8fa021e71b459a4b434921860bd Mon Sep 17 00:00:00 2001 From: Matthew Duggan Date: Mon, 9 Mar 2015 08:06:48 +0900 Subject: [PATCH 18/78] Fix all the unit tests: * Updated various apis * Added Laps to random activities * Removed no-longer-relevant distance calculation test --- tapiriik/testing/gpx.py | 4 +- tapiriik/testing/interchange.py | 100 +---------- tapiriik/testing/statistics.py | 4 +- tapiriik/testing/sync.py | 293 ++++++++++++++++++++------------ tapiriik/testing/testtools.py | 75 +++++--- 5 files changed, 248 insertions(+), 228 deletions(-) diff --git a/tapiriik/testing/gpx.py b/tapiriik/testing/gpx.py index 8bc5281b6..8fa103918 100644 --- a/tapiriik/testing/gpx.py +++ b/tapiriik/testing/gpx.py @@ -9,11 +9,11 @@ def test_constant_representation(self): svcA, other = TestTools.create_mock_services() svcA.SupportsHR = svcA.SupportsCadence = svcA.SupportsTemp = True svcA.SupportsPower = svcA.SupportsCalories = False - act = TestTools.create_random_activity(svcA, tz=True) + act = TestTools.create_random_activity(svcA, tz=True, withPauses=False) mid = GPXIO.Dump(act) - act2 = GPXIO.Parse(bytes(mid,"UTF-8")) + act2 = GPXIO.Parse(bytes(mid, "UTF-8")) act2.TZ = act.TZ # we need to fake this since local TZ isn't defined in GPX files, and TZ discovery will flail with random activities act2.AdjustTZ() act.Stats.Distance = act2.Stats.Distance = None # same here diff --git a/tapiriik/testing/interchange.py b/tapiriik/testing/interchange.py index b86272b97..1df0db820 100644 --- a/tapiriik/testing/interchange.py +++ b/tapiriik/testing/interchange.py @@ -1,14 +1,9 @@ -from unittest import TestCase - from tapiriik.testing.testtools import TestTools, TapiriikTestCase -from tapiriik.sync import Sync from tapiriik.services import Service -from tapiriik.services.interchange import Activity, ActivityType, Waypoint, WaypointType -from tapiriik.sync import Sync +from tapiriik.services.interchange import Activity, ActivityType from datetime import datetime, timedelta -import random class InterchangeTests(TapiriikTestCase): @@ -25,103 +20,24 @@ def test_round_precise_time(self): self.assertEqual(actA.UID, actB.UID) - def test_constant_representation(self): + def test_constant_representation_rk(self): ''' ensures that all services' API clients are consistent through a simulated download->upload cycle ''' # runkeeper rkSvc = Service.FromID("runkeeper") - act = TestTools.create_random_activity(rkSvc, rkSvc.SupportedActivities[0]) + act = TestTools.create_random_activity(rkSvc, rkSvc.SupportedActivities[0], withLaps=False) record = rkSvc._createUploadData(act) + record["has_path"] = act.GPS # RK helpfully adds a "has_path" entry if we have waypoints. returnedAct = rkSvc._populateActivity(record) act.Name = None # RK doesn't have a "name" field, so it's fudged into the notes, but not really rkSvc._populateActivityWaypoints(record, returnedAct) + # RK deliberately doesn't set timezone.. + returnedAct.EnsureTZ() self.assertActivitiesEqual(returnedAct, act) # can't test Strava well this way, the upload and download formats are entirely different - # endomondo - only waypoints at this point, the activity metadata is somewhat out-of-band - eSvc = Service.FromID("endomondo") - - act = TestTools.create_random_activity(eSvc, eSvc.SupportedActivities[0]) - oldWaypoints = act.Waypoints - self.assertEqual(oldWaypoints[0].Calories, None) - record = eSvc._createUploadData(act) - eSvc._populateActivityFromTrackData(act, record) - self.assertEqual(oldWaypoints, act.Waypoints) - - def test_duration_calculation(self): - ''' ensures that true-duration calculation is being reasonable ''' - act = TestTools.create_blank_activity() - act.StartTime = datetime.now() - act.EndTime = act.StartTime + timedelta(hours=3) - - # No waypoints - self.assertRaises(ValueError, act.GetTimerTime) - - # Too few waypoints - act.Waypoints = [Waypoint(timestamp=act.StartTime), Waypoint(timestamp=act.EndTime)] - self.assertRaises(ValueError, act.GetTimerTime) - - # straight-up calculation - act.EndTime = act.StartTime + timedelta(seconds=14) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=10)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=14))] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=14)) - - # pauses - act.EndTime = act.StartTime + timedelta(seconds=14) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=9), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=10), ptType=WaypointType.Resume), - Waypoint(timestamp=act.StartTime + timedelta(seconds=14))] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=10)) - - # laps - NO effect - act.EndTime = act.StartTime + timedelta(seconds=14) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6), ptType=WaypointType.Lap), - Waypoint(timestamp=act.StartTime + timedelta(seconds=9)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=10), ptType=WaypointType.Lap), - Waypoint(timestamp=act.StartTime + timedelta(seconds=14))] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=14)) - - # multiple pauses + ending after pause - act.EndTime = act.StartTime + timedelta(seconds=20) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=9), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=10), ptType=WaypointType.Resume), - Waypoint(timestamp=act.StartTime + timedelta(seconds=12)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=16)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=17), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=20), ptType=WaypointType.End)] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=13)) - - # implicit pauses (>1m5s) - act.EndTime = act.StartTime + timedelta(seconds=20) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=120)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=124)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=130))] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=16)) - - # mixed pauses - would this ever happen?? Either way, the explicit pause should override the implicit one and cause otherwise-ignored time to be counted - act.EndTime = act.StartTime + timedelta(seconds=23) - act.Waypoints = [Waypoint(timestamp=act.StartTime), - Waypoint(timestamp=act.StartTime + timedelta(seconds=2)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=6)), - Waypoint(timestamp=act.StartTime + timedelta(seconds=20), ptType=WaypointType.Pause), - Waypoint(timestamp=act.StartTime + timedelta(seconds=24), ptType=WaypointType.Resume), - Waypoint(timestamp=act.StartTime + timedelta(seconds=30))] - self.assertEqual(act.GetTimerTime(), timedelta(seconds=26)) + # can't test endomondo - upload data all constructed in upload function.. needs refactor? + def test_activity_specificity_resolution(self): # Mountain biking is more specific than just cycling diff --git a/tapiriik/testing/statistics.py b/tapiriik/testing/statistics.py index 38f8775fd..898ab5ef9 100644 --- a/tapiriik/testing/statistics.py +++ b/tapiriik/testing/statistics.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from tapiriik.testing.testtools import TapiriikTestCase from tapiriik.services.interchange import ActivityStatistic, ActivityStatisticUnit @@ -152,4 +150,4 @@ def test_stat_coalesce_multi_missingmixed_multivalued(self): self.assertEqual(stat1.Min, 3) self.assertEqual(stat1.Max, 2) self.assertEqual(stat1.Gain, 3) - self.assertEqual(stat1.Loss, 4) \ No newline at end of file + self.assertEqual(stat1.Loss, 4) diff --git a/tapiriik/testing/sync.py b/tapiriik/testing/sync.py index ffc3a4387..8096505fb 100644 --- a/tapiriik/testing/sync.py +++ b/tapiriik/testing/sync.py @@ -1,13 +1,13 @@ from tapiriik.testing.testtools import TestTools, TapiriikTestCase -from tapiriik.sync import Sync -from tapiriik.services import Service +from tapiriik.sync import SynchronizationTask +from tapiriik.sync.activity_record import ActivityRecord +from tapiriik.services import UserException, UserExceptionType from tapiriik.services.api import APIExcludeActivity from tapiriik.services.interchange import Activity, ActivityType from tapiriik.auth import User from datetime import datetime, timedelta, tzinfo -import random import pytz import copy @@ -44,14 +44,13 @@ def test_svc_level_dupe(self): actA.CalculateUID() actB.CalculateUID() - activities = [] + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) - - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) - - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) def test_svc_level_dupe_tz_uniform(self): ''' check that service-level duplicate activities with the same TZs are caught ''' @@ -68,11 +67,12 @@ def test_svc_level_dupe_tz_uniform(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) def test_svc_level_dupe_tz_nonuniform(self): ''' check that service-level duplicate activities with non-uniform TZs are caught ''' @@ -89,11 +89,12 @@ def test_svc_level_dupe_tz_nonuniform(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) def test_svc_level_dupe_tz_irregular(self): ''' check that service-level duplicate activities with irregular TZs are caught ''' @@ -110,11 +111,12 @@ def test_svc_level_dupe_tz_irregular(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) def test_svc_level_dupe_time_leeway(self): ''' check that service-level duplicate activities within the defined time leeway are caught ''' @@ -133,9 +135,10 @@ def test_svc_level_dupe_time_leeway(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) self.assertIn(actA.UID, actA.UIDs) self.assertIn(actB.UID, actA.UIDs) @@ -148,12 +151,13 @@ def test_svc_level_dupe_time_leeway(self): recA.SynchronizedActivities = [actA.UID] recB.SynchronizedActivities = [actB.UID] - recipientServicesA = Sync._determineRecipientServices(actA, [recA, recB]) - recipientServicesB = Sync._determineRecipientServices(actB, [recA, recB]) + s._serviceConnections = [recA, recB] + recipientServicesA = s._determineRecipientServices(actA) + recipientServicesB = s._determineRecipientServices(actB) self.assertEqual(len(recipientServicesA), 0) self.assertEqual(len(recipientServicesB), 0) - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) def test_svc_supported_activity_types(self): ''' check that only activities are only sent to services which support them ''' @@ -167,38 +171,42 @@ def test_svc_supported_activity_types(self): actA.StartTime = datetime(1, 2, 3, 4, 5, 6, 7) actA.ServiceDataCollection = TestTools.create_mock_servicedatacollection(svcA, record=recA) actA.Type = svcA.SupportedActivities[0] + actA.CalculateUID() + actA.UIDs = set([actA.UID]) + actA.Record = ActivityRecord.FromActivity(actA) + actB = Activity() actB.StartTime = datetime(5, 6, 7, 8, 9, 10, 11) actB.ServiceDataCollection = TestTools.create_mock_servicedatacollection(svcB, record=recB) actB.Type = [x for x in svcB.SupportedActivities if x != actA.Type][0] - - actA.CalculateUID() actB.CalculateUID() + actB.UIDs = set([actB.UID]) + actB.Record = ActivityRecord.FromActivity(actB) - allConns = [recA, recB] - - activities = [] - Sync._accumulateActivities(recA, [actA], activities) - Sync._accumulateActivities(recB, [actB], activities) + s = SynchronizationTask(None) + s._serviceConnections = [recA, recB] + s._activities = [] + s._accumulateActivities(recA, [actA]) + s._accumulateActivities(recB, [actB]) - syncToA = Sync._determineRecipientServices(actA, allConns) - syncToB = Sync._determineRecipientServices(actB, allConns) + syncToA = s._determineRecipientServices(actA) + syncToB = s._determineRecipientServices(actB) self.assertEqual(len(syncToA), 0) self.assertEqual(len(syncToB), 0) svcB.SupportedActivities = svcA.SupportedActivities - syncToA = Sync._determineRecipientServices(actA, allConns) - syncToB = Sync._determineRecipientServices(actB, allConns) + syncToA = s._determineRecipientServices(actA) + syncToB = s._determineRecipientServices(actB) self.assertEqual(len(syncToA), 1) self.assertEqual(len(syncToB), 0) svcB.SupportedActivities = svcA.SupportedActivities = [ActivityType.CrossCountrySkiing, ActivityType.Cycling] - syncToA = Sync._determineRecipientServices(actA, allConns) - syncToB = Sync._determineRecipientServices(actB, allConns) + syncToA = s._determineRecipientServices(actA) + syncToB = s._determineRecipientServices(actB) self.assertEqual(len(syncToA), 1) self.assertEqual(len(syncToB), 1) @@ -207,10 +215,12 @@ def test_accumulate_exclusions(self): svcA, svcB = TestTools.create_mock_services() recA = TestTools.create_mock_svc_record(svcA) - exclusionstore = {recA._id: {}} # regular + s = SynchronizationTask(None) + s._syncExclusions = {recA._id: {}} exc = APIExcludeActivity("Messag!e", activity_id=3.14) - Sync._accumulateExclusions(recA, exc, exclusionstore) + s._accumulateExclusions(recA, exc) + exclusionstore = s._syncExclusions self.assertTrue("3_14" in exclusionstore[recA._id]) self.assertEqual(exclusionstore[recA._id]["3_14"]["Message"], "Messag!e") self.assertEqual(exclusionstore[recA._id]["3_14"]["Activity"], None) @@ -221,7 +231,10 @@ def test_accumulate_exclusions(self): act = TestTools.create_blank_activity(svcA) act.UID = "3_14" # meh exc = APIExcludeActivity("Messag!e2", activity_id=42, permanent=False, activity=act) - Sync._accumulateExclusions(recA, exc, exclusionstore) + s = SynchronizationTask(None) + s._syncExclusions = {recA._id: {}} + s._accumulateExclusions(recA, exc) + exclusionstore = s._syncExclusions self.assertTrue("3_14" in exclusionstore[recA._id]) self.assertEqual(exclusionstore[recA._id]["3_14"]["Message"], "Messag!e2") self.assertNotEqual(exclusionstore[recA._id]["3_14"]["Activity"], None) # Who knows what the string format will be down the road? @@ -231,14 +244,17 @@ def test_accumulate_exclusions(self): # multiple, retaining existing exc2 = APIExcludeActivity("INM", activity_id=13) exc3 = APIExcludeActivity("FNIM", activity_id=37) - Sync._accumulateExclusions(recA, [exc2, exc3], exclusionstore) + s._accumulateExclusions(recA, [exc2, exc3]) + exclusionstore = s._syncExclusions self.assertTrue("3_14" in exclusionstore[recA._id]) self.assertTrue("37" in exclusionstore[recA._id]) self.assertTrue("13" in exclusionstore[recA._id]) # don't allow with no identifiers exc4 = APIExcludeActivity("nooooo") - self.assertRaises(ValueError, Sync._accumulateExclusions, recA, [exc4], exclusionstore) + s = SynchronizationTask(None) + s._syncExclusions = {} + self.assertRaises(ValueError, s._accumulateExclusions, recA, [exc4]) def test_activity_deduplicate_normaltz(self): ''' ensure that we can't deduplicate activities with non-pytz timezones ''' @@ -257,9 +273,10 @@ def test_activity_deduplicate_normaltz(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) - self.assertRaises(ValueError, Sync._accumulateActivities, recA, [copy.deepcopy(actA)], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + self.assertRaises(ValueError, s._accumulateActivities, recA, [copy.deepcopy(actA)]) def test_activity_deduplicate_tzerror(self): ''' Test that probably-duplicate activities with starttimes like 09:12:22 and 15:12:22 (on the same day) are recognized as one ''' @@ -276,27 +293,36 @@ def test_activity_deduplicate_tzerror(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + s._accumulateActivities(recA, [copy.deepcopy(actA)]) - self.assertEqual(len(activities), 1) + self.assertEqual(len(s._activities), 1) - # Ensure that it is an exact match + # Ensure that it is deduplicated on non-exact match actB.StartTime = actA.StartTime.replace(tzinfo=pytz.timezone("America/Denver")) + timedelta(hours=5, seconds=1) - activities = [] - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + s._accumulateActivities(recA, [copy.deepcopy(actA)]) - self.assertEqual(len(activities), 2) + self.assertEqual(len(s._activities), 1) + + # Ensure that it is *not* deduplicated when it really doesn't match + actB.StartTime = actA.StartTime.replace(tzinfo=pytz.timezone("America/Denver")) + timedelta(hours=5, minutes=7) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + s._accumulateActivities(recA, [copy.deepcopy(actA)]) + + self.assertEqual(len(s._activities), 2) # Ensure that overly large differences >38hr - not possible via TZ differences & shamefully bad import/export code on the part of some services - are not deduplicated actB.StartTime = actA.StartTime.replace(tzinfo=pytz.timezone("America/Denver")) + timedelta(hours=50) - activities = [] - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + s._accumulateActivities(recA, [copy.deepcopy(actA)]) - self.assertEqual(len(activities), 2) + self.assertEqual(len(s._activities), 2) def test_activity_coalesce(self): ''' ensure that activity data is getting coalesced by _accumulateActivities ''' @@ -314,51 +340,61 @@ def test_activity_coalesce(self): actA.CalculateUID() actB.CalculateUID() - activities = [] - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) + s = SynchronizationTask(None) + s._activities = [] + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + s._accumulateActivities(recA, [copy.deepcopy(actA)]) - self.assertEqual(len(activities), 1) - act = activities[0] + self.assertEqual(len(s._activities), 1) + act = s._activities[0] self.assertEqual(act.StartTime, actA.StartTime) self.assertEqual(act.EndTime, actA.EndTime) self.assertEqual(act.EndTime.tzinfo, actA.StartTime.tzinfo) self.assertEqual(act.StartTime.tzinfo, actA.StartTime.tzinfo) - self.assertEqual(act.Waypoints, actA.Waypoints) + self.assertLapsListsEqual(act.Laps, actA.Laps) self.assertTrue(act.Private) # Most restrictive setting self.assertEqual(act.Name, actB.Name) # The first activity takes priority. self.assertEqual(act.Type, actB.Type) # Same here. self.assertTrue(list(actB.ServiceDataCollection.keys())[0] in act.ServiceDataCollection) self.assertTrue(list(actA.ServiceDataCollection.keys())[0] in act.ServiceDataCollection) - activities = [] - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) + s._activities = [] + s._accumulateActivities(recA, [copy.deepcopy(actA)]) + s._accumulateActivities(recB, [copy.deepcopy(actB)]) - self.assertEqual(len(activities), 1) - act = activities[0] + self.assertEqual(len(s._activities), 1) + act = s._activities[0] self.assertEqual(act.StartTime, actA.StartTime) self.assertEqual(act.EndTime, actA.EndTime) self.assertEqual(act.EndTime.tzinfo, actA.StartTime.tzinfo) self.assertEqual(act.StartTime.tzinfo, actA.StartTime.tzinfo) - self.assertEqual(act.Waypoints, actA.Waypoints) + self.assertLapsListsEqual(act.Laps, actA.Laps) self.assertEqual(act.Name, actA.Name) # The first activity takes priority. self.assertEqual(act.Type, actB.Type) # Exception: ActivityType.Other does not take priority self.assertTrue(list(actB.ServiceDataCollection.keys())[0] in act.ServiceDataCollection) self.assertTrue(list(actA.ServiceDataCollection.keys())[0] in act.ServiceDataCollection) - actA.Type = ActivityType.CrossCountrySkiing - activities = [] - Sync._accumulateActivities(recA, [copy.deepcopy(actA)], activities) - Sync._accumulateActivities(recB, [copy.deepcopy(actB)], activities) + # Similar activities should be coalesced (Hiking, Walking..).. + actA.Type = ActivityType.Hiking + s._activities = [] + s._accumulateActivities(recA, [copy.deepcopy(actA)]) + s._accumulateActivities(recB, [copy.deepcopy(actB)]) - self.assertEqual(len(activities), 1) - act = activities[0] + self.assertEqual(len(s._activities), 1) + act = s._activities[0] self.assertEqual(act.Type, actA.Type) # Here, it will take priority. + # Dissimilar should not.. + actA.Type = ActivityType.CrossCountrySkiing + s._activities = [] + s._accumulateActivities(recA, [copy.deepcopy(actA)]) + s._accumulateActivities(recB, [copy.deepcopy(actB)]) + self.assertEqual(len(s._activities), 2) + act = s._activities[0] + self.assertEqual(act.Type, actA.Type) def test_eligibility_excluded(self): user = TestTools.create_mock_user() @@ -367,8 +403,13 @@ def test_eligibility_excluded(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) recipientServices = [recA, recB] - excludedServices = [recA] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {recA._id: UserException(UserExceptionType.Private)} + s.user = user + s._serviceConnections = recipientServices + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recB in eligible) self.assertTrue(recA not in eligible) @@ -381,8 +422,13 @@ def test_eligibility_config(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) recipientServices = [recA, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = recipientServices + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recB in eligible) self.assertTrue(recA not in eligible) @@ -393,10 +439,15 @@ def test_eligibility_flowexception(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recA) act.Origin = recA + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) User.SetFlowException(user, recA, recB, flowToTarget=False) recipientServices = [recA, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = recipientServices + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA in eligible) self.assertFalse(recB in eligible) @@ -412,24 +463,33 @@ def test_eligibility_flowexception_shortcircuit(self): # Behaviour with known origin and no override set act.Origin = recA + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) recipientServices = [recC, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB, recC], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = [recA, recB, recC] + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB in eligible) self.assertTrue(recC not in eligible) # Enable alternate routing - recB.SetConfiguration({"allow_activity_flow_exception_bypass_via_self":True}, no_save=True) + # FIXME: This setting doesn't seem to be used anywhere any more?? Test disabled at the end.. + recB.SetConfiguration({"allow_activity_flow_exception_bypass_via_self": True}, no_save=True) self.assertTrue(recB.GetConfiguration()["allow_activity_flow_exception_bypass_via_self"]) # We should now be able to arrive at recC via recB act.Origin = recA + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) recipientServices = [recC, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB, recC], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s._excludedServices = {} + s._serviceConnections = [recA, recB, recC] + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB in eligible) - self.assertTrue(recC in eligible) + # self.assertTrue(recC in eligible) def test_eligibility_flowexception_reverse(self): user = TestTools.create_mock_user() @@ -438,10 +498,15 @@ def test_eligibility_flowexception_reverse(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) act.Origin = recB + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) User.SetFlowException(user, recA, recB, flowToSource=False) recipientServices = [recA, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = recipientServices + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertFalse(recA in eligible) self.assertTrue(recB in eligible) @@ -452,16 +517,21 @@ def test_eligibility_flowexception_both(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) act.Origin = recB + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) User.SetFlowException(user, recA, recB, flowToSource=False, flowToTarget=False) recipientServices = [recA, recB] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = recipientServices + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertFalse(recA in eligible) self.assertTrue(recB in eligible) act.Origin = recA act.ServiceDataCollection = TestTools.create_mock_servicedatacollection(svcA, record=recA) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=recipientServices, recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA in eligible) self.assertFalse(recB in eligible) @@ -472,17 +542,22 @@ def test_eligibility_flowexception_none(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) act.Origin = recB + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) User.SetFlowException(user, recA, recB, flowToSource=False, flowToTarget=False) recipientServices = [recA] - excludedServices = [] - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = [recA, recB] + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB not in eligible) recipientServices = [recB] act.Origin = recA act.ServiceDataCollection = TestTools.create_mock_servicedatacollection(svcA, record=recA) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB not in eligible) @@ -493,13 +568,17 @@ def test_eligibility_flowexception_change(self): recB = TestTools.create_mock_svc_record(svcB) act = TestTools.create_blank_activity(svcA, record=recB) act.Origin = recB + act.UIDs = set([act.UID]) + act.Record = ActivityRecord.FromActivity(act) recipientServices = [recA] - excludedServices = [] - + s = SynchronizationTask(None) + s._excludedServices = {} + s.user = user + s._serviceConnections = [recA, recB] User.SetFlowException(user, recA, recB, flowToSource=False, flowToTarget=True) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB not in eligible) @@ -507,21 +586,21 @@ def test_eligibility_flowexception_change(self): act.Origin = recA act.ServiceDataCollection = TestTools.create_mock_servicedatacollection(svcA, record=recA) User.SetFlowException(user, recA, recB, flowToSource=True, flowToTarget=False) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB not in eligible) User.SetFlowException(user, recA, recB, flowToSource=False, flowToTarget=False) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA not in eligible) self.assertTrue(recB not in eligible) recipientServices = [recA, recB] User.SetFlowException(user, recA, recB, flowToSource=True, flowToTarget=True) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA in eligible) self.assertTrue(recB in eligible) - eligible = Sync._determineEligibleRecipientServices(activity=act, connectedServices=[recA, recB], recipientServices=recipientServices, excludedServices=excludedServices, user=user) + eligible = s._determineEligibleRecipientServices(act, recipientServices) self.assertTrue(recA in eligible) - self.assertTrue(recB in eligible) \ No newline at end of file + self.assertTrue(recB in eligible) diff --git a/tapiriik/testing/testtools.py b/tapiriik/testing/testtools.py index 57fb21d8c..78965021d 100644 --- a/tapiriik/testing/testtools.py +++ b/tapiriik/testing/testtools.py @@ -1,7 +1,7 @@ from unittest import TestCase from tapiriik.services import Service, ServiceRecord, ServiceBase -from tapiriik.services.interchange import Activity, ActivityType, ActivityStatistic, ActivityStatisticUnit, Waypoint, WaypointType, Location +from tapiriik.services.interchange import Activity, ActivityType, ActivityStatistic, ActivityStatisticUnit, Waypoint, WaypointType, Lap, Location from datetime import datetime, timedelta import random @@ -30,31 +30,43 @@ def assertActivitiesEqual(self, a, b): self.assertEqual(a.Type, b.Type) self.assertEqual(a.Stats.Distance, b.Stats.Distance) self.assertEqual(a.Name, b.Name) - self.assertEqual(len(a.Waypoints), len(b.Waypoints)) - for idx in range(0, len(a.Waypoints) - 1): - self.assertEqual(a.Waypoints[idx].Timestamp.astimezone(pytz.utc), b.Waypoints[idx].Timestamp.astimezone(pytz.utc)) - self.assertEqual(a.Waypoints[idx].Location.Latitude, b.Waypoints[idx].Location.Latitude) - self.assertEqual(a.Waypoints[idx].Location.Longitude, b.Waypoints[idx].Location.Longitude) - self.assertEqual(a.Waypoints[idx].Location.Altitude, b.Waypoints[idx].Location.Altitude) - self.assertEqual(a.Waypoints[idx].Type, b.Waypoints[idx].Type) - self.assertEqual(a.Waypoints[idx].HR, b.Waypoints[idx].HR) - self.assertEqual(a.Waypoints[idx].Calories, b.Waypoints[idx].Calories) - self.assertEqual(a.Waypoints[idx].Power, b.Waypoints[idx].Power) - self.assertEqual(a.Waypoints[idx].Cadence, b.Waypoints[idx].Cadence) - self.assertEqual(a.Waypoints[idx].Temp, b.Waypoints[idx].Temp) - - self.assertEqual(a.Waypoints[idx].Location, b.Waypoints[idx].Location) - self.assertEqual(a.Waypoints[idx], b.Waypoints[idx]) - self.assertEqual(a, b) + self.assertLapsListsEqual(a.Laps, b.Laps) + + def assertLapsListsEqual(self, lapsa, lapsb): + self.assertEqual(len(lapsa), len(lapsb)) + for idx in range(len(lapsa)): + la = lapsa[idx] + lb = lapsb[idx] + self.assertLapsEqual(la, lb) + + def assertLapsEqual(self, la, lb): + self.assertEqual(la.StartTime, lb.StartTime) + self.assertEqual(la.EndTime, lb.EndTime) + self.assertEqual(len(la.Waypoints), len(lb.Waypoints)) + for idx in range(len(la.Waypoints)): + wpa = la.Waypoints[idx] + wpb = lb.Waypoints[idx] + self.assertEqual(wpa.Timestamp.astimezone(pytz.utc), wpb.Timestamp.astimezone(pytz.utc)) + self.assertEqual(wpa.Location.Latitude, wpb.Location.Latitude) + self.assertEqual(wpa.Location.Longitude, wpb.Location.Longitude) + self.assertEqual(wpa.Location.Altitude, wpb.Location.Altitude) + self.assertEqual(wpa.Type, wpb.Type) + self.assertEqual(wpa.HR, wpb.HR) + self.assertEqual(wpa.Calories, wpb.Calories) + self.assertEqual(wpa.Power, wpb.Power) + self.assertEqual(wpa.Cadence, wpb.Cadence) + self.assertEqual(wpa.Temp, wpb.Temp) + self.assertEqual(wpa.Location, wpb.Location) + self.assertEqual(wpa, wpb) class TestTools: def create_mock_user(): - db.test.insert({"asd":"asdd"}) + db.test.insert({"asd": "asdd"}) return {"_id": str(random.randint(1, 1000))} def create_mock_svc_record(svc): - return ServiceRecord({"Service": svc.ID, "_id": str(random.randint(1, 1000)), "ExternalID": str(random.randint(1,1000))}) + return ServiceRecord({"Service": svc.ID, "_id": str(random.randint(1, 1000)), "ExternalID": str(random.randint(1, 1000))}) def create_mock_servicedata(svc, record=None): return {"ActivityID": random.randint(1, 1000), "Connection": record} @@ -74,7 +86,7 @@ def create_blank_activity(svc=None, actType=ActivityType.Other, record=None): act.CalculateUID() return act - def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, record=None): + def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, record=None, withPauses=True, withLaps=True): ''' creates completely random activity with valid waypoints and data ''' act = TestTools.create_blank_activity(svc, actType, record=record) @@ -84,7 +96,7 @@ def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, recor elif tz is not False: act.TZ = tz - if len(act.Waypoints) > 0: + if act.CountTotalWaypoints() > 0: raise ValueError("Waypoint list already populated") # this is entirely random in case the testing account already has events in it (API doesn't support delete, etc) act.StartTime = datetime(random.randint(2000, 2020), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)) @@ -99,6 +111,8 @@ def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, recor paused = False waypointTime = act.StartTime backToBackPauses = False + act.Laps = [] + lap = Lap(startTime=act.StartTime) while waypointTime < act.EndTime: wp = Waypoint() if waypointTime == act.StartTime: @@ -119,7 +133,7 @@ def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, recor if svc.SupportsTemp: wp.Temp = float(random.randint(0, 100)) - if (random.randint(40, 50) == 42 or backToBackPauses) and not paused: # pause quite often + if withPauses and (random.randint(40, 50) == 42 or backToBackPauses) and not paused: # pause quite often wp.Type = WaypointType.Pause paused = True @@ -130,12 +144,25 @@ def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, recor waypointTime += timedelta(0, int(random.random() + 9.5)) # 10ish seconds + lap.Waypoints.append(wp) if waypointTime > act.EndTime: wp.Timestamp = act.EndTime wp.Type = WaypointType.End - act.Waypoints.append(wp) - if len(act.Waypoints) == 0: + elif withLaps and wp.Timestamp < act.EndTime and random.randint(40, 60) == 42: + # occasionally start new laps + lap.EndTime = wp.Timestamp + act.Laps.append(lap) + lap = Lap(startTime=waypointTime) + + # Final lap + lap.EndTime = act.EndTime + act.Laps.append(lap) + if act.CountTotalWaypoints() == 0: raise ValueError("No waypoints populated") + + act.CalculateUID() + act.EnsureTZ() + return act def create_mock_service(id): From cea75ce3d7fa8477e3d53b600ef13e7c4c738648 Mon Sep 17 00:00:00 2001 From: Matthew Duggan Date: Mon, 9 Mar 2015 08:13:07 +0900 Subject: [PATCH 19/78] Add a simple travis.yml. --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..793fef0f3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: python +python: + - "3.3" +services: + - mongodb + - redis-server + - rabbitmq +install: + - "pip install -r requirements.txt" + - "cp tapiriik/local_settings.py.example tapiriik/local_settings.py" +script: "python runtests.py" From 53ad7b2eb36e9e81c92a9de9980507fa2db3deb3 Mon Sep 17 00:00:00 2001 From: Dimitrios Kanellopoulos Date: Tue, 10 Mar 2015 10:48:03 +0100 Subject: [PATCH 20/78] Fix issue #108 On Ubuntu 14 guide this endopoint crashed the server --- tapiriik/services/GarminConnect/garminconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index 32ade06b7..d60965b42 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -114,7 +114,7 @@ class GarminConnectService(ServiceBase): def __init__(self): cachedHierarchy = cachedb.gc_type_hierarchy.find_one() if not cachedHierarchy: - rawHierarchy = requests.get("https://connect.garmin.com/modern/proxy/activity-service-1.2/json/activity_types", headers=self._obligatory_headers).text + rawHierarchy = requests.get("https://connect.garmin.com/proxy/activity-service-1.2/json/activity_types", headers=self._obligatory_headers).text self._activityHierarchy = json.loads(rawHierarchy)["dictionary"] cachedb.gc_type_hierarchy.insert({"Hierarchy": rawHierarchy}) else: From 3e34fe55a98c1d4bff43d2ca54bfedb4e7eea9fb Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Tue, 10 Mar 2015 11:39:20 -0400 Subject: [PATCH 21/78] Obligatory travisCI badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e155cfde6..cd47dea01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ tapiriik keeps your fitness in sync ======== +[![Build Status](https://travis-ci.org/cpfair/tapiriik.svg?branch=master)](https://travis-ci.org/cpfair/tapiriik) ## Looking to run tapiriik locally? From 4c29d0418b2f45e37fa0de4dd855fcb109a078fc Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 16 Mar 2015 18:02:05 -0400 Subject: [PATCH 22/78] Move webhook sync triggering out of the web worker --- sync_remote_triggers.py | 32 ++++++++++++++++++++++++++++++++ tapiriik/web/views/sync.py | 9 ++------- 2 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 sync_remote_triggers.py diff --git a/sync_remote_triggers.py b/sync_remote_triggers.py new file mode 100644 index 000000000..88640875f --- /dev/null +++ b/sync_remote_triggers.py @@ -0,0 +1,32 @@ +from tapiriik.database import db, close_connections +from tapiriik.settings import RABBITMQ_BROKER_URL, MONGO_FULL_WRITE_CONCERN +from datetime import datetime +from celery import Celery +from celery.signals import worker_shutdown + +class _celeryConfig: + CELERY_ROUTES = { + "sync_remote_triggers.trigger_remote": {"queue": "tapiriik-remote-trigger"} + } + CELERYD_CONCURRENCY = 1 # Otherwise the GC rate limiting breaks since file locking is per-process. + CELERYD_PREFETCH_MULTIPLIER = 1 # The message queue could use some exercise. + +celery_app = Celery('sync_remote_triggers', broker=RABBITMQ_BROKER_URL) +celery_app.config_from_object(_celeryConfig()) + +@worker_shutdown.connect +def celery_shutdown(): + close_connections() + +@celery_app.task() +def trigger_remote(service_id, affected_connection_external_ids): + from tapiriik.auth import User + from tapiriik.services import Service + svc = Service.FromID(service_id) + db.connections.update({"Service": svc.ID, "ExternalID": {"$in": affected_connection_external_ids}}, {"$set":{"TriggerPartialSync": True, "TriggerPartialSyncTimestamp": datetime.utcnow()}}, multi=True, w=MONGO_FULL_WRITE_CONCERN) + affected_connection_ids = db.connections.find({"Service": svc.ID, "ExternalID": {"$in": affected_connection_external_ids}}, {"_id": 1}) + affected_connection_ids = [x["_id"] for x in affected_connection_ids] + trigger_users_query = User.PaidUserMongoQuery() + trigger_users_query.update({"ConnectedServices.ID": {"$in": affected_connection_ids}}) + trigger_users_query.update({"Config.suppress_auto_sync": {"$ne": True}}) + db.users.update(trigger_users_query, {"$set": {"NextSynchronization": datetime.utcnow()}}, multi=True) # It would be nicer to use the Sync.Schedule... method, but I want to cleanly do this in bulk diff --git a/tapiriik/web/views/sync.py b/tapiriik/web/views/sync.py index fac2bfc2d..b18c1e897 100644 --- a/tapiriik/web/views/sync.py +++ b/tapiriik/web/views/sync.py @@ -96,12 +96,7 @@ def sync_clear_errorgroup(req, service, group): def sync_trigger_partial_sync_callback(req, service): svc = Service.FromID(service) affected_connection_external_ids = svc.ExternalIDsForPartialSyncTrigger(req) - db.connections.update({"Service": svc.ID, "ExternalID": {"$in": affected_connection_external_ids}}, {"$set":{"TriggerPartialSync": True, "TriggerPartialSyncTimestamp": datetime.utcnow()}}, multi=True, w=MONGO_FULL_WRITE_CONCERN) - affected_connection_ids = db.connections.find({"Service": svc.ID, "ExternalID": {"$in": affected_connection_external_ids}}, {"_id": 1}) - affected_connection_ids = [x["_id"] for x in affected_connection_ids] - trigger_users_query = User.PaidUserMongoQuery() - trigger_users_query.update({"ConnectedServices.ID": {"$in": affected_connection_ids}}) - trigger_users_query.update({"Config.suppress_auto_sync": {"$ne": True}}) - db.users.update(trigger_users_query, {"$set": {"NextSynchronization": datetime.utcnow()}}, multi=True) # It would be nicer to use the Sync.Schedule... method, but I want to cleanly do this in bulk + + trigger_remote.apply_async(args=[service, affected_connection_external_ids]) return HttpResponse(status=204) From 37de63e9ff5ce1025aaecd0d7ae09d8b0ec29f40 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 16 Mar 2015 19:01:50 -0400 Subject: [PATCH 23/78] More scheduler debug output --- sync_scheduler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sync_scheduler.py b/sync_scheduler.py index 251be0b80..58e2c691f 100644 --- a/sync_scheduler.py +++ b/sync_scheduler.py @@ -27,7 +27,9 @@ read_preference=ReadPreference.PRIMARY )) scheduled_ids = [x["_id"] for x in users] + print("Found %d users at %s" % (len(scheduled_ids), datetime.utcnow())) db.users.update({"_id": {"$in": scheduled_ids}}, {"$set": {"QueuedAt": queueing_at, "QueuedGeneration": generation}, "$unset": {"NextSynchronization": True}}, multi=True, w=MONGO_FULL_WRITE_CONCERN) + print("Marked %d users as queued at %s" % (len(scheduled_ids), datetime.utcnow())) for user in users: producer.publish({"user_id": str(user["_id"]), "generation": generation}, routing_key=user["SynchronizationHostRestriction"] if "SynchronizationHostRestriction" in user and user["SynchronizationHostRestriction"] else "") print("Scheduled %d users at %s" % (len(scheduled_ids), datetime.utcnow())) From 352281da443838e915f34474f5deecba364fd443 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 16 Mar 2015 19:14:26 -0400 Subject: [PATCH 24/78] Missing import for earlier commit, oops --- tapiriik/web/views/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tapiriik/web/views/sync.py b/tapiriik/web/views/sync.py index b18c1e897..4d811cb0e 100644 --- a/tapiriik/web/views/sync.py +++ b/tapiriik/web/views/sync.py @@ -94,6 +94,7 @@ def sync_clear_errorgroup(req, service, group): @csrf_exempt @require_POST def sync_trigger_partial_sync_callback(req, service): + from sync_remote_triggers import trigger_remote svc = Service.FromID(service) affected_connection_external_ids = svc.ExternalIDsForPartialSyncTrigger(req) From 025ad6d4289cd4ee61a41ba5cf8988fc472b9405 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sat, 21 Mar 2015 10:15:53 -0400 Subject: [PATCH 25/78] Store session cache data in shared redis instance --- .../services/GarminConnect/garminconnect.py | 2 +- tapiriik/services/Motivato/motivato.py | 2 +- tapiriik/services/NikePlus/nikeplus.py | 2 +- tapiriik/services/RideWithGPS/rwgps.py | 2 +- tapiriik/services/SportTracks/sporttracks.py | 2 +- tapiriik/services/sessioncache.py | 44 +++++++------------ 6 files changed, 22 insertions(+), 32 deletions(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index d60965b42..a9c99ec01 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -84,7 +84,7 @@ class GarminConnectService(ServiceBase): SupportsActivityDeletion = True - _sessionCache = SessionCache(lifetime=timedelta(minutes=30), freshen_on_get=True) + _sessionCache = SessionCache("garminconnect", lifetime=timedelta(minutes=120), freshen_on_get=True) _unitMap = { "mph": ActivityStatisticUnit.MilesPerHour, diff --git a/tapiriik/services/Motivato/motivato.py b/tapiriik/services/Motivato/motivato.py index b83750e2d..b75ddd351 100644 --- a/tapiriik/services/Motivato/motivato.py +++ b/tapiriik/services/Motivato/motivato.py @@ -52,7 +52,7 @@ class MotivatoService(ServiceBase): SupportedActivities = list(_reverseActivityMappings.values()) - _sessionCache = SessionCache(lifetime=timedelta(minutes=30), freshen_on_get=True) + _sessionCache = SessionCache("motivato", lifetime=timedelta(minutes=30), freshen_on_get=True) _obligatory_headers = { "Referer": "https://sync.tapiriik.com" } diff --git a/tapiriik/services/NikePlus/nikeplus.py b/tapiriik/services/NikePlus/nikeplus.py index 6cdf50c79..a312e174f 100644 --- a/tapiriik/services/NikePlus/nikeplus.py +++ b/tapiriik/services/NikePlus/nikeplus.py @@ -72,7 +72,7 @@ class NikePlusService(ServiceBase): SupportedActivities = list(_reverseActivityMappings.values()) - _sessionCache = SessionCache(lifetime=timedelta(minutes=45), freshen_on_get=False) + _sessionCache = SessionCache("nikeplus", lifetime=timedelta(minutes=45), freshen_on_get=False) _obligatoryHeaders = { "User-Agent": "NPConnect", diff --git a/tapiriik/services/RideWithGPS/rwgps.py b/tapiriik/services/RideWithGPS/rwgps.py index 73add532d..2310c2e4c 100644 --- a/tapiriik/services/RideWithGPS/rwgps.py +++ b/tapiriik/services/RideWithGPS/rwgps.py @@ -40,7 +40,7 @@ class RideWithGPSService(ServiceBase): SupportsHR = SupportsCadence = True - _sessionCache = SessionCache(lifetime=timedelta(minutes=30), freshen_on_get=True) + _sessionCache = SessionCache("rwgps", lifetime=timedelta(minutes=30), freshen_on_get=True) def _add_auth_params(self, params=None, record=None): """ diff --git a/tapiriik/services/SportTracks/sporttracks.py b/tapiriik/services/SportTracks/sporttracks.py index c7a5a9162..a22dcf9e9 100644 --- a/tapiriik/services/SportTracks/sporttracks.py +++ b/tapiriik/services/SportTracks/sporttracks.py @@ -137,7 +137,7 @@ class SportTracksService(ServiceBase): SupportedActivities = list(_reverseActivityMappings.keys()) - _tokenCache = SessionCache(lifetime=timedelta(minutes=115), freshen_on_get=False) + _tokenCache = SessionCache("sporttracks", lifetime=timedelta(minutes=115), freshen_on_get=False) def WebInit(self): self.UserAuthorizationURL = "https://api.sporttracks.mobi/oauth2/authorize?response_type=code&client_id=%s&state=mobi_api" % SPORTTRACKS_CLIENT_ID diff --git a/tapiriik/services/sessioncache.py b/tapiriik/services/sessioncache.py index 15208a993..c1738f277 100644 --- a/tapiriik/services/sessioncache.py +++ b/tapiriik/services/sessioncache.py @@ -1,36 +1,26 @@ from datetime import datetime +from tapiriik.database import redis +import pickle class SessionCache: - def __init__(self, lifetime, freshen_on_get=False): + def __init__(self, scope, lifetime, freshen_on_get=False): self._lifetime = lifetime self._autorefresh = freshen_on_get - self._cache = {} + self._scope = scope + self._cacheKey = "sessioncache:%s:%s" % (self._scope, "%s") def Get(self, pk, freshen=False): - if pk not in self._cache: - return - record = self._cache[pk] - if record.Expired(): - del self._cache[pk] - return None - if self._autorefresh or freshen: - record.Refresh() - return record.Get() + res = redis.get(self._cacheKey % pk) + if res: + try: + res = pickle.loads(res) + except pickle.UnpicklingError: + redis.delete(self._cacheKey % pk) + res = None + else: + if self._autorefresh or freshen: + redis.expire(self._cacheKey % pk, self._lifetime) + return res def Set(self, pk, value): - self._cache[pk] = SessionCacheRecord(value, self._lifetime) - -class SessionCacheRecord: - def __init__(self, data, lifetime): - self._value = data - self._lifetime = lifetime - self.Refresh() - - def Expired(self): - return self._timestamp < datetime.utcnow() - self._lifetime - - def Refresh(self): - self._timestamp = datetime.utcnow() - - def Get(self): - return self._value + redis.setex(self._cacheKey % pk, pickle.dumps(value), self._lifetime) From d653054f2d34e89dfbcc32597545191e9121374a Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sat, 21 Mar 2015 10:18:45 -0400 Subject: [PATCH 26/78] Diagnostics for extended GC session cache TTL --- tapiriik/services/GarminConnect/garminconnect.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index a9c99ec01..d0e424a82 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -149,6 +149,7 @@ def _get_session(self, record=None, email=None, password=None, skip_cache=False) from tapiriik.auth.credential_storage import CredentialStore cached = self._sessionCache.Get(record.ExternalID if record else email) if cached and not skip_cache: + logger.debug("Using cached credential") return cached if record: # longing for C style overloads... @@ -274,6 +275,7 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): res = session.get("https://connect.garmin.com/modern/proxy/activity-search-service-1.0/json/activities", params={"start": (page - 1) * pageSz, "limit": pageSz}) # It's 10 PM and I have no clue why it's throwing these errors, maybe we just need to log in again? if res.status_code == 403 and not retried_auth: + logger.debug("Retrying auth w/o cache") retried_auth = True session = self._get_session(serviceRecord, skip_cache=True) else: From 7359a178241469054cecbc3dd17a9c00d707faff Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 23 Mar 2015 09:08:23 -0400 Subject: [PATCH 27/78] Retry GC auth on 500 response because that fixes mystery --- tapiriik/services/GarminConnect/garminconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index d0e424a82..c9d6d21aa 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -274,7 +274,7 @@ def DownloadActivityList(self, serviceRecord, exhaustive=False): while True: res = session.get("https://connect.garmin.com/modern/proxy/activity-search-service-1.0/json/activities", params={"start": (page - 1) * pageSz, "limit": pageSz}) # It's 10 PM and I have no clue why it's throwing these errors, maybe we just need to log in again? - if res.status_code == 403 and not retried_auth: + if res.status_code in [500, 403] and not retried_auth: logger.debug("Retrying auth w/o cache") retried_auth = True session = self._get_session(serviceRecord, skip_cache=True) From 1e9fe5c641326d663e4a3a1bf0c4edac04da3531 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 29 Mar 2015 10:10:03 -0400 Subject: [PATCH 28/78] Always retry GC auth on interactive sign-in, allow another login request flow. --- tapiriik/services/GarminConnect/garminconnect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index c9d6d21aa..19c0c9de1 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -221,7 +221,7 @@ def _get_session(self, record=None, email=None, password=None, skip_cache=False) if current_redirect_count >= expected_redirect_count and gcRedeemResp.status_code != 200: raise APIException("GC redeem %d/%d error %s %s" % (current_redirect_count, expected_redirect_count, gcRedeemResp.status_code, gcRedeemResp.text)) - if gcRedeemResp.status_code == 200: + if gcRedeemResp.status_code == 200 or gcRedeemResp.status_code == 404: break current_redirect_count += 1 if current_redirect_count > expected_redirect_count: @@ -238,7 +238,7 @@ def WebInit(self): def Authorize(self, email, password): from tapiriik.auth.credential_storage import CredentialStore - session = self._get_session(email=email, password=password) + session = self._get_session(email=email, password=password, skip_cache=True) # TODO: http://connect.garmin.com/proxy/userprofile-service/socialProfile/ has the proper immutable user ID, not that anyone ever changes this one... self._rate_limit() username = session.get("http://connect.garmin.com/user/username").json()["username"] From cf8237db49d34a2cec67813f09a23a96cfd5ac7a Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 29 Mar 2015 10:19:31 -0400 Subject: [PATCH 29/78] Fix login form not POSTing in Chrome all of a sudden --- tapiriik/web/templates/diag/login.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/web/templates/diag/login.html b/tapiriik/web/templates/diag/login.html index 6f50d0ac7..31c54a102 100644 --- a/tapiriik/web/templates/diag/login.html +++ b/tapiriik/web/templates/diag/login.html @@ -3,7 +3,7 @@ {% block content %}

Super seecreet diagnostics dashboard login (ooooh! aaaaah!)

-
+

From 0d1fd3ea779f85157539d62171fdbe72f725eef1 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 29 Mar 2015 11:24:22 -0400 Subject: [PATCH 30/78] pass != continue; stop fruitlessly attempting to upload activities to Runsense, TR --- tapiriik/sync/sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index b38abc6b4..e58595428 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -384,12 +384,12 @@ def _determineRecipientServices(self, activity): for conn in self._serviceConnections: if not conn.Service.ReceivesActivities: # Nope. - pass + continue if conn._id in activity.ServiceDataCollection: # The activity record is updated earlier for these, blegh. - pass + continue elif hasattr(conn, "SynchronizedActivities") and len([x for x in activity.UIDs if x in conn.SynchronizedActivities]): - pass + continue elif activity.Type not in conn.Service.SupportedActivities: logger.debug("\t...%s doesn't support type %s" % (conn.Service.ID, activity.Type)) activity.Record.MarkAsNotPresentOn(conn, UserException(UserExceptionType.TypeUnsupported)) From ba85ac72efadc0ba05b32a568834609e2112f704 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 30 Mar 2015 12:27:59 -0400 Subject: [PATCH 31/78] Don't just lock up UI if direct login fails due to unforeseen circumstances --- tapiriik/web/static/css/style.css | 1 + tapiriik/web/static/js/tapiriik.js | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tapiriik/web/static/css/style.css b/tapiriik/web/static/css/style.css index 71c62bf76..089a4a3b3 100644 --- a/tapiriik/web/static/css/style.css +++ b/tapiriik/web/static/css/style.css @@ -313,6 +313,7 @@ button.delete:active { } form .error { + text-align: center; display:none; color:#900; border-bottom:1px dotted #900; diff --git a/tapiriik/web/static/js/tapiriik.js b/tapiriik/web/static/js/tapiriik.js index b9ae1d64e..98d70fec7 100644 --- a/tapiriik/web/static/js/tapiriik.js +++ b/tapiriik/web/static/js/tapiriik.js @@ -266,7 +266,7 @@ tapiriik.OpenDeauthDialog = function(svcId){ }; tapiriik.CreateDirectLoginForm = function(svcId){ - var form = $("
There was a problem logging you in


"); + var form = $("
There was a problem logging you in
There was a system error :(


"); if (!tapiriik.ServiceInfo[svcId].UsesExtendedAuth){ $(".persist-controls",form).hide(); } @@ -284,11 +284,15 @@ tapiriik.CreateDirectLoginForm = function(svcId){ $().redirect("trainingpeaks_premium", {personId: data.result.extra, username:$("#email",form).val(), password:$("#password",form).val()}); return; } - $(".error",form).show(); + $(".error", form).hide(); + $("#login-fail", form).show(); $("button",form).removeClass("disabled"); loginPending = false; } - }, "json"); + }, "json").fail(function(){ + $(".error", form).hide(); + $("#login-error", form).show(); + }); return false; }); return form; From fa2a9ee6d355b3ea9b7c7a62425a1d39450fb3fc Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 30 Mar 2015 13:02:04 -0400 Subject: [PATCH 32/78] Fix silly layout issues on dashboard w/ user exceptions & sync settings visible at once --- tapiriik/web/static/css/style.css | 10 ++++++++-- tapiriik/web/templates/dashboard.html | 2 -- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tapiriik/web/static/css/style.css b/tapiriik/web/static/css/style.css index 089a4a3b3..e70ab6039 100644 --- a/tapiriik/web/static/css/style.css +++ b/tapiriik/web/static/css/style.css @@ -654,7 +654,8 @@ p.infotip .close:hover { } .syncButtonBlock { - margin:30px 0; + margin:20px 0; + padding-bottom: 30px; opacity: 0; } @@ -780,7 +781,7 @@ to {transform: rotate(0deg);} width:50%; left:25%; border-radius: 5px; - margin:10px; + margin:20px 0; } .userException p { @@ -1165,6 +1166,7 @@ ul.itemized-clean li { .recentActivitiesTable { width: 50%; + margin: 20px 0; margin-left: 25%; } @@ -1233,6 +1235,10 @@ ul.itemized-clean li { font-size:70%; } +.syncSettingsBlock { + margin: 20px 0; +} + .syncSettingsCloze { font-size:27px; width:500px; diff --git a/tapiriik/web/templates/dashboard.html b/tapiriik/web/templates/dashboard.html index 7b6bb167f..af707d946 100644 --- a/tapiriik/web/templates/dashboard.html +++ b/tapiriik/web/templates/dashboard.html @@ -31,10 +31,8 @@

Substituting user

Queueing to Synchronize
Synchronize Now
Synchronization Settings
Attached info 10%
-
{% include "settings-block.html" %} {% include "recent-sync-activity-block.html" %} -
{% endif %} {% if user.NonblockingSyncErrorCount > 0 %} Some activities could not be synchronized.
From 4681062e26eed12575f06d1b5e018e6de760f3f1 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 30 Mar 2015 19:14:22 -0400 Subject: [PATCH 33/78] Put user exception blocks above the 'recent activities' list on dashboard --- tapiriik/web/templates/dashboard.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tapiriik/web/templates/dashboard.html b/tapiriik/web/templates/dashboard.html index af707d946..d9410b756 100644 --- a/tapiriik/web/templates/dashboard.html +++ b/tapiriik/web/templates/dashboard.html @@ -32,11 +32,13 @@

Substituting user

Queueing to Synchronize
Synchronize Now
Synchronization Settings
Attached info 10%
{% include "settings-block.html" %} + + {% if user.NonblockingSyncErrorCount > 0 %} + Some activities could not be synchronized.
+ {% endif %} + {% include "recent-sync-activity-block.html" %} {% endif %} - {% if user.NonblockingSyncErrorCount > 0 %} - Some activities could not be synchronized.
- {% endif %} {% if user.ConnectedServices|length > 1 %} {% if user|has_active_payment %} From c9e678e3bda71cde4f83735b84a1075d48c7e8fc Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 30 Mar 2015 19:49:22 -0400 Subject: [PATCH 34/78] Continue to rearrange blocks on dashboard --- tapiriik/web/templates/dashboard.html | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tapiriik/web/templates/dashboard.html b/tapiriik/web/templates/dashboard.html index d9410b756..186f5bf49 100644 --- a/tapiriik/web/templates/dashboard.html +++ b/tapiriik/web/templates/dashboard.html @@ -37,25 +37,23 @@

Substituting user

Some activities could not be synchronized.
{% endif %} - {% include "recent-sync-activity-block.html" %} - {% endif %} - - {% if user.ConnectedServices|length > 1 %} {% if user|has_active_payment %} {% else %} {% include "payment.html" %} {% endif %} - {% endif %} - {% for connection in user.ConnectedServices|svc_populate_conns %} - {% with svc=connection.Service %} - {% for error in connection.SyncErrors %} - {% if error.UserException.InterventionRequired and error.Block %} - {% include "service-blockingexception.html" with provider=svc connection=connection exception=error %} - {% endif %} - {% endfor %} - {% endwith %} - {% endfor %} + {% for connection in user.ConnectedServices|svc_populate_conns %} + {% with svc=connection.Service %} + {% for error in connection.SyncErrors %} + {% if error.UserException.InterventionRequired and error.Block %} + {% include "service-blockingexception.html" with provider=svc connection=connection exception=error %} + {% endif %} + {% endfor %} + {% endwith %} + {% endfor %} + + {% include "recent-sync-activity-block.html" %} + {% endif %} {% for provider in user.ConnectedServices|svc_providers_except %} {% if provider.ID not in config.soft_launch and provider.ID not in config.withdrawn_services %} From 5f05bfc59343c76ae341c09d20d36c3ea5b4f412 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Mon, 30 Mar 2015 20:06:32 -0400 Subject: [PATCH 35/78] Put notes element in correct sequence during TCX generation Fixes #120 --- tapiriik/services/tcx.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tapiriik/services/tcx.py b/tapiriik/services/tcx.py index 84b0b2b77..45b55bb56 100644 --- a/tapiriik/services/tcx.py +++ b/tapiriik/services/tcx.py @@ -260,13 +260,6 @@ def Dump(activity): dateFormat = "%Y-%m-%dT%H:%M:%S.000Z" - if activity.Name is not None and activity.Notes is not None: - etree.SubElement(act, "Notes").text = '\n'.join((activity.Name, activity.Notes)) - elif activity.Name is not None: - etree.SubElement(act, "Notes").text = activity.Name - elif activity.Notes is not None: - etree.SubElement(act, "Notes").text = '\n' + activity.Notes - if activity.Type == ActivityType.Cycling: act.attrib["Sport"] = "Biking" elif activity.Type == ActivityType.Running: @@ -376,6 +369,13 @@ def _writeStat(parent, elName, value, wrapValue=False, naturalValue=False, defau if exts is not None: track.addnext(exts) + if activity.Name is not None and activity.Notes is not None: + etree.SubElement(act, "Notes").text = '\n'.join((activity.Name, activity.Notes)) + elif activity.Name is not None: + etree.SubElement(act, "Notes").text = activity.Name + elif activity.Notes is not None: + etree.SubElement(act, "Notes").text = '\n' + activity.Notes + if activity.Device and activity.Device.Identifier: devId = DeviceIdentifier.FindEquivalentIdentifierOfType(DeviceIdentifierType.TCX, activity.Device.Identifier) if devId: From aa434cbb722567ecf3d23287a2b2dfc55eeec30f Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 5 Apr 2015 10:36:46 -0400 Subject: [PATCH 36/78] Don't update recently-synchronized list for failed synchronizations --- tapiriik/sync/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index e58595428..12aca3f58 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -1063,7 +1063,8 @@ def Run(self, exhaustive=False, null_next_sync_on_unlock=False, heartbeat_callba db.sync_stats.update({"ActivityID": activity.UID}, {"$addToSet": {"DestinationServices": destSvc.ID, "SourceServices": activitySource.ID}, "$set": {"Distance": activity.Stats.Distance.asUnits(ActivityStatisticUnit.Meters).Value, "Timestamp": datetime.utcnow()}}, upsert=True) - self._pushRecentSyncActivity(full_activity, successful_destination_service_ids) + if len(successful_destination_service_ids): + self._pushRecentSyncActivity(full_activity, successful_destination_service_ids) del full_activity processedActivities += 1 except ActivityShouldNotSynchronizeException: From 763522f68187c3a2ddc200acd3e2bc12e4f3a0e6 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 5 Apr 2015 13:34:14 -0400 Subject: [PATCH 37/78] Prevent sync_upload_delay from scheduling automatic synchronizations years into the future --- tapiriik/sync/sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index 12aca3f58..af4c07e56 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -916,9 +916,11 @@ def Run(self, exhaustive=False, null_next_sync_on_unlock=False, heartbeat_callba logger.debug(" %s since upload" % time_past) if time_remaining > timedelta(0): activity.Record.MarkAsNotPresentOtherwise(UserException(UserExceptionType.Deferred)) - next_sync = datetime.utcnow() + time_remaining - # Reschedule them so this activity syncs immediately on schedule - sync_result.ForceScheduleNextSyncOnOrBefore(next_sync) + # Only reschedule if it won't slow down their auto-sync timing + if time_remaining < (Sync.SyncInterval + Sync.SyncIntervalJitter): + next_sync = datetime.utcnow() + time_remaining + # Reschedule them so this activity syncs immediately on schedule + sync_result.ForceScheduleNextSyncOnOrBefore(next_sync) logger.info("\t\t...is delayed for %s (out of %s)" % (time_remaining, timedelta(seconds=self._user_config["sync_upload_delay"]))) # We need to ensure we check these again when the sync re-runs From de88f4e3cf20087688e4b55ab99444702cfe2451 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 5 Apr 2015 14:03:33 -0400 Subject: [PATCH 38/78] Timestamp service exceptions --- tapiriik/sync/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index af4c07e56..886408d59 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -54,7 +54,7 @@ def _isWarning(exc): # It's practically an ORM! def _packServiceException(step, e): - res = {"Step": step, "Message": e.Message + "\n" + _formatExc(), "Block": e.Block, "Scope": e.Scope, "TriggerExhaustive": e.TriggerExhaustive} + res = {"Step": step, "Message": e.Message + "\n" + _formatExc(), "Block": e.Block, "Scope": e.Scope, "TriggerExhaustive": e.TriggerExhaustive, "Timestamp": datetime.utcnow()} if e.UserException: res["UserException"] = _packUserException(e.UserException) return res From 8b0ad1f0f14ac7c7ddf3d474654d837534b4f3cf Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Tue, 7 Apr 2015 14:34:34 -0400 Subject: [PATCH 39/78] Trap Strava activity list JSON parse failures --- tapiriik/services/Strava/strava.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tapiriik/services/Strava/strava.py b/tapiriik/services/Strava/strava.py index 28978473a..9aa7f09f5 100644 --- a/tapiriik/services/Strava/strava.py +++ b/tapiriik/services/Strava/strava.py @@ -122,7 +122,10 @@ def DownloadActivityList(self, svcRecord, exhaustive=False): earliestDate = None - reqdata = resp.json() + try: + reqdata = resp.json() + except ValueError: + raise APIException("Failed parsing strava list response %s - %s" % (resp.status_code, resp.text)) if not len(reqdata): break # No more activities to see From 6e6b6d3a84d038c8f971cf431c5f68244ecef885 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 12 Apr 2015 11:33:51 -0400 Subject: [PATCH 40/78] Trap GC maintenance message instead of blocking service for reauth. --- tapiriik/services/GarminConnect/garminconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/services/GarminConnect/garminconnect.py b/tapiriik/services/GarminConnect/garminconnect.py index 19c0c9de1..a70ebb96f 100644 --- a/tapiriik/services/GarminConnect/garminconnect.py +++ b/tapiriik/services/GarminConnect/garminconnect.py @@ -196,7 +196,7 @@ def _get_session(self, record=None, email=None, password=None, skip_cache=False) data["lt"] = re.search("name=\"lt\"\s+value=\"([^\"]+)\"", preResp.text).groups(1)[0] ssoResp = session.post("https://sso.garmin.com/sso/login", params=params, data=data, allow_redirects=False) - if ssoResp.status_code != 200: + if ssoResp.status_code != 200 or "temporarily unavailable" in ssoResp.text: raise APIException("SSO error %s %s" % (ssoResp.status_code, ssoResp.text)) ticket_match = re.search("ticket=([^']+)'", ssoResp.text) From 446ba474226f5966559f71319d7a8d4be2411628 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Wed, 15 Apr 2015 19:46:41 -0400 Subject: [PATCH 41/78] Don't re-send payment confirmations during spurious IPN retries --- tapiriik/web/views/payments/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tapiriik/web/views/payments/__init__.py b/tapiriik/web/views/payments/__init__.py index a13ce092d..64e9ce528 100644 --- a/tapiriik/web/views/payments/__init__.py +++ b/tapiriik/web/views/payments/__init__.py @@ -39,7 +39,8 @@ def payments_ipn(req): user = User.Get(req.POST["custom"]) User.AssociatePayment(user, payment) - payments_send_confirmation(req, req.POST["payer_email"]) + if "_id" not in payment: # Is the payment newly entered? Really should have used an ORM about 3 years ago. + payments_send_confirmation(req, req.POST["payer_email"]) return HttpResponse() From 19aeecea8553e2cbd094698cc415858e4623ceba Mon Sep 17 00:00:00 2001 From: Bert Van Steen Date: Fri, 17 Apr 2015 23:58:41 +0200 Subject: [PATCH 42/78] Strava scope should be comma separated --- tapiriik/services/Strava/strava.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/services/Strava/strava.py b/tapiriik/services/Strava/strava.py index 9aa7f09f5..2d5f6e497 100644 --- a/tapiriik/services/Strava/strava.py +++ b/tapiriik/services/Strava/strava.py @@ -76,7 +76,7 @@ def UserUploadedActivityURL(self, uploadId): return "https://www.strava.com/activities/%d" % uploadId def WebInit(self): - params = {'scope':'write view_private', + params = {'scope':'write,view_private', 'client_id':STRAVA_CLIENT_ID, 'response_type':'code', 'redirect_uri':WEB_ROOT + reverse("oauth_return", kwargs={"service": "strava"})} From da933a95fbbf2c4f442b609fd3e31805b16c0cfa Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 19 Apr 2015 10:28:26 -0400 Subject: [PATCH 43/78] Give payment confirmation emails subject lines, for the sake of my inbox's sanity --- tapiriik/web/views/payments/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/web/views/payments/__init__.py b/tapiriik/web/views/payments/__init__.py index 64e9ce528..bfac35361 100644 --- a/tapiriik/web/views/payments/__init__.py +++ b/tapiriik/web/views/payments/__init__.py @@ -48,7 +48,7 @@ def payments_send_confirmation(request, email): dashboard_url = request.build_absolute_uri(reverse("dashboard")) from tapiriik.web.email import generate_message_from_template, send_email message, plaintext_message = generate_message_from_template("email/payment_confirm.html", {"url": dashboard_url}) - send_email(email, "Thanks!", message, plaintext_message=plaintext_message) + send_email(email, "Thanks, %s!" % request.POST["first_name"], message, plaintext_message=plaintext_message) def payments_return(req): if req.user is None: From 49325c632146711bdc94123f56c8014db7fd4531 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 19 Apr 2015 10:55:03 -0400 Subject: [PATCH 44/78] Revert "Don't re-send payment confirmations during spurious IPN retries" - it stopped all confirmation emails and I'm not sure why. This reverts commit 446ba474226f5966559f71319d7a8d4be2411628. --- tapiriik/web/views/payments/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tapiriik/web/views/payments/__init__.py b/tapiriik/web/views/payments/__init__.py index bfac35361..894160be2 100644 --- a/tapiriik/web/views/payments/__init__.py +++ b/tapiriik/web/views/payments/__init__.py @@ -39,8 +39,7 @@ def payments_ipn(req): user = User.Get(req.POST["custom"]) User.AssociatePayment(user, payment) - if "_id" not in payment: # Is the payment newly entered? Really should have used an ORM about 3 years ago. - payments_send_confirmation(req, req.POST["payer_email"]) + payments_send_confirmation(req, req.POST["payer_email"]) return HttpResponse() From a3623b069aa50bf7c68cc56aaa25a10465c4873d Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sat, 25 Apr 2015 20:49:34 -0400 Subject: [PATCH 45/78] Reduce sync log retention --- tapiriik/sync/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/sync/sync.py b/tapiriik/sync/sync.py index 886408d59..79b8ed5f8 100644 --- a/tapiriik/sync/sync.py +++ b/tapiriik/sync/sync.py @@ -229,7 +229,7 @@ def _updateSyncProgress(self, step, progress): db.users.update({"_id": self.user["_id"]}, {"$set": {"SynchronizationProgress": progress, "SynchronizationStep": step}}) def _initializeUserLogging(self): - self._logging_file_handler = logging.handlers.RotatingFileHandler(USER_SYNC_LOGS + str(self.user["_id"]) + ".log", maxBytes=0, backupCount=10, encoding="utf-8") + self._logging_file_handler = logging.handlers.RotatingFileHandler(USER_SYNC_LOGS + str(self.user["_id"]) + ".log", maxBytes=0, backupCount=5, encoding="utf-8") self._logging_file_handler.setFormatter(logging.Formatter(self._logFormat, self._logDateFormat)) self._logging_file_handler.doRollover() _global_logger.addHandler(self._logging_file_handler) From b70337dae67e067510692f60af050deb81b411a3 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 11:19:09 -0400 Subject: [PATCH 46/78] Update pymongo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 69577b872..f9a02037d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests==2.2.1 -pymongo +pymongo==3.0.1 pytz lxml git+https://github.com/cpfair/dropbox-sdk-python.git From 93102194df934c7ff81c933ff0a6895df2feac61 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 11:27:59 -0400 Subject: [PATCH 47/78] These tests are pretty useless even before they randomly fail --- tapiriik/testing/testtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/testing/testtools.py b/tapiriik/testing/testtools.py index 78965021d..3dfbcd728 100644 --- a/tapiriik/testing/testtools.py +++ b/tapiriik/testing/testtools.py @@ -91,7 +91,7 @@ def create_random_activity(svc=None, actType=ActivityType.Other, tz=False, recor act = TestTools.create_blank_activity(svc, actType, record=record) if tz is True: - tz = pytz.timezone(pytz.all_timezones[random.randint(0, len(pytz.all_timezones) - 1)]) + tz = pytz.timezone("America/Atikokan") act.TZ = tz elif tz is not False: act.TZ = tz From 7faa26bb3e4db61386d5d47ec838f981b42df051 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 11:36:50 -0400 Subject: [PATCH 48/78] Stop performing un-indexed lookups against the sync_workers collection (also fixes glitches if hostname changes at runtime) Also, attempt to bail if server disk is readonly before adding an item in sync_workers --- sync_worker.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/sync_worker.py b/sync_worker.py index cf21b3a6b..eb68cc932 100644 --- a/sync_worker.py +++ b/sync_worker.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime import os # I'm trying to track down where some missing seconds are going in the sync process # Will grep these out of the log at some later date @@ -10,6 +10,7 @@ def worker_message(state): from tapiriik.requests_lib import patch_requests_with_default_timeout, patch_requests_source_address from tapiriik import settings from tapiriik.database import db, close_connections +from pymongo import ReturnDocument import sys import subprocess import socket @@ -21,11 +22,37 @@ def worker_message(state): os.chdir(oldCwd) def sync_heartbeat(state, user=None): - db.sync_workers.update({"Process": os.getpid(), "Host": socket.gethostname()}, {"$set": {"Heartbeat": datetime.utcnow(), "State": state, "User": user}}) + db.sync_workers.update({"_id": heartbeat_rec_id}, {"$set": {"Heartbeat": datetime.utcnow(), "State": state, "User": user}}) worker_message("initialized") -db.sync_workers.update({"Process": os.getpid(), "Host": socket.gethostname()}, {"Process": os.getpid(), "Heartbeat": datetime.utcnow(), "Startup": datetime.utcnow(), "Version": WorkerVersion, "Host": socket.gethostname(), "Index": settings.WORKER_INDEX, "State": "startup"}, upsert=True) + +# Moved this flush before the sync_workers upsert for a rather convoluted reason: +# Some of the sync servers were encountering filesystem corruption, causing the FS to be remounted as read-only. +# Then, when a sync worker would start, it would insert a record in sync_workers then immediately die upon calling flush - since output is piped to a log file on the read-only FS. +# Supervisor would dutifully restart the worker again and again, causing sync_workers to quickly fill up. +# ...which is a problem, since it doesn't have indexes on Process or Host - what later lookups were based on. So, the database would be brought to a near standstill. +# Theoretically, the watchdog would clean up these records soon enough - but since it too logs to a file, it would crash removing only a few stranded records +# By flushing the logs before we insert, it should crash before filling that collection up. +# (plus, we no longer query with Process/Host in sync_hearbeat) + sys.stdout.flush() +heartbeat_rec = db.sync_workers.find_one_and_update( + { + "Process": os.getpid(), + "Host": socket.gethostname() + }, { + "$set": { + "Process": os.getpid(), + "Host": socket.gethostname(), + "Heartbeat": datetime.utcnow(), + "Startup": datetime.utcnow(), + "Version": WorkerVersion, + "Index": settings.WORKER_INDEX, + "State": "startup" + } + }, upsert=True, + return_document=ReturnDocument.AFTER) +heartbeat_rec_id = heartbeat_rec["_id"] patch_requests_with_default_timeout(timeout=60) @@ -48,7 +75,7 @@ def sync_heartbeat(state, user=None): Sync.PerformGlobalSync(heartbeat_callback=sync_heartbeat, version=WorkerVersion, max_users=RecycleInterval) worker_message("shutting down cleanly") -db.sync_workers.remove({"Process": os.getpid(), "Host": socket.gethostname()}) +db.sync_workers.remove({"_id": heartbeat_rec_id}) close_connections() worker_message("shut down") sys.stdout.flush() From be7e8190944ceb627993a3965e99cfe84f3dc076 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 11:57:03 -0400 Subject: [PATCH 49/78] Pymongo 3.x killed the read_preference kwarg on find() :( --- sync_scheduler.py | 5 ++--- tapiriik/auth/__init__.py | 2 +- tapiriik/services/ratelimiting.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sync_scheduler.py b/sync_scheduler.py index 58e2c691f..f75f922db 100644 --- a/sync_scheduler.py +++ b/sync_scheduler.py @@ -15,7 +15,7 @@ while True: generation = str(uuid.uuid4()) queueing_at = datetime.utcnow() - users = list(db.users.find( + users = list(db.users.with_options(read_preference=ReadPreference.PRIMARY).find( { "NextSynchronization": {"$lte": datetime.utcnow()}, "QueuedAt": {"$exists": False} @@ -23,8 +23,7 @@ { "_id": True, "SynchronizationHostRestriction": True - }, - read_preference=ReadPreference.PRIMARY + } )) scheduled_ids = [x["_id"] for x in users] print("Found %d users at %s" % (len(scheduled_ids), datetime.utcnow())) diff --git a/tapiriik/auth/__init__.py b/tapiriik/auth/__init__.py index edff2290e..b27350de2 100644 --- a/tapiriik/auth/__init__.py +++ b/tapiriik/auth/__init__.py @@ -39,7 +39,7 @@ def Logout(req): def Create(creationIP=None): uid = db.users.insert({"Created": datetime.utcnow(), "CreationIP": creationIP}) # will mongodb insert an almost empty doc, i.e. _id? - return db.users.find_one({"_id": uid}, read_preference=ReadPreference.PRIMARY) + return db.users.with_options(read_preference=ReadPreference.PRIMARY).find_one({"_id": uid}) def GetConnectionRecordsByUser(user): return [ServiceRecord(x) for x in db.connections.find({"_id": {"$in": [x["ID"] for x in user["ConnectedServices"]]}})] diff --git a/tapiriik/services/ratelimiting.py b/tapiriik/services/ratelimiting.py index d8238e7fa..e842114f7 100644 --- a/tapiriik/services/ratelimiting.py +++ b/tapiriik/services/ratelimiting.py @@ -25,7 +25,7 @@ def Refresh(key, limits): time_since_midnight = (datetime.utcnow() - midnight) rl_db.limits.remove({"Key": key, "Expires": {"$lt": datetime.utcnow()}}) - current_limits = list(rl_db.limits.find({"Key": key}, {"Duration": 1}, read_preference=ReadPreference.PRIMARY)) + current_limits = list(rl_db.limits.with_options(read_preference=ReadPreference.PRIMARY).find({"Key": key}, {"Duration": 1})) missing_limits = [x for x in limits if x[0].total_seconds() not in [limit["Duration"] for limit in current_limits]] for limit in missing_limits: window_start = midnight + timedelta(seconds=math.floor(time_since_midnight.total_seconds()/limit[0].total_seconds()) * limit[0].total_seconds()) From c413f18e3ac59bdabedd84caac009dec7c0750d3 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 12:06:07 -0400 Subject: [PATCH 50/78] Fix stats_cron for pymongo 3.x, too --- stats_cron.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/stats_cron.py b/stats_cron.py index f8bbc46fa..eb83bc533 100644 --- a/stats_cron.py +++ b/stats_cron.py @@ -2,21 +2,21 @@ from datetime import datetime, timedelta # total distance synced -distanceSyncedAggr = db.sync_stats.aggregate([{"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])["result"] +distanceSyncedAggr = list(db.sync_stats.aggregate([{"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])) if distanceSyncedAggr: distanceSynced = distanceSyncedAggr[0]["total"] else: distanceSynced = 0 # last 24hr, for rate calculation -lastDayDistanceSyncedAggr = db.sync_stats.aggregate([{"$match": {"Timestamp": {"$gt": datetime.utcnow() - timedelta(hours=24)}}}, {"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])["result"] +lastDayDistanceSyncedAggr = list(db.sync_stats.aggregate([{"$match": {"Timestamp": {"$gt": datetime.utcnow() - timedelta(hours=24)}}}, {"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])) if lastDayDistanceSyncedAggr: lastDayDistanceSynced = lastDayDistanceSyncedAggr[0]["total"] else: lastDayDistanceSynced = 0 # similarly, last 1hr -lastHourDistanceSyncedAggr = db.sync_stats.aggregate([{"$match": {"Timestamp": {"$gt": datetime.utcnow() - timedelta(hours=1)}}}, {"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])["result"] +lastHourDistanceSyncedAggr = list(db.sync_stats.aggregate([{"$match": {"Timestamp": {"$gt": datetime.utcnow() - timedelta(hours=1)}}}, {"$group": {"_id": None, "total": {"$sum": "$Distance"}}}])) if lastHourDistanceSyncedAggr: lastHourDistanceSynced = lastHourDistanceSyncedAggr[0]["total"] else: @@ -31,7 +31,7 @@ # sync time utilization db.sync_worker_stats.remove({"Timestamp": {"$lt": datetime.utcnow() - timedelta(hours=1)}}) # clean up old records -timeUsedAgg = db.sync_worker_stats.aggregate([{"$group": {"_id": None, "total": {"$sum": "$TimeTaken"}}}])["result"] +timeUsedAgg = list(db.sync_worker_stats.aggregate([{"$group": {"_id": None, "total": {"$sum": "$TimeTaken"}}}])) totalSyncOps = db.sync_worker_stats.count() if timeUsedAgg: timeUsed = timeUsedAgg[0]["total"] @@ -41,41 +41,41 @@ avgSyncTime = 0 # error/pending/locked stats -lockedSyncRecords = db.users.aggregate([ +lockedSyncRecords = list(db.users.aggregate([ {"$match": {"SynchronizationWorker": {"$ne": None}}}, {"$group": {"_id": None, "count": {"$sum": 1}}} - ]) -if len(lockedSyncRecords["result"]) > 0: - lockedSyncRecords = lockedSyncRecords["result"][0]["count"] + ])) +if len(lockedSyncRecords) > 0: + lockedSyncRecords = lockedSyncRecords[0]["count"] else: lockedSyncRecords = 0 -pendingSynchronizations = db.users.aggregate([ +pendingSynchronizations = list(db.users.aggregate([ {"$match": {"NextSynchronization": {"$lt": datetime.utcnow()}}}, {"$group": {"_id": None, "count": {"$sum": 1}}} - ]) -if len(pendingSynchronizations["result"]) > 0: - pendingSynchronizations = pendingSynchronizations["result"][0]["count"] + ])) +if len(pendingSynchronizations) > 0: + pendingSynchronizations = pendingSynchronizations[0]["count"] else: pendingSynchronizations = 0 -usersWithErrors = db.users.aggregate([ +usersWithErrors = list(db.users.aggregate([ {"$match": {"NonblockingSyncErrorCount": {"$gt": 0}}}, {"$group": {"_id": None, "count": {"$sum": 1}}} - ]) -if len(usersWithErrors["result"]) > 0: - usersWithErrors = usersWithErrors["result"][0]["count"] + ])) +if len(usersWithErrors) > 0: + usersWithErrors = usersWithErrors[0]["count"] else: usersWithErrors = 0 -totalErrors = db.users.aggregate([ +totalErrors = list(db.users.aggregate([ {"$group": {"_id": None, "total": {"$sum": "$NonblockingSyncErrorCount"}}} -]) +])) -if len(totalErrors["result"]) > 0: - totalErrors = totalErrors["result"][0]["total"] +if len(totalErrors) > 0: + totalErrors = totalErrors[0]["total"] else: totalErrors = 0 From 98fb15d4aa8beea8d071626d0f6323cbd73e8a9d Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 12:14:49 -0400 Subject: [PATCH 51/78] Fix empty tag breaking TCX parsing --- tapiriik/services/tcx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tapiriik/services/tcx.py b/tapiriik/services/tcx.py index 45b55bb56..38c6a2b14 100644 --- a/tapiriik/services/tcx.py +++ b/tapiriik/services/tcx.py @@ -54,7 +54,7 @@ def Parse(tcxData, act=None): act.Type = ActivityType.Running xnotes = xact.find("tcx:Notes", namespaces=ns) - if xnotes is not None: + if xnotes is not None and xnotes.text: xnotes_lines = xnotes.text.splitlines() act.Name = xnotes_lines[0] if len(xnotes_lines) > 1: From 9b0e7751d2a7594fb13415c7c34e86e08d2792f9 Mon Sep 17 00:00:00 2001 From: Collin Fair Date: Sun, 26 Apr 2015 14:17:42 -0400 Subject: [PATCH 52/78] Simplify error summary so it actually loads before getting timed out --- tapiriik/web/templates/diag/errors.html | 8 +------- tapiriik/web/views/diagnostics.py | 18 ++++-------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/tapiriik/web/templates/diag/errors.html b/tapiriik/web/templates/diag/errors.html index 42f03f129..482821994 100644 --- a/tapiriik/web/templates/diag/errors.html +++ b/tapiriik/web/templates/diag/errors.html @@ -4,15 +4,9 @@ {% block content %}