Support ingest (#111)

Package.swift

@@ -14,13 +14,15 @@ let package = Package(
             targets: ["TreasureData_iOS_SDK"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/treasure-data/KeenClient-iOS.git", .exact("3.9.0"))
+        .package(url: "https://github.com/treasure-data/KeenClient-iOS.git", .exact("3.9.0")),
+        .package(url: "https://github.com/nicklockwood/GZIP.git", .upToNextMajor(from: "1.3.0"))
     ],
     targets: [
       .target(
             name: "TreasureData_iOS_SDK",
             dependencies: [
-                "KeenClientTD"
+                "KeenClientTD",
+                "GZIP"
             ],
             path: ".",
             exclude: [

Podfile

@@ -2,7 +2,8 @@ platform :ios, '12.0'
 use_frameworks!
 
 target 'TreasureData' do
-  pod 'KeenClientTD', '= 3.9.0'
+  pod 'KeenClientTD', '= 4.0.0'
+  pod 'GZIP', '= 1.3.0'
 end
 
 target 'TreasureDataTests' do

Podfile.lock

@@ -1,18 +1,22 @@
 PODS:
-  - KeenClientTD (3.3.0):
-    - KeenClientTD/keen_sqlite (= 3.3.0)
-  - KeenClientTD/keen_sqlite (3.3.0)
+  - GZIP (1.3.0)
+  - KeenClientTD (4.0.0):
+    - KeenClientTD/keen_sqlite (= 4.0.0)
+  - KeenClientTD/keen_sqlite (4.0.0)
 
 DEPENDENCIES:
-  - KeenClientTD (= 3.3.0)
+  - GZIP (= 1.3.0)
+  - KeenClientTD (= 4.0.0)
 
 SPEC REPOS:
   https://cdn.cocoapods.org/:
+    - GZIP
     - KeenClientTD
 
 SPEC CHECKSUMS:
-  KeenClientTD: 0e0399a8e5030890ace84644cea0bd39463c3ad4
+  GZIP: 416858efbe66b41b206895ac6dfd5493200d95b3
+  KeenClientTD: 5a1c60b2a4393680a1deeec30f7e461f7398f2c6
 
-PODFILE CHECKSUM: 091a6cd77a6be77b4e4e4841fdf03fcf11300360
+PODFILE CHECKSUM: 6e7065dea999cebc611545b16f8397d8b061b0be
 
 COCOAPODS: 1.11.3

README.md

@@ -5,6 +5,14 @@ iOS and tvOS SDK for [Treasure Data](http://www.treasuredata.com/). With this SD
 
 Also, there is an alternative SDK written in Swift [https://github.com/recruit-lifestyle/TreasureDataSDK](https://github.com/recruit-lifestyle/TreasureDataSDK). Note, however, that it does not support current GDPR functionality in the mainstream TD SDKs.
 
+## Migration to version 1
+
+Version 1 has major changes that are not backward compatible with previous versions. If you are upgrading from version 0.9.0 or earlier, your code will not run correctly without doing these following steps:
+- API endpoint has changed to Ingestion Endpoint. The default value is https://us01.records.in.treasuredata.com.
+- `initializeApiEndpoint:` API is no longer available, please use `initializeWithApiKey:apiEndpoint:` instead.
+- Server side upload timestamp feature is removed. If you need this feature, please contact our support team.
+- `uuid` is now reserved column name. If you try to add value to event's `uuid` key, you won't see the column show up in the database.
+
 ## Installation
 
 There are several ways to install the library.
@@ -297,11 +305,10 @@ You can detect if it's the first running or not easily using `isFirstRun` method
 
 ### Endpoint
 
-The API endpoint (default: https://in.treasuredata.com) can be modified using `initializeApiEndpoint` class method. For example,
+The API endpoint (default: https://us01.records.in.treasuredata.com) can be specify using `+[TreasureData initializeWithApiKey:apiEndpoint]`
 
 ```
-[TreasureData initializeApiEndpoint:@"https://specifying-another-endpoint.com"];
-[TreasureData initializeWithApiKey:@"your_api_key"];
+[TreasureData initializeWithApiKey:@"your_api_key" apiEndpoint: @"https://specifying-another-endpoint.com"];
 ```
 
 ### Encryption key
@@ -342,6 +349,20 @@ You can also reset UUID (`td_uuid`) at any time using following API.
 [[TreasureData sharedInstance] resetUniqId];
 ```
 
+### Adding local time to each even record automatically (enabled by default)
+By default, local timestamp will be added to event's `time` key automatically. If you `disableAutoAppendLocalTimestamp` without adding `time` key to the event yourself, the server will add server side timestamp to `time` column. You can also auto track local time with custom column. If so, the `time` column will have server side timestamp.
+
+```
+// Use local time as `time` column
+[[TreasureData sharedInstance] enableAutoAppendLocalTimestamp];
+
+// Add local time as a customized column name
+[[TreasureData sharedInstance] enableAutoAppendLocalTimestamp:@"clientside_time"];
+
+// Disable auto append local time
+[[TreasureData sharedInstance] disableAutoAppendLocalTimestamp];
+```
+
 ### Adding an UUID to each event record automatically
 UUID will be added to each event record automatically if you call `enableAutoAppendRecordUUID`. Each event has different UUID.
 
@@ -466,7 +487,7 @@ TreasureData SDK is able to automatically track IAP `SKPaymentTransactionStatePu
 [[TreasureData sharedInstance] enableInAppPurchaseEvent];
 ```
 
-This is disabled by default. There is a subtle difference between this and `appLifecycleEvent`, `customEvent`. The other two, for a historical reason, are persistent settings, meaning their statuses are saved across app launches. `inAppPurchaseEvent` behaves like an ordinary object option and is not saved. You have to enable it after initialize your new `TreasureData` instance (probably only the `sharedInstance` with `initializeWithApiKey()`).
+This is disabled by default. There is a subtle difference between this and `appLifecycleEvent`, `customEvent`. The other two, for a historical reason, are persistent settings, meaning their statuses are saved across app launches. `inAppPurchaseEvent` behaves like an ordinary object option and is not saved. You have to enable it after initialize your new `TreasureData` instance (probably only the `sharedInstance` with `initializeWithApiKey`).
 
 An example of a IAP event:
 

TreasureData-iOS-SDK.podspec

@@ -14,5 +14,6 @@ Pod::Spec.new do |s|
   s.frameworks   = 'Security', 'StoreKit'
   s.public_header_files = ["TreasureData/TreasureData.h", "TreasureData/TDClient.h", "TreasureData/TDRequestOptionsKey.h"]
   s.dependency "KeenClientTD", '= 3.9.0'
+  s.dependency "GZIP", '= 1.3.0'
   s.requires_arc = true
 end

TreasureData.xcodeproj/project.pbxproj

@@ -10,7 +10,6 @@
 		183E38A18F0332AA400DC7CF /* Pods_TreasureDataTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 84D7672182E6BD476D808297 /* Pods_TreasureDataTests.framework */; };
 		1950CB9871EC3D104BCD112A /* TDIAPObserver.m in Sources */ = {isa = PBXBuildFile; fileRef = 1950CA957CA6115F3C3DD018 /* TDIAPObserver.m */; };
 		2053A39C2047F37F0089A959 /* TDUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = 2053A39B2047F37F0089A959 /* TDUtils.m */; };
-		6B351A76194ABF0F001CBAAD /* Deflate.m in Sources */ = {isa = PBXBuildFile; fileRef = 6B351A75194ABF0F001CBAAD /* Deflate.m */; };
 		6B738092192A09160097D56E /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6B738091192A09160097D56E /* Foundation.framework */; };
 		6B738097192A09160097D56E /* TreasureData.h in CopyFiles */ = {isa = PBXBuildFile; fileRef = 6B738096192A09160097D56E /* TreasureData.h */; };
 		6B738099192A09160097D56E /* TreasureData.m in Sources */ = {isa = PBXBuildFile; fileRef = 6B738098192A09160097D56E /* TreasureData.m */; };
@@ -86,8 +85,6 @@
 		4D4265C575E11B5930549F10 /* Pods-Common-TreasureData.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Common-TreasureData.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Common-TreasureData/Pods-Common-TreasureData.debug.xcconfig"; sourceTree = "<group>"; };
 		57CFB8AE9EF047FBDE70DEA8 /* Pods-Common-TreasureData.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Common-TreasureData.release.xcconfig"; path = "Pods/Target Support Files/Pods-Common-TreasureData/Pods-Common-TreasureData.release.xcconfig"; sourceTree = "<group>"; };
 		644A82FEB7ABBA6EEE825324 /* Pods-TreasureDataTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-TreasureDataTests.release.xcconfig"; path = "Pods/Target Support Files/Pods-TreasureDataTests/Pods-TreasureDataTests.release.xcconfig"; sourceTree = "<group>"; };
-		6B351A74194ABF0F001CBAAD /* Deflate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Deflate.h; sourceTree = "<group>"; };
-		6B351A75194ABF0F001CBAAD /* Deflate.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Deflate.m; sourceTree = "<group>"; };
 		6B73808E192A09160097D56E /* libTreasureData.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libTreasureData.a; sourceTree = BUILT_PRODUCTS_DIR; };
 		6B738091192A09160097D56E /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
 		6B738095192A09160097D56E /* TreasureData-Prefix.pch */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "TreasureData-Prefix.pch"; sourceTree = "<group>"; };
@@ -169,8 +166,6 @@
 		4F99812D269844A000471FD4 /* TreasureDataInternal */ = {
 			isa = PBXGroup;
 			children = (
-				6B351A74194ABF0F001CBAAD /* Deflate.h */,
-				6B351A75194ABF0F001CBAAD /* Deflate.m */,
 				DF259EBD22A5138800697319 /* NSString+Helpers.h */,
 				E5701BA5225755C5000FB17F /* TDClientInternal.h */,
 				2053A39D2047F38B0089A959 /* TDUtils.h */,
@@ -474,10 +469,12 @@
 			);
 			inputPaths = (
 				"${PODS_ROOT}/Target Support Files/Pods-TreasureDataTests/Pods-TreasureDataTests-frameworks.sh",
+				"${BUILT_PRODUCTS_DIR}/GZIP/GZIP.framework",
 				"${BUILT_PRODUCTS_DIR}/KeenClientTD/KeenClientTD.framework",
 			);
 			name = "[CP] Embed Pods Frameworks";
 			outputPaths = (
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/GZIP.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/KeenClientTD.framework",
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -496,7 +493,6 @@
 				6BF8C9341A3E9B76005B1804 /* TDClient.m in Sources */,
 				2053A39C2047F37F0089A959 /* TDUtils.m in Sources */,
 				6B738099192A09160097D56E /* TreasureData.m in Sources */,
-				6B351A76194ABF0F001CBAAD /* Deflate.m in Sources */,
 				1950CB9871EC3D104BCD112A /* TDIAPObserver.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

TreasureData/TDClient.h

@@ -20,7 +20,7 @@
 @property(nonatomic, strong) NSString *apiKey;
 
 /**
- * The targeting API endpoint, default is https://in.treasuredata.com
+ * The targeting API endpoint, default is https://us01.records.in.treasuredata.com
  */
 @property(nonatomic, strong) NSString *apiEndpoint;
 

TreasureData/TDClient.m

@@ -10,8 +10,8 @@
 
 #import <UIKit/UIKit.h>
 #import <CommonCrypto/CommonDigest.h>
-#import "Deflate.h"
 #import "TDClient.h"
+@import GZIP;
 
 static NSString *version = @"0.9.0";
 
@@ -31,7 +31,7 @@ - (id)__initWithApiKey:(NSString *)apiKey apiEndpoint:(NSString*)apiEndpoint {
         if (!NSClassFromString(@"NSUUID")) {
             return @{};
         }
-        return @{@"#UUID": [[NSUUID UUID] UUIDString]};
+        return @{@"uuid": [[NSUUID UUID] UUIDString]};
     };
     /*
      > 5.times.inject(0){|a, i| puts a; x = 4 * (2 ** i); a += x; a}
@@ -64,37 +64,25 @@ - (NSString *)sha256Hash:(NSString *)input {
     return nsString;
 }
 
-- (void)sendEvents:(NSData *)data completionHandler:(void (^)(NSData *data, NSURLResponse *response, NSError *error))completionHandler {
-    NSString *urlString = [NSString stringWithFormat:@"%@/%@", self.apiEndpoint, @"ios/v3/event"];
+- (void)sendEvents:(NSData *)data database:(NSString *)database table:(NSString *)table completionHandler:(void (^)(NSData *data, NSURLResponse *response, NSError *error))completionHandler {
+    NSString *urlString = [NSString stringWithFormat:@"%@/%@/%@", self.apiEndpoint, database, table];
     KCLog(@"Sending events to: %@", urlString);
     NSURL *url = [NSURL URLWithString:urlString];
     NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
     [request setHTTPMethod:@"POST"];
     [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
-    [request setValue:self.apiKey forHTTPHeaderField:@"X-TD-Write-Key"];
-    [request setValue:@"k" forHTTPHeaderField:@"X-TD-Data-Type"];   // means KeenIO data type
+    [request setValue:[NSString stringWithFormat: @"TD1 %@", self.apiKey] forHTTPHeaderField:@"Authorization"];
+    [request setValue:@"application/vnd.treasuredata.v1+json" forHTTPHeaderField:@"Content-Type"];
+    [request setValue:@"application/vnd.treasuredata.v1+json" forHTTPHeaderField:@"Accept"];
     [request setValue:[NSString stringWithFormat:@"TD-iOS-SDK/%@ (%@ %@)", version, [[UIDevice currentDevice] systemName], [[UIDevice currentDevice] systemVersion]] forHTTPHeaderField:@"User-Agent"];
 
     if (_enableEventCompression) {
-        NSData *compressedData = [Deflate deflate:data];
-        if (!compressedData) {
-            KCLog(@"Compression failed");
-        }
-        else {
-            KCLog(@"Compressed: before=%ld, after=%ld", (unsigned long)[data length], (unsigned long)[compressedData length]);
-            data = compressedData;
-            /*
-             Byte* bytes = [data bytes];
-             for (int i=0; i < [data length]; i++) {
-             NSLog(@"byte[%d]: 0x%02x", i, bytes[i]);
-             }
-             */
-            [request setValue:@"deflate" forHTTPHeaderField:@"Content-Encoding"];
-        }
+        [request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
+        [request setHTTPBody:[data gzippedData]];
+    } else {
+        [request setHTTPBody:data];
     }
 
-    [request setHTTPBody:data];
-
     [self __sendHTTPRequest:request retryCounter:0 completionHandler:completionHandler];
 }
 

TreasureData/TreasureData.h

@@ -78,17 +78,24 @@ typedef void (^ErrorHandler)(NSString* _Nonnull errorCode, NSString* _Nullable e
 #pragma mark - Initialization
 
 /**
- * Assign the target API endpoint, default is "https://in.treasuredata.com".
- * Possible values:
- *    AWS East  https://in.treasuredata.com
- *    AWS Tokyo https://tokyo.in.treasuredata.com
- *    AWS EU    https://eu01.in.treasuredata.com
- *    AWS Asia Pacific (Seoul)  https://ap02.in.treasuredata.com
- *    AWS Asia Pacific (Tokyo)  https://ap03.in.treasuredata.com
- * This have to be call before `initializeWithApiKey(apiKey:)`, otherwise it won't have effect.
- * @param apiEndpoint for the in effect endpoint (`+[TreasureData initializeApiEndpoint:]`).
+ * Initialize `TreasureData.sharedInstance` with the default API endpoint "https://us01.records.in.treasuredata.com".
+ * To use endpoint other than the default one, call `+[TreasureData initializeWithApiKey:apiEndpoint:]` instead.
+ *
+ * @param apiKey API Key (only requires `write-only`) for the default API endpoint.
  */
-+ (void)initializeApiEndpoint:(NSString * _Nullable)apiEndpoint;
++ (void)initializeWithApiKey:(NSString * _Nonnull)apiKey;
+
+/**
+ * Initialize `TreasureData.sharedInstance`
+ * Possible apiEndpoint values:
+ *    AWS East https://us01.records.in.treasuredata.com
+ *    AWS EU https://eu01.records.in.treasuredata.com
+ *    AWS Asia Pacific (Tokyo) https://ap01.records.in.treasuredata.com
+ *    AWS Asia Pacific (Seoul) https://ap02.records.in.treasuredata.com
+ * @param apiKey API Key (only requires `write-only`).
+ * @param apiEndpoint a Treasure Data API endpoint.
+ */
++ (void)initializeWithApiKey:(NSString * _Nonnull)apiKey apiEndpoint:(NSString * _Nonnull)apiEndpoint;
 
 /**
  * Encrypted the event data in the local persisted buffer.
@@ -97,26 +104,27 @@ typedef void (^ErrorHandler)(NSString* _Nonnull errorCode, NSString* _Nullable e
 + (void)initializeEncryptionKey:(NSString* _Nullable)encryptionKey;
 
 /**
- * Initialize `TreasureData.sharedInstance` with the current `apiEndpoint` configured via `+[TreasureData initializeApiEndpoint:]`
+ * The default singleton SDK instance.
  *
- * @param apiKey API Key (only requires `write-only`) for the in effect endpoint (`+[TreasureData initializeApiEndpoint:]`).
+ * You could create multiple instances that target different endpoints (and of course apiKey, and default database, table, etc.) with `-[TreasureData initWithApiKey:]` or `-[TreasureData initWithApiKey:apiEndpoint:]`.
  */
-+ (void)initializeWithApiKey:(NSString * _Nonnull)apiKey;
++ (instancetype _Nonnull)sharedInstance;
 
 /**
- * The default singleton SDK instance.
+ * Construct a new `TreasureData` instance with the default API endpoint "https://us01.records.in.treasuredata.com".
+ * To use endpoint other than the default one, call `-[TreasureData initWithApiKey:apiEndpoint:]` instead.
  *
- * You could create multiple instances that target different endpoints (and of course apiKey, and default database, table, etc.) with `-[TreasureData initWithApiKey:]`,
- * but mind that `+[TreasureData initializeApiEndpoint:]` is shared have to be called before `-[TreasureData initWithApiKey:]` to be affected.
+ * @param apiKey API Key (only requires `write-only`) for the default API endpoint.
  */
-+ (instancetype _Nonnull)sharedInstance;
+- (id _Nonnull)initWithApiKey:(NSString * _Nonnull)apiKey;
 
 /**
  * Construct a new `TreasureData` instance.
  *
- * @param apiKey for the in effect endpoint (`+[TreasureData initializeApiEndpoint:]`).
+ * @param apiKey API Key (only requires `write-only`).
+ * @param apiEndpoint a Treasure Data API endpoint.
  */
-- (id _Nonnull)initWithApiKey:(NSString * _Nonnull)apiKey;
+- (id _Nonnull)initWithApiKey:(NSString * _Nonnull)apiKey apiEndpoint:(NSString * _Nonnull)apiEndpoint;
 
 #pragma mark - Tracking events
 
@@ -213,6 +221,25 @@ typedef void (^ErrorHandler)(NSString* _Nonnull errorCode, NSString* _Nullable e
  */
 - (void)resetUniqId;
 
+/**
+ * Automatically append the local time value when the event is added. Enabled by default.
+ *
+ * @param columnName The column to write the local time value
+ */
+- (void)enableAutoAppendLocalTimestamp:(NSString* _Nonnull)columnName;
+
+/**
+ * Automatically append the local time when the event is added. By default, the column's name is "time"
+ *
+ * This is enabled by default.
+ */
+- (void)enableAutoAppendLocalTimestamp;
+
+/**
+ * Disable automatically append the local time when the event is added.
+ */
+- (void)disableAutoAppendLocalTimestamp;
+
 /**
  * Disable these auto appended columns:
  *
@@ -256,25 +283,6 @@ typedef void (^ErrorHandler)(NSString* _Nonnull errorCode, NSString* _Nullable e
  */
 - (void)disableAutoAppendLocaleInformation;
 
-/**
- * Automatically append the time value when the event is received on server. Disabled by default.
- *
- * @param columnName The column to write the uploaded time value
- */
-- (void)enableServerSideUploadTimestamp: (NSString* _Nonnull)columnName;
-
-/**
- * Automatically append the time when the event is received on server. Disabled by default.
- *
- * This is disabled by default.
- */
-- (void)enableServerSideUploadTimestamp;
-
-/**
- * Disable the uploading time column
- */
-- (void)disableServerSideUploadTimestamp;
-
 /**
  * Automatically append a random and unique ID for each event. Disabled by default.
  *