DateTieredCompaction for existing tsdb tables #1065

Signed-off-by: Chris Larsen <[email protected]>

src/core/TsdbQuery.java

@@ -1400,7 +1400,9 @@ protected Scanner getScanner(final int salt_bucket) throws HBaseException {
     if(tsdb.getConfig().use_otsdb_timestamp()) {
       long stTime = (getScanStartTimeSeconds() * 1000);
       long endTime = end_time == UNSET ? -1 : (getScanEndTimeSeconds() * 1000);
-      scanner.setTimeRange(stTime, endTime);
+      if (tsdb.getConfig().get_date_tiered_compaction_start() <= stTime) {
+        scanner.setTimeRange(stTime, endTime);
+      }
     }
     if (tsuids != null && !tsuids.isEmpty()) {
       createAndSetTSUIDFilter(scanner);

src/tools/CliOptions.java

@@ -67,7 +67,7 @@ static void addAutoMetricFlag(final ArgP argp) {
 
   /**
    * Parse the command line arguments with the given options.
-   * @param options Options to parse in the given args.
+   * @param opt,ions Options to parse in the given args.
    * @param args Command line arguments to parse.
    * @return The remainder of the command line or
    * {@code null} if {@code args} were invalid and couldn't be parsed.
@@ -154,6 +154,8 @@ static void overloadConfig(final ArgP argp, final Config config) {
         config.overrideConfig("tsd.network.worker_threads", entry.getValue());
       } else if(entry.getKey().toLowerCase().equals("--use-otsdb-ts")) {
         config.overrideConfig("tsd.storage.use_otsdb_timestamp", "true");
+      } else if (entry.getKey().toLowerCase().equals("--dtc-ts")) {
+        config.overrideConfig("tsd.storage.get_date_tiered_compaction_start", entry.getValue());
       }
     }
   }

src/utils/Config.java

@@ -110,6 +110,15 @@ public class Config {
   /** Sets the HBase cell timestamp equal to metric timestamp */
   private boolean use_otsdb_timestamp = true;
 
+  /** tsd.storage.get_date_tiered_compaction_start */
+  /** Sets the time at which you started using use_otsdb_timestamp
+   * this value is overriden if you have existing data in your tsdb table
+   * but don't want to re-write the cell timestamps, therefore you can start
+   * set this timestamp to when you start using use_otsdb_timestamp
+   * */
+  private long get_date_tiered_compaction_start = 0;
+
+
   /** tsd.storage.use_max_value */
   /** Used for resolving between data coming in at same timestamp */
   /** If set to true, the maximum value will be returned, minimum */
@@ -273,6 +282,11 @@ public boolean use_otsdb_timestamp() {
     return use_otsdb_timestamp;
   }
 
+  /** @return the time at which you started storing data using otsdb_timestamp(), if not set defaults to 0 */
+  public long get_date_tiered_compaction_start() {
+    return get_date_tiered_compaction_start;
+  }
+
   public boolean use_max_value() {
     return use_max_value;
   }
@@ -593,6 +607,7 @@ protected void setDefaults() {
     default_map.put("tsd.query.timeout", "0");
     default_map.put("tsd.storage.use_otsdb_timestamp", "true");
     default_map.put("tsd.storage.use_max_value", "true");
+    default_map.put("tsd.storage.get_date_tiered_compaction_start", "0");
 
     for (Map.Entry<String, String> entry : default_map.entrySet()) {
       if (!properties.containsKey(entry.getKey()))
@@ -707,6 +722,7 @@ public void loadStaticVariables() {
     fix_duplicates = this.getBoolean("tsd.storage.fix_duplicates");
     scanner_max_num_rows = this.getInt("tsd.storage.hbase.scanner.maxNumRows");
     use_otsdb_timestamp = this.getBoolean("tsd.storage.use_otsdb_timestamp");
+    get_date_tiered_compaction_start = this.getLong("tsd.storage.get_date_tiered_compaction_start");
     use_max_value = this.getBoolean("tsd.storage.use_max_value");
   }