Skip to content

Commit

Permalink
Add %{{TIME_NOW}} pattern for sprintf (#16906)
Browse files Browse the repository at this point in the history
* Add a new pattern %{{TIME_NOW}} to `event.sprintf` to generate a fresh timestamp.
The timestamp is represented as a string in the default ISO 8601 format

For example,
```
input {
    heartbeat {
    add_field => { "heartbeat_time" => "%{{TIME_NOW}}" }
    }
}
```

(cherry picked from commit cd729b7)
  • Loading branch information
kaisecheng authored and logstashmachine committed Jan 17, 2025
1 parent b8d2cec commit 813b524
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 16 deletions.
17 changes: 17 additions & 0 deletions docs/static/event-data.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,23 @@ These formats are not directly interchangeable, and we advise you to begin using

NOTE: A Logstash timestamp represents an instant on the UTC-timeline, so using sprintf formatters will produce results that may not align with your machine-local timezone.

You can generate a fresh timestamp by using `%{{TIME_NOW}}` syntax instead of relying on the value in `@timestamp`.
This is particularly useful when you need to estimate the time span of each plugin.

[source,js]
----------------------------------
input {
heartbeat {
add_field => { "heartbeat_time" => "%{{TIME_NOW}}" }
}
}
filter {
mutate {
add_field => { "mutate_time" => "%{{TIME_NOW}}" }
}
}
----------------------------------

[discrete]
[[conditionals]]
==== Conditionals
Expand Down
20 changes: 14 additions & 6 deletions logstash-core/src/main/java/org/logstash/StringInterpolation.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.Optional;

public final class StringInterpolation {

private static final String TIME_NOW = "TIME_NOW";
private static final ThreadLocal<StringBuilder> STRING_BUILDER =
new ThreadLocal<StringBuilder>() {
@Override
Expand Down Expand Up @@ -83,13 +85,19 @@ public static String evaluate(final Event event, final String template) throws J
// JAVA-style @timestamp formatter:
// - `%{{yyyy-MM-dd}}` -> `2021-08-11`
// - `%{{YYYY-'W'ww}}` -> `2021-W32`
// A special pattern to generate a fresh current time
// - `%{{TIME_NOW}}` -> `2025-01-16T16:57:12.488955Z`
close = close + 1; // consume extra closing squiggle
final Timestamp t = event.getTimestamp();
if (t != null) {
final String javaTimeFormatPattern = template.substring(open+3, close-1);
final java.time.format.DateTimeFormatter javaDateTimeFormatter = DateTimeFormatter.ofPattern(javaTimeFormatPattern).withZone(ZoneOffset.UTC);
final String formattedTimestamp = javaDateTimeFormatter.format(t.toInstant());
builder.append(formattedTimestamp);
final String pattern = template.substring(open+3, close-1);
if (pattern.equals(TIME_NOW)) {
builder.append(new Timestamp());
} else {
Optional.ofNullable(event.getTimestamp())
.map(Timestamp::toInstant)
.map(instant -> DateTimeFormatter.ofPattern(pattern)
.withZone(ZoneOffset.UTC)
.format(instant))
.ifPresent(builder::append);
}
} else if (template.charAt(open + 2) == '+') {
// JODA-style @timestamp formatter:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,21 @@
package org.logstash;


import org.awaitility.Awaitility;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.junit.Test;

import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;


public class StringInterpolationTest extends RubyTestBase {
Expand Down Expand Up @@ -71,14 +76,14 @@ public void testDateFormatter() throws IOException {
}

@Test
public void TestMixDateAndFields() throws IOException {
public void testMixDateAndFields() throws IOException {
Event event = getTestEvent();
String path = "/full/%{+YYYY}/weeee/%{bar}";
assertEquals("/full/2015/weeee/foo", StringInterpolation.evaluate(event, path));
}

@Test
public void TestMixDateAndFieldsJavaSyntax() throws IOException {
public void testMixDateAndFieldsJavaSyntax() throws IOException {
Event event = getTestEvent();
String path = "/full/%{{YYYY-DDD}}/weeee/%{bar}";
assertEquals("/full/2015-274/weeee/foo", StringInterpolation.evaluate(event, path));
Expand All @@ -92,28 +97,28 @@ public void testUnclosedTag() throws IOException {
}

@Test
public void TestStringIsOneDateTag() throws IOException {
public void testStringIsOneDateTag() throws IOException {
Event event = getTestEvent();
String path = "%{+YYYY}";
assertEquals("2015", StringInterpolation.evaluate(event, path));
}

@Test
public void TestStringIsJavaDateTag() throws IOException {
public void testStringIsJavaDateTag() throws IOException {
Event event = getTestEvent();
String path = "%{{YYYY-'W'ww}}";
assertEquals("2015-W40", StringInterpolation.evaluate(event, path));
}

@Test
public void TestFieldRef() throws IOException {
public void testFieldRef() throws IOException {
Event event = getTestEvent();
String path = "%{[j][k1]}";
assertEquals("v", StringInterpolation.evaluate(event, path));
}

@Test
public void TestEpochSeconds() throws IOException {
public void testEpochSeconds() throws IOException {
Event event = getTestEvent();
String path = "%{+%ss}";
// `+%ss` bypasses the EPOCH syntax and instead matches the JODA syntax.
Expand All @@ -122,14 +127,14 @@ public void TestEpochSeconds() throws IOException {
}

@Test
public void TestEpoch() throws IOException {
public void testEpoch() throws IOException {
Event event = getTestEvent();
String path = "%{+%s}";
assertEquals("1443657600", StringInterpolation.evaluate(event, path));
}

@Test
public void TestValueIsArray() throws IOException {
public void testValueIsArray() throws IOException {
ArrayList<String> l = new ArrayList<>();
l.add("Hello");
l.add("world");
Expand All @@ -142,13 +147,30 @@ public void TestValueIsArray() throws IOException {
}

@Test
public void TestValueIsHash() throws IOException {
public void testValueIsHash() throws IOException {
Event event = getTestEvent();

String path = "%{j}";
assertEquals("{\"k1\":\"v\"}", StringInterpolation.evaluate(event, path));
}

@Test
public void testPatternTimeNowGenerateFreshTimestamp() throws IOException, InterruptedException {
Event event = getTestEvent();
Timestamp before = new Timestamp();
Awaitility.await("Make sure we sleep enough get another current timestamp")
.atMost(Duration.ofSeconds(1))
.until(() -> Instant.now().isAfter(before.toInstant()));
Timestamp result = new Timestamp(StringInterpolation.evaluate(event, "%{{TIME_NOW}}"));
assertTrue(before.compareTo(result) < 0);
}

@Test
public void testBadPatternTimeNowShouldThrowException() throws IOException {
Event event = getTestEvent();
assertThrows(IllegalArgumentException.class, () -> StringInterpolation.evaluate(event, "%{{BAD_TIME_NOW}}"));
}

public Event getTestEvent() {
Map<String, Object> data = new HashMap<>();
Map<String, String> inner = new HashMap<>();
Expand Down

0 comments on commit 813b524

Please sign in to comment.