diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 72a399744f34..d151eb6862de 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -4,15 +4,21 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.AsyncHttpResponse; +import com.yahoo.container.jdisc.ContentChannelOutputStream; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; +import java.io.IOException; +import java.io.InterruptedIOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; public class LogHandler extends ThreadedHttpRequestHandler { @@ -36,8 +42,6 @@ public AsyncHttpResponse handle(HttpRequest request) { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant to = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); - long maxLines = Optional.ofNullable(request.getProperty("maxLines")) - .map(Long::valueOf).orElse(100_000L); Optional hostname = Optional.ofNullable(request.getProperty("hostname")); return new AsyncHttpResponse(200) { @@ -46,7 +50,7 @@ public AsyncHttpResponse handle(HttpRequest request) { @Override public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) { try (output) { - logReader.writeLogs(output, from, to, maxLines, hostname); + logReader.writeLogs(output, from, to, hostname); } catch (Throwable t) { log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); @@ -58,4 +62,6 @@ public void render(OutputStream output, ContentChannel networkChannel, Completio }; } + + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index dae4b940307a..93881b52eb65 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -59,10 +59,9 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional hostname) { + void writeLogs(OutputStream out, Instant from, Instant to, Optional hostname) { double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; - long linesWritten = 0; BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); for (List logs : getMatchingFiles(from, to)) { List logLineIterators = new ArrayList<>(); @@ -74,7 +73,6 @@ void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Option Iterator lines = Iterators.mergeSorted(logLineIterators, Comparator.comparingDouble(LineWithTimestamp::timestamp)); while (lines.hasNext()) { - if (linesWritten++ >= maxLines) return; String line = lines.next().line(); writer.write(line); writer.newLine(); @@ -189,7 +187,16 @@ private LineWithTimestamp readNext() { } - private record LineWithTimestamp(String line, double timestamp) { } + private static class LineWithTimestamp { + final String line; + final double timestamp; + LineWithTimestamp(String line, double timestamp) { + this.line = line; + this.timestamp = timestamp; + } + String line() { return line; } + double timestamp() { return timestamp; } + } /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ private List> getMatchingFiles(Instant from, Instant to) { diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 8b698b25ab2b..3d9a2360e774 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -51,7 +51,7 @@ class MockLogReader extends LogReader { } @Override - protected void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional hostname) { + protected void writeLogs(OutputStream out, Instant from, Instant to, Optional hostname) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { out.write("newer log".getBytes()); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index e98f8cac2762..106b8cef35fc 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -37,7 +37,7 @@ public class LogReaderTest { @BeforeEach public void setup() throws IOException { - logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs")); + logDirectory = newFolder(folder, "opt/vespa/logs").toPath(); // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. // Multiple entries may exist for each hour. Files.createDirectories(logDirectory.resolve("1970/01/01")); @@ -59,7 +59,7 @@ public void setup() throws IOException { void testThatLogsOutsideRangeAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty()); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty()); assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @@ -68,7 +68,7 @@ void testThatLogsOutsideRangeAreExcluded() { void testThatLogsNotMatchingRegexAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); assertEquals(log101 + logv11, baos.toString(UTF_8)); } @@ -78,7 +78,7 @@ void testThatLogsNotMatchingRegexAreExcluded() { void testZippedStreaming() { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty()); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } @@ -87,20 +87,11 @@ void testZippedStreaming() { void logsForSingeNodeIsRetrieved() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com")); assertEquals(log101 + log100 + log200, baos.toString(UTF_8)); } - @Test - void logsLimitedToMaxLines() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com")); - - assertEquals(log101 + log100, baos.toString(UTF_8)); - } - private byte[] compress1(String input) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream zip = new GZIPOutputStream(baos); @@ -109,9 +100,18 @@ private byte[] compress1(String input) throws IOException { return baos.toByteArray(); } - private byte[] compress2(String input) { + private byte[] compress2(String input) throws IOException { byte[] data = input.getBytes(); return new ZstdCompressor().compress(data, 0, data.length); } + private static File newFolder(File root, String... subDirs) throws IOException { + String subFolder = String.join("/", subDirs); + File result = new File(root, subFolder); + if (!result.mkdirs()) { + throw new IOException("Couldn't create folders " + root); + } + return result; + } + }