Skip to content

Commit

Permalink
Revert "Make it possible to limit number of log lines returned" (#24168)
Browse files Browse the repository at this point in the history
  • Loading branch information
@freva
freva authored Sep 21, 2022
1 parent 1040425 commit d058fe5
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 23 deletions.
12 changes: 9 additions & 3 deletions container-core/src/main/java/com/yahoo/container/handler/LogHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,21 @@
import com.yahoo.component.annotation.Inject;
import com.yahoo.container.core.LogHandlerConfig;
import com.yahoo.container.jdisc.AsyncHttpResponse;
import com.yahoo.container.jdisc.ContentChannelOutputStream;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
import com.yahoo.jdisc.handler.CompletionHandler;
import com.yahoo.jdisc.handler.ContentChannel;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;

public class LogHandler extends ThreadedHttpRequestHandler {
Expand All @@ -36,8 +42,6 @@ public AsyncHttpResponse handle(HttpRequest request) {
.map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN);
Instant to = Optional.ofNullable(request.getProperty("to"))
.map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX);
long maxLines = Optional.ofNullable(request.getProperty("maxLines"))
.map(Long::valueOf).orElse(100_000L);
Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname"));

return new AsyncHttpResponse(200) {
Expand All @@ -46,7 +50,7 @@ public AsyncHttpResponse handle(HttpRequest request) {
@Override
public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) {
try (output) {
logReader.writeLogs(output, from, to, maxLines, hostname);
logReader.writeLogs(output, from, to, hostname);
}
catch (Throwable t) {
log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t);
Expand All @@ -58,4 +62,6 @@ public void render(OutputStream output, ContentChannel networkChannel, Completio
};
}



}
15 changes: 11 additions & 4 deletions container-core/src/main/java/com/yahoo/container/handler/LogReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,9 @@ class LogReader {
this.logFilePattern = logFilePattern;
}

void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) {
void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) {
double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9;
double toSeconds = to.getEpochSecond() + to.getNano() / 1e9;
long linesWritten = 0;
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
for (List<Path> logs : getMatchingFiles(from, to)) {
List<LogLineIterator> logLineIterators = new ArrayList<>();
Expand All @@ -74,7 +73,6 @@ void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Option
Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators,
Comparator.comparingDouble(LineWithTimestamp::timestamp));
while (lines.hasNext()) {
if (linesWritten++ >= maxLines) return;
String line = lines.next().line();
writer.write(line);
writer.newLine();
Expand Down Expand Up @@ -189,7 +187,16 @@ private LineWithTimestamp readNext() {

}

private record LineWithTimestamp(String line, double timestamp) { }
private static class LineWithTimestamp {
final String line;
final double timestamp;
LineWithTimestamp(String line, double timestamp) {
this.line = line;
this.timestamp = timestamp;
}
String line() { return line; }
double timestamp() { return timestamp; }
}

/** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */
private List<List<Path>> getMatchingFiles(Instant from, Instant to) {
Expand Down
2 changes: 1 addition & 1 deletion container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class MockLogReader extends LogReader {
}

@Override
protected void writeLogs(OutputStream out, Instant from, Instant to, long maxLines, Optional<String> hostname) {
protected void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) {
try {
if (to.isAfter(Instant.ofEpochMilli(1000))) {
out.write("newer log".getBytes());
Expand Down
30 changes: 15 additions & 15 deletions container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class LogReaderTest {

@BeforeEach
public void setup() throws IOException {
logDirectory = Files.createDirectories(folder.toPath().resolve("opt/vespa/logs"));
logDirectory = newFolder(folder, "opt/vespa/logs").toPath();
// Log archive paths and file names indicate what hour they contain logs for, with the start of that hour.
// Multiple entries may exist for each hour.
Files.createDirectories(logDirectory.resolve("1970/01/01"));
Expand All @@ -59,7 +59,7 @@ public void setup() throws IOException {
void testThatLogsOutsideRangeAreExcluded() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), 100, Optional.empty());
logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty());

assertEquals(log100 + logv11 + log110, baos.toString(UTF_8));
}
Expand All @@ -68,7 +68,7 @@ void testThatLogsOutsideRangeAreExcluded() {
void testThatLogsNotMatchingRegexAreExcluded() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty());

assertEquals(log101 + logv11, baos.toString(UTF_8));
}
Expand All @@ -78,7 +78,7 @@ void testThatLogsNotMatchingRegexAreExcluded() {
void testZippedStreaming() {
ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.empty());
logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty());

assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8));
}
Expand All @@ -87,20 +87,11 @@ void testZippedStreaming() {
void logsForSingeNodeIsRetrieved() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 100, Optional.of("node2.com"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com"));

assertEquals(log101 + log100 + log200, baos.toString(UTF_8));
}

@Test
void logsLimitedToMaxLines() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*"));
logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), 2, Optional.of("node2.com"));

assertEquals(log101 + log100, baos.toString(UTF_8));
}

private byte[] compress1(String input) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream zip = new GZIPOutputStream(baos);
Expand All @@ -109,9 +100,18 @@ private byte[] compress1(String input) throws IOException {
return baos.toByteArray();
}

private byte[] compress2(String input) {
private byte[] compress2(String input) throws IOException {
byte[] data = input.getBytes();
return new ZstdCompressor().compress(data, 0, data.length);
}

private static File newFolder(File root, String... subDirs) throws IOException {
String subFolder = String.join("/", subDirs);
File result = new File(root, subFolder);
if (!result.mkdirs()) {
throw new IOException("Couldn't create folders " + root);
}
return result;
}

}

0 comments on commit d058fe5

Please sign in to comment.