diff --git a/app/controllers/HomeController.java b/app/controllers/HomeController.java index fb3c32d..283350b 100644 --- a/app/controllers/HomeController.java +++ b/app/controllers/HomeController.java @@ -77,6 +77,7 @@ import modules.IndexComponent; import play.Environment; import play.Logger; +import play.cache.Cached; import play.libs.Json; import play.libs.ws.WSBodyReadables; import play.libs.ws.WSBodyWritables; @@ -735,4 +736,14 @@ private static String findText(JsonNode info, String field) { return node != null ? node.get("value").asText().replace("\n", " ").trim() : ""; } + @Cached(key = "robots", duration = 24 * 60 * 60) // One day + public Result robots() throws IOException { + return ok("User-agent: *\nDisallow: /" + + Files.readAllLines(Paths.get("conf/rppd-export.jsonl")).stream() + .filter(line -> line.contains("doNotCrawl\":true")) + .map(line -> line.replaceAll(".*gndIdentifier\":\"(.+?)\".*", "$1") + .replaceAll("Keine GND-Ansetzung für ", "")) + .collect(Collectors.joining("\nDisallow: /"))); + } + } diff --git a/app/modules/IndexComponent.java b/app/modules/IndexComponent.java index fcfc71c..eab2043 100644 --- a/app/modules/IndexComponent.java +++ b/app/modules/IndexComponent.java @@ -34,6 +34,7 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient; import controllers.HomeController; +import net.sf.ehcache.CacheManager; import play.Logger; import play.inject.ApplicationLifecycle; @@ -74,6 +75,7 @@ public ElasticsearchServer(ApplicationLifecycle lifecycle) { }); lifecycle.addStopHook(() -> { client.close(); + CacheManager.getInstance().shutdown(); return null; }); } diff --git a/build.sbt b/build.sbt index 2b17f47..5599864 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,8 @@ libraryDependencies += guice libraryDependencies += ws +libraryDependencies += ehcache + libraryDependencies += "com.github.jsonld-java" % "jsonld-java" % "0.12.0" libraryDependencies += "javax.mail" % "mail" % "1.4.1" diff --git a/conf/routes b/conf/routes index 0bb78bd..a6bbf8c 100644 --- a/conf/routes +++ b/conf/routes @@ -28,6 +28,8 @@ GET /advanced controllers.HomeController.advanced() GET /cgi-bin/wwwalleg/:name.pl controllers.HomeController.authorityPl(name, db ?= "rnam", index: Int ?= 1, zeilen: Int ?= 1, s1) +GET /robots.txt controllers.HomeController.robots() + GET /:id.:format controllers.HomeController.authorityDotFormat(id, format) GET /:id controllers.HomeController.authority(id, format ?= null)