diff --git a/elasticsearch (copy)/README.md b/elasticsearch (copy)/README.md deleted file mode 100644 index 1e6f3397..00000000 --- a/elasticsearch (copy)/README.md +++ /dev/null @@ -1,54 +0,0 @@ -Start `/bin/elasticsearch` -and `/bin/kibana`. - -Open kibana at `http://localhost:5601`. - -To submit the schema, wrap it in - -```json -PUT habitus - -PUT habitus/_mapping -{ - "add schema here" -} -``` - -To get a list of indices, run -```json -GET _cat/indices -``` - -To add data, wrap it in - -```json -PUT habitus/_doc/ -{ - "add document here" -} - -``` -Skip the to have one generated, but change it to `POST`. -```json -POST habitus/_doc/ -{ - "add document here" -} -``` - -To query for all records, use - -```json -GET habitus - -``` - -Delete a document with -```json -DELETE /habitus/_doc/ -``` - -Remove everything with -```json -DELETE habitus -``` diff --git a/elasticsearch (copy)/build.sbt b/elasticsearch (copy)/build.sbt deleted file mode 100644 index 30d2d683..00000000 --- a/elasticsearch (copy)/build.sbt +++ /dev/null @@ -1,12 +0,0 @@ - -name := "habitus-elasticsearch" - -resolvers += "clulab" at "https://artifactory.clulab.org/artifactory/sbt-release" - -libraryDependencies ++= { - Seq( - "co.elastic.clients" % "elasticsearch-java" % "8.12.0", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.16.1" - ) -} - diff --git a/elasticsearch (copy)/insert.json b/elasticsearch (copy)/insert.json deleted file mode 100644 index e69de29b..00000000 diff --git a/elasticsearch (copy)/project/build.properties b/elasticsearch (copy)/project/build.properties deleted file mode 100644 index 3161d214..00000000 --- a/elasticsearch (copy)/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.6.1 diff --git a/elasticsearch (copy)/schema.json b/elasticsearch (copy)/schema.json deleted file mode 100644 index bdbba615..00000000 --- a/elasticsearch (copy)/schema.json +++ /dev/null @@ -1,154 +0,0 @@ -{ - "properties": { - "dataset": { - "type": "keyword" - }, - "region": { - "type": "keyword" - }, - "url": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "terms": { - "type": "keyword" - }, - "dateline": { - "type": "keyword" - }, - "date": { - "type": "date" - }, - "byline": { - "type": "keyword" - }, - "sentenceIndex": { - "type": "integer" - }, - "sentence": { - "type": "text" - }, - "causalRelations": { - "type": "nested", - "properties": { - "index": { - "type": "integer" - }, - "negationCount": { - "type": "integer" - }, - "relations": { - "type": "nested", - "properties": { - "cause": { - "type": "nested", - "properties": { - "text": { - "type": "text" - }, - "incCount": { - "type": "integer" - }, - "decCount": { - "type": "integer" - }, - "posCount": { - "type": "integer" - }, - "negCount": { - "type": "integer" - } - } - }, - "effect": { - "type": "nested", - "properties": { - "text": { - "type": "text" - }, - "incCount": { - "type": "integer" - }, - "decCount": { - "type": "integer" - }, - "posCount": { - "type": "integer" - }, - "negCount": { - "type": "integer" - } - } - } - } - } - } - }, - "isBelief": { - "type": "boolean" - }, - "sentiment": { - "type": "keyword" - }, - "sentenceLocations": { - "type": "nested", - "properties": { - "name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - }, - "contextBefore": { - "type": "text", - "index": false - }, - "contextAfter": { - "type": "text", - "index": false - }, - "contextLocations": { - "type": "nested", - "properties": { - "name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - }, - "prevLocations": { - "type": "nested", - "properties": { - "name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - }, - "prevDistance": { - "type": "integer" - }, - "nextLocations": { - "type": "nested", - "properties": { - "name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - }, - "nextDistance": { - "type": "integer" - } - } -} \ No newline at end of file diff --git a/elasticsearch (copy)/src/main/scala/org/clulab/habitus/elasticsearch/apps/DatasetToElasticsearchApp.scala b/elasticsearch (copy)/src/main/scala/org/clulab/habitus/elasticsearch/apps/DatasetToElasticsearchApp.scala deleted file mode 100644 index f81799c1..00000000 --- a/elasticsearch (copy)/src/main/scala/org/clulab/habitus/elasticsearch/apps/DatasetToElasticsearchApp.scala +++ /dev/null @@ -1,156 +0,0 @@ -package org.clulab.habitus.elasticsearch.apps - -import co.elastic.clients.elasticsearch.ElasticsearchClient -import co.elastic.clients.elasticsearch.cat.{IndicesRequest, IndicesResponse} -import co.elastic.clients.elasticsearch.core.{IndexRequest, IndexResponse} -import co.elastic.clients.json.jackson.JacksonJsonpMapper -import co.elastic.clients.elasticsearch.indices.{CreateIndexRequest, CreateIndexResponse, ElasticsearchIndicesClient} -import co.elastic.clients.transport.rest_client.RestClientTransport -import org.apache.http.HttpHost -import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} -import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.elasticsearch.client.RestClient -import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback - -case class CauseOrEffect( - text: String, - incCount: Int, - decCount: Int, - posCount: Int, - negCount: Int -) { - -} - -case class Relation(cause: CauseOrEffect, effect: CauseOrEffect) { - -} - -case class CausalRelation( - index: Int, - negationCount: Int, - relations: Seq[Relation] -) { - -} - -case class LatLon(lat: Float, lon: Float) - -case class Location( - val name: String, - val latLon: LatLon -) { - -} - -// TODO: almost all of these are optional -case class DatasetRow( - dataset: String, - region: String, - url: String, - titleOpt: Option[String], - terms: Seq[String], - dateline: String, - date: String, - byline: String, - sentenceIndex: Int, - sentence: String, - causalRelations: Seq[CausalRelation], - isBelief: Boolean, - sentiment: String, - sentenceLocations: Seq[Location], - contextBefore: String, - contextAfter: String, - contextLocations: Seq[Location], - prevLocations: Seq[Location], - prevDistance: Int, - nextLocations: Seq[Location], - nextDistance: Int -) { - -} - -object DatasetToElasticsearchApp extends App { - val credentialsProvider = { - val username = "" - val password = "" - // TODO: Make other users - // TODO: Read credentials from file or environment - val usernamePasswordCredentials = new UsernamePasswordCredentials("elastic", "AJbD8giOsRujhuRS_kVh") - val credentialsProvider = new BasicCredentialsProvider() - - credentialsProvider.setCredentials(AuthScope.ANY, usernamePasswordCredentials) - credentialsProvider - } - - def mkRestClient(): RestClient = { - val restClient = RestClient - .builder(new HttpHost("localhost", 9200)) - // .builder(new HttpHost("elasticsearch.keithalcock.com", 443, "https")) - .setHttpClientConfigCallback(new HttpClientConfigCallback() { - def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = { - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - } - }) - .build() - - restClient - } - - def mkElasticsearchClient(): ElasticsearchClient = { - val restClient = mkRestClient() - val jsonpMapper = new JacksonJsonpMapper() - val elasticsearchTransport = new RestClientTransport(restClient, jsonpMapper) - val elasticsearchClient = new ElasticsearchClient(elasticsearchTransport) - - elasticsearchClient - } - - def createIndex(elasticsearchClient: ElasticsearchClient, indexName: String): CreateIndexResponse = { - val elasticsearchIndicesClient = elasticsearchClient.indices() - val createIndexRequest = new CreateIndexRequest.Builder().index(indexName).build() - val createIndexResponse = elasticsearchIndicesClient.create(createIndexRequest) - - println(createIndexResponse) - createIndexResponse - } - - def index(elasticsearchClient: ElasticsearchClient, indexName: String, datasetRow: DatasetRow): IndexResponse = { - val indexRequest = new IndexRequest.Builder().index(indexName).document(datasetRow).build() - val indexResponse = elasticsearchClient.index(indexRequest) - println(indexResponse) - - indexResponse - } - - def indices(elasticsearchClient: ElasticsearchClient): IndicesResponse = { - val elasticsearchCatClient = elasticsearchClient.cat() - val indicesResponse = elasticsearchCatClient.indices() - - println(indicesResponse) - indicesResponse - } - - def run(): Unit = { - val indexName = "habitus4" - val elasticsearchClient = mkElasticsearchClient() - val datasetRow = DatasetRow( - - ) - - // createIndex(elasticsearchClient, indexName) - // indices(elasticsearchClient) - index(elasticsearchClient, indexName, datasetRow) - - println("Goodbye") - } - - try { - run() - } - catch { - case throwable: Throwable => - println(throwable) - } -}