From 9b9c06117c3296409efd27a89a41bc654060806c Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Wed, 9 Oct 2024 12:05:55 +0100 Subject: [PATCH 1/7] add uax_url_email tokenizer docs Signed-off-by: Anton Rubin --- _analyzers/tokenizers/index.md | 2 +- _analyzers/tokenizers/uax-url-email.md | 80 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 _analyzers/tokenizers/uax-url-email.md diff --git a/_analyzers/tokenizers/index.md b/_analyzers/tokenizers/index.md index d401851f60..1abc5ee7ff 100644 --- a/_analyzers/tokenizers/index.md +++ b/_analyzers/tokenizers/index.md @@ -2,7 +2,7 @@ layout: default title: Tokenizers nav_order: 60 -has_children: false +has_children: true has_toc: false --- diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md new file mode 100644 index 0000000000..c705ac2756 --- /dev/null +++ b/_analyzers/tokenizers/uax-url-email.md @@ -0,0 +1,80 @@ +--- +layout: default +title: UAX URL email tokenizer +parent: Tokenizers +nav_order: 150 +--- + +# UAX URL email tokenizer + +In addition to normal text, the `uax_url_email` tokenizer is designed to handle URLs, email addresses, and domain names. It is based on the Unicode Text Segmentation algorithm ([UAX #29](https://www.unicode.org/reports/tr29/)), which allows it to tokenize complex text correctly, including URLs and email addresses. + +## Example usage + +The following example request creates a new index named `my_index` and configures an analyzer with `uax_url_email` tokenizer: + +``` +PUT /my_index +{ + "settings": { + "analysis": { + "tokenizer": { + "uax_url_email_tokenizer": { + "type": "uax_url_email" + } + }, + "analyzer": { + "my_uax_analyzer": { + "type": "custom", + "tokenizer": "uax_url_email_tokenizer" + } + } + } + }, + "mappings": { + "properties": { + "content": { + "type": "text", + "analyzer": "my_uax_analyzer" + } + } + } +} +``` +{% include copy-curl.html %} + +## Generated tokens + +Use the following request to examine the tokens generated using the created analyzer: + +```json +POST /my_index/_analyze +{ + "analyzer": "my_uax_analyzer", + "text": "Contact us at support@example.com or visit https://example.com for details." +} +``` +{% include copy-curl.html %} + +The response contains the generated tokens: + +```json +{ + "tokens": [ + {"token": "Contact","start_offset": 0,"end_offset": 7,"type": "","position": 0}, + {"token": "us","start_offset": 8,"end_offset": 10,"type": "","position": 1}, + {"token": "at","start_offset": 11,"end_offset": 13,"type": "","position": 2}, + {"token": "support@example.com","start_offset": 14,"end_offset": 33,"type": "","position": 3}, + {"token": "or","start_offset": 34,"end_offset": 36,"type": "","position": 4}, + {"token": "visit","start_offset": 37,"end_offset": 42,"type": "","position": 5}, + {"token": "https://example.com","start_offset": 43,"end_offset": 62,"type": "","position": 6}, + {"token": "for","start_offset": 63,"end_offset": 66,"type": "","position": 7}, + {"token": "details","start_offset": 67,"end_offset": 74,"type": "","position": 8} + ] +} +``` + +## Configuration + +The `uax_url_email` tokenizer can be configured with parameter `max_token_length` which sets the maximum length of the produced token. If this length is exceeded, the token is split into multiple tokens at length configured in `max_token_length`. Default is `255` (Integer, _Optional_) + From ac91f8b5eb3d3f5ecda7282e30ef7ed501dc897d Mon Sep 17 00:00:00 2001 From: AntonEliatra Date: Thu, 10 Oct 2024 11:16:24 +0100 Subject: [PATCH 2/7] Update uax-url-email.md Signed-off-by: AntonEliatra --- _analyzers/tokenizers/uax-url-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index c705ac2756..29fcae19f0 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -1,6 +1,6 @@ --- layout: default -title: UAX URL email tokenizer +title: UAX URL email parent: Tokenizers nav_order: 150 --- From e0dfa6de6151c56092e6c5c4c3093a7c19c17702 Mon Sep 17 00:00:00 2001 From: AntonEliatra Date: Thu, 10 Oct 2024 11:26:08 +0100 Subject: [PATCH 3/7] Update uax-url-email.md Signed-off-by: AntonEliatra --- _analyzers/tokenizers/uax-url-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index 29fcae19f0..7db2859a55 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -13,7 +13,7 @@ In addition to normal text, the `uax_url_email` tokenizer is designed to handle The following example request creates a new index named `my_index` and configures an analyzer with `uax_url_email` tokenizer: -``` +```json PUT /my_index { "settings": { From 1833e57bf59614ca224df5e5e5f0576ea236ba14 Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Wed, 16 Oct 2024 17:22:29 +0100 Subject: [PATCH 4/7] updating parameter table Signed-off-by: Anton Rubin --- _analyzers/tokenizers/uax-url-email.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index c705ac2756..370a766189 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -76,5 +76,9 @@ The response contains the generated tokens: ## Configuration -The `uax_url_email` tokenizer can be configured with parameter `max_token_length` which sets the maximum length of the produced token. If this length is exceeded, the token is split into multiple tokens at length configured in `max_token_length`. Default is `255` (Integer, _Optional_) +The `uax_url_email` tokenizer can be configured with the following parameter. + +Parameter | Required/Optional | Data type | Description +:--- | :--- | :--- | :--- +`max_token_length` | Optional | Integer | Sets the maximum length of the produced token. If this length is exceeded, the token is split into multiple tokens at length configured in `max_token_length`. Default is `255`. From 81e942e181c6d7df6a482e022562b1e5c993ce59 Mon Sep 17 00:00:00 2001 From: Fanit Kolchina Date: Thu, 5 Dec 2024 13:55:22 -0500 Subject: [PATCH 5/7] Doc review Signed-off-by: Fanit Kolchina --- _analyzers/tokenizers/uax-url-email.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index 926bf8cef4..ac39564bf2 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -7,11 +7,11 @@ nav_order: 150 # UAX URL email tokenizer -In addition to normal text, the `uax_url_email` tokenizer is designed to handle URLs, email addresses, and domain names. It is based on the Unicode Text Segmentation algorithm ([UAX #29](https://www.unicode.org/reports/tr29/)), which allows it to tokenize complex text correctly, including URLs and email addresses. +In addition to regular text, the `uax_url_email` tokenizer is designed to handle URLs, email addresses, and domain names. It is based on the Unicode Text Segmentation algorithm ([UAX #29](https://www.unicode.org/reports/tr29/)), which allows it to correctly tokenize complex text, including URLs and email addresses. ## Example usage -The following example request creates a new index named `my_index` and configures an analyzer with `uax_url_email` tokenizer: +The following example request creates a new index named `my_index` and configures an analyzer with a `uax_url_email` tokenizer: ```json PUT /my_index @@ -74,11 +74,11 @@ The response contains the generated tokens: } ``` -## Configuration +## Parameters The `uax_url_email` tokenizer can be configured with the following parameter. Parameter | Required/Optional | Data type | Description :--- | :--- | :--- | :--- -`max_token_length` | Optional | Integer | Sets the maximum length of the produced token. If this length is exceeded, the token is split into multiple tokens at length configured in `max_token_length`. Default is `255`. +`max_token_length` | Optional | Integer | Sets the maximum length of the produced token. If this length is exceeded, the token is split into multiple tokens at the length configured in `max_token_length`. Default is `255`. From 306251442b9ad8b6f8d54863516a2eb7490d8da2 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:58:37 -0500 Subject: [PATCH 6/7] Update _analyzers/tokenizers/uax-url-email.md Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --- _analyzers/tokenizers/uax-url-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index ac39564bf2..9931c083a6 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -45,7 +45,7 @@ PUT /my_index ## Generated tokens -Use the following request to examine the tokens generated using the created analyzer: +Use the following request to examine the tokens generated using the analyzer: ```json POST /my_index/_analyze From 4c491b6a14404a75d09fabb89cd29f7a863750dc Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:12:13 -0500 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --- _analyzers/tokenizers/uax-url-email.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_analyzers/tokenizers/uax-url-email.md b/_analyzers/tokenizers/uax-url-email.md index 9931c083a6..34336a4f55 100644 --- a/_analyzers/tokenizers/uax-url-email.md +++ b/_analyzers/tokenizers/uax-url-email.md @@ -7,7 +7,7 @@ nav_order: 150 # UAX URL email tokenizer -In addition to regular text, the `uax_url_email` tokenizer is designed to handle URLs, email addresses, and domain names. It is based on the Unicode Text Segmentation algorithm ([UAX #29](https://www.unicode.org/reports/tr29/)), which allows it to correctly tokenize complex text, including URLs and email addresses. +In addition to regular text, the `uax_url_email` tokenizer is designed to handle URLs, email addresses, and domain names. It is based on the Unicode Text Segmentation algorithm ([UAX #29](https://www.unicode.org/reports/tr29/)), which allows it to correctly tokenize complex text, including URLs and email addresses. ## Example usage