diff --git a/docs/results/cl-nagoya/ruri-base/summary.json b/docs/results/cl-nagoya/ruri-base/summary.json
new file mode 100644
index 0000000..a7c7b05
--- /dev/null
+++ b/docs/results/cl-nagoya/ruri-base/summary.json
@@ -0,0 +1,62 @@
+{
+ "Classification": {
+ "amazon_counterfactual_classification": {
+ "macro_f1": 0.7665550732749669
+ },
+ "amazon_review_classification": {
+ "macro_f1": 0.5575876111411316
+ },
+ "massive_intent_classification": {
+ "macro_f1": 0.8141210121425055
+ },
+ "massive_scenario_classification": {
+ "macro_f1": 0.8848812917656395
+ }
+ },
+ "Reranking": {
+ "esci": {
+ "ndcg@10": 0.9290942178703699
+ }
+ },
+ "Retrieval": {
+ "jagovfaqs_22k": {
+ "ndcg@10": 0.7455660589538348
+ },
+ "jaqket": {
+ "ndcg@10": 0.5012253145754781
+ },
+ "mrtydi": {
+ "ndcg@10": 0.3545113073009125
+ },
+ "nlp_journal_abs_intro": {
+ "ndcg@10": 0.8689204088388403
+ },
+ "nlp_journal_title_abs": {
+ "ndcg@10": 0.9656989703684407
+ },
+ "nlp_journal_title_intro": {
+ "ndcg@10": 0.7531306059721564
+ }
+ },
+ "STS": {
+ "jsick": {
+ "spearman": 0.8231772134744029
+ },
+ "jsts": {
+ "spearman": 0.8342848039994751
+ }
+ },
+ "Clustering": {
+ "livedoor_news": {
+ "v_measure_score": 0.5427223607801758
+ },
+ "mewsc16": {
+ "v_measure_score": 0.5404099864321413
+ }
+ },
+ "PairClassification": {
+ "paws_x_ja": {
+ "binary_f1": 0.6237623762376238
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/results/cl-nagoya/ruri-large/summary.json b/docs/results/cl-nagoya/ruri-large/summary.json
new file mode 100644
index 0000000..e86c46b
--- /dev/null
+++ b/docs/results/cl-nagoya/ruri-large/summary.json
@@ -0,0 +1,62 @@
+{
+ "Classification": {
+ "amazon_counterfactual_classification": {
+ "macro_f1": 0.8080806321853091
+ },
+ "amazon_review_classification": {
+ "macro_f1": 0.5680171450057119
+ },
+ "massive_intent_classification": {
+ "macro_f1": 0.8255898596881264
+ },
+ "massive_scenario_classification": {
+ "macro_f1": 0.8956410349938264
+ }
+ },
+ "Reranking": {
+ "esci": {
+ "ndcg@10": 0.9298524733536755
+ }
+ },
+ "Retrieval": {
+ "jagovfaqs_22k": {
+ "ndcg@10": 0.7667506664925435
+ },
+ "jaqket": {
+ "ndcg@10": 0.6173871224245404
+ },
+ "mrtydi": {
+ "ndcg@10": 0.3803302462897418
+ },
+ "nlp_journal_abs_intro": {
+ "ndcg@10": 0.8712459719069233
+ },
+ "nlp_journal_title_abs": {
+ "ndcg@10": 0.9657898747088243
+ },
+ "nlp_journal_title_intro": {
+ "ndcg@10": 0.779665053945222
+ }
+ },
+ "STS": {
+ "jsick": {
+ "spearman": 0.8199959693684533
+ },
+ "jsts": {
+ "spearman": 0.8426164139167538
+ }
+ },
+ "Clustering": {
+ "livedoor_news": {
+ "v_measure_score": 0.5139491572866559
+ },
+ "mewsc16": {
+ "v_measure_score": 0.5225025331595674
+ }
+ },
+ "PairClassification": {
+ "paws_x_ja": {
+ "binary_f1": 0.6228813559322034
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/results/cl-nagoya/ruri-small/summary.json b/docs/results/cl-nagoya/ruri-small/summary.json
new file mode 100644
index 0000000..cb591ea
--- /dev/null
+++ b/docs/results/cl-nagoya/ruri-small/summary.json
@@ -0,0 +1,62 @@
+{
+ "Classification": {
+ "amazon_counterfactual_classification": {
+ "macro_f1": 0.7991935990685706
+ },
+ "amazon_review_classification": {
+ "macro_f1": 0.556129066893332
+ },
+ "massive_intent_classification": {
+ "macro_f1": 0.8148895285345188
+ },
+ "massive_scenario_classification": {
+ "macro_f1": 0.8787774569382543
+ }
+ },
+ "Reranking": {
+ "esci": {
+ "ndcg@10": 0.9300177985352138
+ }
+ },
+ "Retrieval": {
+ "jagovfaqs_22k": {
+ "ndcg@10": 0.736494039429321
+ },
+ "jaqket": {
+ "ndcg@10": 0.484437639428696
+ },
+ "mrtydi": {
+ "ndcg@10": 0.3342716158897666
+ },
+ "nlp_journal_abs_intro": {
+ "ndcg@10": 0.8768878489670099
+ },
+ "nlp_journal_title_abs": {
+ "ndcg@10": 0.9716879343439146
+ },
+ "nlp_journal_title_intro": {
+ "ndcg@10": 0.7608660955794895
+ }
+ },
+ "STS": {
+ "jsick": {
+ "spearman": 0.8343927017558587
+ },
+ "jsts": {
+ "spearman": 0.8213297790184827
+ }
+ },
+ "Clustering": {
+ "livedoor_news": {
+ "v_measure_score": 0.5096442244018489
+ },
+ "mewsc16": {
+ "v_measure_score": 0.5141045788711239
+ }
+ },
+ "PairClassification": {
+ "paws_x_ja": {
+ "binary_f1": 0.6211267605633802
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/results/pkshatech/GLuCoSE-base-ja-v2/summary.json b/docs/results/pkshatech/GLuCoSE-base-ja-v2/summary.json
new file mode 100644
index 0000000..7318aab
--- /dev/null
+++ b/docs/results/pkshatech/GLuCoSE-base-ja-v2/summary.json
@@ -0,0 +1,62 @@
+{
+ "Classification": {
+ "amazon_counterfactual_classification": {
+ "macro_f1": 0.7492232749031491
+ },
+ "amazon_review_classification": {
+ "macro_f1": 0.5530707609927811
+ },
+ "massive_intent_classification": {
+ "macro_f1": 0.7979144461303402
+ },
+ "massive_scenario_classification": {
+ "macro_f1": 0.8683641924034757
+ }
+ },
+ "Reranking": {
+ "esci": {
+ "ndcg@10": 0.9301469431250418
+ }
+ },
+ "Retrieval": {
+ "jagovfaqs_22k": {
+ "ndcg@10": 0.6979374757372254
+ },
+ "jaqket": {
+ "ndcg@10": 0.6729417850207029
+ },
+ "mrtydi": {
+ "ndcg@10": 0.41858579533990486
+ },
+ "nlp_journal_abs_intro": {
+ "ndcg@10": 0.9029337913460675
+ },
+ "nlp_journal_title_abs": {
+ "ndcg@10": 0.9511153967130517
+ },
+ "nlp_journal_title_intro": {
+ "ndcg@10": 0.7580448576047344
+ }
+ },
+ "STS": {
+ "jsick": {
+ "spearman": 0.849637366944316
+ },
+ "jsts": {
+ "spearman": 0.8095684318108997
+ }
+ },
+ "Clustering": {
+ "livedoor_news": {
+ "v_measure_score": 0.5151536908540161
+ },
+ "mewsc16": {
+ "v_measure_score": 0.45782610528001805
+ }
+ },
+ "PairClassification": {
+ "paws_x_ja": {
+ "binary_f1": 0.623716814159292
+ }
+ }
+}
diff --git a/docs/results/pkshatech/RoSEtta-base-ja/summary.json b/docs/results/pkshatech/RoSEtta-base-ja/summary.json
new file mode 100644
index 0000000..d82af4b
--- /dev/null
+++ b/docs/results/pkshatech/RoSEtta-base-ja/summary.json
@@ -0,0 +1,62 @@
+{
+ "Classification": {
+ "amazon_counterfactual_classification": {
+ "macro_f1": 0.7005147244958231
+ },
+ "amazon_review_classification": {
+ "macro_f1": 0.5263680453119501
+ },
+ "massive_intent_classification": {
+ "macro_f1": 0.7983787583297884
+ },
+ "massive_scenario_classification": {
+ "macro_f1": 0.8709593192703351
+ }
+ },
+ "Reranking": {
+ "esci": {
+ "ndcg@10": 0.9268625513429571
+ }
+ },
+ "Retrieval": {
+ "jagovfaqs_22k": {
+ "ndcg@10": 0.6595934642903105
+ },
+ "jaqket": {
+ "ndcg@10": 0.6533452086105761
+ },
+ "mrtydi": {
+ "ndcg@10": 0.36731170141136216
+ },
+ "nlp_journal_abs_intro": {
+ "ndcg@10": 0.9553567926226499
+ },
+ "nlp_journal_title_abs": {
+ "ndcg@10": 0.940828991756893
+ },
+ "nlp_journal_title_intro": {
+ "ndcg@10": 0.8163161967769845
+ }
+ },
+ "STS": {
+ "jsick": {
+ "spearman": 0.8383455453168481
+ },
+ "jsts": {
+ "spearman": 0.7895388048564987
+ }
+ },
+ "Clustering": {
+ "livedoor_news": {
+ "v_measure_score": 0.5861760622672214
+ },
+ "mewsc16": {
+ "v_measure_score": 0.4784844036038961
+ }
+ },
+ "PairClassification": {
+ "paws_x_ja": {
+ "binary_f1": 0.6173974540311173
+ }
+ }
+}
diff --git a/leaderboard.md b/leaderboard.md
index b07dbca..4b05e46 100644
--- a/leaderboard.md
+++ b/leaderboard.md
@@ -3,38 +3,48 @@ This leaderboard shows the results stored under `docs/results`. The scores are a
## Summary
-The summary shows the average scores within each task.
+The summary shows the average scores within each task. The average score is the average of scores by dataset.
| Model | Avg. | Retrieval | STS | Classification | Reranking | Clustering | PairClassification |
|:----------------------------------------------|:----------|:------------|:----------|:-----------------|:------------|:-------------|:---------------------|
-| OpenAI/text-embedding-3-large | **73.97** | **74.48** | 82.52 | **77.58** | **93.58** | **53.32** | 62.35 |
-| intfloat/multilingual-e5-large | 71.65 | 70.98 | 79.70 | 72.89 | 92.96 | 51.24 | 62.15 |
-| OpenAI/text-embedding-3-small | 70.86 | 66.39 | 79.46 | 73.06 | 92.92 | 51.06 | 62.27 |
-| pkshatech/GLuCoSE-base-ja | 70.44 | 59.02 | 78.71 | 76.82 | 91.90 | 49.78 | **66.39** |
-| intfloat/multilingual-e5-base | 70.12 | 68.21 | 79.84 | 69.30 | 92.85 | 48.26 | 62.26 |
-| intfloat/multilingual-e5-small | 69.52 | 67.27 | 80.07 | 67.62 | 93.03 | 46.91 | 62.19 |
-| OpenAI/text-embedding-ada-002 | 69.48 | 64.38 | 79.02 | 69.75 | 93.04 | 48.30 | 62.40 |
-| cl-nagoya/sup-simcse-ja-base | 68.56 | 49.64 | 82.05 | 73.47 | 91.83 | 51.79 | 62.57 |
-| MU-Kindai/Japanese-SimCSE-BERT-large-unsup | 66.89 | 47.38 | 78.99 | 73.13 | 91.30 | 48.25 | 62.27 |
-| oshizo/sbert-jsnli-luke-japanese-base-lite | 66.75 | 43.00 | 76.60 | 76.61 | 91.56 | 50.33 | 62.38 |
-| cl-nagoya/sup-simcse-ja-large | 66.51 | 37.62 | **83.18** | 73.73 | 91.48 | 50.56 | 62.51 |
-| cl-nagoya/unsup-simcse-ja-large | 66.27 | 40.53 | 80.56 | 74.66 | 90.95 | 48.41 | 62.49 |
-| MU-Kindai/Japanese-SimCSE-BERT-base-unsup | 66.23 | 46.36 | 77.49 | 73.30 | 91.16 | 46.68 | 62.38 |
-| MU-Kindai/Japanese-SimCSE-BERT-large-sup | 65.28 | 40.82 | 78.28 | 73.47 | 90.95 | 45.81 | 62.35 |
-| MU-Kindai/Japanese-MixCSE-BERT-base | 65.14 | 42.59 | 77.05 | 72.90 | 91.01 | 44.95 | 62.33 |
-| cl-nagoya/unsup-simcse-ja-base | 65.07 | 40.23 | 78.72 | 73.07 | 91.16 | 44.77 | 62.44 |
-| MU-Kindai/Japanese-DiffCSE-BERT-base | 64.77 | 41.79 | 75.50 | 73.77 | 90.95 | 44.22 | 62.38 |
-| sentence-transformers/LaBSE | 64.70 | 40.12 | 76.56 | 72.66 | 91.63 | 44.88 | 62.33 |
-| pkshatech/simcse-ja-bert-base-clcmlp | 64.42 | 37.00 | 76.80 | 71.30 | 91.49 | 47.53 | 62.40 |
-| MU-Kindai/Japanese-SimCSE-BERT-base-sup | 64.15 | 41.32 | 74.66 | 72.76 | 90.66 | 43.11 | 62.37 |
-| colorfulscoop/sbert-base-ja | 58.85 | 16.52 | 70.42 | 69.07 | 89.97 | 44.81 | 62.31 |
-| sentence-transformers/stsb-xlm-r-multilingual | 58.01 | 21.00 | 75.40 | 71.84 | 90.20 | 27.46 | 62.20 |
+| OpenAI/text-embedding-3-large | **74.05** | **74.48** | 82.52 | **77.58** | **93.58** | 53.32 | 62.35 |
+| cl-nagoya/ruri-large | 73.31 | 73.02 | 83.13 | 77.43 | 92.99 | 51.82 | 62.29 |
+| pkshatech/GLuCoSE-base-ja-v2 | 72.23 | 73.36 | 82.96 | 74.21 | 93.01 | 48.65 | 62.37 |
+| pkshatech/RoSEtta-base-ja | 72.04 | 73.21 | 81.39 | 72.41 | 92.69 | 53.23 | 61.74 |
+| cl-nagoya/ruri-base | 71.91 | 69.82 | 82.87 | 75.58 | 92.91 | **54.16** | 62.38 |
+| cl-nagoya/ruri-small | 71.53 | 69.41 | 82.79 | 76.22 | 93.00 | 51.19 | 62.11 |
+| intfloat/multilingual-e5-large | 70.90 | 70.98 | 79.70 | 72.89 | 92.96 | 51.24 | 62.15 |
+| OpenAI/text-embedding-3-small | 69.18 | 66.39 | 79.46 | 73.06 | 92.92 | 51.06 | 62.27 |
+| intfloat/multilingual-e5-base | 68.61 | 68.21 | 79.84 | 69.30 | 92.85 | 48.26 | 62.26 |
+| intfloat/multilingual-e5-small | 67.71 | 67.27 | 80.07 | 67.62 | 93.03 | 46.91 | 62.19 |
+| pkshatech/GLuCoSE-base-ja | 67.29 | 59.02 | 78.71 | 76.82 | 91.90 | 49.78 | **66.39** |
+| OpenAI/text-embedding-ada-002 | 67.21 | 64.38 | 79.02 | 69.75 | 93.04 | 48.30 | 62.40 |
+| cl-nagoya/sup-simcse-ja-base | 63.36 | 49.64 | 82.05 | 73.47 | 91.83 | 51.79 | 62.57 |
+| MU-Kindai/Japanese-SimCSE-BERT-large-unsup | 61.55 | 47.38 | 78.99 | 73.13 | 91.30 | 48.25 | 62.27 |
+| MU-Kindai/Japanese-SimCSE-BERT-base-unsup | 60.83 | 46.36 | 77.49 | 73.30 | 91.16 | 46.68 | 62.38 |
+| oshizo/sbert-jsnli-luke-japanese-base-lite | 60.77 | 43.00 | 76.60 | 76.61 | 91.56 | 50.33 | 62.38 |
+| cl-nagoya/unsup-simcse-ja-large | 59.58 | 40.53 | 80.56 | 74.66 | 90.95 | 48.41 | 62.49 |
+| MU-Kindai/Japanese-MixCSE-BERT-base | 59.03 | 42.59 | 77.05 | 72.90 | 91.01 | 44.95 | 62.33 |
+| cl-nagoya/sup-simcse-ja-large | 58.88 | 37.62 | **83.18** | 73.73 | 91.48 | 50.56 | 62.51 |
+| MU-Kindai/Japanese-SimCSE-BERT-large-sup | 58.77 | 40.82 | 78.28 | 73.47 | 90.95 | 45.81 | 62.35 |
+| MU-Kindai/Japanese-DiffCSE-BERT-base | 58.66 | 41.79 | 75.50 | 73.77 | 90.95 | 44.22 | 62.38 |
+| cl-nagoya/unsup-simcse-ja-base | 58.39 | 40.23 | 78.72 | 73.07 | 91.16 | 44.77 | 62.44 |
+| sentence-transformers/LaBSE | 58.01 | 40.12 | 76.56 | 72.66 | 91.63 | 44.88 | 62.33 |
+| MU-Kindai/Japanese-SimCSE-BERT-base-sup | 57.97 | 41.32 | 74.66 | 72.76 | 90.66 | 43.11 | 62.37 |
+| pkshatech/simcse-ja-bert-base-clcmlp | 56.86 | 37.00 | 76.80 | 71.30 | 91.49 | 47.53 | 62.40 |
+| sentence-transformers/stsb-xlm-r-multilingual | 48.21 | 21.00 | 75.40 | 71.84 | 90.20 | 27.46 | 62.20 |
+| colorfulscoop/sbert-base-ja | 47.38 | 16.52 | 70.42 | 69.07 | 89.97 | 44.81 | 62.31 |
## Retrieval
| Model | Avg. | jagovfaqs_22k
(ndcg@10) | jaqket
(ndcg@10) | mrtydi
(ndcg@10) | nlp_journal_abs_intro
(ndcg@10) | nlp_journal_title_abs
(ndcg@10) | nlp_journal_title_intro
(ndcg@10) |
|:----------------------------------------------|:----------|:-----------------------------|:----------------------|:----------------------|:-------------------------------------|:-------------------------------------|:---------------------------------------|
-| OpenAI/text-embedding-3-large | **74.48** | **72.41** | 48.21 | 34.88 | **99.33** | **96.55** | **95.47** |
-| intfloat/multilingual-e5-large | 70.98 | 70.30 | **58.78** | **43.63** | 86.00 | 94.70 | 72.48 |
+| OpenAI/text-embedding-3-large | **74.48** | 72.41 | 48.21 | 34.88 | **99.33** | 96.55 | **95.47** |
+| pkshatech/GLuCoSE-base-ja-v2 | 73.36 | 69.79 | **67.29** | 41.86 | 90.29 | 95.11 | 75.80 |
+| pkshatech/RoSEtta-base-ja | 73.21 | 65.96 | 65.33 | 36.73 | 95.54 | 94.08 | 81.63 |
+| cl-nagoya/ruri-large | 73.02 | **76.68** | 61.74 | 38.03 | 87.12 | 96.58 | 77.97 |
+| intfloat/multilingual-e5-large | 70.98 | 70.30 | 58.78 | **43.63** | 86.00 | 94.70 | 72.48 |
+| cl-nagoya/ruri-base | 69.82 | 74.56 | 50.12 | 35.45 | 86.89 | 96.57 | 75.31 |
+| cl-nagoya/ruri-small | 69.41 | 73.65 | 48.44 | 33.43 | 87.69 | **97.17** | 76.09 |
| intfloat/multilingual-e5-base | 68.21 | 65.34 | 50.67 | 38.38 | 87.10 | 94.73 | 73.05 |
| intfloat/multilingual-e5-small | 67.27 | 64.11 | 49.97 | 36.05 | 85.21 | 95.26 | 72.99 |
| OpenAI/text-embedding-3-small | 66.39 | 64.02 | 33.94 | 20.03 | 98.47 | 91.70 | 90.17 |
@@ -59,9 +69,14 @@ The summary shows the average scores within each task.
## STS
| Model | Avg. | jsick
(spearman) | jsts
(spearman) |
|:----------------------------------------------|:----------|:----------------------|:---------------------|
-| cl-nagoya/sup-simcse-ja-large | **83.18** | **83.80** | 82.57 |
-| OpenAI/text-embedding-3-large | 82.52 | 81.27 | **83.77** |
+| cl-nagoya/sup-simcse-ja-large | **83.18** | 83.80 | 82.57 |
+| cl-nagoya/ruri-large | 83.13 | 82.00 | **84.26** |
+| pkshatech/GLuCoSE-base-ja-v2 | 82.96 | **84.96** | 80.96 |
+| cl-nagoya/ruri-base | 82.87 | 82.32 | 83.43 |
+| cl-nagoya/ruri-small | 82.79 | 83.44 | 82.13 |
+| OpenAI/text-embedding-3-large | 82.52 | 81.27 | 83.77 |
| cl-nagoya/sup-simcse-ja-base | 82.05 | 82.83 | 81.27 |
+| pkshatech/RoSEtta-base-ja | 81.39 | 83.83 | 78.95 |
| cl-nagoya/unsup-simcse-ja-large | 80.56 | 80.15 | 80.98 |
| intfloat/multilingual-e5-small | 80.07 | 81.50 | 78.65 |
| intfloat/multilingual-e5-base | 79.84 | 81.28 | 78.39 |
@@ -85,10 +100,14 @@ The summary shows the average scores within each task.
## Classification
| Model | Avg. | amazon_counterfactual
(macro_f1) | amazon_review
(macro_f1) | massive_intent
(macro_f1) | massive_scenario
(macro_f1) |
|:----------------------------------------------|:----------|:--------------------------------------|:------------------------------|:-------------------------------|:---------------------------------|
-| OpenAI/text-embedding-3-large | **77.58** | 77.90 | **60.44** | **80.91** | **91.08** |
+| OpenAI/text-embedding-3-large | **77.58** | 77.90 | **60.44** | 80.91 | **91.08** |
+| cl-nagoya/ruri-large | 77.43 | 80.81 | 56.80 | **82.56** | 89.56 |
| pkshatech/GLuCoSE-base-ja | 76.82 | **82.44** | 58.07 | 78.85 | 87.94 |
| oshizo/sbert-jsnli-luke-japanese-base-lite | 76.61 | 79.95 | 57.48 | 80.26 | 88.75 |
+| cl-nagoya/ruri-small | 76.22 | 79.92 | 55.61 | 81.49 | 87.88 |
+| cl-nagoya/ruri-base | 75.58 | 76.66 | 55.76 | 81.41 | 88.49 |
| cl-nagoya/unsup-simcse-ja-large | 74.66 | 76.79 | 55.37 | 79.13 | 87.36 |
+| pkshatech/GLuCoSE-base-ja-v2 | 74.21 | 74.92 | 55.31 | 79.79 | 86.84 |
| MU-Kindai/Japanese-DiffCSE-BERT-base | 73.77 | 78.10 | 51.56 | 78.79 | 86.63 |
| cl-nagoya/sup-simcse-ja-large | 73.73 | 73.21 | 54.76 | 79.23 | 87.72 |
| MU-Kindai/Japanese-SimCSE-BERT-large-sup | 73.47 | 77.25 | 53.42 | 76.83 | 86.39 |
@@ -101,6 +120,7 @@ The summary shows the average scores within each task.
| intfloat/multilingual-e5-large | 72.89 | 70.66 | 56.54 | 75.78 | 88.59 |
| MU-Kindai/Japanese-SimCSE-BERT-base-sup | 72.76 | 76.20 | 52.06 | 77.89 | 84.90 |
| sentence-transformers/LaBSE | 72.66 | 73.61 | 51.70 | 76.99 | 88.35 |
+| pkshatech/RoSEtta-base-ja | 72.41 | 70.05 | 52.64 | 79.84 | 87.10 |
| sentence-transformers/stsb-xlm-r-multilingual | 71.84 | 75.65 | 51.32 | 74.28 | 86.10 |
| pkshatech/simcse-ja-bert-base-clcmlp | 71.30 | 67.49 | 50.85 | 79.67 | 87.20 |
| OpenAI/text-embedding-ada-002 | 69.75 | 64.42 | 53.13 | 74.57 | 86.89 |
@@ -114,9 +134,14 @@ The summary shows the average scores within each task.
| OpenAI/text-embedding-3-large | **93.58** | **93.58** |
| OpenAI/text-embedding-ada-002 | 93.04 | 93.04 |
| intfloat/multilingual-e5-small | 93.03 | 93.03 |
+| pkshatech/GLuCoSE-base-ja-v2 | 93.01 | 93.01 |
+| cl-nagoya/ruri-small | 93.00 | 93.00 |
+| cl-nagoya/ruri-large | 92.99 | 92.99 |
| intfloat/multilingual-e5-large | 92.96 | 92.96 |
| OpenAI/text-embedding-3-small | 92.92 | 92.92 |
+| cl-nagoya/ruri-base | 92.91 | 92.91 |
| intfloat/multilingual-e5-base | 92.85 | 92.85 |
+| pkshatech/RoSEtta-base-ja | 92.69 | 92.69 |
| pkshatech/GLuCoSE-base-ja | 91.90 | 91.90 |
| cl-nagoya/sup-simcse-ja-base | 91.83 | 91.83 |
| sentence-transformers/LaBSE | 91.63 | 91.63 |
@@ -137,13 +162,18 @@ The summary shows the average scores within each task.
## Clustering
| Model | Avg. | livedoor_news
(v_measure_score) | mewsc16
(v_measure_score) |
|:----------------------------------------------|:----------|:-------------------------------------|:-------------------------------|
-| OpenAI/text-embedding-3-large | **53.32** | 57.09 | 49.55 |
+| cl-nagoya/ruri-base | **54.16** | 54.27 | **54.04** |
+| OpenAI/text-embedding-3-large | 53.32 | 57.09 | 49.55 |
+| pkshatech/RoSEtta-base-ja | 53.23 | **58.62** | 47.85 |
+| cl-nagoya/ruri-large | 51.82 | 51.39 | 52.25 |
| cl-nagoya/sup-simcse-ja-base | 51.79 | 52.67 | 50.91 |
-| intfloat/multilingual-e5-large | 51.24 | **57.13** | 45.34 |
+| intfloat/multilingual-e5-large | 51.24 | 57.13 | 45.34 |
+| cl-nagoya/ruri-small | 51.19 | 50.96 | 51.41 |
| OpenAI/text-embedding-3-small | 51.06 | 54.57 | 47.55 |
| cl-nagoya/sup-simcse-ja-large | 50.56 | 50.75 | 50.38 |
-| oshizo/sbert-jsnli-luke-japanese-base-lite | 50.33 | 46.77 | **53.89** |
+| oshizo/sbert-jsnli-luke-japanese-base-lite | 50.33 | 46.77 | 53.89 |
| pkshatech/GLuCoSE-base-ja | 49.78 | 49.89 | 49.68 |
+| pkshatech/GLuCoSE-base-ja-v2 | 48.65 | 51.52 | 45.78 |
| cl-nagoya/unsup-simcse-ja-large | 48.41 | 50.90 | 45.92 |
| OpenAI/text-embedding-ada-002 | 48.30 | 49.67 | 46.92 |
| intfloat/multilingual-e5-base | 48.26 | 55.03 | 41.49 |
@@ -171,18 +201,23 @@ The summary shows the average scores within each task.
| pkshatech/simcse-ja-bert-base-clcmlp | 62.40 | 62.40 |
| OpenAI/text-embedding-ada-002 | 62.40 | 62.40 |
| MU-Kindai/Japanese-SimCSE-BERT-base-unsup | 62.38 | 62.38 |
+| cl-nagoya/ruri-base | 62.38 | 62.38 |
| oshizo/sbert-jsnli-luke-japanese-base-lite | 62.38 | 62.38 |
| MU-Kindai/Japanese-DiffCSE-BERT-base | 62.38 | 62.38 |
+| pkshatech/GLuCoSE-base-ja-v2 | 62.37 | 62.37 |
| MU-Kindai/Japanese-SimCSE-BERT-base-sup | 62.37 | 62.37 |
| MU-Kindai/Japanese-SimCSE-BERT-large-sup | 62.35 | 62.35 |
| OpenAI/text-embedding-3-large | 62.35 | 62.35 |
| MU-Kindai/Japanese-MixCSE-BERT-base | 62.33 | 62.33 |
| sentence-transformers/LaBSE | 62.33 | 62.33 |
| colorfulscoop/sbert-base-ja | 62.31 | 62.31 |
+| cl-nagoya/ruri-large | 62.29 | 62.29 |
| OpenAI/text-embedding-3-small | 62.27 | 62.27 |
| MU-Kindai/Japanese-SimCSE-BERT-large-unsup | 62.27 | 62.27 |
| intfloat/multilingual-e5-base | 62.26 | 62.26 |
| sentence-transformers/stsb-xlm-r-multilingual | 62.20 | 62.20 |
| intfloat/multilingual-e5-small | 62.19 | 62.19 |
| intfloat/multilingual-e5-large | 62.15 | 62.15 |
+| cl-nagoya/ruri-small | 62.11 | 62.11 |
+| pkshatech/RoSEtta-base-ja | 61.74 | 61.74 |
diff --git a/make_leaderboard.py b/make_leaderboard.py
index ff3a330..0e43ccf 100644
--- a/make_leaderboard.py
+++ b/make_leaderboard.py
@@ -62,7 +62,14 @@ def format_score(score: float) -> str:
table_list: list[list[str | float]] = []
for model_signature, dataset_scores in task_results.items():
model_scores = [dataset_scores[k] for k in dataset_keys]
- average_score = sum(model_scores) / len(model_scores)
+ if task_name == SUMMARY_KEY:
+ scores_by_dataset = []
+ for _task_name, _task_results in all_results.items():
+ if _task_name != SUMMARY_KEY:
+ scores_by_dataset.extend(list(_task_results[model_signature].values()))
+ average_score = sum(scores_by_dataset) / len(scores_by_dataset)
+ else:
+ average_score = sum(model_scores) / len(model_scores)
table_list.append([model_signature, average_score, *model_scores])
# sort by the average score
@@ -97,7 +104,10 @@ def format_score(score: float) -> str:
f.write(f"## {task_name}\n")
if task_name == SUMMARY_KEY:
- f.write("\nThe summary shows the average scores within each task.\n\n")
+ f.write(
+ "\nThe summary shows the average scores within each task. "
+ "The average score is the average of scores by dataset.\n\n"
+ )
f.write(markdown_table)
f.write("\n\n")
diff --git a/pyproject.toml b/pyproject.toml
index 57e4f74..743b6bd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,7 +12,7 @@ description = "The evaluation scripts for JMTEB (Japanese Massive Text Embedding
name = "JMTEB"
packages = [{from = "src", include = "jmteb"}]
readme = "README.md"
-version = "1.3.1"
+version = "1.3.2"
[tool.poetry.dependencies]
python = ">=3.10,<4.0"