From ebae7fc61b23ad6250c7e38b0a347476fcdaa50f Mon Sep 17 00:00:00 2001 From: LikyThu Date: Tue, 30 May 2023 14:13:26 +0800 Subject: [PATCH] feat: result file and docker --- .../_4paradigm/benchmark/BenchmarkConfig.java | 4 ++ .../OpenMLDBPerfBenchmark_Cluster.java | 9 ++- .../java/com/_4paradigm/benchmark/Util.java | 27 +++++++++ README.md | 58 +++++++++++++++++-- .../org/example/join/BenchmarkConfig.java | 5 ++ .../org/example/join/flink_benchmark.java | 2 + 6 files changed, 97 insertions(+), 8 deletions(-) diff --git a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/BenchmarkConfig.java b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/BenchmarkConfig.java index 350442e..4e6102b 100644 --- a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/BenchmarkConfig.java +++ b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/BenchmarkConfig.java @@ -47,6 +47,8 @@ public class BenchmarkConfig { public static String DROP_SQL_C[]; + public static String logfile; + private static SqlExecutor executor = null; private static SdkOption option = null; @@ -91,6 +93,8 @@ public class BenchmarkConfig { PK_NUM = Integer.valueOf(prop.getProperty("PK_NUM", "100000")); PK_MAX = Integer.valueOf(prop.getProperty("PK_MAX", "0")); + logfile = "logs/" + System.currentTimeMillis() + ".json"; + } catch (Exception e) { e.printStackTrace(); } diff --git a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/OpenMLDBPerfBenchmark_Cluster.java b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/OpenMLDBPerfBenchmark_Cluster.java index eca6030..6a67204 100644 --- a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/OpenMLDBPerfBenchmark_Cluster.java +++ b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/OpenMLDBPerfBenchmark_Cluster.java @@ -23,6 +23,7 @@ import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.results.format.ResultFormatType; import org.slf4j.*; import org.apache.log4j.PropertyConfigurator; @@ -34,6 +35,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.FileInputStream; +import java.io.PrintStream; @@ -51,7 +53,7 @@ -@BenchmarkMode(Mode.SampleTime) +@BenchmarkMode({Mode.SampleTime, Mode.Throughput}) @OutputTimeUnit(TimeUnit.MICROSECONDS) @State(Scope.Benchmark) @@ -537,7 +539,7 @@ public void initEnv() { @TearDown public void cleanEnv() { - + // Util.reportGenerate(BenchmarkConfig.logfile); } @@ -657,6 +659,7 @@ public void executeDeployment() { } public static void main(String[] args) { + int dataSetID = BenchmarkConfig.DATASET_ID; int dataSetNUM = BenchmarkConfig.DATASET_NUM; @@ -671,6 +674,8 @@ public static void main(String[] args) { Options opt = new OptionsBuilder() .include(OpenMLDBPerfBenchmark_Cluster.class.getSimpleName()) .forks(1) + .resultFormat(ResultFormatType.JSON) + .result(BenchmarkConfig.logfile) .build(); new Runner(opt).run(); } catch (Exception e) { diff --git a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/Util.java b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/Util.java index 7e7b97f..cdb6c4f 100644 --- a/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/Util.java +++ b/OpenMLDB/src/src/main/java/com/_4paradigm/benchmark/Util.java @@ -38,6 +38,15 @@ import org.slf4j.*; import org.apache.log4j.PropertyConfigurator; +// import org.apache.pdfbox.pdmodel.PDDocument; +// import org.apache.pdfbox.pdmodel.PDPage; +// import org.apache.pdfbox.pdmodel.PDPageContentStream; + +// import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.File; + public class Util { @@ -398,4 +407,22 @@ public static Map extractResultSet(ResultSet resultSet) { return val; } + // public static boolean reportGenerate(String filepath) { + // ObjectMapper objectMapper = new ObjectMapper(); + + // try { + // File jsonFile = new File(filepath); + // // Deserialize JSON file into Java object + // MyDataObject dataObject = objectMapper.readValue(jsonFile, MyDataObject.class); + + // // Access data from the object + // System.out.println("Name: " + dataObject.getprimaryMetric()); + // System.out.println("Age: " + dataObject.getAge()); + // // ... + + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + } diff --git a/README.md b/README.md index b395987..b72f18e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,15 @@
+ ----- + A Benchmark for Real-Time Relational Data Feature Extraction. [**What is FEBench?**](#-what-is-febench) | [**Data and Query**](#-data-and-query) | [**Getting Started**](#%EF%B8%8F-quickstart) +| [**Run in Docker**](#-run-in-docker) +| [**Result Uploading**](#-result-uploading) | [**Contributing**](#%EF%B8%8F-contributing)
@@ -29,6 +33,7 @@ However, there is no any study about the workload characteristics and benchmarks See the detailed [technical report](https://github.com/decis-bench/febench/blob/main/report/febench.pdf) and [Standard Specification](https://github.com/decis-bench/febench/blob/main/report/Feature_Extraction_Benchmark_Standard_Specification.pdf)! + ## 📜 Data and Query We have conducted an analysis of both the schema of our datasets and the characteristics of the queries. Please refer to our detailed [data schema analysis](https://github.com/decis-bench/febench/blob/main/report/tableSchema.md) and [query analysis](https://github.com/decis-bench/febench/blob/main/report/queryAnalysis.md) for further information. @@ -56,6 +61,7 @@ Step 2: Download the datasets and move the data files to the dataset directory Note that, the above server is located in China, if you are experiencing slow connection, you may try to download from OneDrive (this copy is compressed, please decompress after downloading): https://1drv.ms/f/s!At2bMwG7v7Dngbg21F0ELbZrhC7NBA?e=atHwQy Step 3: [Start the cluster](https://openmldb.ai/docs/zh/main/deploy/install_deploy.html) and enter the `./OpenMLDB` folder in this project. For a quick start, you can use the [docker](https://openmldb.ai/docs/zh/main/quickstart/openmldb_quickstart.html#id4), but note that the performance may not be optimal since all the components are deployed on a single physical machine. + > Please be aware that the default values for `spark.driver.memory` and `spark.executor.memory` may not be enough for your needs. If you encounter a `java.lang.OutOfMemoryError: Java heap space` error, you may need to increase them. You can refer to [this document](https://openmldb.ai/docs/zh/main/maintain/faq.html#java-lang-outofmemoryerror-java-heap-space) for guidance. One acceptable size for these parameters is 32G/32G. Step 4: Please modify the `conf.properties.template` file to create your own `conf.properties` file in the `./conf` directory, and update the configuration settings in the file accordingly, including the OpenMLDB cluster and the locations of data and queries. @@ -64,10 +70,6 @@ Step 4: Please modify the `conf.properties.template` file to create your own `co export FEBENCH_ROOT=`pwd` sed s#\#$FEBENCH_ROOT# ./OpenMLDB/conf/conf.properties.template > ./OpenMLDB/conf/conf.properties sed s#\#$FEBENCH_ROOT# ./flink/conf/conf.properties.template > ./flink/conf/conf.properties -``` - - - ```sh DATASET_ID=5 # set by compile_test.sh/test.sh DATASET_NUM=6 ... @@ -83,7 +85,7 @@ CREATE_SQL_C3=/OpenMLDB/fequery/Q3/Q3_create_benchmark.sql DROP_SQL_C3=/OpenMLDB/fequery/Q3/Q3_drop_benchmark.sql ... - ``` +``` Step 5: Run the testing script @@ -115,10 +117,53 @@ Repeat the 1-5 steps in *OpenMLDB Evaluation*. And there are a few new issues: ./compile_test.sh 3 // compile and run the test of task3 ./test.sh // rerun the test of task3 ``` + 3. You need to rerun compile_test.sh if you modify the conf.properties. This is not needed for *OpenMLDB Evaluation*. ![image](./imgs/flink-jmh.png) + + +## 🐳 Run in Docker + +1. Download docker image. + +```bash +docker pull lucky20020327/febench:v2.0 +``` + +2. Run the image. + +```bash +# note that you need download the data in advance and mount it into the container. +docker run -it -v :/home/febench/dataset +``` + +3. Enter the `env` directory and start the clusters. + +```bash +cd /home/env/bin +./recover.sh +./start-all.sh +``` + +4. Enter the `febench` directory and init the configuration. + +```bash +cd /home/febench +export FEBENCH_ROOT=`pwd` +sed s#\#$FEBENCH_ROOT# ./OpenMLDB/conf/conf.properties.template > ./OpenMLDB/conf/conf.properties +sed s#\#$FEBENCH_ROOT# ./flink/conf/conf.properties.template > ./flink/conf/conf.properties +``` + +5. Run the benchmark. + + + +## 📧 Result Uploading + +The result of benchmark is stored at \/logs. Email us if you want report your result and tell us .... + ## Citation If you use FEBench in your research, please cite: @@ -142,4 +187,5 @@ If you use FEBench in your research, please cite: ``` ## ✉️ Contributing -FEBench is developed as an open platform to attract industry and academia to collaborate on the benchmark and further development of RTFE. Please leave your comments in the [Issues](https://github.com/decis-bench/febench/issues) if you would like to get involved or contribute! + +FEBench is developed as an open platform to attract industry and academia to collaborate on the benchmark and further development of RTFE. Please leave your comments in the [Issues](https://github.com/decis-bench/febench/issues) if you would like to get involved or contribute! \ No newline at end of file diff --git a/flink/src/src/main/java/org/example/join/BenchmarkConfig.java b/flink/src/src/main/java/org/example/join/BenchmarkConfig.java index 554c9ee..572e2ea 100755 --- a/flink/src/src/main/java/org/example/join/BenchmarkConfig.java +++ b/flink/src/src/main/java/org/example/join/BenchmarkConfig.java @@ -43,6 +43,8 @@ public class BenchmarkConfig { // public static String DROP_SQL_C[]; + public static String logfile; + public static int PK_NUM = 1; public static int PK_MAX = 0; static { @@ -84,6 +86,9 @@ public class BenchmarkConfig { PK_NUM = Integer.valueOf(prop.getProperty("PK_NUM", "100000")); PK_MAX = Integer.valueOf(prop.getProperty("PK_MAX", "0")); + logfile = "logs/" + System.currentTimeMillis() + ".json"; + + } catch (Exception e) { e.printStackTrace(); } diff --git a/flink/src/src/main/java/org/example/join/flink_benchmark.java b/flink/src/src/main/java/org/example/join/flink_benchmark.java index 29cf1fd..7cb378e 100755 --- a/flink/src/src/main/java/org/example/join/flink_benchmark.java +++ b/flink/src/src/main/java/org/example/join/flink_benchmark.java @@ -875,6 +875,8 @@ public static void main(String[] args) { try { Options opt = new OptionsBuilder() .include(flink_benchmark.class.getSimpleName()) + .resultFormat(ResultFormatType.JSON) + .result(BenchmarkConfig.logfile) .forks(1) .build(); new Runner(opt).run();