it works!

Multi-V-VM · Jan 11, 2024 · 92c1451 · 92c1451
1 parent d7a362c
commit 92c1451
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 74 deletions.
diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt
@@ -83,7 +83,7 @@ ExternalProject_Add(ftp
         CONFIGURE_COMMAND ${CMAKE_COMMAND}
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
-        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
+        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
         -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/ftp
@@ -97,7 +97,7 @@ ExternalProject_Add(nas
         CONFIGURE_COMMAND ${CMAKE_COMMAND}
         -DCMAKE_BUILD_TYPE=Release
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
-        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
+        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
         -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
         ${CMAKE_CURRENT_SOURCE_DIR}/nas
@@ -132,7 +132,7 @@ ExternalProject_Add(linpack
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ${CMAKE_COMMAND}
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
-        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
+        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1
@@ -146,7 +146,7 @@ ExternalProject_Add(hdastar
         PATCH_COMMAND ""
         CONFIGURE_COMMAND ${CMAKE_COMMAND}
         -GNinja -DWASI_SDK_DIR=${WASI_SDK} -DWASI_SDK_PREFIX=${WASI_SDK}
-        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk-pthread.cmake
+        -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK}/share/cmake/wasi-sdk.cmake
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_EXE_LINKER_FLAGS="-Wl,--export-all -Wl,--allow-undefined-file=${WASI_SDK}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt"
         -DCMAKE_SYSROOT=${WASI_SDK}/share/wasi-sysroot -DCMAKE_C_COMPILER_WORK=1 -DCMAKE_CXX_COMPILER_WORK=1

diff --git a/include/wamr.h b/include/wamr.h
@@ -43,6 +43,7 @@ class WAMRInstance {
     // add offset to pair->tuple, 3rd param 'int'
     std::map<int, SocketMetaData> socket_fd_map_;
     // lwcp is LightWeight CheckPoint
+    std::map<ssize_t, int> lwcp_list;
     size_t ready = 0;
     std::mutex as_mtx;
 

diff --git a/include/wamr_memory_instance.h b/include/wamr_memory_instance.h
@@ -13,7 +13,7 @@ struct WAMRMemoryInstance {
     /* Module type */
     uint32 module_type;
     /* Shared memory flag */
-    bool is_shared;
+    uint16 ref_count;
     /* Number bytes per page */
     uint32 num_bytes_per_page;
     /* Current page count */
@@ -33,7 +33,8 @@ struct WAMRMemoryInstance {
 
     void dump_impl(WASMMemoryInstance *env) {
         module_type = env->module_type;
-        is_shared = env->ref_count;
+        ref_count = env->ref_count+1;
+        LOGV(ERROR)<< "ref_count:" << ref_count;
         num_bytes_per_page = env->num_bytes_per_page;
         cur_page_count = env->cur_page_count;
         max_page_count = env->max_page_count;
@@ -48,7 +49,8 @@ struct WAMRMemoryInstance {
     };
     void restore_impl(WASMMemoryInstance *env) {
         env->module_type = module_type;
-        env->ref_count = is_shared;
+        env->ref_count = ref_count;
+        LOGV(ERROR)<< "ref_count:" << env->ref_count;
         env->num_bytes_per_page = num_bytes_per_page;
         env->cur_page_count = cur_page_count;
         env->max_page_count = max_page_count;

diff --git a/lib/wasm-micro-runtime b/lib/wasm-micro-runtime
diff --git a/src/checkpoint.cpp b/src/checkpoint.cpp
@@ -144,46 +144,57 @@ void serialize_to_file(WASMExecEnv *instance) {
         // Clean up
         close(fd);
     }
-#if !defined(_WIN32)
-    auto cluster = wasm_exec_env_get_cluster(instance);
-    // wasm_cluster_suspend_all_except_self(cluster, instance);
-    auto all_count = bh_list_length(&cluster->exec_env_list);
-    int cur_count = 0;
-    if (all_count > 1) {
-        auto elem = (WASMExecEnv *)bh_list_first_elem(&cluster->exec_env_list);
-        while (elem) {
-            if (elem == instance) {
-                break;
-            }
-            cur_count++;
-            elem = (WASMExecEnv *)bh_list_elem_next(elem);
-        }
-    } // gets the element index
-#else
     auto all_count = 1;
-#endif
-    auto a = new WAMRExecEnv();
-    dump(a, instance);
-    a->cur_count = gettid();
-
-    std::unique_lock as_ul(as_mtx);
-    as.emplace_back(a);
-    if (as.size() == all_count - 1) {
+    auto cur_count = 0;
+    // fill vector
 #if !defined(_WIN32)
-        kill(getpid(), SIGINT);
-#else
-        raise(SIGINT);
-#endif
+    std::unique_lock as_ul(wamr->as_mtx);
+    auto cluster = wasm_exec_env_get_cluster(instance);
+    all_count = bh_list_length(&cluster->exec_env_list);
+    printf("get lock\n");
+    wamr->ready++;
+    wamr->lwcp_list[gettid()]++;
+    //If we're not all ready
+    printf("thread %d, with %ld ready out of %d total\n", gettid(), wamr->ready, all_count);
+    if(wamr->ready < all_count){
+	// Then wait for someone else to get here and finish the job
+        std::condition_variable as_cv;
+        as_cv.wait(as_ul);
     }
-    if (as.size() == all_count) {
-        struct_pack::serialize_to(*writer, as);
-        exit(EXIT_SUCCESS);
+    // If we're all ready
+    // double check
+    {
+        auto ready_count = 0;
+	for(auto [k, v] : wamr->lwcp_list){
+		printf("%ld: %d\n", k, v);
+            if(v>0) ready_count++;
+	}
+	if(ready_count != all_count){
+	    printf("we have a discrepancy between ready count and number of threads that say they are\n");
+	    printf("ready: %d, all: %d\n", ready_count, all_count);
+	    //not actually ready
+            std::condition_variable as_cv;
+            as_cv.wait(as_ul);
+	}
     }
+    // wasm_cluster_suspend_all_except_self(cluster, instance);
+    auto elem = (WASMExecEnv *)bh_list_first_elem(&cluster->exec_env_list);
+    while (elem) {
+	instance = elem;
+#endif // windows has no threads so only does it once
+        auto a = new WAMRExecEnv();
+        dump(a, instance);
+        a->cur_count = cur_count;
+        as.emplace_back(a);
+        cur_count++;
 #if !defined(_WIN32)
-    // Is there some better way to sleep until exit?
-    std::condition_variable as_cv;
-    as_cv.wait(as_ul);
+        elem = (WASMExecEnv *)bh_list_elem_next(elem);
+    }
+    // finish filling vector
 #endif
+
+    struct_pack::serialize_to(*writer, as);
+    exit(EXIT_SUCCESS);
 }
 
 int main(int argc, char *argv[]) {
@@ -251,7 +262,7 @@ int main(int argc, char *argv[]) {
     wamr->set_wasi_args(dir, map_dir, env, arg, addr, ns_pool);
     wamr->instantiate();
     wamr->get_int3_addr();
-    wamr->replace_int3_with_nop();
+    //wamr->replace_int3_with_nop();
 
     // freopen("output.txt", "w", stdout);
 #if defined(_WIN32)

diff --git a/src/wamr.cpp b/src/wamr.cpp
@@ -587,6 +587,8 @@ void WAMRInstance::recover(std::vector<std::unique_ptr<WAMRExecEnv>> *execEnv) {
     main_env->restore_call_chain = nullptr;
 
     invoke_init_c();
+    invoke_preopen(1, "/dev/stdout");
+
 #if !defined(_WIN32)
     for (auto [idx, exec_] : *execEnv | enumerate) {
         if (idx + 1 == execEnv->size()) {
@@ -696,13 +698,15 @@ void WAMRInstance::set_wasi_args(WAMRWASIContext &context) {
 extern WAMRInstance *wamr;
 extern "C" { // stop name mangling so it can be linked externally
 void wamr_wait(wasm_exec_env_t exec_env) {
+    fprintf(stderr, "child getting ready to wait\n");
     // register thread id mapping
     wamr->register_tid_map();
     thread_init.release(1);
     fprintf(stderr, "finish child restore\n");
     wakeup.acquire();
     fprintf(stderr, "go child!! %d\n", gettid());
     wamr->replay_sync_ops(false, exec_env);
+    fprintf(stderr, "finish syncing\n");
 }
 WASMExecEnv *restore_env() {
     auto exec_env = wasm_exec_env_create_internal(wamr->module_inst, wamr->stack_size);

diff --git a/src/wamr_export.cpp b/src/wamr_export.cpp
@@ -230,9 +230,12 @@ void lightweight_checkpoint(WASMExecEnv *exec_env) {
         LOGV(DEBUG) << "skip checkpoint";
         return;
     }
+
     std::unique_lock as_ul(wamr->as_mtx);
+    wamr->lwcp_list[gettid()]++;
     wamr->ready++;
 }
+
 void lightweight_uncheckpoint(WASMExecEnv *exec_env) {
     int fid = -1;
     if (((AOTFrame *)exec_env->cur_frame)) {
@@ -244,6 +247,14 @@ void lightweight_uncheckpoint(WASMExecEnv *exec_env) {
         return;
     }
     std::unique_lock as_ul(wamr->as_mtx);
+    if(wamr->lwcp_list[gettid()] == 0){
+        // someone has reset our counter
+	// which means we've been serialized
+	// so we shouldn't return back to the wasm state
+        std::condition_variable cv;
+        cv.wait(as_ul);
+    }
+    wamr->lwcp_list[gettid()]--;
     wamr->ready--;
 }
 

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -7,11 +7,11 @@ endif ()
 set(WAMR_DIR ${PROJECT_SOURCE_DIR}/lib/wasm-micro-runtime/)
 set(HOST_PREFIX ${WASI_SDK_DIR}/bin/)
 enable_testing()
-function(wamr_app input)
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
-            COMMAND ${HOST_PREFIX}clang -fopenmp=libomp -lomp -Wno-implicit-function-declaration -O3 -Wno-int-conversion --target=wasm32-wasi-threads -lwasi-emulated-getpid  -lwasi-emulated-process-clocks -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS -g -pthread -Wl,--max-memory=67108864 -z stack-size=65536 -Wl,--export-all -Wl,--allow-undefined -Wl,--allow-undefined-file=${WASI_SDK_DIR}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
-            DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c
-    )
+function(wamr_openmp_app input)
+      add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+        COMMAND ${HOST_PREFIX}clang -fopenmp=libomp -lomp -Wno-implicit-function-declaration -O3 -Wno-int-conversion --target=wasm32-wasi-threads -lwasi-emulated-getpid  -lwasi-emulated-process-clocks -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS -g -pthread -Wl,--max-memory=67108864 -z stack-size=65536 -Wl,--export-all -Wl,--allow-undefined -Wl,--allow-undefined-file=${WASI_SDK_DIR}/share/wasi-sysroot/share/wasm32-wasi-threads/undefined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c
+      )
     add_custom_target(${input}_wamr ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
     if (WAMR_BUILD_AOT)
         add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}-pure.aot
@@ -31,15 +31,37 @@ function(wamr_app input)
                 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
         )
         add_custom_target(${input}_compile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
+        add_test(NAME ${input}_checkpoint COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_checkpoint --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
+        add_test(NAME ${input}_restore COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_restore --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
+    else ()
+        add_test(NAME ${input}_checkpoint COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_checkpoint --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
+        add_test(NAME ${input}_restore COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_restore --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
+endif ()
+endfunction()
+function(wamr_app input)
+    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+            COMMAND ${HOST_PREFIX}clang -Wno-implicit-function-declaration -Wno-int-conversion --target=wasm32-unknown-wasi -g -pthread -z stack-size=65536 -Wl,--initial-memory=655360 -Wl,--max-memory=655360 -Wl,--export-all -Wl,--allow-undefined -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+            DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c
+    )
+    add_custom_target(${input}_wamr ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
+    if (WAMR_BUILD_AOT)
+        add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}-pure.aot
+                COMMAND ${CMAKE_SOURCE_DIR}/lib/wasm-micro-runtime/wamr-compiler/build/wamrc -o ${CMAKE_CURRENT_BINARY_DIR}/${input}-pure.aot ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+                DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+        )
+        add_custom_target(${input}_pure_compile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}-pure.aot)
 
-        # add_custom_target(${input}_ckpt_br_compile ALL DEPENDS ${input}_wamr COMMAND ${CMAKE_SOURCE_DIR}/lib/wasm-micro-runtime/wamr-compiler/build/wamrc --disable-aux-stack-check --enable-br-checkpoint -o ${CMAKE_CURRENT_BINARY_DIR}/${input}-ckpt-br.aot ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
-        add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}-ckpt-br.aot
-                COMMAND ${CMAKE_SOURCE_DIR}/lib/wasm-micro-runtime/wamr-compiler/build/wamrc --disable-aux-stack-check --enable-br-checkpoint -o ${CMAKE_CURRENT_BINARY_DIR}/${input}-ckpt-br.aot ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+        add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}-stack.aot
+                COMMAND ${CMAKE_SOURCE_DIR}/lib/wasm-micro-runtime/wamr-compiler/build/wamrc --disable-aux-stack-check --enable-dump-call-stack -o ${CMAKE_CURRENT_BINARY_DIR}/${input}-stack.aot ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
                 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
         )
-        add_custom_target(${input}_ckpt_br_compile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}-ckpt-br.aot)
+        add_custom_target(${input}_stack_compile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}-stack.aot)
 
-        add_custom_target(${input}_compile_all ALL DEPENDS ${input}_pure_compile ${input}_stack_compile ${input}_compile ${input}_ckpt_br_compile)
+        add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot
+                COMMAND ${CMAKE_SOURCE_DIR}/lib/wasm-micro-runtime/wamr-compiler/build/wamrc --disable-aux-stack-check --enable-checkpoint -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+                DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+        )
+        add_custom_target(${input}_compile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
 
         add_test(NAME ${input}_checkpoint COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_checkpoint --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
         add_test(NAME ${input}_restore COMMAND ${CMAKE_CURRENT_BINARY_DIR}/../MVVM_restore --target ${CMAKE_CURRENT_BINARY_DIR}/${input}.aot)
@@ -55,11 +77,11 @@ wamr_app(mutex)
 wamr_app(read-file)
 wamr_app(gups)
 wamr_app(gemm)
-wamr_app(openmp)
+wamr_openmp_app(openmp)
 
 function(wamr_socket_app input)
     add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
-            COMMAND ${HOST_PREFIX}clang --target=wasm32-unknown-wasi -g -O3 -pthread -Wl,--export-all -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${WAMR_DIR}/samples/socket-api/wasm-app-prefix/src/wasm-app-build/libsocket_wasi_ext.a -I${WAMR_DIR}/core/iwasm/libraries/lib-socket/inc/ ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
+            COMMAND ${HOST_PREFIX}clang --target=wasm32-unknown-wasi -g -pthread -Wl,--export-all -Wl,--allow-undefined-file=${WAMR_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt --sysroot=${HOST_PREFIX}/../share/wasi-sysroot ${WAMR_DIR}/samples/socket-api/wasm-app-prefix/src/wasm-app-build/libsocket_wasi_ext.a -I${WAMR_DIR}/core/iwasm/libraries/lib-socket/inc/ ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c -o ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm
             DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c
     )
     add_custom_target(${input}_wamr ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input}.wasm)
@@ -104,12 +126,4 @@ endfunction()
 wamr_socket_app(server)
 wamr_socket_app(client)
 wamr_socket_app(tcp_server)
-wamr_socket_app(tcp_client)
-
-# function(wamr_mpi_app input)
-#     add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${input}
-#         COMMAND mpicc -L${CMAKE_BINARY_DIR} -I${CMAKE_SOURCE_DIR}/include -lMVVM_export -O3 -o ${CMAKE_CURRENT_BINARY_DIR}/${input} ${CMAKE_CURRENT_SOURCE_DIR}/${input}.c
-#     )  
-#     add_custom_target(${input}_wamr ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${input})
-# endfunction()
-# wamr_mpi_app(mpi)
+wamr_socket_app(tcp_client)
diff --git a/test/gemm.c b/test/gemm.c
@@ -1,20 +1,19 @@
-#include <stdio.h>
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <time.h>
 
-void dgemm(int32_t d, int32_t e, int32_t f,
-    double g, double *h, int32_t i, double *j, int32_t k, double l, double *m, int32_t n);
+void dgemm(double g, double *h, int32_t i, double *j, int32_t k, double l, double *m, int32_t n);
 
-void init(double* matrix, int row, int column) {
+void init(double *matrix, int row, int column) {
     for (int j = 0; j < column; j++) {
         for (int i = 0; i < row; i++) {
             matrix[j * row + i] = i;
         }
     }
 }
 
-int main(int argc, char* argv[]) {
+int main(int argc, char *argv[]) {
     int rowsA, colsB, common;
     int i, j, k;
 
@@ -31,8 +30,7 @@ int main(int argc, char* argv[]) {
     for (i = 0; i < 1e8; i++) {
         init(A, rowsA, common);
         init(B, common, colsB);
-        dgemm(rowsA, colsB, common,
-                    1.0, A, rowsA, B, common, 0.0, C, rowsA);
+        dgemm(1.0, A, rowsA, B, common, 0.0, C, rowsA);
     }
 
     return 0;