From a3c0f1f153d5dd76446ac64cd6ebbb230d8ec63b Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Mon, 19 Aug 2024 06:21:43 +0200 Subject: [PATCH 1/3] update libfuzzer to sane version --- fuzzers/libfuzzer/builder.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/libfuzzer/builder.Dockerfile b/fuzzers/libfuzzer/builder.Dockerfile index 7fe80447e..75705600e 100644 --- a/fuzzers/libfuzzer/builder.Dockerfile +++ b/fuzzers/libfuzzer/builder.Dockerfile @@ -17,7 +17,7 @@ FROM $parent_image RUN git clone https://github.com/llvm/llvm-project.git /llvm-project && \ cd /llvm-project && \ - git checkout 5cda4dc7b4d28fcd11307d4234c513ff779a1c6f && \ + git checkout 3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff && \ cd compiler-rt/lib/fuzzer && \ (for f in *.cpp; do \ clang++ -stdlib=libc++ -fPIC -O2 -std=c++11 $f -c & \ From 652394b17f5021755d733d6788478010e38b517a Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Mon, 19 Aug 2024 06:57:52 +0200 Subject: [PATCH 2/3] use the correct std version --- fuzzers/libfuzzer/builder.Dockerfile | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fuzzers/libfuzzer/builder.Dockerfile b/fuzzers/libfuzzer/builder.Dockerfile index 75705600e..8d03d9b84 100644 --- a/fuzzers/libfuzzer/builder.Dockerfile +++ b/fuzzers/libfuzzer/builder.Dockerfile @@ -15,12 +15,21 @@ ARG parent_image FROM $parent_image +# Install dependencies. +RUN apt-get update && \ + apt-get remove -y llvm-10 && \ + apt-get install -y \ + build-essential \ + lsb-release wget software-properties-common gnupg && \ + wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 18 + RUN git clone https://github.com/llvm/llvm-project.git /llvm-project && \ cd /llvm-project && \ - git checkout 3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff && \ - cd compiler-rt/lib/fuzzer && \ + git checkout 3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff + +RUN cd /llvm-project/compiler-rt/lib/fuzzer && \ (for f in *.cpp; do \ - clang++ -stdlib=libc++ -fPIC -O2 -std=c++11 $f -c & \ + clang++ -stdlib=libc++ -fPIC -O2 -std=c++17 $f -c & \ done && wait) && \ ar r libFuzzer.a *.o && \ cp libFuzzer.a /usr/lib From e3253ce4060bce5d65d727cbc700779804d16ab6 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Mon, 19 Aug 2024 18:00:31 +0200 Subject: [PATCH 3/3] purge llvm better, remove old patch, ignore container overflow --- fuzzers/libfuzzer/builder.Dockerfile | 2 +- fuzzers/libfuzzer/fuzzer.py | 5 ++ fuzzers/libfuzzer/patch.diff | 100 --------------------------- 3 files changed, 6 insertions(+), 101 deletions(-) delete mode 100644 fuzzers/libfuzzer/patch.diff diff --git a/fuzzers/libfuzzer/builder.Dockerfile b/fuzzers/libfuzzer/builder.Dockerfile index 8d03d9b84..b2f7f0902 100644 --- a/fuzzers/libfuzzer/builder.Dockerfile +++ b/fuzzers/libfuzzer/builder.Dockerfile @@ -17,7 +17,7 @@ FROM $parent_image # Install dependencies. RUN apt-get update && \ - apt-get remove -y llvm-10 && \ + apt-get remove -y llvm-* clang-* && \ apt-get install -y \ build-essential \ lsb-release wget software-properties-common gnupg && \ diff --git a/fuzzers/libfuzzer/fuzzer.py b/fuzzers/libfuzzer/fuzzer.py index 78a1e2f8f..5a7c9aff3 100755 --- a/fuzzers/libfuzzer/fuzzer.py +++ b/fuzzers/libfuzzer/fuzzer.py @@ -53,6 +53,11 @@ def run_fuzzer(input_corpus, output_corpus, target_binary, extra_flags=None): os.makedirs(crashes_dir) os.makedirs(output_corpus) + if 'ASAN_OPTIONS' in os.environ: + os.environ['ASAN_OPTIONS'] += ':detect_container_overflow=0' + else: + os.environ['ASAN_OPTIONS'] = 'detect_container_overflow=0' + # Enable symbolization if needed. # Note: if the flags are like `symbolize=0:..:symbolize=1` then # only symbolize=1 is respected. diff --git a/fuzzers/libfuzzer/patch.diff b/fuzzers/libfuzzer/patch.diff deleted file mode 100644 index a31cc301e..000000000 --- a/fuzzers/libfuzzer/patch.diff +++ /dev/null @@ -1,100 +0,0 @@ -diff --git a/compiler-rt/lib/fuzzer/FuzzerFork.cpp b/compiler-rt/lib/fuzzer/FuzzerFork.cpp -index 84725d2..4e1a506 100644 ---- a/compiler-rt/lib/fuzzer/FuzzerFork.cpp -+++ b/compiler-rt/lib/fuzzer/FuzzerFork.cpp -@@ -26,6 +26,8 @@ - #include - #include - #include -+#include -+#include - - namespace fuzzer { - -@@ -70,6 +72,8 @@ struct FuzzJob { - std::string SeedListPath; - std::string CFPath; - size_t JobId; -+ bool Executing = false; -+ Vector CopiedSeeds; - - int DftTimeInSeconds = 0; - -@@ -124,7 +128,6 @@ struct GlobalEnv { - Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. - Cmd.addFlag("print_final_stats", "1"); - Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. -- Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); - Cmd.addFlag("stop_file", StopFile()); - if (!DataFlowBinary.empty()) { - Cmd.addFlag("data_flow_trace", DFTDir); -@@ -133,11 +136,10 @@ struct GlobalEnv { - } - auto Job = new FuzzJob; - std::string Seeds; -- if (size_t CorpusSubsetSize = -- std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { -+ if (size_t CorpusSubsetSize = Files.size()) { - auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { -- auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; -+ auto &SF = Files[i]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); - } -@@ -213,11 +215,20 @@ struct GlobalEnv { - Set NewFeatures, NewCov; - CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, - &NewFeatures, Cov, &NewCov, Job->CFPath, false); -+ RemoveFile(Job->CFPath); - for (auto &Path : FilesToAdd) { -- auto U = FileToVector(Path); -- auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); -- WriteToFile(U, NewPath); -- Files.push_back(NewPath); -+ // Only merge files that have not been merged already. -+ if (std::find(Job->CopiedSeeds.begin(), Job->CopiedSeeds.end(), Path) == Job->CopiedSeeds.end()) { -+ // NOT THREAD SAFE: Fast check whether file still exists. -+ struct stat buffer; -+ if (stat (Path.c_str(), &buffer) == 0) { -+ auto U = FileToVector(Path); -+ auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); -+ WriteToFile(U, NewPath); -+ Files.push_back(NewPath); -+ Job->CopiedSeeds.push_back(Path); -+ } -+ } - } - Features.insert(NewFeatures.begin(), NewFeatures.end()); - Cov.insert(NewCov.begin(), NewCov.end()); -@@ -271,10 +282,19 @@ struct JobQueue { - } - }; - --void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { -+void WorkerThread(GlobalEnv *Env, JobQueue *FuzzQ, JobQueue *MergeQ) { - while (auto Job = FuzzQ->Pop()) { -- // Printf("WorkerThread: job %p\n", Job); -+ Job->Executing = true; -+ int Sleep_ms = 5 * 60 * 1000; -+ std::thread([=]() { -+ std::this_thread::sleep_for(std::chrono::milliseconds(Sleep_ms / 5)); -+ while (Job->Executing) { -+ Env->RunOneMergeJob(Job); -+ std::this_thread::sleep_for(std::chrono::milliseconds(Sleep_ms)); -+ } -+ }).detach(); - Job->ExitCode = ExecuteCommand(Job->Cmd); -+ Job->Executing = false; - MergeQ->Push(Job); - } - } -@@ -335,7 +355,7 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - size_t JobId = 1; - Vector Threads; - for (int t = 0; t < NumJobs; t++) { -- Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); -+ Threads.push_back(std::thread(WorkerThread, &Env, &FuzzQ, &MergeQ)); - FuzzQ.Push(Env.CreateNewJob(JobId++)); - } -