From b3bd57246fbbb80da5a76223d35f57589725cc48 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 10 Sep 2024 14:10:30 +0200 Subject: [PATCH] ci: mov from cifuzz to clusterfuzzlite To better support main7 CI fuzzing Ticket: 7253 --- .clusterfuzzlite/Dockerfile | 20 +++++++ .clusterfuzzlite/build.sh | 105 ++++++++++++++++++++++++++++++++++ .clusterfuzzlite/project.yaml | 15 +++++ .github/workflows/cifuzz.yml | 24 ++++---- 4 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 .clusterfuzzlite/Dockerfile create mode 100755 .clusterfuzzlite/build.sh create mode 100644 .clusterfuzzlite/project.yaml diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 000000000000..93e2a7380959 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,20 @@ +FROM gcr.io/oss-fuzz-base/base-builder-rust +RUN apt-get update && apt-get install -y build-essential autoconf automake libtool make pkg-config python flex bison zlib1g-dev libpcre3-dev cmake tshark + +# TODO libmagic, liblzma and other optional libraries +ADD https://github.com/PhilipHazel/pcre2/releases/download/pcre2-10.44/pcre2-10.44.tar.gz pcre2-10.44.tar.gz +ADD https://www.tcpdump.org/release/libpcap-1.10.5.tar.gz libpcap-1.10.5.tar.gz +ADD https://github.com/akheron/jansson/releases/download/v2.14/jansson-2.14.tar.gz jansson-2.14.tar.gz +RUN git clone --depth=1 https://github.com/yaml/libyaml +ADD https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz lz4-1.10.0.tar.gz +RUN git clone --depth 1 -b develop https://github.com/madler/zlib.git +RUN git clone --depth=1 https://github.com/catenacyber/fuzzpcap + +ENV RUSTUP_TOOLCHAIN nightly +RUN cargo install --force cbindgen + +RUN git clone --depth 1 https://github.com/OISF/libhtp.git libhtp + +COPY . $SRC/suricata +WORKDIR $SRC/suricata +COPY ./.clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 000000000000..db611be44b33 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,105 @@ +#!/bin/bash -eu + +cd $SRC/ +# build dependencies statically +if [ "$SANITIZER" = "memory" ] +then + ( + cd zlib + ./configure --static + make -j$(nproc) clean + make -j$(nproc) all + make -j$(nproc) install + ) +fi + +( +tar -xvzf pcre2-10.44.tar.gz +cd pcre2-10.44 +./configure --disable-shared +make -j$(nproc) clean +make -j$(nproc) all +make -j$(nproc) install +) + +tar -xvzf lz4-1.10.0.tar.gz +cd lz4-1.10.0 +make liblz4.a +cp lib/liblz4.a /usr/local/lib/ +cp lib/lz4*.h /usr/local/include/ +cd .. + +tar -xvzf jansson-2.14.tar.gz +cd jansson-2.14 +./configure --disable-shared +make -j$(nproc) +make install +cd .. + +tar -xvzf libpcap-1.10.5.tar.gz +cd libpcap-1.10.5 +./configure --disable-shared +make -j$(nproc) +make install +cd .. + +cd fuzzpcap +mkdir build +cd build +cmake .. +make install +cd ../.. + +cd libyaml +./bootstrap +./configure --disable-shared +make -j$(nproc) +make install +cd .. + +export CARGO_BUILD_TARGET="x86_64-unknown-linux-gnu" +# cf https://github.com/google/sanitizers/issues/1389 +export MSAN_OPTIONS=strict_memcmp=false + +#run configure with right options +if [ "$SANITIZER" = "address" ] +then + export RUSTFLAGS="$RUSTFLAGS -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-trace-compares -Cllvm-args=-sanitizer-coverage-inline-8bit-counters -Cllvm-args=-sanitizer-coverage-pc-table -Clink-dead-code -Cllvm-args=-sanitizer-coverage-stack-depth -Ccodegen-units=1" + export RUSTFLAGS="$RUSTFLAGS -Cdebug-assertions=yes" +fi + +rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + +#we did not put libhtp there before so that cifuzz does not remove it +cp -r libhtp suricata/ +# build project + +cd suricata +sh autogen.sh + +./src/tests/fuzz/oss-fuzz-configure.sh +make -j$(nproc) + +./src/suricata --list-app-layer-protos | tail -n +2 | while read i; do cp src/fuzz_applayerparserparse $OUT/fuzz_applayerparserparse""_$i; done + +( +cd src +ls fuzz_* | while read i; do + cp $i $OUT/$i + # download oss-fuzz public corpuses + wget "https://storage.googleapis.com/suricata-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/suricata_$i/public.zip" --output-file=$OUT/"$i"_seed_corpus.zip || true +done +) +# dictionaries +./src/suricata --list-keywords | grep "\- " | sed 's/- //' | awk '{print "\""$0"\""}' > $OUT/fuzz_siginit.dict + +echo \"SMB\" > $OUT/fuzz_applayerparserparse""_smb.dict + +echo "\"FPC0\"" > $OUT/fuzz_sigpcap_aware.dict +echo "\"FPC0\"" > $OUT/fuzz_predefpcap_aware.dict + +git grep tag rust | grep '"' | cut -d '"' -f2 | sort | uniq | awk 'length($0) > 2' | awk '{print "\""$0"\""}' | grep -v '\\' > generic.dict +cat generic.dict >> $OUT/fuzz_siginit.dict +cat generic.dict >> $OUT/fuzz_applayerparserparse.dict +cat generic.dict >> $OUT/fuzz_sigpcap.dict +cat generic.dict >> $OUT/fuzz_sigpcap_aware.dict diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 000000000000..2606abaaeeca --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1,15 @@ +homepage: "https://suricata.io" +language: rust +primary_contact: "vjulien@openinfosecfoundation.org" +auto_ccs: +- "jish@openinfosecfoundation.org" +- "p.antoine@catenacyber.fr" +sanitizers: + - address + - memory + - undefined +fuzzing_engines: + - afl + - honggfuzz + - libfuzzer +main_repo: 'https://github.com/OISF/suricata.git' diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index d6019cd80599..4d130f38c664 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -1,4 +1,5 @@ -name: CIFuzz +# CIFuzz did not support fuzzing of branch main7 well +name: ClusterFuzzLite on: pull_request: @@ -25,21 +26,18 @@ jobs: sudo rm -rf /usr/share/dotnet/ /usr/share/swift /usr/local/.ghcup/ /usr/local/share/powershell /usr/local/share/chromium /usr/local/lib/android /usr/local/lib/node_modules df - name: Build Fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@v1 with: - oss-fuzz-project-name: 'suricata' - dry-run: false + language: rust + github-token: ${{ secrets.GITHUB_TOKEN }} sanitizer: ${{ matrix.sanitizer }} - name: Run Fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 with: - oss-fuzz-project-name: 'suricata' + github-token: ${{ secrets.GITHUB_TOKEN }} fuzz-seconds: 600 - dry-run: false + mode: 'code-change' sanitizer: ${{ matrix.sanitizer }} - - name: Upload Crash - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 - if: failure() - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts + output-sarif: true