#!/usr/bin/env bash set -e shopt -s extglob #----------------------------------------------------------------------------- # RediSearch Build Script # # This script handles building the RediSearch module and running tests. # It supports various build configurations and test types. #----------------------------------------------------------------------------- # Get the absolute path to script directory ROOT="$(cd "$(dirname "$0")" && pwd)" BINROOT="$ROOT/bin" #----------------------------------------------------------------------------- # Default configuration values #----------------------------------------------------------------------------- COORD="oss" # Coordinator type: oss or rlec DEBUG=0 # Debug build flag PROFILE=0 # Profile build flag FORCE=0 # Force clean build flag VERBOSE=0 # Verbose output flag QUICK=${QUICK:-0} # Quick test mode (subset of tests) COV=${COV:-0} # Coverage mode (for building and testing) BUILD_INTEL_SVS_OPT=${BUILD_INTEL_SVS_OPT:-0} # Use SVS pre-compiled library # Enable Rust/C LTO. Requires Clang and lld (Linux only). # Clang needs to have the same version as the LLVM version used by Rust. # Check using `clang --version` and `rustc --version --verbose`. LTO=0 # If set to `1`, C headers for Rust modules are (re)generated from the Rust source. # If set to `0`, (re)generation is skipped and the committed C headers are used. REDISEARCH_GENERATE_HEADERS=${REDISEARCH_GENERATE_HEADERS:-1} # Inline LSE atomics on Linux AArch64 (Armv8.1-a+). Set to 0 on pre-Armv8.1-a # cores (Cortex-A72, AWS Graviton1, Raspberry Pi 4) to avoid SIGILL on load. INLINE_LSE_ATOMICS=${INLINE_LSE_ATOMICS:-1} # Test configuration (0=disabled, 1=enabled) BUILD_TESTS=0 # Build test binaries RUN_UNIT_TESTS=0 # Run C/C++ unit tests RUN_RUST_TESTS=0 # Run Rust tests RUN_RUST_VALGRIND=0 # Run Valgrind Rust tests RUN_PYTEST=0 # Run Python tests RUN_ALL_TESTS=0 # Run all test types RUN_MICRO_BENCHMARKS=0 # Run micro-benchmarks # Rust configuration RUST_PROFILE="" # Which profile should be used to build/test Rust code # If unspecified, the correct profile will be determined based # the operations to be performed RUN_MIRI=0 # Run Rust tests through miri to catch undefined behavior RUST_DENY_WARNS=0 # Deny all Rust compiler warnings RUST_TOOLCHAIN_MODIFIER="" # Rust toolchain to use (e.g., +nightly) # Rust code is built first, so exclude benchmarking crates that link C code, # since the static libraries they depend on haven't been built yet. EXCLUDE_RUST_BENCHING_CRATES_LINKING_C="--exclude inverted_index_bencher --exclude rqe_iterators_bencher --exclude iterators_ffi --exclude top_k_bencher --exclude trie_bencher --exclude triemap_ffi --exclude ttl_table_bencher" # Retrieve our pinned nightly version. NIGHTLY_VERSION=$(cat ${ROOT}/.rust-nightly) #----------------------------------------------------------------------------- # Function: parse_arguments # Parse command-line arguments and set configuration variables # Requires extglob to be enabled #----------------------------------------------------------------------------- parse_arguments() { for arg in "$@"; do # macOS only has bash 3.2 built-in, which doesn't support the more modern ${arg^^} syntax. upper_arg=$(printf '%s' "$arg" | tr '[:lower:]' '[:upper:]') case $upper_arg in COORD=*) COORD="${arg#*=}" ;; DEBUG?(=1)) DEBUG=1 ;; PROFILE?(=1)) PROFILE=1 ;; TESTS?(=1)) BUILD_TESTS=1 ;; RUN_TESTS?(=1)) RUN_ALL_TESTS=1 ;; RUN_UNIT_TESTS?(=1)) RUN_UNIT_TESTS=1 ;; RUN_RUST_TESTS?(=1)) RUN_RUST_TESTS=1 ;; RUN_RUST_VALGRIND?(=1)) RUN_RUST_VALGRIND=1 ;; RUN_MICRO_BENCHMARKS?(=1)) RUN_MICRO_BENCHMARKS=1 ;; COV=*) COV="${arg#*=}" ;; RUN_PYTEST?(=1)) RUN_PYTEST=1 ;; EXT=*) EXT="${arg#*=}" ;; EXT_HOST=*) if [[ "${arg#*=}" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then EXT_HOST="${arg#*=}" else echo "Invalid IP address: ${arg#*=}" exit 1 fi ;; EXT_PORT=*) EXT_PORT="${arg#*=}" ;; TEST=*) TEST_FILTER="${arg#*=}" ;; REDISEARCH_GENERATE_HEADERS=*) REDISEARCH_GENERATE_HEADERS="${arg#*=}" ;; RUST_PROFILE=*) RUST_PROFILE="${arg#*=}" ;; RUST_DYN_CRT=*) RUST_DYN_CRT="${arg#*=}" ;; RUN_MIRI=*) RUN_MIRI="${arg#*=}" ;; RUST_DENY_WARNS=*) RUST_DENY_WARNS="${arg#*=}" ;; SAN=*) SAN="${arg#*=}" ;; FORCE?(=1)) FORCE=1 ;; VERBOSE?(=1)) VERBOSE=1 ;; QUICK=*) QUICK="${arg#*=}" ;; TEST_TIMEOUT=*) TEST_TIMEOUT="${arg#*=}" ;; SA=*) SA="${arg#*=}" ;; REDIS_STANDALONE=*) REDIS_STANDALONE="${arg#*=}" ;; BUILD_INTEL_SVS_OPT=*) BUILD_INTEL_SVS_OPT="${arg#*=}" ;; LTO?(=1)) LTO=1 ;; INLINE_LSE_ATOMICS=*) INLINE_LSE_ATOMICS="${arg#*=}" ;; *) # Pass all other arguments directly to CMake CMAKE_ARGS="$CMAKE_ARGS -D${arg}" ;; esac done } #----------------------------------------------------------------------------- # Function: setup_test_configuration # Configure test settings based on input arguments #----------------------------------------------------------------------------- setup_test_configuration() { # If any tests will be run, ensure BUILD_TESTS is enabled if [[ "$RUN_ALL_TESTS" == "1" || "$RUN_UNIT_TESTS" == "1" || "$RUN_RUST_TESTS" == "1" || "$RUN_RUST_VALGRIND" == "1" || "$RUN_PYTEST" == "1" || "$RUN_MICRO_BENCHMARKS" == "1" ]]; then if [[ "$BUILD_TESTS" != "1" ]]; then echo "Test execution requested, enabling test build automatically" BUILD_TESTS="1" fi fi # If RUN_ALL_TESTS is enabled, enable all test types if [[ "$RUN_ALL_TESTS" == "1" ]]; then RUN_UNIT_TESTS=1 RUN_RUST_TESTS=1 RUN_PYTEST=1 fi } #----------------------------------------------------------------------------- # Function: setup_build_environment # Configure the build environment variables #----------------------------------------------------------------------------- setup_build_environment() { # Determine Rust toolchain if [[ -n "$SAN" || "$COV" == "1" || "$RUN_MIRI" == "1" ]]; then # For coverage, we use the `nightly` compiler in order to include doc tests in the coverage computation. # See https://github.com/taiki-e/cargo-llvm-cov/issues/2 for more details. echo "Using nightly version: ${NIGHTLY_VERSION}" RUST_TOOLCHAIN_MODIFIER="+$NIGHTLY_VERSION" fi # Determine build flavor if [ "$SAN" == "address" ]; then FLAVOR="debug-asan" elif [[ "$RUN_MIRI" == "1" ]]; then FLAVOR="debug-miri" elif [[ "$DEBUG" == "1" ]]; then FLAVOR="debug" elif [[ "$COV" == "1" ]]; then FLAVOR="debug-cov" elif [[ "$PROFILE" == "1" ]]; then FLAVOR="release-profile" else FLAVOR="release" fi # We must set the build target explicitly when running with a sanitizer to prevent the Rust flags from being applied to build # scripts and procedural macros. # # See https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#build-scripts-and-procedural-macros if [ "$SAN" == "address" ]; then export CARGO_BUILD_TARGET="$(rustc $RUST_TOOLCHAIN_MODIFIER -vV | sed -n 's/host: //p')" fi # Disable ODR violation detection when building tests with ASAN. This is needed because both the # shared redisearch.so and the test binaries link to the same static libraries, causing false # positives (mostly in the Rust's compiler_builtins for `RSQRT_TAB`). if [[ "$SAN" == "address" && "$BUILD_TESTS" == "1" ]]; then export ASAN_OPTIONS=detect_odr_violation=0 fi # Determine the correct Rust profile for both build and tests # Only set RUST_PROFILE if it wasn't already set by the user if [[ -z "$RUST_PROFILE" ]]; then if [[ "$BUILD_TESTS" == "1" ]]; then if [[ "$DEBUG" == "1" || -n "$SAN" || "$COV" == "1" || "$RUN_MIRI" == "1" ]]; then RUST_PROFILE="dev" else if [[ "$RUN_MICRO_BENCHMARKS" == "1" ]]; then # We don't want debug assertions to be enabled in microbenchmarks RUST_PROFILE="release" else RUST_PROFILE="optimised_test" fi fi else if [[ "$DEBUG" == "1" ]]; then RUST_PROFILE="dev" else RUST_PROFILE="release" fi fi fi # Get OS and architecture OS_NAME=$(uname) # Convert OS name to lowercase and convert Darwin to macos if [[ "$OS_NAME" == "Darwin" ]]; then OS_NAME="macos" else OS_NAME=$(echo "$OS_NAME" | tr '[:upper:]' '[:lower:]') fi # Get architecture and convert arm64 to aarch64 ARCH=$(uname -m) if [[ "$ARCH" == "arm64" ]]; then ARCH="aarch64" elif [[ "$ARCH" == "x86_64" ]]; then ARCH="x64" fi # Create full variant string for the build directory FULL_VARIANT="${OS_NAME}-${ARCH}-${FLAVOR}" # Set BINDIR based on configuration and FULL_VARIANT if [[ "$COORD" == "oss" ]]; then OUTDIR="search-community" elif [[ "$COORD" == "rlec" ]]; then OUTDIR="search-enterprise" else echo "COORD should be either oss or rlec" exit 1 fi # Set the final BINDIR using the full variant path BINDIR="${BINROOT}/${FULL_VARIANT}/${OUTDIR}" # Create compatibility symlink for aarch64 -> arm64v8 if needed if [[ "$ARCH" == "aarch64" ]]; then export ARM64V8_VARIANT="${OS_NAME}-arm64v8-${FLAVOR}" export ARM64V8_BINROOT="${BINROOT}/${ARM64V8_VARIANT}" fi } start_group() { if [[ -n $GITHUB_ACTIONS ]]; then echo "::group::$1" fi } end_group() { if [[ -n $GITHUB_ACTIONS ]]; then echo "::endgroup::" fi } #----------------------------------------------------------------------------- # Function: prepare_coverage_capture # Run lcov preparations before testing for coverage #----------------------------------------------------------------------------- prepare_coverage_capture() { start_group "Code Coverage Preparation" lcov --zerocounters --directory $BINROOT --base-directory $ROOT lcov --capture --initial --directory $BINROOT --base-directory $ROOT -o $BINROOT/base.info \ --ignore-errors inconsistent,corrupt,mismatch \ --exclude '*/_deps/*' end_group } #----------------------------------------------------------------------------- # Function: capture_coverage # Capture coverage collected since `prepare_coverage_capture` was invoked #----------------------------------------------------------------------------- capture_coverage() { NAME=${1:-cov} # Get output name. Defaults to `cov.info` start_group "Code Coverage Capture ($NAME)" # Capture coverage collected while running tests previously. # Threaded code (e.g. tiered SVS index) can produce gcov data where a function # is reported "not hit" while its line is hit; lcov flags this as an # 'inconsistent' data error, which becomes fatal (surfaced as 'corrupt') when a # later command re-reads the affected trace file. Demote these to warnings so # coverage post-processing doesn't flake the job. 'mismatch' is kept as well to # cover lcov versions that classify the same disagreement under that name. lcov --capture --directory $BINROOT --base-directory $ROOT -o $BINROOT/test.info \ --ignore-errors inconsistent,corrupt,mismatch \ --exclude '*/_deps/*' # Accumulate results with the baseline captured before the test lcov --add-tracefile $BINROOT/base.info --add-tracefile $BINROOT/test.info -o $BINROOT/full.info \ --ignore-errors inconsistent,corrupt,mismatch # Extract only the coverage of the project source files lcov --output-file $BINROOT/source.info --extract $BINROOT/full.info \ --ignore-errors inconsistent,corrupt,mismatch \ "$ROOT/src/*" \ "$ROOT/deps/thpool/*" \ # Remove coverage for directories we don't want (ignore if no file matches) lcov -o $BINROOT/$NAME.info --ignore-errors inconsistent,corrupt,mismatch,unused --remove $BINROOT/source.info \ "*/tests/*" \ end_group # Clean up temporary files rm $BINROOT/base.info $BINROOT/test.info $BINROOT/full.info $BINROOT/source.info } #----------------------------------------------------------------------------- # Function: prepare_cmake_arguments # Prepare arguments to pass to CMake #----------------------------------------------------------------------------- prepare_cmake_arguments() { # Initialize with base arguments CMAKE_BASIC_ARGS="-DCOORD_TYPE=$COORD" # A reused build directory keeps the LTO configuration it was created with: # the CMake cache retains CMAKE_INTERPROCEDURAL_OPTIMIZATION=true and the # static libraries already built from it contain LLVM bitcode, which # non-LTO consumers reject at link time (the Rust test binaries linking # libredisearch_all.a through the default cc/bfd linker fail with # "file format not recognized"). Inherit LTO from the cache so a re-drive # without the LTO argument (e.g. `make test` after an LTO `make build`) # keeps the matching clang/lld toolchain. FORCE wipes the directory in # run_cmake(), so a forced clean build is exempt. if [[ "$LTO" != "1" && "$FORCE" != "1" && -f "$BINDIR/CMakeCache.txt" ]] && \ grep -qE '^CMAKE_INTERPROCEDURAL_OPTIMIZATION:[^=]*=(true|TRUE|ON|1)$' "$BINDIR/CMakeCache.txt"; then echo "Reusing LTO-configured build directory: enabling LTO" LTO=1 fi if [[ "$LTO" == "1" ]]; then # LTO is only supported on Linux if [[ "$OS_NAME" != "linux" ]]; then echo "Error: LTO is only supported on Linux" echo "Current OS: $OS_NAME" exit 1 fi # Enable Rust/C LTO by using clang and lld # Determine compilers and linker: # 1. Use CC/CXX/LD if set by user # 2. Otherwise, try clang-$VERSION matching Rust's LLVM version # 3. Otherwise, fall back to clang/clang++/lld RUSTC_LLVM_VERSION=$(rustc --version --verbose | grep "LLVM version" | awk '{print $3}' | cut -d. -f1) if [[ -z "$CC" ]]; then if command -v "clang-$RUSTC_LLVM_VERSION" &>/dev/null; then C_COMPILER="clang-$RUSTC_LLVM_VERSION" else C_COMPILER="clang" fi else C_COMPILER="$CC" if [[ ! "$C_COMPILER" =~ clang(-[0-9]+)?$ ]]; then echo "Error: LTO requires clang as the C compiler" echo "Current CC: $C_COMPILER" echo "Please set CC to a clang-based compiler (e.g. clang, clang-nn)" exit 1 fi fi if [[ -z "$CXX" ]]; then if command -v "clang++-$RUSTC_LLVM_VERSION" &>/dev/null; then CXX_COMPILER="clang++-$RUSTC_LLVM_VERSION" else CXX_COMPILER="clang++" fi else CXX_COMPILER="$CXX" if [[ ! "$CXX_COMPILER" =~ clang([+][+])?(-[0-9]+)?$ ]]; then echo "Error: LTO requires clang++ as the C++ compiler" echo "Current CXX: $CXX_COMPILER" echo "Please set CXX to a clang-based compiler (e.g. clang++, clang++-nn)" exit 1 fi fi if [[ -z "$LD" ]]; then if command -v "lld-$RUSTC_LLVM_VERSION" &>/dev/null; then LINKER="lld-$RUSTC_LLVM_VERSION" else LINKER="lld" fi else LINKER="$LD" if [[ ! "$LINKER" =~ lld(-[0-9]+)?$ ]]; then echo "Error: LTO requires lld as the linker" echo "Current LD: $LINKER" echo "Please set LD to lld (e.g. lld, lld-nn)" exit 1 fi fi # Check LLVM version compatibility between rustc and clang # Use 'sed -E' for compatibility with both GNU sed and BSD sed CLANG_LLVM_VERSION=$($C_COMPILER --version | head -n1 | sed -En 's/.*version ([0-9]+).*/\1/p' | head -n1) if [[ -z "$RUSTC_LLVM_VERSION" || -z "$CLANG_LLVM_VERSION" ]]; then echo "Error: Could not detect LLVM versions for rustc and clang." echo "Cross-language LTO requires matching LLVM major versions." echo "Rust LLVM version: $RUSTC_LLVM_VERSION" echo "Clang LLVM version: $CLANG_LLVM_VERSION" exit 1 fi if [[ "$RUSTC_LLVM_VERSION" != "$CLANG_LLVM_VERSION" ]]; then echo "Error: LLVM version mismatch between rustc and clang" echo "Rust uses LLVM $RUSTC_LLVM_VERSION (from: rustc --version --verbose)" echo "Clang uses LLVM $CLANG_LLVM_VERSION (from: $C_COMPILER --version)" echo "" echo "Cross-language LTO requires matching LLVM major versions." echo "Please either:" echo " 1. Install clang-$RUSTC_LLVM_VERSION" echo " 2. Or build without LTO by removing the 'LTO' argument" exit 1 fi echo "Enabling C/Rust LTO" # LLVM version alignment with the Rust compiler forces us to build with # a rather recent version of clang (>=21.x.y). # This can cause issues on older Linux distributions: if we're not careful, # the .so we produce may rely on C++ symbols that don't exist in the # C++ header files available at runtime (i.e. the ones provided by the # system-level `gcc`/`g++` toolchain). # To prevent this compile-time/runtime header mismatch, we force clang # to use the C++ headers provided by the system's `g++` installation. # This requires us to combine a few different flags and guardrails: # * `--gcc-install-dir`, to point `clang` at artefacts (crtbegin.o, libgcc, etc.) # for a _specific_ version of `gcc` # * `-nostdinc++`, to disable standard `#include` directives for the C++ # standard library # * `-isystem `, to control what paths are included in search space for # C++ standard headers # * An after-the-fact check to ensure we haven't included unwanted headers in # the search GCC_COMMON_FLAGS="" GCC_CXX_FLAGS="" if command -v g++ &>/dev/null; then # Extract the C++ include paths that system g++ actually uses. _cxx_includes=$(g++ -E -x c++ -v /dev/null 2>&1 | \ sed -n '/#include <\.\.\.>/,/^End/{ /^ /p }' | \ sed 's/^ *//' | \ grep -E '(/c\+\+/|/backward)') || true # Point clang at g++'s include paths, disabling its default `#include`s if [[ -n "$_cxx_includes" ]]; then GCC_CXX_FLAGS="-nostdinc++" while IFS= read -r dir; do GCC_CXX_FLAGS+=" -isystem ${dir}" done <<< "$_cxx_includes" else echo "Error: failed to extract C++ include paths from the system g++" >&2 exit 1 fi # Pin `gcc`'s internal library dir (for crtbegin.o, libgcc, etc.) GCC_INSTALL_DIR=$(gcc -print-search-dirs | sed -n 's/^install: //p') if [[ -n "$GCC_INSTALL_DIR" ]]; then GCC_COMMON_FLAGS="--gcc-install-dir=${GCC_INSTALL_DIR}" fi # --- Diagnostic: verify C++ header pinning --- echo ">>> GCC common flags: ${GCC_COMMON_FLAGS}" echo ">>> GCC C++ flags: ${GCC_CXX_FLAGS}" echo ">>> Installed C++ header directories:" ls -d /usr/include/c++/*/ 2>/dev/null || echo " (none found)" echo ">>> Clang C++ include search paths:" _search_paths=$($CXX_COMPILER ${GCC_COMMON_FLAGS} ${GCC_CXX_FLAGS} -x c++ -v -fsyntax-only /dev/null 2>&1 \ | sed -n '/#include <\.\.\.>/,/^End/p') echo "$_search_paths" # Fail if clang's actual search paths include C++ headers from a GCC other than system _sys_gcc_major=$(gcc -dumpversion | cut -d. -f1) _bad_paths=$(echo "$_search_paths" | grep -E "/c\+\+/[0-9]+" | grep -vE "/c\+\+/${_sys_gcc_major}(\.[0-9]+){0,2}(/|$)" || true) if [[ -n "$_bad_paths" ]]; then echo "ERROR: Clang sees C++ headers from a GCC version other than system GCC ${_sys_gcc_major}:" echo "$_bad_paths" echo " C++ header pinning is not working correctly." echo " This will cause GLIBCXX symbol mismatch at link time." exit 1 fi fi # Pass LTO C/CXX flags to CMake via CFLAGS/CXXFLAGS env vars so CMake picks them # up without word-splitting issues. # Note: we assume there are no spaces in system C++ include paths. # The environment may carry GCC-only -W options (e.g. CI exports # -Wno-error=maybe-uninitialized for the gcc-built redis core). This # branch always compiles with clang, which flags those as # -Wunknown-warning-option — a hard error in vendored targets that add # -Werror (e.g. VectorSimilarity spaces). Silence that diagnostic. export CFLAGS="${CFLAGS:+${CFLAGS} }-Wno-unknown-warning-option ${GCC_COMMON_FLAGS}" export CXXFLAGS="${CXXFLAGS:+${CXXFLAGS} }-Wno-unknown-warning-option ${GCC_COMMON_FLAGS}${GCC_CXX_FLAGS:+ ${GCC_CXX_FLAGS}}" # Export CC/CXX so that Rust's cc crate also uses clang, matching the # clang-specific flags in CFLAGS/CXXFLAGS (e.g. --gcc-install-dir). export CC="$C_COMPILER" export CXX="$CXX_COMPILER" CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS \ -DCMAKE_C_COMPILER=$C_COMPILER \ -DCMAKE_CXX_COMPILER=$CXX_COMPILER \ -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=$LINKER \ -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=$LINKER \ -DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=$LINKER \ -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=true" # Include LLVM bitcode information for cross-language LTO RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C linker-plugin-lto -C linker=$C_COMPILER -C link-arg=-fuse-ld=$LINKER" if [[ -n "$GCC_INSTALL_DIR" ]]; then RUSTFLAGS="$RUSTFLAGS -C link-arg=--gcc-install-dir=${GCC_INSTALL_DIR}" fi fi if [[ "$BUILD_TESTS" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DBUILD_SEARCH_UNIT_TESTS=ON" fi if [[ "$REDISEARCH_GENERATE_HEADERS" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DREDISEARCH_GENERATE_HEADERS=ON" else CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DREDISEARCH_GENERATE_HEADERS=OFF" fi if [[ -n "$SAN" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DSAN=$SAN" DEBUG="1" fi if [[ "$COV" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCOV=1" DEBUG=1 fi if [[ "$PROFILE" != 0 ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DPROFILE=$PROFILE" # We shouldn't run profile with debug - so we fail the build if [[ "$DEBUG" == "1" ]]; then echo "Error: Cannot run profile with debug/sanitizer/coverage" exit 1 fi fi # Set build type if [[ "$DEBUG" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_BUILD_TYPE=Debug" else CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_BUILD_TYPE=RelWithDebInfo" fi # Ensure output file is always .so even on macOS CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_SHARED_LIBRARY_SUFFIX=.so" # Enable sccache for C/C++ compilation caching if available. # Prefer SCCACHE_PATH (set by sccache-action in CI with the full path), otherwise look on PATH. SCCACHE="${SCCACHE_PATH:-$(command -v sccache 2>/dev/null || true)}" if [[ -n "$SCCACHE" && -x "$SCCACHE" ]]; then # The binary being present is not enough: if the sccache server fails to # start (e.g. the remote S3 cache is unreachable), every compile invocation # errors out and the whole build fails. Probe that the server actually # comes up first, and fall back to a regular uncached build if it does not. # --show-stats is idempotent: it spawns the server if needed, connects to # an already-running one otherwise, and fails only if it cannot start. if SCCACHE_PROBE_OUTPUT=$("$SCCACHE" --show-stats 2>&1); then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_C_COMPILER_LAUNCHER=$SCCACHE -DCMAKE_CXX_COMPILER_LAUNCHER=$SCCACHE" echo "Using sccache for C/C++ compilation caching" else # Surface the underlying sccache error (e.g. "Timed out waiting for # server startup") so the cause is visible in CI logs. echo "WARNING: sccache server failed to start; building without sccache" >&2 echo "$SCCACHE_PROBE_OUTPUT" >&2 # run_cmake() reuses an existing build directory unless FORCE=1, so a # previously cached CMAKE_*_COMPILER_LAUNCHER would otherwise keep the # broken sccache active on reconfigure. Pass empty values to clear it. CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_C_COMPILER_LAUNCHER= -DCMAKE_CXX_COMPILER_LAUNCHER=" fi fi # Add caching flags to prevent using old configurations CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -UCMAKE_TOOLCHAIN_FILE" if [[ "$OS_NAME" == "macos" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" fi if [[ "$BUILD_INTEL_SVS_OPT" == "yes" || "$BUILD_INTEL_SVS_OPT" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DBUILD_INTEL_SVS_OPT=ON" else CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DSVS_SHARED_LIB=OFF" fi # Forward INLINE_LSE_ATOMICS to CMake (controls the C/C++ side). if [[ "$INLINE_LSE_ATOMICS" == "1" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DINLINE_LSE_ATOMICS=ON" else CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DINLINE_LSE_ATOMICS=OFF" fi # Handle RUST_DYN_CRT flag for Alpine Linux compatibility if [[ "$RUST_DYN_CRT" == "1" ]]; then # Add the dynamic C runtime flag to RUSTFLAGS RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C target-feature=-crt-static" # Export so child processes (CMake → regen_headers.sh) can apply the # same flag to their own cargo invocations. See regen_headers.sh. export RUST_DYN_CRT fi # MOD-14916: inline LSE atomics for Rust on AArch64. `-C target-cpu=neoverse-n1` # implies +lse so rustc emits LDADDH/LDADD instead of an ldxrh/stxrh LL/SC loop. # macOS is excluded to match the CMake NOT APPLE gate: Apple Silicon's default # target-cpu (apple-m1) is already ≥Armv8.5-a with inline LSE, so overriding it # with neoverse-n1 would only downgrade scheduling. Gated by INLINE_LSE_ATOMICS # so users on pre-Armv8.1-a cores (Cortex-A72, Graviton1, RPi4) can opt out. if [[ "$ARCH" == "aarch64" && "$OS_NAME" != "macos" && "$INLINE_LSE_ATOMICS" == "1" ]]; then RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C target-cpu=neoverse-n1" fi # Set up RUSTFLAGS for warnings if [[ "$RUST_DENY_WARNS" == "1" ]]; then RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-D warnings" fi # Ensure we can compute coverage across the FFI boundary if [[ $OS_NAME != "macos" && $COV == "1" ]]; then # Needs the C code to link on gcov RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} } -C link-args=-lgcov" # Doctests are compiled by rustdoc, which uses RUSTDOCFLAGS rather than # RUSTFLAGS for its link step. Without this, doctests that pull in # gcov-instrumented C objects (via transitive deps on FFI crates) fail # to link with undefined `__gcov_*` symbols. RUSTDOCFLAGS="${RUSTDOCFLAGS:+${RUSTDOCFLAGS} }-C link-args=-lgcov" export RUSTDOCFLAGS fi if [[ $SAN == "address" ]]; then # Add ASAN flags to RUSTFLAGS (following RedisJSON pattern) # -Zsanitizer=address enables ASAN in Rust RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-Zsanitizer=address" fi # Workaround for macOS 14: # Apple's ld (through Apple clang 16 / Xcode 16.2) has an ARM64 bug that # misaligns symbols, causing "not 8-byte aligned" LDR/STR errors. Fixed in # Apple clang 17 (Xcode 16.4+). Use LLVM's lld as a workaround when needed. if [[ "$OS_NAME" == "macos" ]]; then APPLE_CLANG_MAJOR=$(cc --version 2>/dev/null | head -1 | grep -oE 'version [0-9]+' | grep -oE '[0-9]+') if [[ -n "$APPLE_CLANG_MAJOR" && "$APPLE_CLANG_MAJOR" -lt 17 ]]; then # llvm@17 provides ld64.lld; the project's llvm@21 doesn't include lld. local lld_path="$(brew --prefix)/opt/llvm@17/bin/ld64.lld" if [[ -x "$lld_path" ]]; then echo "Apple clang $APPLE_CLANG_MAJOR < 17: using llvm@17's ld64.lld to work around ARM64 alignment bug" RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C link-arg=-fuse-ld=${lld_path}" else echo "WARNING: Apple clang $APPLE_CLANG_MAJOR has a known ARM64 linker bug but ld64.lld is not installed at ${lld_path}" fi fi fi # Export RUSTFLAGS so it's available to the Rust build process export RUSTFLAGS # RUSTFLAGS will be passed as environment variable to avoid quoting issues # This prevents CMake argument parsing from truncating complex flag values if [[ "$RUST_PROFILE" != "" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DRUST_PROFILE=$RUST_PROFILE" fi if [[ -n "$CARGO_BUILD_TARGET" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCARGO_BUILD_TARGET=$CARGO_BUILD_TARGET" fi if [[ -n "$RUST_TOOLCHAIN_MODIFIER" ]]; then CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DRUST_TOOLCHAIN_MODIFIER=$RUST_TOOLCHAIN_MODIFIER" fi } #----------------------------------------------------------------------------- # Function: run_cmake # Run CMake to configure the build #----------------------------------------------------------------------------- run_cmake() { # Full wipe: nested CMakeFiles/*.dir/*.o under subdirectories aren't reached # by removing only the top-level CMakeCache.txt and CMakeFiles/. if [[ "$FORCE" == "1" ]]; then echo "Cleaning build directory: $BINDIR" rm -rf "$BINDIR" fi # Create build directory and ensure any parent directories exist mkdir -p "$BINDIR" cd "$BINDIR" # Create compatibility symlink for aarch64 -> arm64v8 if needed if [[ "$ARCH" == "aarch64" && -n "$ARM64V8_BINROOT" ]]; then if [[ ! -e "$ARM64V8_BINROOT" ]]; then echo "Creating compatibility symlink: $ARM64V8_BINROOT -> ${BINROOT}/${FULL_VARIANT}" ln -sf "${FULL_VARIANT}" "$ARM64V8_BINROOT" fi fi echo "Configuring CMake..." echo "Build directory: $BINDIR" # Always reconfigure so -D changes (BUILD_SEARCH_UNIT_TESTS, DEBUG, COV, SAN, ...) # take effect on subsequent invocations without requiring FORCE. CMAKE_CMD="cmake $ROOT $CMAKE_BASIC_ARGS $CMAKE_ARGS" echo "$CMAKE_CMD" if [[ "$VERBOSE" == "1" ]]; then echo "Running CMake with verbose output..." RUSTFLAGS="$RUSTFLAGS" $CMAKE_CMD --trace-expand else RUSTFLAGS="$RUSTFLAGS" $CMAKE_CMD fi } #----------------------------------------------------------------------------- # Function: build_project # Build the RediSearch project using Make #----------------------------------------------------------------------------- build_project() { # redisearch_rs is now built automatically by CMake # Determine number of parallel jobs for make if command -v nproc &> /dev/null; then NPROC=$(nproc) elif command -v sysctl &> /dev/null && [[ "$OS_NAME" == "macos" ]]; then NPROC=$(sysctl -n hw.physicalcpu) else NPROC=4 # Default if we can't determine fi echo "Building RediSearch with $NPROC parallel jobs..." make -j "$NPROC" # Build test dependencies if needed build_test_dependencies # Report build success echo "Build complete. Artifacts in $BINDIR" } #----------------------------------------------------------------------------- # Function: build_test_dependencies # Build additional dependencies needed for tests #----------------------------------------------------------------------------- build_test_dependencies() { if [[ "$BUILD_TESTS" == "1" ]]; then # Ensure ext-example binary gets compiled if [[ -d "$ROOT/tests/ctests/ext-example" ]]; then echo "Building ext-example for unit tests..." # Check if we're already in the build directory if [[ "$PWD" != "$BINDIR" ]]; then cd "$BINDIR" fi # The example_extension target is created by CMake in the build directory # First check if the target exists in this build if grep -q "example_extension" Makefile 2>/dev/null || (make -q example_extension 2>/dev/null); then make example_extension else # If the target doesn't exist, we need to ensure the test was properly configured echo "Warning: 'example_extension' target not found in Makefile" echo "Checking for extension binary..." # Check if extension was already built by a previous run EXTENSION_PATH="$BINDIR/example_extension/libexample_extension.so" if [[ -f "$EXTENSION_PATH" ]]; then echo "Extension binary already exists at: $EXTENSION_PATH" else echo "Extension binary not found. Some tests may fail." echo "Try running 'make example_extension' manually in $BINDIR" fi fi # Export extension path for tests EXTENSION_PATH="$BINDIR/example_extension/libexample_extension.so" if [[ -f "$EXTENSION_PATH" ]]; then echo "Example extension located at: $EXTENSION_PATH" export EXT_TEST_PATH="$EXTENSION_PATH" else echo "Warning: Could not find example extension at $EXTENSION_PATH" echo "Some tests may fail if they depend on this extension" fi fi fi } #----------------------------------------------------------------------------- # Function: run_unit_tests # Run C/C++ unit tests #----------------------------------------------------------------------------- run_unit_tests() { if [[ "$RUN_UNIT_TESTS" != "1" ]]; then return 0 fi echo "Running unit tests..." # Set test environment variables if needed if [[ "$OS_NAME" == "macos" ]]; then echo "Running unit tests on macOS" # On macOS, we may need to set DYLD_LIBRARY_PATH or similar # Uncomment if needed: # export DYLD_LIBRARY_PATH="$BINDIR:$DYLD_LIBRARY_PATH" fi # Set up environment variables for the unit-tests script export BINDIR # Set up test filter if provided if [[ -n "$TEST_FILTER" ]]; then echo "Running tests matching: $TEST_FILTER" export TEST="$TEST_FILTER" fi if [[ $COV == 1 ]]; then prepare_coverage_capture fi # Set verbose mode if requested if [[ "$VERBOSE" == "1" ]]; then export VERBOSE=1 fi # Set sanitizer mode if requested if [[ "$SAN" == "address" ]]; then export SAN="address" fi # Call the unit-tests script from the sbin directory echo "Calling $ROOT/sbin/unit-tests" # Capture exit code without triggering set -e, so capture_coverage runs even on failure. "$ROOT/sbin/unit-tests" && UNIT_TEST_RESULT=0 || UNIT_TEST_RESULT=$? if [[ $UNIT_TEST_RESULT -eq 0 ]]; then echo "All unit tests passed!" else echo "Some unit tests failed. Check the test logs above for details." HAS_FAILURES=1 fi if [[ $COV == 1 ]]; then capture_coverage unit fi } #----------------------------------------------------------------------------- # Function: run_rust_tests # Run Rust tests #----------------------------------------------------------------------------- run_rust_tests() { if [[ "$RUN_RUST_TESTS" != "1" ]]; then return 0 fi echo "Running Rust tests..." # Tell Rust build scripts where to find the compiled static libraries export BINDIR # Set Rust test environment RUST_DIR="$ROOT/src/redisearch_rs" CARGO_BUILD_FLAGS="" # Add Rust test extensions if [[ $COV == 1 ]]; then RUST_TEST_COMMAND="llvm-cov test" # We exclude Rust benchmarking crates that link to C code when computing coverage. # On one side, we aren't interested in coverage of those utilities. # On top of that, it causes linking issues since, when computing coverage, it seems to # require C symbols to be defined even if they aren't invoked at runtime. RUST_TEST_OPTIONS=" --profile=$RUST_PROFILE --doctests $EXCLUDE_RUST_BENCHING_CRATES_LINKING_C --codecov --ignore-filename-regex="varint_bencher/*,trie_bencher/*,inverted_index_bencher/*,top_k_bencher/*" --output-path=$BINROOT/rust_cov.info " elif [[ "$RUN_MIRI" == "1" ]]; then RUST_TEST_COMMAND="miri nextest run" RUST_TEST_OPTIONS="--cargo-profile=$RUST_PROFILE" elif [[ "$SAN" == "address" ]]; then # We must rebuild the Rust standard library to get sanitizer coverage # for its functions. # Since --build-std is a cargo flag (not rustc), we set it separately CARGO_BUILD_FLAGS="${CARGO_BUILD_FLAGS:+${CARGO_BUILD_FLAGS} }-Zbuild-std" RUST_TEST_COMMAND="nextest run" # The doc tests are disabled under ASAN to avoid issues with linking to the sanitizer runtime # in doc tests. RUST_TEST_OPTIONS="--tests --cargo-profile=$RUST_PROFILE $EXCLUDE_RUST_BENCHING_CRATES_LINKING_C" else RUST_TEST_COMMAND="nextest run" RUST_TEST_OPTIONS="--cargo-profile=$RUST_PROFILE" fi # Run cargo test with the appropriate filter cd "$RUST_DIR" RUSTFLAGS="${RUSTFLAGS}" cargo $RUST_TOOLCHAIN_MODIFIER $CARGO_BUILD_FLAGS $RUST_TEST_COMMAND $RUST_TEST_OPTIONS --workspace $TEST_FILTER # Check test results RUST_TEST_RESULT=$? if [[ $RUST_TEST_RESULT -eq 0 ]]; then echo "All Rust tests passed!" else echo "Some Rust tests failed. Check the test logs above for details." HAS_FAILURES=1 fi } #----------------------------------------------------------------------------- # Function: run_rust_valgrind_tests # Run Rust Valgrind tests (to detect memory leaks) #----------------------------------------------------------------------------- run_rust_valgrind_tests() { if [[ "$RUN_RUST_VALGRIND" != "1" ]]; then return 0 fi echo "Running Rust tests..." # Set Rust test environment RUST_DIR="$ROOT/src/redisearch_rs" cd "$RUST_DIR" if [[ "$OS_NAME" == "macos" ]]; then # no valgrind on apple silicon... so... echo "The valgrind test target is only supported on Linux" HAS_FAILURES=1 return 0 else # Run cargo valgrind with the appropriate filter VALGRINDFLAGS=--suppressions=$PWD/valgrind.supp \ RUSTFLAGS="${RUSTFLAGS}" \ cargo valgrind test \ --profile=$RUST_PROFILE \ --workspace $TEST_FILTER \ -- --nocapture fi # Check test results RUST_TEST_RESULT=$? if [[ $RUST_TEST_RESULT -eq 0 ]]; then echo "Rust Valgrind test passed!" else echo "Some Rust valgrind tests failed. Check the test logs above for details." HAS_FAILURES=1 fi } #----------------------------------------------------------------------------- # Function: run_python_tests # Run Python behavioral tests #----------------------------------------------------------------------------- run_python_tests() { if [[ "$RUN_PYTEST" != "1" ]]; then return 0 fi echo "Running Python behavioral tests..." # Locate the built module MODULE_PATH="$BINDIR/redisearch.so" if [[ ! -f "$MODULE_PATH" && -f "$BINDIR/module-enterprise.so" ]]; then MODULE_PATH="$BINDIR/module-enterprise.so" fi if [[ ! -f "$MODULE_PATH" ]]; then echo "Error: Cannot find RediSearch module binary in $BINDIR" exit 1 fi # Set up environment variables required by runtests.sh export MODULE="$(realpath "$MODULE_PATH")" export BINROOT export FULL_VARIANT export BINDIR export REJSON="${REJSON:-1}" export REJSON_BRANCH="${REJSON_BRANCH:-master}" export REJSON_PATH export REJSON_ARGS export TEST export FORCE export PARALLEL="${PARALLEL:-1}" export LOG_LEVEL="${LOG_LEVEL:-debug}" export TEST_TIMEOUT export REDIS_STANDALONE="${REDIS_STANDALONE:-1}" export SA="${SA:-$REDIS_STANDALONE}" export COV export EXT=${EXT-"RUN"} export EXT_HOST=${EXT_HOST-"127.0.0.1"} export EXT_PORT=${EXT_PORT-6379} # Set up test filter if provided if [[ -n "$TEST_FILTER" ]]; then export TEST="$TEST_FILTER" echo "Running Python tests matching: $TEST_FILTER" fi # Enable quick mode if requested (run only a subset of tests) if [[ "$QUICK" == "1" ]]; then echo "Running in QUICK mode - using a subset of tests" export QUICK=1 fi # Enable verbose mode if requested if [[ "$VERBOSE" == "1" ]]; then export VERBOSE=1 export RLTEST_VERBOSE=1 fi if [[ $COV == 1 ]]; then prepare_coverage_capture fi # Use the runtests.sh script for Python tests TESTS_SCRIPT="$ROOT/tests/pytests/runtests.sh" echo "Running Python tests with module at: $MODULE" # Run the tests from the ROOT directory with the requested params cd "$ROOT" # Capture exit code without triggering set -e, so capture_coverage runs even on failure. $TESTS_SCRIPT && PYTHON_TEST_RESULT=0 || PYTHON_TEST_RESULT=$? if [[ $PYTHON_TEST_RESULT -eq 0 ]]; then echo "All Python tests passed!" else echo "Some Python tests failed. Check the test logs above for details." HAS_FAILURES=1 fi if [[ $COV == 1 ]]; then if [[ "$REDIS_STANDALONE" == "1" ]]; then DEPLOYMENT_TYPE="standalone" else DEPLOYMENT_TYPE="coordinator" fi capture_coverage flow_$DEPLOYMENT_TYPE fi } #----------------------------------------------------------------------------- # Function: run_tests # Run all requested tests and check results #----------------------------------------------------------------------------- run_tests() { HAS_FAILURES=0 # Run each test type as requested run_unit_tests run_rust_tests run_python_tests # Exit with failure if any test suite failed if [[ "$HAS_FAILURES" == "1" ]]; then echo "One or more test suites had failures" exit 1 fi } #----------------------------------------------------------------------------- # Function: run_micro_benchmarks # Run micro-benchmarks #----------------------------------------------------------------------------- run_micro_benchmarks() { if [[ "$RUN_MICRO_BENCHMARKS" != "1" ]]; then return 0 fi echo "Running micro-benchmarks..." # Check if micro-benchmarks directory exists MICRO_BENCH_DIR="$BINDIR/micro-benchmarks" # Run each benchmark executable echo "Running benchmarks from $MICRO_BENCH_DIR" cd "$MICRO_BENCH_DIR" for benchmark in benchmark_*; do if [[ -x "$benchmark" ]]; then benchmark_name=${benchmark#benchmark_} echo "Running $benchmark..." if ./"$benchmark" --benchmark_out_format=json --benchmark_out="${benchmark_name}_results.json"; then echo "✓ $benchmark completed successfully" else echo "✗ $benchmark FAILED" HAS_FAILURES=1 fi fi done if [[ "$HAS_FAILURES" == "1" ]]; then echo "Some micro-benchmarks failed. Check the logs above for details." exit 1 else echo "All micro-benchmarks completed successfully." echo "Results saved to $MICRO_BENCH_DIR/*_results.json" fi } #----------------------------------------------------------------------------- # Main execution flow #----------------------------------------------------------------------------- # Parse command line arguments parse_arguments "$@" # Set up test configuration based on input parameters setup_test_configuration # Set up the build environment setup_build_environment # Prepare CMake arguments prepare_cmake_arguments # Run CMake to configure the build run_cmake # Build the project build_project # Run tests if requested run_tests # Run Rust valgrind tests if requested run_rust_valgrind_tests # Run micro-benchmarks if requested run_micro_benchmarks exit 0