Makefile

# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# --------------------------------------------------------------------------
# Copyright (c) ByteDance Ltd. and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
#
# This file has been modified by ByteDance Ltd. and/or its affiliates on
# 2025-11-11.
#
# Original file was released under the Apache License 2.0,
# with the full license text available at:
#     http://www.apache.org/licenses/LICENSE-2.0
#
# This modified file is released under the same license.
# --------------------------------------------------------------------------

# --- 1. Standard Lifecycle Targets ---
# all: Default build; clean: Remove artifacts; help: Show usage; install: Install package
.PHONY: all clean help system_info install

# --- 2. Development Tools & Setup ---
# Includes code formatting, Conan dependency installation/build, and compilation DB generation
.PHONY: clang-format-check _conan_prepare conan_install conan_build _compile_db compile_db_all

# --- 3. Conan Package Export ---
# Export the built package to the local Conan cache
.PHONY: export_base export_debug export_release

# --- 4. Build Configurations & Variants ---
# Covers Debug/Release, Spark compatibility, ASAN checks, and builds with test utilities
.PHONY: debug release RelWithDebInfo debug-with-asan
.PHONY: debug_spark release_spark
.PHONY: release_with_test release_with_debug_info_with_test
.PHONY: debug_with_test debug_with_test_spark debug_with_test_cov
.PHONY: debug_spark_with_test release_spark_with_test
.PHONY: python-wheel

# --- 5. Benchmark Build Targets ---
.PHONY: benchmarks-basic-build benchmarks-build
.PHONY: benchmarks-build-spark benchmarks-build-relwithdebinfo

# --- 6. Test Execution & Coverage ---
# Targets for running CTest and generating code coverage reports
.PHONY: ctest_debug ctest_release
.PHONY: unittest unittest_debug unittest_release
.PHONY: unittest_release_spark unittest_debug_spark unittest_coverage

# -----------------------------------------------------------------
# Interfaces to control CMake options via Makefile.
# Usage:
# To pass any options to Bolt via Makefile in shell script, you can
# ```shell
#  make conan_build BUILD_VERSION="main" BUILD_TYPE=Release \
#        CONAN_OPTIONS="-o bolt/*:enable_hdfs=True" \
#        CONAN_CONFIG="-c bolt/*:tools.build:skip_test=False"
# ```

# for passing conan options
CONAN_OPTIONS ?=

# for passing skip_test
CONAN_CONFIG ?=

# for passing high priority options.
# options in CONAN_OVERRIDE will override the options in CONAN_OPTIONS
CONAN_OVERRIDE ?=

BUILD_VERSION ?= main
PROFILE ?= default
BUILD_TYPE=Release
# Build third-party dependencies in Release unless explicitly overridden.
DEPENDENCY_BUILD_TYPE ?= Release

# TODO: remove `BUILD_USER` and `BUILD_CHANNEL`
BUILD_USER ?=
BUILD_CHANNEL ?=

# Shared Conan arguments used by commands that run from the build directory.
CONAN_PACKAGE_ARGS = \
	--name=bolt \
	--version=${BUILD_VERSION} \
	--user=${BUILD_USER} \
	--channel=${BUILD_CHANNEL}

CONAN_HOST_PROFILE_ARGS = \
	-pr:h ${PROFILE}

# Package-scoped settings only apply when the matching dependency is in the graph.
CONAN_BUILD_SETTINGS = \
	-s "&:build_type=${BUILD_TYPE}" \
	-s build_type=${DEPENDENCY_BUILD_TYPE}

# Reusable Conan option fragments for public build targets.
CONAN_TEST_CONFIG = -c bolt/*:tools.build:skip_test=False
CONAN_PRESTO_OPTIONS = -o bolt/*:spark_compatible=False
CONAN_SPARK_OPTIONS = -o bolt/*:spark_compatible=True
CONAN_TESTUTIL_OPTIONS = -o bolt/*:enable_testutil=True
CONAN_PERF_OPTIONS = -o bolt/*:enable_perf=True

# Controls the complete unit-test runtime linkage. If this differs from the
# exported Bolt library, Conan provides the opposite gflags/glog variant while
# CMake reuses the existing Bolt object files. Supported values: static, shared.
BOLT_TEST_LINKAGE ?= shared
ifneq ($(BOLT_TEST_LINKAGE),static)
ifneq ($(BOLT_TEST_LINKAGE),shared)
$(error Unsupported BOLT_TEST_LINKAGE '$(BOLT_TEST_LINKAGE)'; expected 'static' or 'shared')
endif
endif
export BOLT_TEST_LINKAGE

# Pass an explicitly selected linker to the Conan recipe.
ifneq ($(origin BOLT_LINKER), undefined)
export BOLT_LINKER
endif

# Note that, `benchmarks` and `test coverage` shouldn't  be included in conan's options/configs,
# Control whether to build benchmarks
BOLT_BUILD_BENCHMARKS ?= "OFF"
# Control whether to build only basic benchmarks
BOLT_BUILD_BENCHMARKS_BASIC ?= "OFF"
# Control whether to build tests with coverage instrumentation
BOLT_BUILD_TESTING_WITH_COVERAGE ?= "OFF"
# -----------------------------------------------------------------

# temporary variables for build scripts, not intended for users to set directly
BUILD_BASE_DIR=_build
BENCHMARKS_DUMP_DIR=dumps

UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
    # Linux
    MEMORY ?= $(shell free -g | grep 'Mem:' | awk '{print $$2}')
    FREE_MEMORY ?= $(shell free -g | grep 'Mem:' | awk '{print $$4}')
    CPU_CORES ?= $(shell grep -c 'processor' /proc/cpuinfo)
else ifeq ($(UNAME_S),Darwin)
    # macOS
    MEMORY ?= $(shell sysctl -n hw.memsize | awk '{print int($$1/1024/1024/1024)}')
    FREE_MEMORY ?= $(shell vm_stat | grep "Pages free" | awk '{print int($$3*4096/1024/1024/1024)}')
    CPU_CORES ?= $(shell sysctl -n hw.ncpu)
else
    MEMORY ?= 8
    FREE_MEMORY ?= 4
    CPU_CORES ?= 4
endif

# collect system info
ifeq ($(UNAME_S),Darwin)
    OS_DETAILED   := $(shell sw_vers -productName) $(shell sw_vers -productVersion) ($(shell sw_vers -buildVersion))
    CPU_MODEL     := $(shell sysctl -n machdep.cpu.brand_string)
    MEM_TOTAL     := $(shell sysctl -n hw.memsize | awk '{print int($$1/1024/1024/1024) " GB"}')
else
    OS_DETAILED   := $(shell uname -o 2>/dev/null || uname -s) $(shell uname -r)
    DISTRO_NAME   := $(shell grep -E '^(PRETTY_NAME)=' /etc/os-release 2>/dev/null | cut -d '"' -f 2)
    ifneq ($(DISTRO_NAME),)
        OS_DETAILED += [$(DISTRO_NAME)]
    endif
    CPU_MODEL     := $(shell grep "model name" /proc/cpuinfo | head -n1 | cut -d: -f2 | xargs)
    MEM_TOTAL     := $(shell grep MemTotal /proc/meminfo | awk '{print int($$2/1024/1024) " GB"}')
endif
CONAN_EXE := $(shell command -v conan 2> /dev/null)
CMAKE_EXE := $(shell command -v cmake 2> /dev/null)

export GTEST_COLOR=1

OS:=$(shell uname -s)

ifndef CI_NUM_THREADS
# Make sure each core has 4G memory
	_NUM_THREADS_CALC := $(shell echo $$(( $(CPU_CORES) < $(MEMORY) / 4 ? $(CPU_CORES) : $(MEMORY) / 4 )) )
	_NUM_THREADS_FINAL := $(shell echo $$(( $(_NUM_THREADS_CALC) < 1 ? 1 : $(_NUM_THREADS_CALC) )) )
	NUM_THREADS ?= $(_NUM_THREADS_FINAL)
else
	NUM_THREADS ?= $(CI_NUM_THREADS)
endif

ifndef CI_NUM_LINK_JOB
	_NUM_LINK_JOB_CALC := $(shell echo $$(( $(FREE_MEMORY) / 10 )) )
	_NUM_LINK_JOB_FINAL := $(shell echo $$(( $(_NUM_LINK_JOB_CALC) < 4 ? 4 : $(_NUM_LINK_JOB_CALC) )) )
	NUM_LINK_JOB ?= $(_NUM_LINK_JOB_FINAL)
else
	NUM_LINK_JOB ?= $(CI_NUM_LINK_JOB)
endif

CONAN_BUILD_ENV = \
	NUM_THREADS=$(NUM_THREADS) \
	NUM_LINK_JOB=$(NUM_LINK_JOB) \
	BOLT_BUILD_BENCHMARKS=${BOLT_BUILD_BENCHMARKS} \
	BOLT_BUILD_BENCHMARKS_BASIC=${BOLT_BUILD_BENCHMARKS_BASIC}

CPU_TARGET ?= "avx"

PYTHON_EXECUTABLE ?= $(shell which python3)

PYTHON_PACKAGE_DIR = $(CURDIR)/bolt/python
PYTHON_DIST_DIR = $(CURDIR)/dist

all: 			#: Build the release version
	$(MAKE) release

clean:					#: Delete all build artifacts
	rm -rf $(BUILD_BASE_DIR)/Rel* && rm -rf $(BUILD_BASE_DIR)/Debug* && rm -rf CMakeUserPresets.json

# only used in CI
clang-format-check:
	find bolt \( -name "*.cpp" -o -name "*.h" \) -type f > files.txt
	cat files.txt | xargs -I{} -P $(CPU_CORES) clang-format -style=file --dry-run {} > log.txt 2>&1
	cat log.txt && echo -e "You can use clang-format -i -style=file path_to_file command to format file"
	if grep -q 'warning' log.txt; then false; fi
	@rm -f files.txt log.txt

_conan_prepare:
	if [ ! -d "_build" ]; then \
		mkdir _build; \
	fi; \
	git rev-parse HEAD && \
	mkdir -p _build/${BUILD_TYPE} && \
	cd _build/${BUILD_TYPE} && \
	set -f && \
	echo " \
	$(CONAN_HOST_PROFILE_ARGS) \
	${CONAN_OPTIONS} ${CONAN_OVERRIDE}" > new_conan.options && \
	set -x && \
	if [ -f conan.options ] && [ -f ../.build_type ] && cmp -s new_conan.options conan.options && [ "`cat ../.build_type`" = "${BUILD_TYPE}" ]; then \
	  echo "Conan options and build type unchanged; preserving CMakeCache.txt"; \
	else \
	  echo "Conan options and build type changed! deleting CMakeCache.txt"; \
	  rm -f CMakeCache.txt; \
	fi && \
	mv new_conan.options conan.options && \
	echo ${BUILD_TYPE} > ../.build_type && \
	read ALL_CONAN_OPTIONS < conan.options && \
	conan graph info ../.. $(CONAN_PACKAGE_ARGS) \
	   $(CONAN_BUILD_SETTINGS) \
	   $${ALL_CONAN_OPTIONS} ${CONAN_CONFIG} --build=missing \
	   --format=html > bolt.conan.graph.html && \
	cd -

conan_install: _conan_prepare
	cd _build/${BUILD_TYPE} && \
	set -f && \
	read ALL_CONAN_OPTIONS < conan.options && \
	NUM_LINK_JOB=$(NUM_LINK_JOB) \
	conan install ../.. $(CONAN_PACKAGE_ARGS) \
	   $(CONAN_BUILD_SETTINGS) \
	$${ALL_CONAN_OPTIONS} ${CONAN_CONFIG} --build=missing  &&\
	cd -

conan_build: _conan_prepare
	cd _build/${BUILD_TYPE} && \
	set -f && \
	read ALL_CONAN_OPTIONS < conan.options && \
	$(CONAN_BUILD_ENV) \
	BOLT_BUILD_TESTING_WITH_COVERAGE=${BOLT_BUILD_TESTING_WITH_COVERAGE} \
	conan build ../.. $(CONAN_PACKAGE_ARGS) \
	   $(CONAN_BUILD_SETTINGS) \
	   --build=missing $${ALL_CONAN_OPTIONS} ${CONAN_CONFIG} && \
	cd -

_compile_db: _conan_prepare
	cd _build/${BUILD_TYPE} && \
	set -f && \
	read ALL_CONAN_OPTIONS < conan.options && \
	$(CONAN_BUILD_ENV) \
	BOLT_CONAN_CONFIGURE_ONLY=1 \
	conan build ../.. $(CONAN_PACKAGE_ARGS) \
	   $(CONAN_BUILD_SETTINGS) \
	   --build=missing $${ALL_CONAN_OPTIONS} ${CONAN_CONFIG} && \
	cd - && \
	cmake --build --preset conan-$$(echo "${BUILD_TYPE}" | tr [A-Z] [a-z]) --target generate_parquet_thrift && \
	cmake --build --preset conan-$$(echo "${BUILD_TYPE}" | tr [A-Z] [a-z]) --target bolt_dwio_dwrf_proto


compile_db_all:
	$(MAKE) _compile_db \
	BUILD_TYPE=Release \
	BOLT_BUILD_BENCHMARKS="ON" \
	CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS) -o bolt/*:enable_s3=True \
						-o bolt/*:enable_gcs=True -o bolt/*:enable_abfs=True" \
	CONAN_CONFIG="$(CONAN_TEST_CONFIG)"

export_base:
	cd _build/${BUILD_TYPE} && \
	set -f && \
	read ALL_CONAN_OPTIONS < conan.options && \
	conan export-pkg $(CONAN_PACKAGE_ARGS) \
	 $${ALL_CONAN_OPTIONS} \
	 $(CONAN_BUILD_SETTINGS) ${CONAN_CONFIG}\
	 ../.. && \
	cd -

export_debug:
	$(MAKE) export_base BUILD_TYPE=Debug

export_release:
	$(MAKE) export_base BUILD_TYPE=Release

install:
	$(MAKE) export_base BUILD_TYPE=$(shell cat _build/.build_type)

debug:      	#: Build with debugging symbols
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS)"

debug-with-asan:  #: Build the debug version with address sanitizer enabled
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_OPTIONS=" -o enable_asan=True "

release:  	#: Build the release version
	$(MAKE) conan_build BUILD_TYPE=Release CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS)"

RelWithDebInfo:
	$(MAKE) conan_build BUILD_TYPE=RelWithDebInfo

release_with_test:
	$(MAKE) conan_build BUILD_TYPE=Release CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

release_with_debug_info_with_test:
	$(MAKE) conan_build BUILD_TYPE=RelWithDebInfo CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

debug_with_test:
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

debug_with_test_spark:
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

debug_with_test_cov:
	$(MAKE) conan_build BUILD_TYPE=Debug BOLT_BUILD_TESTING_WITH_COVERAGE="ON" CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

debug_spark:
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS)"

release_spark:
	$(MAKE) conan_build BUILD_TYPE=Release CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS)"

release_spark_with_test:
	$(MAKE) conan_build BUILD_TYPE=Release CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

debug_spark_with_test:
	$(MAKE) conan_build BUILD_TYPE=Debug CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS)"

# Wheels statically include the Bolt engine and test utilities.
python-wheel: BOLT_TEST_LINKAGE = static

python-wheel:
	NUM_THREADS="$(NUM_THREADS)" NUM_LINK_JOB="$(NUM_LINK_JOB)" \
	  "$(PYTHON_EXECUTABLE)" \
	  "$(PYTHON_PACKAGE_DIR)/packaging/build_dev_wheel.py" \
	  --build-type "$(BUILD_TYPE)" --out-dir "$(PYTHON_DIST_DIR)" \
	  --profile "$(PROFILE)" --dependency-build-type "$(DEPENDENCY_BUILD_TYPE)"

benchmarks-basic-build:
	$(MAKE) conan_build BUILD_TYPE=Release BOLT_BUILD_BENCHMARKS_BASIC="ON" CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS) $(CONAN_PERF_OPTIONS)"

benchmarks-build:
	$(MAKE) conan_build BUILD_TYPE=Release BOLT_BUILD_BENCHMARKS="ON" CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS) $(CONAN_PERF_OPTIONS)"

benchmarks-build-spark:
	$(MAKE) conan_build BUILD_TYPE=Release BOLT_BUILD_BENCHMARKS="ON" CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_SPARK_OPTIONS) $(CONAN_TESTUTIL_OPTIONS) $(CONAN_PERF_OPTIONS)"

benchmarks-build-relwithdebinfo:
	$(MAKE) conan_build BUILD_TYPE=RelWithDebInfo BOLT_BUILD_BENCHMARKS="ON" CONAN_CONFIG="$(CONAN_TEST_CONFIG)" CONAN_OPTIONS="$(CONAN_PRESTO_OPTIONS) $(CONAN_TESTUTIL_OPTIONS) $(CONAN_PERF_OPTIONS)"

ctest_debug:
	ctest --test-dir $(BUILD_BASE_DIR)/Debug --timeout 7200 -j $(NUM_THREADS) --output-on-failure

ctest_release:
	ctest --test-dir $(BUILD_BASE_DIR)/Release --timeout 7200 -j $(NUM_THREADS) --output-on-failure

unittest_debug: unittest
unittest: debug_with_test
	$(MAKE) ctest_debug

unittest_release: release_with_test
	$(MAKE) ctest_release

unittest_release_spark: release_spark_with_test
	$(MAKE) ctest_release

unittest_debug_spark: debug_spark_with_test
	$(MAKE) ctest_debug

unittest_coverage: debug_with_test_cov		#: Build with debugging and run unit tests
	cd $(BUILD_BASE_DIR)/Debug && \
	lcov --no-external --capture --initial --directory . --output-file coverage_base.info && \
	ctest --timeout 7200 -j $(NUM_THREADS) --output-on-failure && \
	lcov --capture --directory . --output-file coverage_test.info && \
	lcov --add-tracefile coverage_base.info --add-tracefile coverage_test.info --output-file coverage.info && \
	lcov --remove coverage.info '/usr/*' '*/.conan/data/*' '*/_build/*' '*/tests/*' '*/test/*' --output-file coverage_striped.info && \
	genhtml --ignore-errors source coverage_striped.info --output-directory coverage

system_info:
	@echo "----------------------------------------------------------------"
	@echo "## System Configuration Report"
	@echo "*(Auto-generated via 'make report')*"
	@echo ""
	@echo "### 1. Hardware & OS (Performance Context)"
	@echo "- **OS**: $(OS_DETAILED)"
	@echo "- **Arch**: $(shell uname -m)"
	@echo "- **CPU**: $(CPU_MODEL)"
	@echo "- **Cores**: $(CPU_CORES)"
	@echo "- **RAM**: $(MEM_TOTAL)"
	@echo ""
	@echo "### 2. Compiler Toolchain (Build Context)"
	@echo "- **C Compiler**: $(CC)"
	@echo "  - Version: \`$(shell $(CC) --version | head -n 1)\`"
	@echo "- **CXX Compiler**: $(CXX)"
	@echo "  - Version: \`$(shell $(CXX) --version | head -n 1)\`"
ifneq ($(CMAKE_EXE),)
	@echo "- **CMake**: $(shell cmake --version | head -n 1 | awk '{print $$3}')"
	@echo "- **Ninja**: $(shell ninja --version)"
endif
	@echo ""
	@echo "### 3. Conan Dependency Manager"
ifneq ($(CONAN_EXE),)
	@echo "- **Conan Version**: $(shell conan --version)"
	@echo "- **Conan Profile (default)**:"
	@echo "\`\`\`ini"
	@conan profile show default 2>/dev/null || conan profile show 2>/dev/null || echo "Unable to read profile"
	@echo "\`\`\`"
else
	@echo "*Conan not found in PATH*"
endif
	@echo "----------------------------------------------------------------"

help:					#: Show the help messages
	@cat $(firstword $(MAKEFILE_LIST)) | \
	awk '/^[-a-z]+:/' | \
	awk -F: '{ printf("%-20s   %s\n", $$1, $$NF) }'