Skip to content

Commit

Permalink
Update build scripts
Browse files Browse the repository at this point in the history 
* Support for Ubuntu 22.04
* Remove support for Python2
* Better separation between sanitizer builds and non-sanitizer builds
* Fix build of metaSMT on newer Ubuntu versions
* Use ninja to build LLVM
* Simplifying building arbitrary LLVM configurations, e.g. different
  LLVM sanitizer builds (MemSan, UBSan, ASan)
* Use MemSan with origin tracking
* Build sqlite3 container correctly
* Add support to provide sqlite3 version number
  • Loading branch information
@MartinNowack @ccadar
MartinNowack authored and ccadar committed Mar 17, 2023
1 parent dd492f8 commit ca60811
Show file tree
Hide file tree
Showing 31 changed files with 641 additions and 311 deletions.
18 changes: 11 additions & 7 deletions scripts/build/d-klee-linux-ubuntu.inc
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# TODO remove adding sudo package
# Create ``klee`` user for container with password ``klee``.
# and give it password-less sudo access (temporarily so we can use the CI scripts)
RUN apt update && DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install sudo emacs-nox vim-nox file python3-dateutil && \
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install sudo emacs-nox vim-nox file python3-dateutil && \
rm -rf /var/lib/apt/lists/* && \
useradd -m klee && \
echo klee:klee | chpasswd && \
Expand All @@ -11,19 +11,23 @@ RUN apt update && DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends
# Copy across source files needed for build
COPY --chown=klee:klee . /tmp/klee_src/

USER klee
WORKDIR /home/klee
# Build and set klee user to be owner
RUN /tmp/klee_src/scripts/build/build.sh --debug --install-system-deps klee && chown -R klee:klee /tmp/klee_build* && pip3 install flask wllvm && \
rm -rf /var/lib/apt/lists/*
RUN /tmp/klee_src/scripts/build/build.sh --debug --install-system-deps klee && pip3 install flask wllvm && \
sudo rm -rf /var/lib/apt/lists/*


ENV PATH="$PATH:/tmp/llvm-60-install_O_D_A/bin:/home/klee/klee_build/bin"
ENV PATH="$PATH:/tmp/llvm-110-install_O_D_A/bin:/home/klee/klee_build/bin:/home/klee/.local/bin"
ENV BASE=/tmp

USER klee
WORKDIR /home/klee
# Add KLEE header files to system standard include folder
RUN sudo /bin/bash -c 'ln -s /tmp/klee_src/include/klee /usr/include/'

ENV LD_LIBRARY_PATH /home/klee/klee_build/lib/

# Add KLEE binary directory to PATH
RUN /bin/bash -c 'ln -s ${BASE}/klee_src /home/klee/ && ln -s ${BASE}/klee_build* /home/klee/klee_build'

# TODO Remove when STP is fixed
RUN /bin/bash -c 'echo "export LD_LIBRARY_PATH=$(cd ${BASE}/metaSMT-*-deps/stp-git-basic/lib/ && pwd):$LD_LIBRARY_PATH" >> /home/klee/.bashrc'
RUN /bin/bash -c 'echo "export LD_LIBRARY_PATH=$(cd ${BASE}/metaSMT-*-deps/stp-git-basic/lib/ && pwd):$LD_LIBRARY_PATH" >> /home/klee/.bashrc'
8 changes: 8 additions & 0 deletions scripts/build/p-clang-linux-ubuntu-22.04.inc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
get_docker_config_id_clang() {
if [[ "${LLVM_VERSION_SHORT}" -ge "11" ]]; then
echo ""
return 0;
else
return 1;
fi
}
69 changes: 44 additions & 25 deletions scripts/build/p-clang-linux-ubuntu.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,40 +3,59 @@ install_binary_artifact_clang() {
local version="-${LLVM_VERSION_MAJOR}"

source "${DIR}/common-functions"
with_sudo apt-get update -y

# Check if package in standard repository otherwise use upstream
if ! apt-cache show "llvm${version}"; then
if [[ -z "$(which wget)" ]]; then
# Add certificate
with_sudo apt-get update -y
dependencies=(
ca-certificates
wget
lsb-release
gnupg
)
with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}"
fi

# Add certificate
with_sudo apt update -y
dependencies=(
ca-certificates
wget
lsb-release
gnupg
)
with_sudo apt -y --no-install-recommends install "${dependencies[@]}"

# Add LLVM upstream repository if available
codename="$(lsb_release --codename --short)"
if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add -

apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main"
if ! grep -rq "${apt_entry}" /etc/apt; then
echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list
with_sudo apt update -y
# Add LLVM upstream repository if available
codename="$(lsb_release --codename --short)"
if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then
apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main"
if ! grep -rq "${apt_entry}" /etc/apt; then
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add -
echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list
with_sudo apt-get update -y
fi
fi
fi


with_sudo apt update -y
dependencies=(
"llvm${version}"
"clang${version}"
)

#Install essential dependencies
with_sudo apt -y --no-install-recommends install "${dependencies[@]}" || return 1
with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" || return 1
}

get_docker_config_id_clang() {
return 0
}
setup_artifact_variables_clang() {
local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}"
local version="-${LLVM_VERSION_MAJOR}"

# Only set LLVM_CONFIG if not set yet
if [[ -z "${LLVM_CONFIG:-}" ]]; then
LLVM_CONFIG=$(which "llvm-config${version}")
fi

local bin_path=""
bin_path=$(which "clang${version}")
[[ -z "${bin_path}" ]] && return 1

bin_path="$(dirname "$(readlink -f "${bin_path}")")"
[[ -z "${bin_path}" ]] && return 1

BITCODE_CC="${bin_path}/clang"
BITCODE_CXX="${bin_path}/clang++"
}
7 changes: 5 additions & 2 deletions scripts/build/p-clang-linux.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@ setup_artifact_variables_clang() {
LLVM_CONFIG=$(which "llvm-config${version}")
fi


local bin_path=""
bin_path="$(dirname "$(readlink -f "$(which "clang${version}")")")"
bin_path=$(which "clang${version}")
[[ -z "${bin_path}" ]] && return 1

bin_path="$(dirname "$(readlink -f "${bin_path}")")"
[[ -z "${bin_path}" ]] && return 1

BITCODE_CC="${bin_path}/clang"
BITCODE_CXX="${bin_path}/clang++"
Expand Down
62 changes: 62 additions & 0 deletions scripts/build/p-clang.inc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
local_setup_build_variables_clang() {
source "${DIR}/common-functions"
LLVM_SUFFIX=""

LLVM_SUFFIX+="_O"
LLVM_SUFFIX+="_ND"
LLVM_SUFFIX+="_NA"
LLVM_SUFFIX+="_RTTI"

LLVM_SRC_BASE="${BASE}/llvm-${LLVM_VERSION_SHORT}"
LLVM_BUILD="${LLVM_SRC_BASE}-build${LLVM_SUFFIX}"
LLVM_INSTALL="${LLVM_SRC_BASE}-install${LLVM_SUFFIX}"
LLVM_BIN="${LLVM_INSTALL}/bin"
LLVM_BUILD_BIN="${LLVM_BUILD}/bin"

LLVM_CONFIG="${LLVM_BIN}/llvm-config"
BITCODE_CC="${LLVM_BIN}/clang"
BITCODE_CXX="${LLVM_BIN}/clang++"
}

get_docker_config_id_clang() {
(
local_setup_build_variables_clang
echo "${LLVM_VERSION_SHORT}${LLVM_SUFFIX}"
)
}

get_build_artifacts_clang() {
(
local_setup_build_variables_clang
echo "${LLVM_INSTALL}"
)
}

# Check if the binary artifact is installed
is_installed_clang() {
local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}"
local version="${LLVM_VERSION_MAJOR}"

(
local_setup_build_variables_clang
# Check if clang is installed at a potential location
[[ -f "${LLVM_BIN}/clang" ]]
)
}

setup_artifact_variables_clang() {
is_installed_clang || return 1

local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}"
local version="${LLVM_VERSION_MAJOR}"
local_setup_build_variables_clang
echo "${LLVM_INSTALL}"
}

install_build_dependencies_clang() {
return 0
}

setup_build_variables_clang() {
return 0
}
8 changes: 7 additions & 1 deletion scripts/build/p-klee-linux-ubuntu.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ install_build_dependencies_klee() {
python3-wheel
)

if [[ "${SOLVERS:-}" == "metaSMT" ]]; then
dependencies+=(
libboost-dev
libgmp-dev
)
fi
if [[ $(to_bool "${COVERAGE}") -eq 1 ]]; then
dependencies+=(lcov curl git)
fi
Expand All @@ -20,7 +26,7 @@ install_build_dependencies_klee() {
dependencies+=(doxygen graphviz)
fi

with_sudo apt -y --no-install-recommends install "${dependencies[@]}"
with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}"

pip3 install --user lit
pip3 install --user tabulate
Expand Down
10 changes: 7 additions & 3 deletions scripts/build/p-klee.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ build_klee() {
CMAKE_PREFIX_PATH=("")

local CMAKE_ARGUMENTS=(
"-DLLVM_CONFIG_BINARY=${LLVM_CONFIG}"
"-DLLVMCC=${BITCODE_CC}"
"-DLLVMCXX=${BITCODE_CXX}"
"-DLLVM_DIR=${LLVM_INSTALL}"
"-DGTEST_SRC_DIR=${GTEST_INSTALL_PATH}"
"-DENABLE_UNIT_TESTS=TRUE"
"-DENABLE_SYSTEM_TESTS=TRUE"
Expand Down Expand Up @@ -160,12 +158,18 @@ fi
CXX_FLAGS+=("${SANITIZER_CXX_FLAGS[@]}")
LD_FLAGS+=("${SANITIZER_LD_FLAGS[@]}")

SANITIZER_DIR="$(cd "$(dirname "${SANITIZER_BITCODE_CC}")" && pwd)"

CMAKE_ARGUMENTS+=(
"-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}"
)
fi

CMAKE_ARGUMENTS+=(
"-DLLVMCC=${BITCODE_CC}"
"-DLLVMCXX=${BITCODE_CXX}"
)


mkdir -p "${KLEE_BUILD_DIR}" || return 1
Expand Down
1 change: 1 addition & 0 deletions scripts/build/p-libcxx.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ get_build_artifacts_libcxx() {
(
setup_build_variables_libcxx
echo "${LIBCXX_INSTALL}"
echo "${LIBCXX_SRC}"
)
}

Expand Down
Loading

0 comments on commit ca60811

Please sign in to comment.