Use dependencies from a shared repo
Boost still gets picked from there, but it is installed from a release
tarball instead of a random git subtree -- mainly because this method is
much more efficient than dealing with a gazillion of git repositories.
Depends-on: https://gerrit.cesnet.cz/c/CzechLight/dependencies/+/1521
Change-Id: Ia85eac63bdc0e118824075bad847e0ab347d7f81
diff --git a/.gitmodules b/.gitmodules
index 9b5f9e4..f9bf878 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,27 +1,3 @@
-[submodule "submodules/Catch"]
- path = submodules/Catch
- url = ../../github/philsquared/Catch
-[submodule "submodules/trompeloeil"]
- path = submodules/trompeloeil
- url = ../../github/rollbear/trompeloeil
-[submodule "submodules/docopt.cpp"]
- path = submodules/docopt.cpp
- url = ../../github/docopt/docopt.cpp
-[submodule "submodules/spdlog"]
- path = submodules/spdlog
- url = ../../github/gabime/spdlog
-[submodule "submodules/boost"]
- path = submodules/boost
- url = ../../github/boostorg/boost
-[submodule "submodules/libyang"]
- path = submodules/libyang
- url = ../../github/CESNET/libyang
-[submodule "submodules/sysrepo"]
- path = submodules/sysrepo
- url = ../../github/sysrepo/sysrepo
-[submodule "submodules/libredblack"]
- path = submodules/libredblack
- url = ../../github/sysrepo/libredblack
-[submodule "submodules/replxx"]
- path = submodules/replxx
- url = ../../github/AmokHuginnsson/replxx
+[submodule "submodules/dependencies"]
+ path = submodules/dependencies
+ url = ../dependencies
diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000..b6e739e
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,11 @@
+- project:
+ check:
+ jobs:
+ - clang-format:
+ voting: false
+ - f29-gcc:
+ requires: CzechLight-deps-f29-gcc
+ - f29-gcc-asan-ubsan:
+ requires: CzechLight-deps-f29-gcc-asan-ubsan
+ - f29-clang-asan:
+ requires: CzechLight-deps-f29-clang-asan
diff --git a/ci/build.sh b/ci/build.sh
index ae7beda..acc4eea 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -1,3 +1,81 @@
#!/bin/bash
-BUILD_TIMEOUT=30m
-timeout -k $BUILD_TIMEOUT $BUILD_TIMEOUT $TH_GIT_PATH/ci/do-build.sh
+
+set -eux -o pipefail
+shopt -s failglob
+
+ZUUL_JOB_NAME=$(jq < ~/zuul-env.json -r '.job')
+ZUUL_PROJECT_SRC_DIR=$HOME/$(jq < ~/zuul-env.json -r '.project.src_dir')
+ZUUL_PROJECT_SHORT_NAME=$(jq < ~/zuul-env.json -r '.project.short_name')
+
+CI_PARALLEL_JOBS=$(grep -c '^processor' /proc/cpuinfo)
+CMAKE_OPTIONS=""
+CFLAGS=""
+CXXFLAGS=""
+LDFLAGS=""
+
+if [[ $ZUUL_JOB_NAME =~ .*-clang.* ]]; then
+ export CC=clang
+ export CXX=clang++
+ export LD=clang
+ export CXXFLAGS="-stdlib=libc++"
+fi
+
+if [[ $ZUUL_JOB_NAME =~ .*-ubsan ]]; then
+ export CFLAGS="-fsanitize=undefined ${CFLAGS}"
+ export CXXFLAGS="-fsanitize=undefined ${CXXFLAGS}"
+ export LDFLAGS="-fsanitize=undefined ${LDFLAGS}"
+fi
+
+if [[ $ZUUL_JOB_NAME =~ .*-asan ]]; then
+ export CFLAGS="-fsanitize=address ${CFLAGS}"
+ export CXXFLAGS="-fsanitize=address ${CXXFLAGS}"
+ export LDFLAGS="-fsanitize=address ${LDFLAGS}"
+fi
+
+PREFIX=~/target
+mkdir ${PREFIX}
+BUILD_DIR=~/build
+mkdir ${BUILD_DIR}
+export PATH=${PREFIX}/bin:$PATH
+export LD_LIBRARY_PATH=${PREFIX}/lib64:${PREFIX}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+export PKG_CONFIG_PATH=${PREFIX}/lib64/pkgconfig:${PREFIX}/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}
+
+ARTIFACT_URL=""
+
+if [[ "$(jq < ~/zuul-env.json -r '.artifacts[0].project')" = "CzechLight/dependencies" &&
+ "$(jq < ~/zuul-env.json -r '.artifacts[0].name')" = "tarball" ]]; then
+ # FIXME: Zuul feeds us with wrong artifact (from a different provider) for some reason :(
+ # so let's look into builds from the change pipeline and fetch their artifact directly
+ #ARTIFACT_URL=$(jq < ~/zuul-env.json -r '.artifacts[0].url')
+
+ CHANGE_BUILD_QUERY=$(jq -r '.items | map(select(.project.name == "CzechLight/dependencies"))[-1] | ("change=" + .change + "&patchset=" + .patchset)' < ~/zuul-env.json)
+ if [[ "${CHANGE_BUILD_QUERY}" != "null" ]]; then
+ # We depend on some change from project CzechLight/dependencies, let's look at the latest one
+ ZUUL_TENANT=$(jq < ~/zuul-env.json -r '.tenant')
+ ARTIFACT_URL=$(curl "https://zuul.gerrit.cesnet.cz/api/tenant/${ZUUL_TENANT}/builds?pipeline=check&job_name=${ZUUL_JOB_NAME}&${CHANGE_BUILD_QUERY}" | jq -r '.[].artifacts[0].url')
+ fi
+fi
+
+DEP_SUBMODULE_COMMIT=$(git ls-tree -l master submodules/dependencies | cut -d ' ' -f 3)
+
+if [[ -z "${ARTIFACT_URL}" ]]; then
+ # fallback to a promoted artifact
+ ARTIFACT_URL=https://ci-logs.gerrit.cesnet.cz/t/public/artifacts/${ZUUL_JOB_NAME}/czechlight-dependencies-${DEP_SUBMODULE_COMMIT}.tar.xz
+fi
+
+ARTIFACT_FILE=$(basename ${ARTIFACT_URL})
+DEP_HASH_FROM_ARTIFACT=$(echo "${ARTIFACT_FILE}" | sed -e 's/^czechlight-dependencies-//' -e 's/\.tar\.xz$//')
+if [[ "${DEP_HASH_FROM_ARTIFACT}" != "${DEP_SUBMODULE_COMMIT}" ]]; then
+ echo "Mismatched artifact: HEAD of ./submodules/dependencies does not match artifact commit ref"
+ exit 1
+fi
+curl ${ARTIFACT_URL} --output ${ARTIFACT_FILE}
+tar -C ${PREFIX} -xf ${ARTIFACT_FILE}
+rm ${ARTIFACT_FILE}
+
+cd ${BUILD_DIR}
+cmake -GNinja -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -DCMAKE_INSTALL_PREFIX=${PREFIX} ${CMAKE_OPTIONS} ${ZUUL_PROJECT_SRC_DIR}
+ninja-build
+ctest -j${CI_PARALLEL_JOBS} --output-on-failure
+ninja-build doc
+mv html ~/zuul-output/docs/
diff --git a/ci/do-build.sh b/ci/do-build.sh
deleted file mode 100755
index 5b870dc..0000000
--- a/ci/do-build.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-
-set -eux -o pipefail
-shopt -s failglob
-
-# We're reusing our artifacts, so we absolutely need a stable destdir.
-# Turbo-hipster takes care of cleaning up the mess betweeb builds.
-PREFIX=~/target
-mkdir ${PREFIX}
-export PATH=${PREFIX}/bin:$PATH
-export LD_LIBRARY_PATH=${PREFIX}/lib64:${PREFIX}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
-export PKG_CONFIG_PATH=${PREFIX}/lib64/pkgconfig:${PREFIX}/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}
-
-if [[ $TH_JOB_NAME =~ .*-sanitizers-.* ]]; then
- CMAKE_OPTIONS="${CMAKE_OPTIONS} -DUSE_SR_MEM_MGMT:BOOL=OFF"
- # https://gitlab.kitware.com/cmake/cmake/issues/16609
- CMAKE_OPTIONS="${CMAKE_OPTIONS} -DTHREADS_HAVE_PTHREAD_ARG:BOOL=ON"
-fi
-
-# force-enable tests for packages which use, eh, interesting setup
-# - libyang and libnetconf2 copmare CMAKE_BUILD_TYPE to lowercase "debug"...
-CMAKE_OPTIONS="${CMAKE_OPTIONS} -DENABLE_BUILD_TESTS=ON -DENABLE_VALGRIND_TESTS=OFF"
-
-# nuke python2 builds because we cannot write to the site_path
-CMAKE_OPTIONS="${CMAKE_OPTIONS} -DGEN_PYTHON_BINDINGS=OFF"
-
-build_dep_cmake() {
- pushd ${TH_JOB_WORKING_DIR}
- mkdir build-$1
- pushd build-$1
- ${CMAKE} -GNinja ${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -DCMAKE_INSTALL_PREFIX=${PREFIX} ${TH_GIT_PATH}/submodules/$1
- ninja-build install
- popd
- popd
-}
-
-build_dep_autoconf() {
- pushd ${TH_GIT_PATH}/submodules/$1
- shift
- ./configure --prefix=${PREFIX} "$@"
- make -j4
- make install
- popd
-}
-
-do_test_dep_cmake() {
- pushd ${TH_JOB_WORKING_DIR}/build-$1
- shift
- ${CTEST} --output-on-failure "$@"
- popd
-}
-
-emerge_dep() {
- if [[ -f ${TH_GIT_PATH}/submodules/$1/CMakeLists.txt ]]; then
- build_dep_cmake $1
- elif [[ -f ${TH_GIT_PATH}/submodules/$1/configure ]]; then
- build_dep_autoconf "$@"
- else
- echo "Unrecognized buildsystem for $1"
- exit 1
- fi
-}
-
-ARTIFACT=netconf-cli-$(git --git-dir ${TH_GIT_PATH}/.git rev-parse HEAD:submodules/).tar.xz
-
-scp th-ci-logs@ci-logs.gerrit.cesnet.cz:artifacts/${TH_JOB_NAME}/${ARTIFACT} . \
- || true # ignore network errors
-
-if [[ -f ${TH_JOB_WORKING_DIR}/${ARTIFACT} ]]; then
- tar -C ~/target -xvJf ${TH_JOB_WORKING_DIR}/${ARTIFACT}
-else
- # rebuild everything from scratch
-
- CMAKE_OPTIONS="${CMAKE_OPTIONS} -DGEN_LANGUAGE_BINDINGS=ON -DGEN_PYTHON_BINDINGS=OFF -DGEN_JAVA_BINDINGS=OFF" emerge_dep libyang
- do_test_dep_cmake libyang -j${CI_PARALLEL_JOBS}
-
- emerge_dep libredblack --with-pic
-
- # sysrepo needs to use a persistent repo location
- CMAKE_OPTIONS="${CMAKE_OPTIONS} -DREPOSITORY_LOC=${PREFIX}/etc-sysrepo" emerge_dep sysrepo
- # These tests are only those which can run on the global repo.
- # They also happen to fail when run in parallel. That's expected, they manipulate a shared repository.
- do_test_dep_cmake sysrepo
-
- emerge_dep Catch
- do_test_dep_cmake Catch -j${CI_PARALLEL_JOBS}
-
- # Trompeloeil is a magic snowflake because it attempts to download and build Catch and kcov when building in a debug mode...
- CMAKE_BUILD_TYPE=Release emerge_dep trompeloeil
-
- emerge_dep docopt.cpp
- do_test_dep_cmake docopt.cpp -j${CI_PARALLEL_JOBS}
-
- emerge_dep spdlog
- do_test_dep_cmake spdlog -j${CI_PARALLEL_JOBS}
-
- # boost-spirit doesn't require installation
- pushd ${TH_GIT_PATH}/submodules/boost
- git submodule update --init
- ./bootstrap.sh --prefix=${PREFIX}
- ./b2 --ignore-site-config headers
- cp -LR boost ${PREFIX}/include/
- popd
-
- # examples are broked on clang+ubsan because of their STL override
- CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_SHARED_LIBS=ON -DREPLXX_BuildExamples=OFF" emerge_dep replxx
- do_test_dep_cmake replxx -j${CI_PARALLEL_JOBS}
-
- tar -C ~/target -cvJf ${TH_JOB_WORKING_DIR}/${ARTIFACT} .
- ssh th-ci-logs@ci-logs.gerrit.cesnet.cz mkdir -p artifacts/${TH_JOB_NAME} \
- || true # ignore network errors
- rsync ${TH_JOB_WORKING_DIR}/${ARTIFACT} th-ci-logs@ci-logs.gerrit.cesnet.cz:artifacts/${TH_JOB_NAME}/ \
- || true # ignore network errors
-fi
-
-mkdir -p ${ZUUL_PROJECT}/build
-cd ${ZUUL_PROJECT}/build
-${CMAKE} -GNinja \
- -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} \
- -DCMAKE_INSTALL_PREFIX=${PREFIX} \
- ${CMAKE_OPTIONS} \
- ${TH_GIT_PATH}
-ninja-build
-
-${CTEST} --output-on-failure
diff --git a/submodules/Catch b/submodules/Catch
deleted file mode 160000
index b68484e..0000000
--- a/submodules/Catch
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b68484eff2846825961c5f4cf22c02d6b77213f9
diff --git a/submodules/boost b/submodules/boost
deleted file mode 160000
index 5980152..0000000
--- a/submodules/boost
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 59801528be781d6093bae306e01f452c00276497
diff --git a/submodules/dependencies b/submodules/dependencies
new file mode 160000
index 0000000..8b8cf77
--- /dev/null
+++ b/submodules/dependencies
@@ -0,0 +1 @@
+Subproject commit 8b8cf77404a9bf0dd01651517fc0d0513aa3ae6d
diff --git a/submodules/docopt.cpp b/submodules/docopt.cpp
deleted file mode 160000
index a46ae7f..0000000
--- a/submodules/docopt.cpp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a46ae7fbb91a72a6d81842a4ea114caf1c65ad2e
diff --git a/submodules/libredblack b/submodules/libredblack
deleted file mode 160000
index a399310..0000000
--- a/submodules/libredblack
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a399310d99b61eec4d3c0677573ab5dddcf9395d
diff --git a/submodules/libyang b/submodules/libyang
deleted file mode 160000
index 1a5a4ee..0000000
--- a/submodules/libyang
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 1a5a4ee461b21d7429d607a04176a5d06625cdb7
diff --git a/submodules/replxx b/submodules/replxx
deleted file mode 160000
index 12f2ade..0000000
--- a/submodules/replxx
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 12f2adee7f9123880db1f870c360a88c1f7ba182
diff --git a/submodules/spdlog b/submodules/spdlog
deleted file mode 160000
index c060a10..0000000
--- a/submodules/spdlog
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c060a10c102a2e25f1d2fcfb46e1148d73b0f7d5
diff --git a/submodules/sysrepo b/submodules/sysrepo
deleted file mode 160000
index fbbd0e6..0000000
--- a/submodules/sysrepo
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit fbbd0e6bdf58f88f92d5cfae148714f29d68d11b
diff --git a/submodules/trompeloeil b/submodules/trompeloeil
deleted file mode 160000
index 3f3effd..0000000
--- a/submodules/trompeloeil
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3f3effde6fcc2ac44c354281738db18d4ce9a9a5