diff --git a/ci/build.sh b/ci/build.sh
index 20b203f..f51d271 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -37,16 +37,6 @@
         git config remote.origin.url file:///dev/null
         popd
     fi
-
-    for PROJECT in cla-sysrepo netconf-cli gammarus; do
-        # If there's a change for ${PROJECT} queued ahead, ensure it gets used.
-        # This means that if our submodules still pin, say, `cla-sysrepo` to some ancient version and we're testing a `netconf-cli` change,
-        # then we will keep using that ancient `cla-sysrepo`. Hopefully this reduces the number of false alerts.
-        DEPSRCDIR=$(jq < ~/zuul-env.json -r "[.items[]? | select(.project.name == \"CzechLight/${PROJECT}\")][-1]?.project.src_dir + \"\"")
-        if [[ ! -z "${DEPSRCDIR}" ]]; then
-            sed -i "s|${ZUUL_PROJECT_SRC_DIR}/submodules/${PROJECT}|${HOME}/${DEPSRCDIR}|g" local.mk
-        fi
-    done
 fi
 
 if [[ "${ZUUL_JOB_NAME}" =~ clearfog ]]; then
@@ -59,6 +49,31 @@
 fi
 
 echo BR2_PRIMARY_SITE=\"https://object-store.cloud.muni.cz/swift/v1/ci-artifacts-public/mirror/buildroot\" >> .config
+
+if [[ ${TRIGGERED_VIA_DEP} == 1 ]]; then
+    ARTIFACT_URL=$(jq < ~/zuul-env.json -r '[.artifacts[]? | select(.name == "br2-work-dir") | select(.project == "CzechLight/br2-external")][-1]?.url + ""')
+    if [[ -z "${ARTIFACT_URL}" ]]; then
+        # no job ahead, try to use git commit ID
+        ARTIFACT_URL="https://object-store.cloud.muni.cz/swift/v1/ci-artifacts-${ZUUL_TENANT}/${ZUUL_GERRIT_HOSTNAME}/CzechLight/br2-external/${ZUUL_JOB_NAME}/br2-work-dir-${BR2_EXTERNAL_COMMIT}.tar.zst"
+    fi
+    # We don't use gating, so there's a risk that there's no prebuilt artifact, so don't die if we cannot download that file
+    curl ${ARTIFACT_URL} | unzstd --stdout | tar -xf - || echo "No Buildroot prebuilt tarball found, will build from scratch"
+
+    for PROJECT in cla-sysrepo netconf-cli gammarus; do
+        # If there's a change for ${PROJECT} queued ahead, ensure it gets used.
+        # This means that if our submodules still pin, say, `cla-sysrepo` to some ancient version and we're testing a `netconf-cli` change,
+        # then we will keep using that ancient `cla-sysrepo`. Hopefully this reduces the number of false alerts.
+        DEPSRCDIR=$(jq < ~/zuul-env.json -r "[.items[]? | select(.project.name == \"CzechLight/${PROJECT}\")][-1]?.project.src_dir + \"\"")
+        if [[ ! -z "${DEPSRCDIR}" ]]; then
+            sed -i "s|${ZUUL_PROJECT_SRC_DIR}/submodules/${PROJECT}|${HOME}/${DEPSRCDIR}|g" local.mk
+
+            # `make ${pkg}-reconfigure` is *not* enough with BR2_PER_PACKAGE_DIRECTORIES=y
+            # Even this is possibly fragile if these packages were not the "leaf" ones (in terms of BR-level dependencies).
+            rm -rf build/${PROJECT}-custom/ per-package/${PROJECT}/
+        fi
+    done
+fi
+
 make source -j${CI_PARALLEL_JOBS} --output-sync=target
 
 make -j${CI_PARALLEL_JOBS} --output-sync=target rootfs-czechlight-rauc