150 lines
5 KiB
Text
150 lines
5 KiB
Text
# SPDX-License-Identifier: GPL-2.0
|
|
# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv)
|
|
|
|
THREADCOUNT=${THREADCOUNT:-100%}
|
|
|
|
# This function is passed a list of package.mk paths to be processed.
|
|
# Each package.mk is sourced with relevant variables output in JSON format.
|
|
json_worker() {
|
|
local packages="$@"
|
|
local pkgpath hierarchy exited
|
|
|
|
exit() { exited=1; }
|
|
|
|
. config/options ""
|
|
|
|
for pkgpath in ${packages}; do
|
|
pkgpath="${pkgpath%%@*}"
|
|
|
|
exited=0
|
|
if ! source_package "${pkgpath}/package.mk" &>/dev/null; then
|
|
unset -f exit
|
|
die "$(print_color CLR_ERROR "FAILURE: sourcing package ${pkgpath}/package.mk")"
|
|
fi
|
|
|
|
[ ${exited} -eq 1 ] && continue
|
|
|
|
[[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local"
|
|
|
|
cat <<EOF
|
|
{
|
|
"name": "${PKG_NAME}",
|
|
"hierarchy": "${hierarchy}",
|
|
"section": "${PKG_SECTION}",
|
|
"bootstrap": "${PKG_DEPENDS_BOOTSTRAP}",
|
|
"init": "${PKG_DEPENDS_INIT}",
|
|
"host": "${PKG_DEPENDS_HOST}",
|
|
"target": "${PKG_DEPENDS_TARGET}"
|
|
},
|
|
EOF
|
|
done
|
|
}
|
|
export -f json_worker
|
|
|
|
# This function is passed the build instruction for a single job.
|
|
# The function will run either "build <package>" or "install <package>".
|
|
# ${slot} is the job slot number, ie. 1-8 when THREADCOUNT=8.
|
|
# ${job} is the sequence within the total number of ${jobs}.
|
|
package_worker() {
|
|
local slot=$1 job=$2 jobs=$3 args="$4"
|
|
local task pkgname result status
|
|
local addon istarget isaddon
|
|
|
|
export MTJOBID=${slot} MTMAXJOBS=${jobs}
|
|
|
|
read -r task pkgname <<< "${args}"
|
|
|
|
. config/options "${pkgname}"
|
|
|
|
[ ! -f "${THREAD_CONTROL}/parallel.pid" ] && echo "${PARALLEL_PID}" >"${THREAD_CONTROL}/parallel.pid"
|
|
|
|
${SCRIPTS}/${task} ${pkgname} 2>&1 && result=0 || result=1
|
|
|
|
[[ ${pkgname} =~ :target$ || "${pkgname//:/}" = "${pkgname}" ]] && istarget="yes" || istarget="no"
|
|
|
|
[[ "${MTADDONBUILD}" = "yes" && ( "${PKG_IS_ADDON}" = "yes" || "${PKG_IS_ADDON}" = "embedded" ) ]] && isaddon="yes" || isaddon="no"
|
|
|
|
if [ "${isaddon}" = "yes" -a "${istarget}" = "yes" ]; then
|
|
if [ ${result} -eq 0 ]; then
|
|
${SCRIPTS}/install_addon ${pkgname} 2>&1 && result=0 || result=1
|
|
fi
|
|
|
|
if [ ${result} -ne 0 ]; then
|
|
echo "${PKG_NAME}" >>"${THREAD_CONTROL}/addons.failed"
|
|
fi
|
|
fi
|
|
|
|
(
|
|
flock --exclusive 95
|
|
[ ${result} -eq 0 ] && status="DONE" || status="FAIL"
|
|
num=$(< "${THREAD_CONTROL}/progress")
|
|
mv "${THREAD_CONTROL}/progress" "${THREAD_CONTROL}/progress.prev"
|
|
num=$((num + 1))
|
|
echo ${num} >"${THREAD_CONTROL}/progress"
|
|
printf "[%0*d/%0*d] [%-4s] %-7s %s\n" ${#jobs} ${num} ${#jobs} ${jobs} "${status}" "${task}" "${pkgname}" >&2
|
|
) 95>"${THREAD_CONTROL}/locks/.progress"
|
|
|
|
if [ ${result} -eq 0 ]; then
|
|
pkg_lock_status "IDLE"
|
|
else
|
|
pkg_lock_status "FAILED" "${pkgname}" "${task}"
|
|
|
|
print_color CLR_ERROR "FAILURE: ${SCRIPTS}/${task} ${pkgname} has failed!\n"
|
|
fi
|
|
|
|
return ${result}
|
|
}
|
|
export -f package_worker
|
|
|
|
start_multithread_build() {
|
|
local singlethread buildopts result=0
|
|
|
|
# init thread control folder
|
|
rm -rf "${THREAD_CONTROL}"
|
|
mkdir -p "${THREAD_CONTROL}/locks"
|
|
echo -1 >"${THREAD_CONTROL}/progress.prev"
|
|
echo 0 >"${THREAD_CONTROL}/progress"
|
|
echo 0 >"${THREAD_CONTROL}/status.max"
|
|
touch "${THREAD_CONTROL}/status"
|
|
|
|
# Increase file descriptors if building one thread/package
|
|
[ "${THREADCOUNT}" = "0" ] && ulimit -n ${ULIMITN:-10240}
|
|
|
|
# Bootstrap GNU parallel
|
|
MTWITHLOCKS=no ${SCRIPTS}/build parallel:host 2>&1 || die "Unable to bootstrap parallel package"
|
|
|
|
# determine number of available slots for the given THREADCOUNT - optimise logging for single threaded builds
|
|
[ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && singlethread=yes || singlethread=no
|
|
|
|
# create a single log file by default for a single threaded build (or the builder is a masochist)
|
|
if [ "${singlethread}" = "yes" -a "${ONELOG,,}" != "no" ] || [ "${ONELOG,,}" = "yes" ]; then
|
|
buildopts+=" --ungroup"
|
|
else
|
|
buildopts+=" --group"
|
|
fi
|
|
|
|
# When building addons, don't halt on error - keep building all packages/addons
|
|
[ "${MTADDONBUILD}" = "yes" ] && buildopts+=" --halt never" || buildopts+=" --halt now,fail=1"
|
|
|
|
# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status
|
|
set -o pipefail
|
|
|
|
cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
|
|
${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \
|
|
${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || result=1
|
|
|
|
if [ ${result} -eq 0 ]; then
|
|
save_build_config
|
|
|
|
cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \
|
|
MTBUILDSTART=$(date +%s) MTWITHLOCKS=yes ${TOOLCHAIN}/bin/parallel \
|
|
--plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" --plus ${buildopts} \
|
|
package_worker {%} {#} {##} {} || result=1
|
|
|
|
rm -f "${THREAD_CONTROL}/parallel.pid"
|
|
fi
|
|
|
|
set +o pipefail
|
|
|
|
return ${result}
|
|
}
|