diff --git a/.gitignore b/.gitignore index 4b7714e..f35b614 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ src/ web/ pkg/ +testing.log +*.pkg.tar.xz diff --git a/Makefile b/Makefile index bbb98de..0805b2f 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ systemdList = ./maat-builder.timer ./maat-builder.service serverRoot = /srv/maat/ owner=root perms=0755 +fileperms=0644 compile: /bin/bash @echo Nothing to compile. @@ -12,6 +13,7 @@ install: ${list} mkdir -p ${installDir} mkdir -p ${pkgdir}${serverRoot}/src mkdir -p ${pkgdir}${serverRoot}/pkg + mkdir -p ${pkgdir}/usr/lib/systemd/system for i in ${list}; do install -m ${perms} -o ${owner} $$i ${installDir}; done for i in ${systemdList}; do install -m 0644 -o root $$i ${pkgdir}/usr/lib/systemd/system; done @@ -24,9 +26,24 @@ uninstall: rmdir -p ${pkgdir}${serverRoot}/src rmdir -p ${pkgdir}${serverRoot}/pkg -test: - ./maat-builder -T -U ${USER} - make clean +test: clean + ./maat-builder -T + test -d src + test -d src/cower + test -d src/cower/.git + test -d src/HelloWorld + test -d src/HelloWorld/.git + test -d pkg + test -d pkg/aur + test -f pkg/AniNIX.db + test -f pkg/AniNIX.db.sig + test -f pkg/helloworld*.tar.xz + test -f pkg/aur/aur.db + test -f pkg/aur/aur.db.sig + test -f pkg/aur/cower*.tar.xz + [ `wc -l pkg/index.html | cut -f 1 -d ' '` -eq 24 ] + @echo + @echo Tests passed successfully. diff: for i in ${list}; do if [ -f ${installDir}/$$i ]; then diff "$$i" "${installDir}/$$i"; fi done diff --git a/PKGBUILD b/PKGBUILD index 3c95bcd..5f7f2b4 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,20 +1,22 @@ -# Maintainer: Shikoba Kage -pkgname=maat -pkgver=0.1."$(git rev-parse HEAD)" -pkgrel=1 -epoch= -pkgdesc="AniNIX::Maat \\\\ Quality Assurance and Reporting Framework" -arch=("x86_64") -url="https://aninix.net/wiki/Maat" -license=('custom') -groups=() -depends=('bash>=4.4' 'wget' 'grep' 'make' 'git') +depends=('bash>=4.4' 'git' 'pacman') makedepends=('make>=4.2') checkdepends=() optdepends=() -provides=('maat') +pkgname="$(git config remote.origin.url | rev | cut -f 1 -d '/' | rev | sed 's/.git$//')" +pkgver="$(git describe --tag --abbrev=0)"."$(git rev-parse --short HEAD)" +pkgrel=1 +pkgrel() { + echo $(( `git log "$(git describe --tag --abbrev=0)"..HEAD | grep -c commit` + 1 )) +} +epoch="$(git log | grep -c commit)" +pkgdesc="$(head -n 1 README)" +arch=("x86_64") +url="https://aninix.net/foundation/${pkgname}" +license=('custom') +groups=() +provides=("${pkgname}") conflicts=() -replaces=() +replaces=("${pkgname,,}", "aninix-${pkgname,,}") backup=() options=() install= @@ -26,16 +28,13 @@ validpgpkeys=() prepare() { git pull + make -C .. test } build() { make -C .. } -check() { - make -C "${srcdir}/.." test -} - package() { export pkgdir="${pkgdir}" make -C .. install diff --git a/maat-builder b/maat-builder index f493b6b..2f07e6f 100755 --- a/maat-builder +++ b/maat-builder @@ -15,33 +15,38 @@ passCell="PASS"; failCell="FAIL"; warnCell="N/A"; +tableHead="\n"; ### Add helptext. function Usage() { echo "Usage: $0" - echo " $0 [ -b basedir ] [ -c AUR.git.list ] [ -u https://base.url/ ]" - echo " $0 [ -T ]" + echo " $0 [ -b basedir ] [ -c AUR.git.list ] [ -T ] [ -u https://base.url/ ]" echo " $0 -h" echo echo 'Add -v to increase verbosity or -h for help. Add the -l LOGFILE flags to log to a file' - echo "By default, $0 uses the depriv user to build packages. Override with -U" + echo 'Add the -r REPOCMD, -C make-package-command, -p PKGBUILDName, -t test-command, and' + echo 'the -e package-extension flags for compatibility with non-ArchLinux repos.' } ### Put the initial content in the webfile function SeedWebFile() { - printf '\n\nAniNIX::Maat \\\\ Build Results\n\n\n\n\n\n\n\n

AniNIX::Maat \\\\ Build Status

\nWEBSTATSGOHERE\n' > "$webfile" + printf '\n\nAniNIX::Maat \\\\ Build Results\n\n\n\n\n\n\n\n

AniNIX::Maat \\\\ Build Status

\nWEBSTATSGOHERE\n

AnINIX Packages

\n

These are packages written by the AniNIX. Their source is in AniNIX::Foundation.

\n' > "$webfile" + printf "$tableHead" >> "$webfile" } ### Update the webfile to close up table tags and add stats. function UpdateWebFile() { - sed -i "s#WEBSTATSGOHERE#

These are the AniNIX testing results. We found $passcount passing and $failcount failing packages, with $warncount warnings. It took $runtime seconds to finish.

PackageTesting StatusBuild StatusLatest BuildTime and Log of Run
#" "$webfile" - echo '
PackageTesting StatusBuild StatusLatest BuildTime of Run
' >> "$webfile" - + sed -i "s#WEBSTATSGOHERE#

These are the AniNIX testing results. We found $passcount passing and $failcount failing packages, with $warncount warnings. It took $runtime seconds to finish.

#" "$webfile" + printf '\n\n\n' >> "$webfile" + mv "$webfile" "$webfilefinal" } ### Build the package. Assumes a PKGBUILD is resent in the repo. +# param suffix: where to store the final package function BuildPackage() { - timeout --preserve-status 30s sudo -u "$user" makepkg -s --noconfirm &>/dev/null + suffix="$1" + [ `pgrep -afc pacman` -eq 0 ] && rm -Rf /var/lib/pacman/db.lck + nice -n 10 timeout --preserve-status 20m sudo -u "depriv" /usr/sbin/makepkg -sfc --noconfirm --sign &>> "$pkgdir"/"$repodir".txt if [ $? -ne 0 ]; then # Build failed. printf "$failCell""$warnCell" >> "$webfile" @@ -49,16 +54,21 @@ function BuildPackage() { # Build passed. printf "$passCell""" >> "$webfile" # List passing versions - ls -1 *.pkg.tar.xz | tr '\n' '`' | sed 's#`#
#g' >> "$webfile" + ls -1 *".pkg.tar.xz" | tr '\n' '`' | sed 's#`#
#g' >> "$webfile" printf "" >> "$webfile" + for pkg in `find . -type f | egrep ".pkg.tar.xz$"`; do + mv "$pkg" "$pkgdir"/"$suffix"; + mv "$pkg"".sig" "$pkgdir"/"$suffix"; + done fi - for pkg in `find . -type f | egrep '.tar.xz$'`; do mv "$pkg" "$pkgdir"; done } ### Build the repo passed as argument # param repo: the repo to build. +# param suffix: where to store the final package function BuildRepo() { repo="$1" + suffix="$2" cd "$srcdir" if [ -z "$repo" ]; then continue; fi repodir="$(basename "$repo" | sed 's/\.git$//')" @@ -66,12 +76,12 @@ function BuildRepo() { if [ ! -d "$repodir" ]; then git clone "$repo" fi - chown -R "$user": "$repodir" cd "$repodir" + git reset --hard origin &>/dev/null output="$(git pull 2>&1)" - if [ $? -eq 0 ] && [ -n "$incremental" ] && [[ "$output" =~ Already\ up\ to\ date. ]] && [ -f *.tar.xz ]; then - return; - fi + chown -R "depriv": . + + echo "$output" > "$pkgdir"/"$repodir".txt # Find the PKGBuilds in the repo for pkgbuild in `find . -type f -name PKGBUILD`; do cd "$(dirname "$pkgbuild")" @@ -79,22 +89,22 @@ function BuildRepo() { printf ''"$repodir -- $pkgbuild"'' >> "$webfile" if [ -f Makefile ] && [ `egrep -c '^test:' Makefile` -ge 1 ]; then # Check test status. - timeout --preserve-status 30s /bin/bash -c "sudo -u \'$user\' make test" &>/dev/null + timeout --preserve-status "$timeout" sudo -u "depriv" /bin/bash -l -c 'make test' &>> "$pkgdir"/"$repodir".txt if [ $? -ne 0 ]; then # Testing failed. printf "$failCell""$warnCell""$warnCell" >> "$webfile" else # Testing passed. printf "$passCell" >> "$webfile" - BuildPackage + BuildPackage "$suffix" fi else # Can't test -- usually from non-AniNIX repos. printf "$warnCell" >> "$webfile" - BuildPackage + BuildPackage "$suffix" fi # Timestamp - printf "$(date +%F-%R)\n" >> "$webfile" + printf "$(date +%F-%R)\n" >> "$webfile" cd "$cwd" if [ ! -z "$testing" ]; then break; fi done @@ -103,58 +113,85 @@ function BuildRepo() { ### Update the local repo function UpdateLocalRepo() { + set -x cd "$pkgdir" + chown -R "depriv": . # TODO Add deduplication of updated files -- keep latest 3 versions. - repo-add --new ./AniNIX::Maat.db.tar.xz `ls -1 *.tar.xz` + rm -Rf AniNIX.[db,files]* + sudo -u "depriv" repo-add --sign ./AniNIX.db.tar.xz `ls -1 *".pkg.tar.xz"` + cd aur/ + rm -Rf aur.[db,files]* + sudo -u "depriv" repo-add --sign ./aur.db.tar.xz `ls -1 *".pkg.tar.xz"` + set +x } -# Production defaults -baseurl='https://aninix.net/foundation/' +# Clear variables aurconf='/usr/local/etc/Maat/aur.list' -homedir=/srv/maat/ -unset upload -unset testing +baseurl='https://aninix.net/foundation/' +homedir="/srv/maat/" unset incremental -user="depriv" +unset skipPatching +unset testing +timeout="90s" # Stat tracking starttime=`date +%s` # Parse arguments -while getopts 'b:c:hil:Tu:U:v' OPTION; do +while getopts 'b:c:hil:st:Tv' OPTION; do case "${OPTION}" in b) homedir="${OPTARG}" ;; c) aurconf="${OPTARG}" ;; h) Usage; exit 0 ;; i) incremental=1 ;; - l) cmdstring="$0"; for arg in $@; do if [ "$arg" != "-l" ] && [ "$arg" != "${OPTARG}" ]; then cmdstring="$cmdstring \"${arg}\""; fi; done; exec /bin/bash -c "$0 $cmdstring | tee -a \"${OPTARG}\"" ;; - T) homedir='.'; testing=1 ;; - u) baseurl="${OPTARG}" ;; - U) user="${OPTARG}" ;; + l) cmdstring="$0"; for arg in $@; do if [ "$arg" != "-l" ] && [ "$arg" != "${OPTARG}" ]; then cmdstring="$cmdstring \"${arg}\""; fi; done; exec /bin/bash -c "$cmdstring | tee -a \"${OPTARG}\""; ;; + s) skipPatching=1 ;; + t) timeout="${OPTARG}" ;; + T) export MAATTESTINGVAR=1; exec $0 -l ./testing.log -v -s -c <(echo https://aur.archlinux.org/cower.git) -b . ;; v) set -x ;; *) usage; exit 1 ;; esac done +# Ensure we are up to date -- otherwise, building is not a good plan. +if [ -x `which pacman` ] && [ -z "$skipPatching" ]; then + pacman -Syu --noconfirm; + if [ $? -ne 0 ]; then + echo "Self patching failed -- please investigate!" 1>&2 + exit 1 + fi +fi + # Ensure work directories live if [ $( echo "$homedir" | egrep -c '^/') -ne 1 ]; then homedir="${PWD}/${homedir}" fi + +# Setup srcdir="${homedir}/src" && mkdir -p "${srcdir}" pkgdir="${homedir}/pkg" && mkdir -p "${pkgdir}" +mkdir -p "${pkgdir}"/aur webdir="$pkgdir" -webfile="$webdir"/index.html - -# For each repo listed at the CGIT URL and conf file +webfilefinal="$webdir"/index.html +webfile="$webdir"/index.html.tmp cwd="$(pwd)" SeedWebFile -for aninixrepo in `wget -q -O - "$baseurl" | grep toplevel-repo | cut -f 4 -d \' | sed "s#^#$baseurl#"`; do - BuildRepo "$aninixrepo" - if [ ! -z "$testing" ]; then break; fi -done + +# Build AniNIX Repo +if [ -n "$MAATTESTINGVAR" ]; then + BuildRepo "$baseurl"/HelloWorld +else + for AniNIXrepo in `wget -q -O - "$baseurl" | grep toplevel-repo | cut -f 4 -d \' | sed "s#^#$baseurl#"`; do + BuildRepo "$AniNIXrepo" '.' + done +fi +printf '\n

AUR Packages

\n

These are packages made by other ArchLinux users and uploaded to the AUR.

\n' >> "$webfile" +printf "$tableHead" >> "$webfile" + +# Build AUR for repo in `cat "$aurconf"`; do - BuildRepo "$repo"; - if [ ! -z "$testing" ]; then break; fi + BuildRepo "$repo" aur/; + if [ ! -z "$MAATTESTINGVAR" ]; then break; fi done UpdateLocalRepo