AniNIX/Maat
AniNIX/Maat
3
0
Fork 0
Code Issues 4 Pull Requests 1 Releases Wiki Activity

Testing updates; website usability; GPG signing improvements

Browse Source
This commit is contained in:
DarkFeather 2019-07-25 13:01:53 -05:00
parent 795a4ac0b2
commit 6439ec4b70
4 changed files with 114 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
src/ src/
web/ web/
pkg/ pkg/
testing.log
*.pkg.tar.xz

23
Makefile
View File

@ -4,6 +4,7 @@ systemdList = ./maat-builder.timer ./maat-builder.service
serverRoot = /srv/maat/ serverRoot = /srv/maat/
owner=root owner=root
perms=0755 perms=0755
fileperms=0644
compile: /bin/bash compile: /bin/bash
@echo Nothing to compile. @echo Nothing to compile.
@ -12,6 +13,7 @@ install: ${list}
mkdir -p ${installDir} mkdir -p ${installDir}
mkdir -p ${pkgdir}${serverRoot}/src mkdir -p ${pkgdir}${serverRoot}/src
mkdir -p ${pkgdir}${serverRoot}/pkg mkdir -p ${pkgdir}${serverRoot}/pkg
mkdir -p ${pkgdir}/usr/lib/systemd/system
for i in ${list}; do install -m ${perms} -o ${owner} $$i ${installDir}; done for i in ${list}; do install -m ${perms} -o ${owner} $$i ${installDir}; done
for i in ${systemdList}; do install -m 0644 -o root $$i ${pkgdir}/usr/lib/systemd/system; done for i in ${systemdList}; do install -m 0644 -o root $$i ${pkgdir}/usr/lib/systemd/system; done
@ -24,9 +26,24 @@ uninstall:
rmdir -p ${pkgdir}${serverRoot}/src rmdir -p ${pkgdir}${serverRoot}/src
rmdir -p ${pkgdir}${serverRoot}/pkg rmdir -p ${pkgdir}${serverRoot}/pkg
test: test: clean
./maat-builder -T -U ${USER} ./maat-builder -T
make clean test -d src
test -d src/cower
test -d src/cower/.git
test -d src/HelloWorld
test -d src/HelloWorld/.git
test -d pkg
test -d pkg/aur
test -f pkg/AniNIX.db
test -f pkg/AniNIX.db.sig
test -f pkg/helloworld*.tar.xz
test -f pkg/aur/aur.db
test -f pkg/aur/aur.db.sig
test -f pkg/aur/cower*.tar.xz
[ `wc -l pkg/index.html | cut -f 1 -d ' '` -eq 24 ]
@echo
@echo Tests passed successfully.
diff: diff:
for i in ${list}; do if [ -f ${installDir}/$$i ]; then diff "$$i" "${installDir}/$$i"; fi done for i in ${list}; do if [ -f ${installDir}/$$i ]; then diff "$$i" "${installDir}/$$i"; fi done

33
PKGBUILD
View File

@ -1,20 +1,22 @@
# Maintainer: Shikoba Kage <darkfeather@aninix.net> depends=('bash>=4.4' 'git' 'pacman')
pkgname=maat
pkgver=0.1."$(git rev-parse HEAD)"
pkgrel=1
epoch=
pkgdesc="AniNIX::Maat \\\\ Quality Assurance and Reporting Framework"
arch=("x86_64")
url="https://aninix.net/wiki/Maat"
license=('custom')
groups=()
depends=('bash>=4.4' 'wget' 'grep' 'make' 'git')
makedepends=('make>=4.2') makedepends=('make>=4.2')
checkdepends=() checkdepends=()
optdepends=() optdepends=()
provides=('maat') pkgname="$(git config remote.origin.url | rev | cut -f 1 -d '/' | rev | sed 's/.git$//')"
pkgver="$(git describe --tag --abbrev=0)"."$(git rev-parse --short HEAD)"
pkgrel=1
pkgrel() {
echo $(( `git log "$(git describe --tag --abbrev=0)"..HEAD | grep -c commit` + 1 ))
}
epoch="$(git log | grep -c commit)"
pkgdesc="$(head -n 1 README)"
arch=("x86_64")
url="https://aninix.net/foundation/${pkgname}"
license=('custom')
groups=()
provides=("${pkgname}")
conflicts=() conflicts=()
replaces=() replaces=("${pkgname,,}", "aninix-${pkgname,,}")
backup=() backup=()
options=() options=()
install= install=
@ -26,16 +28,13 @@ validpgpkeys=()
prepare() { prepare() {
git pull git pull
make -C .. test
} }
build() { build() {
make -C .. make -C ..
} }
check() {
make -C "${srcdir}/.." test
}
package() { package() {
export pkgdir="${pkgdir}" export pkgdir="${pkgdir}"
make -C .. install make -C .. install

113
maat-builder
View File

@ -15,33 +15,38 @@
passCell="<td style='color:green;'>PASS</td>"; passCell="<td style='color:green;'>PASS</td>";
failCell="<td style='color:red;'>FAIL</td>"; failCell="<td style='color:red;'>FAIL</td>";
warnCell="<td style='color:yellow;'>N/A</td>"; warnCell="<td style='color:yellow;'>N/A</td>";
tableHead="<table style='text-align: left;'>\n<tr><th>Package</th><th>Testing Status</th><th>Build Status</th><th>Latest Build</th><th>Time and Log of Run</th></tr>";
### Add helptext. ### Add helptext.
function Usage() { function Usage() {
echo "Usage: $0" echo "Usage: $0"
echo " $0 [ -b basedir ] [ -c AUR.git.list ] [ -u https://base.url/ ]" echo " $0 [ -b basedir ] [ -c AUR.git.list ] [ -T ] [ -u https://base.url/ ]"
echo " $0 [ -T ]"
echo " $0 -h" echo " $0 -h"
echo echo
echo 'Add -v to increase verbosity or -h for help. Add the -l LOGFILE flags to log to a file' echo 'Add -v to increase verbosity or -h for help. Add the -l LOGFILE flags to log to a file'
echo "By default, $0 uses the depriv user to build packages. Override with -U" echo 'Add the -r REPOCMD, -C make-package-command, -p PKGBUILDName, -t test-command, and'
echo 'the -e package-extension flags for compatibility with non-ArchLinux repos.'
} }
### Put the initial content in the webfile ### Put the initial content in the webfile
function SeedWebFile() { function SeedWebFile() {
printf '<html>\n<head>\n<title>AniNIX::Maat \\\\ Build Results</title>\n<link rel="icon" type="image/png" href=https://aninix.net/mediawiki/images/9/90/MaatIcon.png" />\n<link rel="icon" type="image/png" href="/mediawiki/images/9/90/MaatIcon.png">\n<meta name="apple-mobile-web-app-capable" content="yes" />\n<link rel="stylesheet" type="text/css" href="/style.css">\n<link rel="apple-touch-icon" sizes="180x180" href="mediawiki/images/9/90/MaatIcon.png" />\n</head>\n<body>\n<h1>AniNIX::Maat \\\\ Build Status</h1>\nWEBSTATSGOHERE\n' > "$webfile" printf '<html>\n<head>\n<title>AniNIX::Maat \\\\ Build Results</title>\n<link rel="icon" type="image/png" href="https://aninix.net/mediawiki/images/9/90/MaatIcon.png" />\n<link rel="icon" type="image/png" href="/mediawiki/images/9/90/MaatIcon.png">\n<meta name="apple-mobile-web-app-capable" content="yes" />\n<link rel="stylesheet" type="text/css" href="/style.css">\n<link rel="apple-touch-icon" sizes="180x180" href="mediawiki/images/9/90/MaatIcon.png" />\n</head>\n<body>\n<h1>AniNIX::Maat \\\\ Build Status</h1>\nWEBSTATSGOHERE\n<h2>AnINIX Packages</h2>\n<p>These are packages written by the AniNIX. Their source is in <a href="https://aninix.net/foundation/" alt=AniNIX::Foundation>AniNIX::Foundation</a>.</p>\n' > "$webfile"
printf "$tableHead" >> "$webfile"
} }
### Update the webfile to close up table tags and add stats. ### Update the webfile to close up table tags and add stats.
function UpdateWebFile() { function UpdateWebFile() {
sed -i "s#WEBSTATSGOHERE#<p>These are the AniNIX testing results. We found $passcount passing and $failcount failing packages, with $warncount warnings. It took $runtime seconds to finish.</p><table style='text-align: left;'><tr><th>Package</th><th>Testing Status</th><th>Build Status</th><th>Latest Build</th><th>Time of Run</th></tr>#" "$webfile" sed -i "s#WEBSTATSGOHERE#<p>These are the AniNIX testing results. We found $passcount passing and $failcount failing packages, with $warncount warnings. It took $runtime seconds to finish.</p>#" "$webfile"
echo '</table></body></html>' >> "$webfile" printf '</table>\n</body>\n</html>\n' >> "$webfile"
mv "$webfile" "$webfilefinal"
} }
### Build the package. Assumes a PKGBUILD is resent in the repo. ### Build the package. Assumes a PKGBUILD is resent in the repo.
# param suffix: where to store the final package
function BuildPackage() { function BuildPackage() {
timeout --preserve-status 30s sudo -u "$user" makepkg -s --noconfirm &>/dev/null suffix="$1"
[ `pgrep -afc pacman` -eq 0 ] && rm -Rf /var/lib/pacman/db.lck
nice -n 10 timeout --preserve-status 20m sudo -u "depriv" /usr/sbin/makepkg -sfc --noconfirm --sign &>> "$pkgdir"/"$repodir".txt
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
# Build failed. # Build failed.
printf "$failCell""$warnCell" >> "$webfile" printf "$failCell""$warnCell" >> "$webfile"
@ -49,16 +54,21 @@ function BuildPackage() {
# Build passed. # Build passed.
printf "$passCell""<td>" >> "$webfile" printf "$passCell""<td>" >> "$webfile"
# List passing versions # List passing versions
ls -1 *.pkg.tar.xz | tr '\n' '`' | sed 's#`#<br/>#g' >> "$webfile" ls -1 *".pkg.tar.xz" | tr '\n' '`' | sed 's#`#<br/>#g' >> "$webfile"
printf "</td>" >> "$webfile" printf "</td>" >> "$webfile"
for pkg in `find . -type f | egrep ".pkg.tar.xz$"`; do
mv "$pkg" "$pkgdir"/"$suffix";
mv "$pkg"".sig" "$pkgdir"/"$suffix";
done
fi fi
for pkg in `find . -type f | egrep '.tar.xz$'`; do mv "$pkg" "$pkgdir"; done
} }
### Build the repo passed as argument ### Build the repo passed as argument
# param repo: the repo to build. # param repo: the repo to build.
# param suffix: where to store the final package
function BuildRepo() { function BuildRepo() {
repo="$1" repo="$1"
suffix="$2"
cd "$srcdir" cd "$srcdir"
if [ -z "$repo" ]; then continue; fi if [ -z "$repo" ]; then continue; fi
repodir="$(basename "$repo" | sed 's/\.git$//')" repodir="$(basename "$repo" | sed 's/\.git$//')"
@ -66,12 +76,12 @@ function BuildRepo() {
if [ ! -d "$repodir" ]; then if [ ! -d "$repodir" ]; then
git clone "$repo" git clone "$repo"
fi fi
chown -R "$user": "$repodir"
cd "$repodir" cd "$repodir"
git reset --hard origin &>/dev/null
output="$(git pull 2>&1)" output="$(git pull 2>&1)"
if [ $? -eq 0 ] && [ -n "$incremental" ] && [[ "$output" =~ Already\ up\ to\ date. ]] && [ -f *.tar.xz ]; then chown -R "depriv": .
return;
fi echo "$output" > "$pkgdir"/"$repodir".txt
# Find the PKGBuilds in the repo # Find the PKGBuilds in the repo
for pkgbuild in `find . -type f -name PKGBUILD`; do for pkgbuild in `find . -type f -name PKGBUILD`; do
cd "$(dirname "$pkgbuild")" cd "$(dirname "$pkgbuild")"
@ -79,22 +89,22 @@ function BuildRepo() {
printf '<tr style="border: 1px solid #FFF;"><td>'"<a href='$repo'>$repodir</a> -- $pkgbuild"'</td>' >> "$webfile" printf '<tr style="border: 1px solid #FFF;"><td>'"<a href='$repo'>$repodir</a> -- $pkgbuild"'</td>' >> "$webfile"
if [ -f Makefile ] && [ `egrep -c '^test:' Makefile` -ge 1 ]; then if [ -f Makefile ] && [ `egrep -c '^test:' Makefile` -ge 1 ]; then
# Check test status. # Check test status.
timeout --preserve-status 30s /bin/bash -c "sudo -u \'$user\' make test" &>/dev/null timeout --preserve-status "$timeout" sudo -u "depriv" /bin/bash -l -c 'make test' &>> "$pkgdir"/"$repodir".txt
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
# Testing failed. # Testing failed.
printf "$failCell""$warnCell""$warnCell" >> "$webfile" printf "$failCell""$warnCell""$warnCell" >> "$webfile"
else else
# Testing passed. # Testing passed.
printf "$passCell" >> "$webfile" printf "$passCell" >> "$webfile"
BuildPackage BuildPackage "$suffix"
fi fi
else else
# Can't test -- usually from non-AniNIX repos. # Can't test -- usually from non-AniNIX repos.
printf "$warnCell" >> "$webfile" printf "$warnCell" >> "$webfile"
BuildPackage BuildPackage "$suffix"
fi fi
# Timestamp # Timestamp
printf "<td>$(date +%F-%R)</td></tr>\n" >> "$webfile" printf "<td><a href='/maat/$repodir.txt'>$(date +%F-%R)</a></td></tr>\n" >> "$webfile"
cd "$cwd" cd "$cwd"
if [ ! -z "$testing" ]; then break; fi if [ ! -z "$testing" ]; then break; fi
done done
@ -103,58 +113,85 @@ function BuildRepo() {
### Update the local repo ### Update the local repo
function UpdateLocalRepo() { function UpdateLocalRepo() {
set -x
cd "$pkgdir" cd "$pkgdir"
chown -R "depriv": .
# TODO Add deduplication of updated files -- keep latest 3 versions. # TODO Add deduplication of updated files -- keep latest 3 versions.
repo-add --new ./AniNIX::Maat.db.tar.xz `ls -1 *.tar.xz` rm -Rf AniNIX.[db,files]*
sudo -u "depriv" repo-add --sign ./AniNIX.db.tar.xz `ls -1 *".pkg.tar.xz"`
cd aur/
rm -Rf aur.[db,files]*
sudo -u "depriv" repo-add --sign ./aur.db.tar.xz `ls -1 *".pkg.tar.xz"`
set +x
} }
# Production defaults # Clear variables
baseurl='https://aninix.net/foundation/'
aurconf='/usr/local/etc/Maat/aur.list' aurconf='/usr/local/etc/Maat/aur.list'
homedir=/srv/maat/ baseurl='https://aninix.net/foundation/'
unset upload homedir="/srv/maat/"
unset testing
unset incremental unset incremental
user="depriv" unset skipPatching
unset testing
timeout="90s"
# Stat tracking # Stat tracking
starttime=`date +%s` starttime=`date +%s`
# Parse arguments # Parse arguments
while getopts 'b:c:hil:Tu:U:v' OPTION; do while getopts 'b:c:hil:st:Tv' OPTION; do
case "${OPTION}" in case "${OPTION}" in
b) homedir="${OPTARG}" ;; b) homedir="${OPTARG}" ;;
c) aurconf="${OPTARG}" ;; c) aurconf="${OPTARG}" ;;
h) Usage; exit 0 ;; h) Usage; exit 0 ;;
i) incremental=1 ;; i) incremental=1 ;;
l) cmdstring="$0"; for arg in $@; do if [ "$arg" != "-l" ] && [ "$arg" != "${OPTARG}" ]; then cmdstring="$cmdstring \"${arg}\""; fi; done; exec /bin/bash -c "$0 $cmdstring | tee -a \"${OPTARG}\"" ;; l) cmdstring="$0"; for arg in $@; do if [ "$arg" != "-l" ] && [ "$arg" != "${OPTARG}" ]; then cmdstring="$cmdstring \"${arg}\""; fi; done; exec /bin/bash -c "$cmdstring | tee -a \"${OPTARG}\""; ;;
T) homedir='.'; testing=1 ;; s) skipPatching=1 ;;
u) baseurl="${OPTARG}" ;; t) timeout="${OPTARG}" ;;
U) user="${OPTARG}" ;; T) export MAATTESTINGVAR=1; exec $0 -l ./testing.log -v -s -c <(echo https://aur.archlinux.org/cower.git) -b . ;;
v) set -x ;; v) set -x ;;
*) usage; exit 1 ;; *) usage; exit 1 ;;
esac esac
done done
# Ensure we are up to date -- otherwise, building is not a good plan.
if [ -x `which pacman` ] && [ -z "$skipPatching" ]; then
pacman -Syu --noconfirm;
if [ $? -ne 0 ]; then
echo "Self patching failed -- please investigate!" 1>&2
exit 1
fi
fi
# Ensure work directories live # Ensure work directories live
if [ $( echo "$homedir" | egrep -c '^/') -ne 1 ]; then if [ $( echo "$homedir" | egrep -c '^/') -ne 1 ]; then
homedir="${PWD}/${homedir}" homedir="${PWD}/${homedir}"
fi fi
# Setup
srcdir="${homedir}/src" && mkdir -p "${srcdir}" srcdir="${homedir}/src" && mkdir -p "${srcdir}"
pkgdir="${homedir}/pkg" && mkdir -p "${pkgdir}" pkgdir="${homedir}/pkg" && mkdir -p "${pkgdir}"
mkdir -p "${pkgdir}"/aur
webdir="$pkgdir" webdir="$pkgdir"
webfile="$webdir"/index.html webfilefinal="$webdir"/index.html
webfile="$webdir"/index.html.tmp
# For each repo listed at the CGIT URL and conf file
cwd="$(pwd)" cwd="$(pwd)"
SeedWebFile SeedWebFile
for aninixrepo in `wget -q -O - "$baseurl" | grep toplevel-repo | cut -f 4 -d \' | sed "s#^#$baseurl#"`; do
BuildRepo "$aninixrepo" # Build AniNIX Repo
if [ ! -z "$testing" ]; then break; fi if [ -n "$MAATTESTINGVAR" ]; then
BuildRepo "$baseurl"/HelloWorld
else
for AniNIXrepo in `wget -q -O - "$baseurl" | grep toplevel-repo | cut -f 4 -d \' | sed "s#^#$baseurl#"`; do
BuildRepo "$AniNIXrepo" '.'
done done
fi
printf '</table>\n<h2>AUR Packages</h3>\n<p>These are packages made by other ArchLinux users and uploaded to the <a href="https://aur.archlinux.org/" alt="ArchLinux AUR">AUR</a>.</p>\n' >> "$webfile"
printf "$tableHead" >> "$webfile"
# Build AUR
for repo in `cat "$aurconf"`; do for repo in `cat "$aurconf"`; do
BuildRepo "$repo"; BuildRepo "$repo" aur/;
if [ ! -z "$testing" ]; then break; fi if [ ! -z "$MAATTESTINGVAR" ]; then break; fi
done done
UpdateLocalRepo UpdateLocalRepo