Update to .gitignore so clean step does not break CI/CD

.gitignore

@@ -1,5 +1,9 @@
-src/
-web/
+# Testing outputs
+src/ascii-invaders**
+src/HelloWorld**
 pkg/
 testing.log
-*.pkg.tar.xz
+
+# Packaging outputs
+*.pkg.tar.zst
+*.pkg.tar.zst.sig

Makefile

@@ -1,24 +1,27 @@
 installDir = ${pkgdir}/usr/local/bin
-list = ./maat-builder
-systemdList = ./maat-builder.timer ./maat-builder.service
+list = ./maat
+systemdList = ./maat.timer ./maat.service
 serverRoot = /srv/maat/
-owner=root
+owner=maat
 perms=0755
 fileperms=0644
 
 compile: /bin/bash
 	@echo Nothing to compile.
 
-install: ${list}
+install: ${list} MaatIcon.png
 	mkdir -p ${installDir}
 	mkdir -p ${pkgdir}${serverRoot}/src
 	mkdir -p ${pkgdir}${serverRoot}/pkg
+	install -m ${fileperms} -o ${owner} ./MaatIcon.png ${pkgdir}${serverRoot}/pkg
 	mkdir -p ${pkgdir}/usr/lib/systemd/system
-	for i in ${list}; do install -m ${perms} -o ${owner} $$i ${installDir}; done
+	for i in ${list}; do install -m ${perms} -o root $$i ${installDir}; done
+	install -m ${perms} -o ${owner} -d ${pkgdir}/usr/local/etc/Maat/
+	touch ${pkgdir}/usr/local/etc/Maat/aur.list; chown ${owner} ${pkgdir}/usr/local/etc/Maat/aur.list; chmod ${fileperms} ${pkgdir}/usr/local/etc/Maat/aur.list
 	for i in ${systemdList}; do install -m 0644 -o root $$i ${pkgdir}/usr/lib/systemd/system; done
 
 clean:
-	cat .gitignore | xargs rm -Rf
+	cat .gitignore | sudo xargs rm -Rf
 
 uninstall:
 	for i in ${list}; do rm -Rf ${installDir}/$$i; done
@@ -27,31 +30,32 @@ uninstall:
 	rmdir -p ${pkgdir}${serverRoot}/pkg
 
 test: clean
-	./maat-builder -T 
+	sudo ./maat -u $$USER -T
 	test -d src
-	test -d src/cower
-	test -d src/cower/.git
+	test -d src/ascii-invaders
+	test -d src/ascii-invaders/.git
 	test -d src/HelloWorld
 	test -d src/HelloWorld/.git
 	test -d pkg
 	test -d pkg/aur
 	test -f pkg/AniNIX.db
 	test -f pkg/AniNIX.db.sig
-	test -f pkg/helloworld*.tar.xz
+	test -f pkg/HelloWorld*.tar.zst
 	test -f pkg/aur/aur.db
 	test -f pkg/aur/aur.db.sig
-	test -f pkg/aur/cower*.tar.xz
+	test -f pkg/aur/ascii-invaders*.tar.zst
 	[ `wc -l pkg/index.html | cut -f 1 -d ' '` -eq 24 ]
+	make clean
 	@echo
 	@echo Tests passed successfully.
 
 diff:
-	for i in ${list}; do if [ -f ${installDir}/$$i ]; then diff "$$i" "${installDir}/$$i"; fi done 
+	for i in ${list}; do if [ -f ${installDir}/$$i ]; then diff "$$i" "${installDir}/$$i"; fi done
 
 reverse:
-	for i in ${list}; do if [ -f ${installDir}/$$i ]; then cp "${installDir}/$$i" "$$i"; fi done 
+	for i in ${list}; do if [ -f ${installDir}/$$i ]; then cp "${installDir}/$$i" "$$i"; fi done
 
 checkperm:
 	chown -R ${owner}: ${pkgdir}${serverRoot}/src
 	chown -R ${owner}: ${pkgdir}${serverRoot}/pkg
-	for i in ${list}; do chown -R ${owner}: "${installDir}/$$i"; chmod ${perms} "${installDir}/$$i"; done
+	for i in ${list}; do chown -R ${owner}: "${installDir}/$$i"; sudo chmod ${perms} "${installDir}/$$i"; done

PKGBUILD

@@ -17,9 +17,9 @@ groups=()
 provides=("${pkgname}")
 conflicts=()
 replaces=("${pkgname,,}", "aninix-${pkgname,,}")
-backup=()
+backup=(usr/local/etc/Maat/aur.list)
 options=()
-install=
+install=installscript
 changelog=
 source=()
 noextract=()
@@ -28,7 +28,7 @@ validpgpkeys=()
 
 prepare() {
     git pull
-	make -C .. test
+	make -C .. test && make -C .. clean
 }
 
 build() {

README

@@ -1,2 +0,0 @@
-AniNIX::Maat \\ Quality Assurance and Reporting Framework
-This package allows automation around code packaging and deployment. It will pull from both AniNIX::Foundation and ArchLinux AUR.

README.md

@@ -0,0 +1,15 @@
+Continuous integration and continuous delivery (CI/CD) are integral to current DevOps mentality -- a step further is GitOps, wherein developers only need to interact with Git and their artifacts are magically available downstream. AniNIX/Maat is intended to provide this for AniNIX and [ArchLinux AUR](https://wiki.archlinux.org/title/Arch_User_Repository) packages.
+
+# Etymology
+Maat is named for [the Egyptian goddess of truth and order](https://en.wikipedia.org/wiki/Maat) -- Maat was deeply tied into the idea of honorable contribution to community and the flow of the Nile, from which came the lifeblood of Egypt. Truth is reflected in the Maat service's use of testing and GPG signing prior to delivering packages, and the communal aspect is that Maat is how we deliver AniNIX and AUR packages for the world to use.
+
+# Relevant Files and Software
+
+Maat runs as a [systemd.timer](https://man.archlinux.org/man/systemd.timer.5) nightly, invoking our build script. Adhoc builds can be requested from admins. Output gets published on the [landing page](https://maat.aninix.net) from our build pipeline -- there you can download packages, view build logs, see testing status, etc.
+
+Our CI/CD pipeline attempts to pull directly from Git sources, either from the AniNIX or the curated AUR, build those packages, and make the available to systems downstream. The `/usr/local/etc/Maat/aur.list` file allows the admin to control which AUR packages are pulled and built -- output files from builds can help identify issues like missing public GPG keys or dependency issues.
+
+Maat also consumes a GPG key. The admin will need to set up this key and publish it. You will need to add [our GPG key](https://aninix.net/AniNIX/ShadowArch/src/branch/main/EtcFiles/aninix.gpg) to use the AniNIX's CI/CD.
+
+# Available Clients
+As Maat is a pipeline for ArchLinux systems, the primary client is [Pacman](https://wiki.archlinux.org/title/Pacman). You can see how to subscribe your host to Maat in [this configuration snippet](https://aninix.net/AniNIX/Ubiqtorate/src/branch/main/roles/ShadowArch/files/pacman.conf#L103).

aur.list

@@ -0,0 +1,81 @@
+https://aur.archlinux.org/acidrip.git
+https://aur.archlinux.org/animecheck-git.git
+https://aur.archlinux.org/ascii-invaders.git
+https://aur.archlinux.org/autopsy.git
+https://aur.archlinux.org/brother-mfc-j430w.git
+https://aur.archlinux.org/brscan4.git
+https://aur.archlinux.org/castnow-git.git
+https://aur.archlinux.org/ccrypt.git
+https://aur.archlinux.org/chromium-pepper-flash.git
+https://aur.archlinux.org/defcon.git
+https://aur.archlinux.org/discord-cli-git.git
+https://aur.archlinux.org/downgrader.git
+https://aur.archlinux.org/dotnet-core-bin.git
+https://aur.archlinux.org/dotnet-runtime-bin.git
+https://aur.archlinux.org/emby-server-beta.git
+https://aur.archlinux.org/freeme2.git
+https://aur.archlinux.org/gnome-alsamixer.git
+https://aur.archlinux.org/google-chrome.git
+https://aur.archlinux.org/google-earth.git
+https://aur.archlinux.org/googlecl.git
+https://aur.archlinux.org/googler.git
+https://aur.archlinux.org/gstreamer0.10-ffmpeg.git
+https://aur.archlinux.org/gtk-xfce-engine.git
+https://aur.archlinux.org/gyazo.git
+https://aur.archlinux.org/helloworld.git
+https://aur.archlinux.org/injection.git
+https://aur.archlinux.org/inspircd.git
+https://aur.archlinux.org/jmtpfs.git
+https://aur.archlinux.org/jpcsp.git
+https://aur.archlinux.org/js.git
+https://aur.archlinux.org/keepassx.git
+https://aur.archlinux.org/kpcli.git
+https://aur.archlinux.org/lib32-glib.git
+https://aur.archlinux.org/libc++.git
+https://aur.archlinux.org/libc++abi.git
+https://aur.archlinux.org/libc++experimental.git
+https://aur.archlinux.org/libdaq.git
+https://aur.archlinux.org/libdwarf-20140413.git
+https://aur.archlinux.org/libiconv.git
+https://aur.archlinux.org/maya.git
+https://aur.archlinux.org/meshlab.git
+https://aur.archlinux.org/nagios.git
+https://aur.archlinux.org/oinkmaster.git
+https://aur.archlinux.org/openvisualtraceroute.git
+https://aur.archlinux.org/ossec-local.git
+https://aur.archlinux.org/pcmciautils.git
+https://aur.archlinux.org/pdfshuffler.git
+https://aur.archlinux.org/pear-net-ldap2.git
+https://aur.archlinux.org/perl-crypt-rijndael.git
+https://aur.archlinux.org/perl-expect.git
+https://aur.archlinux.org/perl-file-keepass.git
+https://aur.archlinux.org/perl-net-sftp-foreign.git
+https://aur.archlinux.org/perl-php-serialization.git
+https://aur.archlinux.org/perl-sys-mmap.git
+https://aur.archlinux.org/perl-term-shellui.git
+https://aur.archlinux.org/php-pear.git
+https://aur.archlinux.org/plex-media-server.git
+https://aur.archlinux.org/plex-media-server-plexpass.git
+https://aur.archlinux.org/plex-media-player.git
+https://aur.archlinux.org/pm-utils.git
+https://aur.archlinux.org/powerpanel.git
+https://aur.archlinux.org/python-dill.git
+https://aur.archlinux.org/python-vincenty.git
+https://aur.archlinux.org/python2-astral.git
+https://aur.archlinux.org/python2-dill.git
+https://aur.archlinux.org/rarcrack.git
+https://aur.archlinux.org/savage.git
+https://aur.archlinux.org/shadowarch.git
+https://aur.archlinux.org/snort.git
+https://aur.archlinux.org/suricata.git
+https://aur.archlinux.org/swfdec.git
+https://aur.archlinux.org/swfdec-gnome.git
+https://aur.archlinux.org/trid.git
+https://aur.archlinux.org/tt-rss-auth-ldap-git.git
+https://aur.archlinux.org/udisks.git
+https://aur.archlinux.org/undvd.git
+https://aur.archlinux.org/uniglot.git
+https://aur.archlinux.org/urbanterror.git
+https://aur.archlinux.org/vbam-gtk.git
+https://aur.archlinux.org/xfce4-mixer.git
+https://aur.archlinux.org/xorg-server-utils.git

installscript

@@ -0,0 +1,9 @@
+pre_install() {
+  groupadd maat &>/dev/null
+  useradd -g maat -d /srv/maat -s /bin/false maat &> /dev/null
+  chown -R maat:maat /srv/maat &> /dev/null
+}
+
+pre_remove() {
+  getent passwd maat &>/dev/null && userdel maat &> /dev/null
+}

maat

@@ -3,13 +3,17 @@
 # File: maat-builder
 #
 # Description: This file allows a Maat host to build source packages
-#              and optionally upload to the AniNIX::Foundation
+#              and optionally upload to the AniNIX/Foundation
 #
-# Package: ConfigPackages/Maat
+# Package: Maat
 # Copyright: WTFPL
 #
-# Author: darkfeather@aninix.net
-# 
+# Author: DarkFeather
+#
+
+# Arch keys to look for
+pkgExt=".pkg.tar.zst"
+deprivuser="maat"
 
 ### Printing defaults
 passCell="<td style='color:green;'>PASS</td>";
@@ -30,11 +34,11 @@ function Usage() {
 
 ### Put the initial content in the webfile
 function SeedWebFile() {
-    printf '<html>\n<head>\n<title>AniNIX::Maat \\\\ Build Results</title>\n<link rel="icon" type="image/png" href="https://aninix.net/mediawiki/images/9/90/MaatIcon.png" />\n<link rel="icon" type="image/png" href="/mediawiki/images/9/90/MaatIcon.png">\n<meta name="apple-mobile-web-app-capable" content="yes" />\n<link rel="stylesheet" type="text/css" href="/style.css">\n<link rel="apple-touch-icon" sizes="180x180" href="mediawiki/images/9/90/MaatIcon.png" />\n</head>\n<body>\n<h1>AniNIX::Maat \\\\ Build Status</h1>\nWEBSTATSGOHERE\n<h2>AnINIX Packages</h2>\n<p>These are packages written by the AniNIX. Their source is in <a href="https://aninix.net/foundation/" alt=AniNIX::Foundation>AniNIX::Foundation</a>.</p>\n' > "$webfile"
+    printf '<html>\n<head>\n<title>AniNIX/Maat -- Build Results</title>\n<link rel="icon" type="image/png" href="/MaatIcon.png" />\n<link rel="icon" type="image/png" href="/MaatIcon.png">\n<meta name="apple-mobile-web-app-capable" content="yes" />\n<link rel="stylesheet" type="text/css" href="https://aninix.net/assets/css/theme-aninix.css">\n<link rel="apple-touch-icon" sizes="180x180" href="/MaatIcon.png" />\n</head>\n<body>\n<h1>AniNIX/Maat -- Build Status</h1>\nWEBSTATSGOHERE\n<h2>AnINIX Packages</h2>\n<p>These are packages written by the AniNIX. Their source is in <a href="https://aninix.net/" alt=AniNIX/Foundation>AniNIX/Foundation</a>.</p>\n' > "$webfile"
     printf "$tableHead" >> "$webfile"
 }
 
-### Update the webfile to close up table tags and add stats. 
+### Update the webfile to close up table tags and add stats.
 function UpdateWebFile() {
     sed -i "s#WEBSTATSGOHERE#<p>These are the AniNIX testing results. We found $passcount passing and $failcount failing packages, with $warncount warnings. It took $runtime seconds to finish.</p>#" "$webfile"
     printf '</table>\n</body>\n</html>\n' >> "$webfile"
@@ -43,23 +47,25 @@ function UpdateWebFile() {
 
 ### Build the package. Assumes a PKGBUILD is resent in the repo.
 # param suffix: where to store the final package
-function BuildPackage() { 
+function BuildPackage() {
     suffix="$1"
     [ `pgrep -afc pacman` -eq 0 ] && rm -Rf /var/lib/pacman/db.lck
-    nice -n 10 timeout --preserve-status 20m sudo -u "depriv" /usr/sbin/makepkg -sfc --noconfirm --sign &>> "$pkgdir"/"$repodir".txt
+    nice -n 10 timeout --preserve-status 60m sudo -u "$deprivuser" /usr/sbin/makepkg -sfc --noconfirm --sign &>> "$pkgdir"/"$repodir".txt
     if [ $? -ne 0 ]; then
         # Build failed.
-        printf "$failCell""$warnCell" >> "$webfile" 
+        printf "$failCell""$warnCell" >> "$webfile"
     else
         # Build passed.
         printf "$passCell""<td>"  >> "$webfile"
         # List passing versions
-        ls -1 *".pkg.tar.xz" | tr '\n' '`' | sed 's#`#<br/>#g' >> "$webfile" 
-        printf "</td>" >> "$webfile"
-        for pkg in `find . -type f | egrep ".pkg.tar.xz$"`; do 
-            mv "$pkg" "$pkgdir"/"$suffix"; 
-            mv "$pkg"".sig" "$pkgdir"/"$suffix"; 
+        for pkg in `find . -type f | grep -E "${pkgExt}""\$"`; do
+            printf "<a href=\"/$suffix/$pkg\">$pkg</a><br/>" >> "$webfile"
+            pkgname="$(basename "$pkg" | cut -f 1 -d '.' | sed 's/-[[:digit:]]\+$//')"
+            find "${pkgdir}/${suffix}/" -name "${pkgname}-[0-9]*" -exec rm {} \;
+            mv "$pkg" "$pkgdir"/"$suffix";
+            mv "$pkg"".sig" "$pkgdir"/"$suffix";
         done
+        printf "</td>" >> "$webfile"
     fi
 }
 
@@ -76,20 +82,27 @@ function BuildRepo() {
     if [ ! -d "$repodir" ]; then
         git clone "$repo"
     fi
-    cd "$repodir" 
+    cd "$repodir"
     git reset --hard origin &>/dev/null
     output="$(git pull 2>&1)"
-    chown -R "depriv": .
-    
+    if [ -n "$incremental" ] && [ $( echo "$output" | grep -c 'Already up to date.' ) -eq 1 ]; then
+        return;
+    fi
+
+    chown -R "$deprivuser": .
     echo "$output" > "$pkgdir"/"$repodir".txt
     # Find the PKGBuilds in the repo
     for pkgbuild in `find . -type f -name PKGBUILD`; do
         cd "$(dirname "$pkgbuild")"
         # Tell the status file about it.
         printf '<tr style="border: 1px solid #FFF;"><td>'"<a href='$repo'>$repodir</a> -- $pkgbuild"'</td>' >> "$webfile"
-        if [ -f Makefile ] && [ `egrep -c '^test:' Makefile` -ge 1 ]; then
+        if [ -f Makefile ] && [ `grep -E -c '^test:' Makefile` -ge 1 ]; then
+            # Have to try to install dependencies first
+            for dep in $(grep makedepends PKGBUILD | cut -f 2 -d '(' | cut -f 1 -d ')' | sed "s/'//g"); do
+                pacman -Sy "$dep" --noconfirm --needed
+            done
             # Check test status.
-            timeout --preserve-status "$timeout" sudo -u "depriv" /bin/bash -l -c 'make test' &>> "$pkgdir"/"$repodir".txt
+            timeout --preserve-status "$timeout" sudo -u "$deprivuser" /bin/bash -l -c "cd $PWD; make test" &>> "$pkgdir"/"$repodir".txt
             if [ $? -ne 0 ]; then
                 # Testing failed.
                 printf "$failCell""$warnCell""$warnCell" >> "$webfile"
@@ -98,36 +111,36 @@ function BuildRepo() {
                 printf "$passCell" >> "$webfile"
                 BuildPackage "$suffix"
             fi
-        else 
+        else
             # Can't test -- usually from non-AniNIX repos.
             printf "$warnCell" >> "$webfile"
             BuildPackage "$suffix"
         fi
         # Timestamp
-        printf "<td><a href='/maat/$repodir.txt'>$(date +%F-%R)</a></td></tr>\n" >> "$webfile"
+        printf "<td><a href='/$repodir.txt'>$(date +%F-%R)</a></td></tr>\n" >> "$webfile"
         cd "$cwd"
         if [ ! -z "$testing" ]; then break; fi
     done
-    cd "$cwd" 
+    cd "$cwd"
 }
 
-### Update the local repo 
-function UpdateLocalRepo() { 
+### Update the local repo
+function UpdateLocalRepo() {
     set -x
-    cd "$pkgdir" 
-    chown -R "depriv": .
+    cd "$pkgdir"
+    chown -R "$deprivuser": .
     # TODO Add deduplication of updated files -- keep latest 3 versions.
-    rm -Rf AniNIX.[db,files]* 
-    sudo -u "depriv" repo-add --sign ./AniNIX.db.tar.xz `ls -1 *".pkg.tar.xz"`
+    rm -Rf AniNIX.[db,files]*
+    sudo -u "$deprivuser" repo-add --sign ./AniNIX.db.tar.zst `ls -1 *"${pkgExt}"`
     cd aur/
     rm -Rf aur.[db,files]*
-    sudo -u "depriv" repo-add --sign ./aur.db.tar.xz `ls -1 *".pkg.tar.xz"`
+    sudo -u "$deprivuser" repo-add --sign ./aur.db.tar.zst `ls -1 *"${pkgExt}"`
     set +x
 }
 
 # Clear variables
 aurconf='/usr/local/etc/Maat/aur.list'
-baseurl='https://foundation.aninix.net/AniNIX'
+baseurl='https://aninix.net/AniNIX'
 homedir="/srv/maat/"
 unset incremental
 unset skipPatching
@@ -138,7 +151,7 @@ timeout="90s"
 starttime=`date +%s`
 
 # Parse arguments
-while getopts 'b:c:hil:st:Tv' OPTION; do
+while getopts 'b:c:hil:st:Tu:v' OPTION; do
     case "${OPTION}" in
         b) homedir="${OPTARG}" ;;
         c) aurconf="${OPTARG}" ;;
@@ -147,15 +160,17 @@ while getopts 'b:c:hil:st:Tv' OPTION; do
         l) cmdstring="$0"; for arg in $@; do if [ "$arg" != "-l" ] && [ "$arg" != "${OPTARG}" ]; then cmdstring="$cmdstring \"${arg}\""; fi; done; exec /bin/bash -c "$cmdstring | tee -a \"${OPTARG}\""; ;;
         s) skipPatching=1 ;;
         t) timeout="${OPTARG}" ;;
-        T) export MAATTESTINGVAR=1; exec $0 -l ./testing.log -v -s -c <(echo https://aur.archlinux.org/cower.git) -b . ;;
+        T) export MAATTESTINGVAR=1; exec $0 -l ./testing.log -u "$deprivuser" -v -s -c <(echo https://aur.archlinux.org/ascii-invaders.git) -b . ;;
+        u) deprivuser="${OPTARG}" ;;
         v) set -x ;;
         *) usage; exit 1 ;;
     esac
 done
 
 # Ensure we are up to date -- otherwise, building is not a good plan.
-if [ -x `which pacman` ] && [ -z "$skipPatching" ]; then 
-    pacman -Syu --noconfirm;
+if [ -x `which pacman` ] && [ -z "$skipPatching" ]; then
+    pacman -Sc --noconfirm
+    pacman -Syu --noconfirm
     if [ $? -ne 0 ]; then
         echo "Self patching failed -- please investigate!" 1>&2
         exit 1
@@ -163,7 +178,7 @@ if [ -x `which pacman` ] && [ -z "$skipPatching" ]; then
 fi
 
 # Ensure work directories live
-if [ $( echo "$homedir" | egrep -c '^/') -ne 1 ]; then 
+if [ $( echo "$homedir" | grep -E -c '^/') -ne 1 ]; then
     homedir="${PWD}/${homedir}"
 fi
 
@@ -190,7 +205,7 @@ printf "$tableHead" >> "$webfile"
 
 # Build AUR
 for repo in `cat "$aurconf"`; do
-    BuildRepo "$repo" aur/;
+    BuildRepo "$repo" aur;
     if [ ! -z "$MAATTESTINGVAR" ]; then break; fi
 done
 

maat.service

@@ -1,12 +1,12 @@
 [Unit]
-Description=AniNIX::Maat \\ Builder Service
+Description=AniNIX/Maat | Builder Service
 ConditionPathExists=|!/srv/maat/pkg/.lock
 
 [Service]
 ExecStartPre=/usr/sbin/mkdir /srv/maat/pkg/.lock
-ExecStart=/usr/local/bin/maat-builder
+ExecStart=/usr/local/bin/maat
 ExecStopPost=/usr/sbin/rmdir /srv/maat/pkg/.lock
-Type=oneshot
+Type=simple
 RemainAfterExit=no
 User=root
 Group=root

maat.timer

@@ -1,5 +1,5 @@
 [Unit]
-Description=AniNIX::Maat \\ Builder Timer
+Description=AniNIX/Maat | Builder Timer
 
 [Timer]
 OnCalendar=daily