AniNIX/WolfPack
AniNIX/WolfPack
3
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Modernizing WolfPack to use systemd.timer and systemd-cat

Browse Source
This commit is contained in:
DarkFeather 2020-12-22 09:46:21 -06:00
parent dfa79db9c7
commit ac28f8e28f
Signed by: DarkFeather
GPG Key ID: 1CC1E3F4ED06F296
12 changed files with 164 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*.tar.zst
pkg
src

53
Makefile
View File

@ -1,45 +1,30 @@
INSTALLER != curl -s https://aninix.net/foundation/installer-test.bash | /bin/bash
compile:
@echo Nothing to compile.
compile: ./tpb.torrentengine.sample
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi
git -C /usr/local/src/SharedLibraries pull;
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
if [ ! -x /usr/sbin/deluged ] || [ ! -x /usr/sbin/locate ]; then ${INSTALLER} deluge mlocate; fi
install: compile /usr/bin/wget /usr/bin/deluge-console /usr/bin/googler ./wolfpack /usr/local/bin/whatismyip /usr/bin/lynx
cp -p /usr/local/src/WolfPack/wolfpack /usr/local/bin/wolfpack
install: compile ./wolfpack ./offload-wolfpack ./1337x.torrentengine.sample
mkdir -p ${pkgdir}/usr/local/bin/
for i in wolfpack offload-wolfpack; do install -o root -g root -m 0755 $$i ${pkgdir}/usr/local/bin/$$i; done
mkdir -p ${pkgdir}/usr/lib/systemd/system
for i in *.service *.timer; do install -o root -g root -m 0755 $$i ${pkgdir}/usr/lib/systemd/system/$$i; done
mkdir -p ${pkgdir}/usr/local/etc/WolfPack
chmod 0770 ${pkgdir}/usr/local/etc/WolfPack
cp ./*.pup ${pkgdir}/usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample ${pkgdir}/usr/local/etc/WolfPack/torrentengine;
make checkperm
systemctl enable deluged
systemctl start deluged
@echo
@echo "Add pups to /usr/local/etc/WolfPack/ to set up your searches."
@echo "Add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
@echo WolfPack installed\!
clean:
@echo Nothing to do
test:
./test-torrent-engine 'S01E01 Tech-TV'
./wolfpack --member ./example-download.pup
./wolfpack --member ./example-search.pup
./wolfpack --member ./example-shows.pup
./wolfpack --member ./example-torrent.pup
@./test-torrent-engine 'S01E01 Tech-TV' ./1337x.torrentengine.sample
./wolfpack --help
reverse:
cp /usr/local/bin/wolfpack* .
cp /root/bin/offload-wolfpack .
for i in wolfpack offload-wolfpack; do cp /usr/local/bin/$$i .; done
for i in *.service *.timer; do cp /usr/lib/systemd/system/$$i .; done
checkperm: /usr/local/etc/WolfPack/
chmod -R ug+rw /usr/local/etc/WolfPack/
chmod 0770 /usr/local/etc/WolfPack/
chmod 0755 /usr/local/bin/wolfpack
chown root:root /usr/local/bin/wolfpack*
offload: offload-wolfpack /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
cp offload-wolfpack /root/bin
chmod 0700 /root/bin/offload-wolfpack
chown root:root /root/bin/offload-wolfpack
diff: /usr/local/bin/wolfpack
@echo wolfpack:; diff ./wolfpack /usr/local/bin/wolfpack; echo; if [ -f /root/bin/offload-wolfpack ]; then echo offload-wolfpack:; diff ./offload-wolfpack /root/bin/offload-wolfpack; echo; fi;
chmod -R ug+rw ${pkgdir}/usr/local/etc/WolfPack/
chmod 0770 ${pkgdir}/usr/local/etc/WolfPack/
chmod 0755 ${pkgdir}/usr/local/bin/*wolfpack
chown root:root ${pkgdir}/usr/local/bin/*wolfpack
chmod 0660 ${pkgdir}/usr/local/etc/WolfPack/*;

46
PKGBUILD Normal file
View File

@ -0,0 +1,46 @@
depends=('bash>=4.4' 'wget>=1.20' 'deluge>=2.0.4' 'elinks>=0.13' 'Uniglot')
makedepends=('make>=4.2')
checkdepends=()
optdepends=()
pkgname="$(git config remote.origin.url | rev | cut -f 1 -d '/' | rev | sed 's/.git$//')"
pkgver="$(git describe --tag --abbrev=0)"."$(git rev-parse --short HEAD)"
pkgrel=1
pkgrel() {
echo $(( `git log "$(git describe --tag --abbrev=0)"..HEAD | grep -c commit` + 1 ))
}
epoch="$(git log | grep -c commit)"
pkgdesc="$(head -n 1 README.md)"
arch=("x86_64")
url="$(git config remote.origin.url | sed 's/.git$//')"
license=('custom')
groups=()
provides=("${pkgname}")
conflicts=()
replaces=("${pkgname,,}", "aninix-${pkgname,,}")
backup=()
options=()
install=
changelog=
source=()
noextract=()
md5sums=()
validpgpkeys=()
prepare() {
git pull
}
build() {
make -C ..
}
check() {
chmod -R u+r ../pkg
make -C .. test
}
package() {
export pkgdir="${pkgdir}"
make -C .. install
install -D -m644 ../LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
}

7
README → README.md
View File

@ -1,5 +1,4 @@
=== AniNIX::WolfPack ===
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line.
Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices.
@ -9,5 +8,7 @@ Some tools included are:
** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation.
* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination.
We have also included example "pups" and some helper scripts for updating torrent blocklists.
We have also included example "pups" in this source.

12
offload-wolfpack
View File

@ -1,7 +1,6 @@
#!/bin/bash
set -x
logfile=/var/log/wolfpack.log
target="$1"
dir=`sudo -u deluge deluge-console config download_location 2>/dev/null | head -n 1 | sed 's/download_location: //'`
@ -10,22 +9,23 @@ if [ -z "$target" ]; then # || [ "$(ping -c 2 "$target" &>/dev/null; echo $?)" -
exit 1
fi
ssh $target mkdir -p /srv/yggdrasil/new_acquisition
sudo -u deluge deluge-console info 2>/dev/null | egrep '^\[S\]' | (
timeout 60 sudo -u deluge /bin/bash -l -c 'deluge-console info' 2>/dev/null | egrep '^\[S\]' | (
while read line; do
id="$(echo "$line" | awk '{ print $4; }')"
name="$(echo "$line" | awk '{ print $3 }')"
sudo -u deluge deluge-console 2>/dev/null rm "$id";
# Clean up downloads before sending.
rm -Rf "$dir"/"$name"/*.nfo "$dir"/"$name"/*.txt "$dir"/"$name"/*.png "$dir"/"$name"/*.jpg
rsync -avzz "$dir"'/'"$name" $target:/srv/yggdrasil/new_acquisition
ret="$?"
if [ "$ret" -eq 0 ]; then
# Hack because `deluged rm -c $id` isn't working
sudo systemctl stop deluged
sudo rm -Rf /srv/deluge/.config/deluge/state"$id";
sudo systemctl start deluged
rm -Rf "$dir"'/'"$name"
else
echo "ERROR: Return was $ret" >> "$dir"/"$name"/error.log
fi
unset name; unset id;
done
)
cat /var/log/wolfpack.log | ssh $target /bin/bash -c 'cat >> /var/log/wolfpack.log'
cat /dev/null > /var/log/wolfpack.log
) | systemd-cat -t "offload-wolfpack"

11
offload-wolfpack@.service Normal file
View File

@ -0,0 +1,11 @@
[Unit]
Description=AniNIX::WolfPack \\ Data Collection Botnet, Offload to %I
[Service]
ExecStartPre=/usr/sbin/sudo /usr/bin/systemctl start wolfpack.service
ExecStart=/usr/local/bin/offload-wolfpack %I
KillMode=process
Type=oneshot
RemainAfterExit=no
User=depriv
Group=depriv

10
offload-wolfpack@.timer Normal file
View File

@ -0,0 +1,10 @@
[Unit]
Description=AniNIX::WolfPack \\ Data Collection Botnet, Offload to %I (timer)
[Timer]
OnCalendar=00/4:00
AccuracySec=12h
Persistent=true
[Install]
WantedBy=timers.target

2
test-torrent-engine
View File

@ -4,7 +4,7 @@ if [ -z "$1" ]; then
exit 1;
fi
source /usr/local/src/SharedLibraries/Bash/header
source /opt/aninix/Uniglot/Bash/header
header Configuring...
if [ -z "$2" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine

18
update-transmission-blocklist
View File

@ -1,18 +0,0 @@
#!/bin/bash
if [ $(pgrep -afc $0) -ne 0 ]; then
echo Already running!
exit 1;
fi
cd /root/.config/transmission/blocklists/
rm -Rf *
wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz
gunzip blocklist.gz;
while [ "$?" -ne 0 ]; do
echo Error $?: Couldn\'t get blocklist.
wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz
gunzip blocklist.gz;
done
/usr/bin/transmission-cli -b notorrent # Update the blocklist to a .bin format

85
wolfpack
View File

@ -1,15 +1,10 @@
#!/bin/bash
source /usr/local/src/SharedLibraries/Bash/header
source /opt/aninix/Uniglot/Bash/header
export downloadtorrent="sudo -u deluge deluge-console 2>/dev/null add"
if [ -z "$3" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine
else
configuretorrent "$3"
fi
export IFS="
"
export logfile='/var/log/wolfpack.log'
export servicename='wolfpack'
function helptext {
echo '
@ -30,8 +25,6 @@ wolfpack --torrent-monitor ~~ Pull up the Deluge console
for torrents.
wolfpack --torrent-lookup ~~ Look up and add a torrent
if user confirms.
wolfpack --reset-log ~~ Clear the wolfpack log. This
is in /var/log/wolfpack.log.
wolfpack --help ~~ Show this helptext
Available pup types (all pups are line-delimited with the type and
@ -57,7 +50,7 @@ search ~~ Uses the Google search
function privacycheck {
ip="$(/usr/local/bin/whatismyip)"
if [ -z "$ip" ] || [ "$ip" == "$(nslookup aninix.net | grep Address | cut -f 2 -d ' ' | tail -n 1)" ]; then
echo "ERROR: Should not torrent directly from the AniNIX main IP for privacy reasons" 2>&1 | tee -a "$logfile"
echo "ERROR: Should not torrent directly from the AniNIX main IP for privacy reasons" 2>&1 | systemd-cat -t "$servicename"
exit 1
fi
}
@ -67,15 +60,25 @@ function nextairdate {
echo "Need a show to search for"
exit 1;
fi
lynx -read_timeout=5 -connect_timeout=5 --dump "https://www.episodate.com/tv-show/$(echo $1 | tr '[:upper:]' '[:lower:]' | tr ' ' '-')?season=99" 2>&1 | grep 'and airs on' | sed 's# Season [0-9]\+ / Episode [0-9]\+ and airs on ##'
timeout 5 elinks --dump "https://www.episodate.com/tv-show/$(echo $1 | tr '[:upper:]' '[:lower:]' | tr ' ' '-')?season=99" 2>&1 | grep 'and airs on' | sed 's# Season [0-9]\+ / Episode [0-9]\+ and airs on ##'
return
}
function torrentConfigure {
if [ -z "$3" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine
else
configuretorrent "$3"
fi
}
function torrent {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
return;
fi
torrentConfigure
privacycheck;
export downloaddir=$(head -n 2 "$1" | tail -n 1)
sudo -u deluge deluge-console config -s autoadd_location "$downloaddir" 2>/dev/null
@ -85,11 +88,11 @@ function torrent {
if [ "$magnetlink" != "" ]; then
/bin/bash -c "$downloadtorrent $magnetlink"
sed -i "/$i/d" "$1"
echo $magnetlink | tee -a "$logfile"
echo Added magnet to deluged and removed $i | tee -a "$logfile"
echo $magnetlink | systemd-cat -t "$servicename"
echo Added magnet to deluged and removed $i | systemd-cat -t "$servicename"
fi
done
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi
}
function shows {
@ -116,12 +119,12 @@ function shows {
fi
if [ "$magnetlink" != "" ]; then
# We found an episode.
echo $i | tee -a "$logfile"
echo $magnetlink | tee -a "$logfile"
echo $i | systemd-cat -t "$servicename"
echo $magnetlink | systemd-cat -t "$servicename"
/bin/bash -c "$downloadtorrent $magnetlink"
sed -i "s/$i/S${season}E${newep}#${showName}#${modifier}/" "$1"
echo Downloaded and pup updated. | tee -a "$logfile"
echo | tee -a "$logfile"
echo Downloaded and pup updated. | systemd-cat -t "$servicename"
echo | systemd-cat -t "$servicename"
else
# We didn't -- is there a new season?
if [ -z "$modifier" ]; then
@ -130,24 +133,24 @@ function shows {
getmagnetlink "$showName S${newseason}E01 $modifier"
fi
if [ "$magnetlink" != "" ]; then
echo "S${newseason}E01" $showName | tee -a "$logfile"
echo "$magnetlink" | tee -a "$logfile"
echo "S${newseason}E01" $showName | systemd-cat -t "$servicename"
echo "$magnetlink" | systemd-cat -t "$servicename"
/bin/bash -c "$downloadtorrent $magnetlink"
sed -i "s/$i/S${newseason}E02#$showName#$modifier/" "$1"
else
nextAirDate="$(nextairdate "$showName")"
if [ ! -z "$nextAirDate" ]; then
echo "No new season found for $showName -- next airs $(nextairdate "$showName")" | tee -a "$logfile"
echo "No new season found for $showName -- next airs $(nextairdate "$showName")" | systemd-cat -t "$servicename"
if [ "$(date -d "$nextAirDate" -u +%s)" -lt "$(date -u +%s)" ]; then
echo ERROR: Should have found a new episode for $searchterms and did not -- check your provider\! | tee -a "$logfile"
echo ERROR: Should have found a new episode for $searchterms and did not -- check your provider\! | systemd-cat -t "$servicename"
fi
else
echo "No new season found for $showName" | tee -a "$logfile"
echo "No new season found for $showName" | systemd-cat -t "$servicename"
fi
fi
fi
done
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi
}
function download {
@ -170,15 +173,15 @@ function download {
export filename="$downloaddir"/"$(echo "$nexturl" | rev | cut -f 1 -d \/ | rev)"
#if file doesn't already exist....
if [ ! -f "$filename" ]; then
echo Downloading $nexturl to $filename ... | tee -a "$logfile"
echo Downloading $nexturl to $filename ... | systemd-cat -t "$servicename"
for i in $(find "$downloaddir" -type f | grep -i "$dedupterm"); do
echo Deduped\! Removed $i from $downloaddir | tee -a "$logfile"
rm -f "$i" 2>&1 | tee -a "$logfile"
echo Deduped\! Removed $i from $downloaddir | systemd-cat -t "$servicename"
rm -f "$i" 2>&1 | systemd-cat -t "$servicename"
done
wget --timeout=5 -q -O "$filename" "$nexturl"
echo Downloaded | tee -a "$logfile"
echo Downloaded | systemd-cat -t "$servicename"
fi
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi
if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi
}
function search {
@ -191,7 +194,7 @@ function search {
export searchstring=$(echo $i | sed 's/ /+/g')
export file="$downloaddir"/$searchstring'.txt'
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
echo "Ran search for $searchstring to $file" | tee -a "$logfile"
echo "Ran search for $searchstring to $file" | systemd-cat -t "$servicename"
done
}
@ -200,25 +203,25 @@ if [ "$1" == "" ]; then
exit 1;
fi
echo '---' $(date) '---' | tee -a "$logfile"
echo '---' $(date) '---' | systemd-cat -t "$servicename"
case "$1" in
"--alpha")
echo Started alpha on `uname -n` | tee -a "$logfile"
echo Started alpha on `uname -n` | systemd-cat -t "$servicename"
if [ "$2" == "" ]; then
export basedir="/usr/local/etc/WolfPack"
else
export basedir="$2";
fi
echo "Starting members from pup in "$basedir | tee -a "$logfile"
echo "Starting members from pup in "$basedir | systemd-cat -t "$servicename"
for i in $(find "$basedir" -type f | grep '.pup'); do
/usr/local/bin/wolfpack --member "$i" "$3";
done
;;
"--member")
echo "Spawning a $(head -n 1 "$2") member for $2 on `uname -n`" | tee -a "$logfile"
(printf "IP: "; /usr/local/bin/whatismyip) | tee -a /var/log/wolfpack.log
echo "Spawning a $(head -n 1 "$2") member for $2 on `uname -n`" | systemd-cat -t "$servicename"
(printf "IP: "; /usr/local/bin/whatismyip) | systemd-cat -t "$servicename"
case "$(head -n 1 "$2")" in
"shows")
shows "$2"
@ -237,16 +240,12 @@ case "$1" in
;;
esac
;;
"--reset-log")
cat /dev/null > "$logfile"
exit 0;
;;
"--torrent-monitor")
exec sudo -u deluge deluge-console 2>/dev/null;
exit 0;
;;
"--torrent-lookup")
configuretorrent /usr/local/etc/WolfPack/torrentengine
torrentConfigure
getmagnetlink "$(echo $2 | tr '#' "$searchunifier")"
if [ -z "$magnetlink" ]; then
echo Nothing found;
@ -269,6 +268,6 @@ case "$1" in
;;
esac
echo "Exited "$1 | tee -a "$logfile"
echo '---' "$(date)" '---' | tee -a "$logfile"
echo | tee -a "$logfile"
echo "Exited "$1 | systemd-cat -t "$servicename"
echo '---' "$(date)" '---' | systemd-cat -t "$servicename"
echo | systemd-cat -t "$servicename"

11
wolfpack.service Normal file
View File

@ -0,0 +1,11 @@
[Unit]
Description=AniNIX::WolfPack \\ Data Collection Botnet
[Service]
ExecStart=/usr/local/bin/wolfpack --alpha
KillMode=process
Type=oneshot
TimeoutStartSec=1800
RemainAfterExit=no
User=depriv
Group=depriv

10
wolfpack.timer Normal file
View File

@ -0,0 +1,10 @@
[Unit]
Description=AniNIX::WolfPack \\ Data Collection Botnet (timer)
[Timer]
OnCalendar=00/4:00
AccuracySec=12h
Persistent=true
[Install]
WantedBy=timers.target