Major fixes for using standard libraries, protecting variable use, and torrent engine specification.
This commit is contained in:
cxford
parent
7425722bed
commit
02ab50f585
12
1337x.torrentengine.sample
Normal file → Executable file
12
1337x.torrentengine.sample
Normal file → Executable file
@ -1,6 +1,6 @@
|
||||
torrentengine="https://1337x.to/search/%s/1/"
|
||||
torrentterms='printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"'
|
||||
magnetposition='4'
|
||||
searchunifier='+'
|
||||
magnetseparator=\"
|
||||
torrentdebugging='true'
|
||||
torrentengine=https://1337x.to/search/%s/1/
|
||||
torrentterms=printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"
|
||||
magnetposition=4
|
||||
torrentdebugging=true
|
||||
searchunifier=+
|
||||
magnetseparator="
|
||||
|
||||
7
Makefile
7
Makefile
@ -1,6 +1,7 @@
|
||||
compile: ./1337x.torrentengine.sample
|
||||
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; else git -C /usr/local/src/SharedLibraries pull; fi
|
||||
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
|
||||
compile: ./tpb.torrentengine.sample
|
||||
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi
|
||||
git -C /usr/local/src/SharedLibraries pull;
|
||||
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
|
||||
@echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
|
||||
|
||||
install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip
|
||||
|
||||
13
README
Normal file
13
README
Normal file
@ -0,0 +1,13 @@
|
||||
=== AniNIX::WolfPack ===
|
||||
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line
|
||||
|
||||
Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices.
|
||||
|
||||
Some tools included are:
|
||||
* wolfpack: This is the core script, using .pup files to search for various things on the Internet. Pup types include shows, torrents, downloads, and searches.
|
||||
* test-torrent-torrent-engine: A number of sample torrent engines have been provided to look at major sites.
|
||||
** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation.
|
||||
* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination.
|
||||
|
||||
We have also included example "pups" and some helper scripts for updating torrent blocklists.
|
||||
|
||||
17
test-torrent-engine
Executable file
17
test-torrent-engine
Executable file
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Need search terms"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
source /usr/local/src/SharedLibraries/Bash/header
|
||||
header Configuring...
|
||||
if [ -z "$2" ]; then
|
||||
configuretorrent /usr/local/etc/WolfPack/torrentengine
|
||||
else
|
||||
configuretorrent "$2"
|
||||
fi
|
||||
|
||||
header Getting magnet link.
|
||||
getmagnetlink "$(echo $1 | tr '#' "$searchunifier")"
|
||||
@ -1,6 +1,6 @@
|
||||
torrentengine='http://torrentproject.se/?t=%s'
|
||||
torrentterms="printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1"
|
||||
magnetposition='2'
|
||||
torrentdebugging='true'
|
||||
searchunifier='+'
|
||||
magnetseparator=\'
|
||||
torrentengine=http://torrentproject.se/?t=%s
|
||||
torrentterms=printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1
|
||||
magnetposition=2
|
||||
torrentdebugging=true
|
||||
searchunifier=+
|
||||
magnetseparator='
|
||||
|
||||
6
tpb.torrentengine.sample
Normal file
6
tpb.torrentengine.sample
Normal file
@ -0,0 +1,6 @@
|
||||
torrentengine=https://thepiratebay.org/search/%s/0/99/0/
|
||||
torrentterms=printf https://thepiratebay.org; grep "/torrent/" | head -n 1 | cut -f 4 -d \"
|
||||
magnetposition=4
|
||||
torrentdebugging=true
|
||||
searchunifier=%20
|
||||
magnetseparator="
|
||||
100
wolfpack
100
wolfpack
@ -1,46 +1,46 @@
|
||||
#!/bin/bash
|
||||
|
||||
source /usr/local/etc/WolfPack/torrentengine
|
||||
source /usr/local/src/SharedLibraries/Bash/header
|
||||
configuretorrent /usr/local/etc/WolfPack/torrentengine
|
||||
export IFS="
|
||||
"
|
||||
export logfile='/var/log/wolfpack.log'
|
||||
|
||||
|
||||
function helptext {
|
||||
echo '
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
/\____ xxxxxxxxxxxxx AniNIX::WolfPack
|
||||
// ~ / _\_____ Let loose the wolves to hunt, run,
|
||||
/ \ .. \/ and be free. They shall repay you
|
||||
// /~_____/ with their prey.
|
||||
/// \\/ / ~dev@aninix.net
|
||||
// /~_____/ with their prey.
|
||||
/// \\/ / ~dev@aninix.net
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
wolfpack --alpha ~~ Tell the alpha to send the
|
||||
wolfpack --alpha ~~ Tell the alpha to send the
|
||||
pack members hunting.
|
||||
wolfpack --member <pup> ~~ Send a member to train a pup
|
||||
wolfpack --member <pup> ~~ Send a member to train a pup
|
||||
and bring back results.
|
||||
wolfpack --stop-torrent ~~ Stop any transmission-cli
|
||||
wolfpack --stop-torrent ~~ Stop any transmission-cli
|
||||
processes
|
||||
|
||||
Available pup types (all pups are line-delimited with the type and
|
||||
result location as the first two lines):
|
||||
torrent ~~ Torrent the result of the
|
||||
Available pup types (all pups are line-delimited with the type and
|
||||
result location as the first two lines):
|
||||
torrent ~~ Torrent the result of the
|
||||
search term.
|
||||
Will remove search line on
|
||||
success.
|
||||
shows ~~ Download the episode in a
|
||||
shows ~~ Download the episode in a
|
||||
show and increment. Shows
|
||||
are listed in format
|
||||
S01E01#Show terms
|
||||
download ~~ Takes a list of commands to
|
||||
download ~~ Takes a list of commands to
|
||||
run to arrive at a URL to
|
||||
grab and downloads the last
|
||||
URL.
|
||||
search ~~ Uses the Google search
|
||||
search ~~ Uses the Google search
|
||||
engine to return the top
|
||||
five results for the search
|
||||
terms on each line.
|
||||
'
|
||||
'
|
||||
}
|
||||
|
||||
function privacycheck {
|
||||
@ -57,7 +57,6 @@ function transmissioncheck {
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function torrent {
|
||||
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
|
||||
echo "Need a valid argument."
|
||||
@ -71,17 +70,7 @@ function torrent {
|
||||
return;
|
||||
fi
|
||||
for i in $(tail -n +3 "$1"); do
|
||||
export searchstring="$(echo $i | sed 's/ /+/g')"
|
||||
export searchlink="$(printf "$torrentengine" "$searchstring")"
|
||||
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
|
||||
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | head -n 1 |cut -f $magnetposition -d $magnetseparator)
|
||||
if [ $torrentdebugging == "true" ]; then
|
||||
echo $searchstring
|
||||
echo $searchlink
|
||||
echo $searchresult
|
||||
echo $magnetlink
|
||||
read
|
||||
fi
|
||||
getmagnetlink "$i"
|
||||
if [ "$magnetlink" != "" ]; then
|
||||
/usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
|
||||
sed -i "/$i/d" "$1"
|
||||
@ -100,27 +89,18 @@ function shows {
|
||||
export downloaddir=$(head -n 2 "$1" | tail -n 1)
|
||||
echo "Would download to "$downloaddir
|
||||
for i in $(tail -n +3 "$1"); do
|
||||
export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')"
|
||||
export searchlink="$(printf "$torrentengine" "$searchsegment")"
|
||||
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
|
||||
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
|
||||
if [ $torrentdebugging == "true" ]; then
|
||||
echo $searchstring
|
||||
echo $searchlink
|
||||
echo $searchresult
|
||||
echo $magnetlink
|
||||
read
|
||||
fi
|
||||
if [ "$magnetlink" != "" ]; then
|
||||
# We found an episode.
|
||||
echo $i | tee -a $logfile
|
||||
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
|
||||
getmagnetlink "$(echo $i | tr '#' "$searchunifier")"
|
||||
export sepisode=$(echo $i | cut -f 1 -d '#')
|
||||
export searchterms="$(echo $i | cut -f 2 -d '#')"
|
||||
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
|
||||
export episode=$(echo $sepisode | cut -f 2 -d 'E')
|
||||
export ep=$(printf "%02d\n" $((${episode#0} + 1)))
|
||||
sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1"
|
||||
export newep=$(printf "%02d\n" $((${episode#0} + 1)))
|
||||
export newseason="$(printf "%02d\n" $((${season#0} + 1)))"
|
||||
if [ "$magnetlink" != "" ]; then
|
||||
# We found an episode.
|
||||
echo $i | tee -a $logfile
|
||||
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
|
||||
sed -i "s/$i/S"$season"E"$newep"#"$searchterms"/" "$1"
|
||||
echo Downloaded and pup updated. | tee -a $logfile
|
||||
rm -Rf ~/.config/transmission/resume/*
|
||||
rm -Rf ~/.config/transmission/torrents/*
|
||||
@ -129,27 +109,13 @@ function shows {
|
||||
echo | tee -a $logfile
|
||||
else
|
||||
# We didn't -- is there a new season?
|
||||
export sepisode=$(echo $i | cut -f 1 -d '#')
|
||||
export searchterms=$(echo $i | cut -f 2 -d '#')
|
||||
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
|
||||
export episode=$(echo $sepisode | cut -f 2 -d 'E')
|
||||
export newseason=$(printf "%02d\n" $((${season#0} + 1)))
|
||||
export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')"
|
||||
export searchresult=$(wget -q -O - "$newsearch" | bash -c "$torrentterms")
|
||||
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
|
||||
if [ $torrentdebugging == "true" ]; then
|
||||
echo $newseason
|
||||
echo $newsearch
|
||||
echo $searchresult
|
||||
echo $magnetlink
|
||||
read
|
||||
fi
|
||||
getmagnetlink "$(echo $i | sed 's/S'$season'E'$episode'/S'$newseason'E01/' | tr '#' "$searchunifier")"
|
||||
if [ "$magnetlink" != "" ]; then
|
||||
echo "S"$newseason"E01" $searchterms | tee -a $logfile
|
||||
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
|
||||
sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1"
|
||||
else
|
||||
echo "No new season found for" $searchterms | tee -a $logfile
|
||||
echo "No new season found for $searchterms" | tee -a $logfile
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@ -194,14 +160,10 @@ function search {
|
||||
export searchstring=$(echo $i | sed 's/ /+/g')
|
||||
export file="$downloaddir"/$searchstring'.txt'
|
||||
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
|
||||
echo 'Ran search for '$searchstring' to '$file | tee -a $logfile
|
||||
echo "Ran search for $searchstring to $file" | tee -a $logfile
|
||||
done
|
||||
}
|
||||
|
||||
function stoptorrent {
|
||||
# TODO This is destructive to all other transmission-CLI clients on the box.
|
||||
killall transmission-cli
|
||||
}
|
||||
|
||||
if [ "$1" == "" ]; then
|
||||
helptext;
|
||||
@ -212,7 +174,7 @@ echo '---' $(date) '---' | tee -a $logfile
|
||||
echo Started "$1" "$2" on $(uname -n) | tee -a $logfile
|
||||
|
||||
case "$1" in
|
||||
"--alpha")
|
||||
"--alpha")
|
||||
if [ "$2" == "" ]; then
|
||||
export basedir="/usr/local/etc/WolfPack"
|
||||
else
|
||||
@ -224,7 +186,7 @@ case "$1" in
|
||||
done
|
||||
;;
|
||||
|
||||
"--member")
|
||||
"--member")
|
||||
echo "Spawning a member for "$2 | tee -a $logfile
|
||||
whatismyip | tee -a /var/log/wolfpack.log
|
||||
case "$(head -n 1 "$2")" in
|
||||
@ -246,11 +208,7 @@ case "$1" in
|
||||
esac
|
||||
;;
|
||||
|
||||
"--stoptorrent")
|
||||
stoptorrent
|
||||
;;
|
||||
|
||||
*)
|
||||
*)
|
||||
helptext
|
||||
;;
|
||||
esac
|
||||
|
||||
Loading…
Reference in New Issue
Block a user