From 02ab50f585e2ee53cfac0251aa840acb86732bfb Mon Sep 17 00:00:00 2001 From: cxford Date: Wed, 2 Nov 2016 20:31:21 -0500 Subject: [PATCH] Major fixes for using standard libraries, protecting variable use, and torrent engine specification. --- 1337x.torrentengine.sample | 12 +- Makefile | 7 +- README | 13 ++ test-torrent-engine | 17 +++ torrentproject.torrentengine.sample | 12 +- tpb.torrentengine.sample | 6 + wolfpack | 208 +++++++++++----------------- 7 files changed, 135 insertions(+), 140 deletions(-) mode change 100644 => 100755 1337x.torrentengine.sample create mode 100644 README create mode 100755 test-torrent-engine create mode 100644 tpb.torrentengine.sample diff --git a/1337x.torrentengine.sample b/1337x.torrentengine.sample old mode 100644 new mode 100755 index 8d4090f..19461f7 --- a/1337x.torrentengine.sample +++ b/1337x.torrentengine.sample @@ -1,6 +1,6 @@ -torrentengine="https://1337x.to/search/%s/1/" -torrentterms='printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"' -magnetposition='4' -searchunifier='+' -magnetseparator=\" -torrentdebugging='true' +torrentengine=https://1337x.to/search/%s/1/ +torrentterms=printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \" +magnetposition=4 +torrentdebugging=true +searchunifier=+ +magnetseparator=" diff --git a/Makefile b/Makefile index cb1bbca..e1ac7ee 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ -compile: ./1337x.torrentengine.sample - if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; else git -C /usr/local/src/SharedLibraries pull; fi - if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi +compile: ./tpb.torrentengine.sample + if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi + git -C /usr/local/src/SharedLibraries pull; + if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi @echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection." install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip diff --git a/README b/README new file mode 100644 index 0000000..cc54939 --- /dev/null +++ b/README @@ -0,0 +1,13 @@ +=== AniNIX::WolfPack === +WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line + +Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices. + +Some tools included are: +* wolfpack: This is the core script, using .pup files to search for various things on the Internet. Pup types include shows, torrents, downloads, and searches. +* test-torrent-torrent-engine: A number of sample torrent engines have been provided to look at major sites. +** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation. +* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination. + +We have also included example "pups" and some helper scripts for updating torrent blocklists. + diff --git a/test-torrent-engine b/test-torrent-engine new file mode 100755 index 0000000..025a2ca --- /dev/null +++ b/test-torrent-engine @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "Need search terms" + exit 1; +fi + +source /usr/local/src/SharedLibraries/Bash/header +header Configuring... +if [ -z "$2" ]; then + configuretorrent /usr/local/etc/WolfPack/torrentengine +else + configuretorrent "$2" +fi + +header Getting magnet link. +getmagnetlink "$(echo $1 | tr '#' "$searchunifier")" diff --git a/torrentproject.torrentengine.sample b/torrentproject.torrentengine.sample index d2e064f..c8aceda 100644 --- a/torrentproject.torrentengine.sample +++ b/torrentproject.torrentengine.sample @@ -1,6 +1,6 @@ -torrentengine='http://torrentproject.se/?t=%s' -torrentterms="printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1" -magnetposition='2' -torrentdebugging='true' -searchunifier='+' -magnetseparator=\' +torrentengine=http://torrentproject.se/?t=%s +torrentterms=printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1 +magnetposition=2 +torrentdebugging=true +searchunifier=+ +magnetseparator=' diff --git a/tpb.torrentengine.sample b/tpb.torrentengine.sample new file mode 100644 index 0000000..030138a --- /dev/null +++ b/tpb.torrentengine.sample @@ -0,0 +1,6 @@ +torrentengine=https://thepiratebay.org/search/%s/0/99/0/ +torrentterms=printf https://thepiratebay.org; grep "/torrent/" | head -n 1 | cut -f 4 -d \" +magnetposition=4 +torrentdebugging=true +searchunifier=%20 +magnetseparator=" diff --git a/wolfpack b/wolfpack index 2bf97c4..98386f1 100755 --- a/wolfpack +++ b/wolfpack @@ -1,46 +1,46 @@ #!/bin/bash -source /usr/local/etc/WolfPack/torrentengine +source /usr/local/src/SharedLibraries/Bash/header +configuretorrent /usr/local/etc/WolfPack/torrentengine export IFS=" " export logfile='/var/log/wolfpack.log' - function helptext { echo ' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - /\____ xxxxxxxxxxxxx AniNIX::WolfPack - // ~ / _\_____ Let loose the wolves to hunt, run, - / \ .. \/ and be free. They shall repay you - // /~_____/ with their prey. - /// \\/ / ~dev@aninix.net + /\____ xxxxxxxxxxxxx AniNIX::WolfPack + // ~ / _\_____ Let loose the wolves to hunt, run, + / \ .. \/ and be free. They shall repay you +// /~_____/ with their prey. +/// \\/ / ~dev@aninix.net ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -wolfpack --alpha ~~ Tell the alpha to send the - pack members hunting. -wolfpack --member ~~ Send a member to train a pup - and bring back results. -wolfpack --stop-torrent ~~ Stop any transmission-cli - processes + wolfpack --alpha ~~ Tell the alpha to send the + pack members hunting. + wolfpack --member ~~ Send a member to train a pup + and bring back results. + wolfpack --stop-torrent ~~ Stop any transmission-cli + processes -Available pup types (all pups are line-delimited with the type and -result location as the first two lines): -torrent ~~ Torrent the result of the - search term. - Will remove search line on - success. -shows ~~ Download the episode in a - show and increment. Shows - are listed in format - S01E01#Show terms -download ~~ Takes a list of commands to - run to arrive at a URL to - grab and downloads the last - URL. -search ~~ Uses the Google search - engine to return the top - five results for the search - terms on each line. -' + Available pup types (all pups are line-delimited with the type and + result location as the first two lines): + torrent ~~ Torrent the result of the + search term. + Will remove search line on + success. + shows ~~ Download the episode in a + show and increment. Shows + are listed in format + S01E01#Show terms + download ~~ Takes a list of commands to + run to arrive at a URL to + grab and downloads the last + URL. + search ~~ Uses the Google search + engine to return the top + five results for the search + terms on each line. + ' } function privacycheck { @@ -57,7 +57,6 @@ function transmissioncheck { fi } - function torrent { if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then echo "Need a valid argument." @@ -71,17 +70,7 @@ function torrent { return; fi for i in $(tail -n +3 "$1"); do - export searchstring="$(echo $i | sed 's/ /+/g')" - export searchlink="$(printf "$torrentengine" "$searchstring")" - export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms") - export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | head -n 1 |cut -f $magnetposition -d $magnetseparator) - if [ $torrentdebugging == "true" ]; then - echo $searchstring - echo $searchlink - echo $searchresult - echo $magnetlink - read - fi + getmagnetlink "$i" if [ "$magnetlink" != "" ]; then /usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" sed -i "/$i/d" "$1" @@ -100,27 +89,18 @@ function shows { export downloaddir=$(head -n 2 "$1" | tail -n 1) echo "Would download to "$downloaddir for i in $(tail -n +3 "$1"); do - export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')" - export searchlink="$(printf "$torrentengine" "$searchsegment")" - export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms") - export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator) - if [ $torrentdebugging == "true" ]; then - echo $searchstring - echo $searchlink - echo $searchresult - echo $magnetlink - read - fi + getmagnetlink "$(echo $i | tr '#' "$searchunifier")" + export sepisode=$(echo $i | cut -f 1 -d '#') + export searchterms="$(echo $i | cut -f 2 -d '#')" + export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') + export episode=$(echo $sepisode | cut -f 2 -d 'E') + export newep=$(printf "%02d\n" $((${episode#0} + 1))) + export newseason="$(printf "%02d\n" $((${season#0} + 1)))" if [ "$magnetlink" != "" ]; then # We found an episode. echo $i | tee -a $logfile /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" - export sepisode=$(echo $i | cut -f 1 -d '#') - export searchterms="$(echo $i | cut -f 2 -d '#')" - export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') - export episode=$(echo $sepisode | cut -f 2 -d 'E') - export ep=$(printf "%02d\n" $((${episode#0} + 1))) - sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1" + sed -i "s/$i/S"$season"E"$newep"#"$searchterms"/" "$1" echo Downloaded and pup updated. | tee -a $logfile rm -Rf ~/.config/transmission/resume/* rm -Rf ~/.config/transmission/torrents/* @@ -129,30 +109,16 @@ function shows { echo | tee -a $logfile else # We didn't -- is there a new season? - export sepisode=$(echo $i | cut -f 1 -d '#') - export searchterms=$(echo $i | cut -f 2 -d '#') - export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') - export episode=$(echo $sepisode | cut -f 2 -d 'E') - export newseason=$(printf "%02d\n" $((${season#0} + 1))) - export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')" - export searchresult=$(wget -q -O - "$newsearch" | bash -c "$torrentterms") - export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator) - if [ $torrentdebugging == "true" ]; then - echo $newseason - echo $newsearch - echo $searchresult - echo $magnetlink - read - fi + getmagnetlink "$(echo $i | sed 's/S'$season'E'$episode'/S'$newseason'E01/' | tr '#' "$searchunifier")" if [ "$magnetlink" != "" ]; then echo "S"$newseason"E01" $searchterms | tee -a $logfile /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1" else - echo "No new season found for" $searchterms | tee -a $logfile + echo "No new season found for $searchterms" | tee -a $logfile fi - fi - done + fi + done } function download { @@ -194,65 +160,57 @@ function search { export searchstring=$(echo $i | sed 's/ /+/g') export file="$downloaddir"/$searchstring'.txt' echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file" - echo 'Ran search for '$searchstring' to '$file | tee -a $logfile + echo "Ran search for $searchstring to $file" | tee -a $logfile done } -function stoptorrent { - # TODO This is destructive to all other transmission-CLI clients on the box. - killall transmission-cli -} if [ "$1" == "" ]; then - helptext; - exit 1; + helptext; + exit 1; fi echo '---' $(date) '---' | tee -a $logfile echo Started "$1" "$2" on $(uname -n) | tee -a $logfile case "$1" in -"--alpha") - if [ "$2" == "" ]; then - export basedir="/usr/local/etc/WolfPack" - else - export basedir="$2"; - fi - echo "Starting members from pup in "$basedir | tee -a $logfile - for i in $(find "$basedir" -type f | grep '.pup'); do - /usr/local/bin/wolfpack --member "$i"; - done - ;; + "--alpha") + if [ "$2" == "" ]; then + export basedir="/usr/local/etc/WolfPack" + else + export basedir="$2"; + fi + echo "Starting members from pup in "$basedir | tee -a $logfile + for i in $(find "$basedir" -type f | grep '.pup'); do + /usr/local/bin/wolfpack --member "$i"; + done + ;; -"--member") - echo "Spawning a member for "$2 | tee -a $logfile - whatismyip | tee -a /var/log/wolfpack.log - case "$(head -n 1 "$2")" in - "shows") - shows "$2" - ;; - "torrent") - torrent "$2" - ;; - "search") - search "$2" - ;; - "download") - download "$2" - ;; - *) - echo "Bad pup type." - ;; - esac - ;; + "--member") + echo "Spawning a member for "$2 | tee -a $logfile + whatismyip | tee -a /var/log/wolfpack.log + case "$(head -n 1 "$2")" in + "shows") + shows "$2" + ;; + "torrent") + torrent "$2" + ;; + "search") + search "$2" + ;; + "download") + download "$2" + ;; + *) + echo "Bad pup type." + ;; + esac + ;; -"--stoptorrent") - stoptorrent - ;; - -*) - helptext - ;; + *) + helptext + ;; esac echo "Exited "$1 | tee -a $logfile