commit 1b596efb635dbd67beae9ed5499d41af422b5d7f Author: jonnyboy Date: Sun Dec 30 21:58:53 2012 -0500 first commit diff --git a/README.md b/README.md new file mode 100644 index 000000000..a902b8626 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# SETUP + + * Please backup your database first. Something like this should do it. + + `mysqldump --opt -u root -p newznab > ~/newznab_backup.sql` + + * This first step will modify your setup using [andrewit's github](https://github.com/itandrew/Newznab-InnoDB-Dropin.git) + * convert database to innodb, edit path if necessary + + `cd /var/www/newznab/misc/testing/` + + `git clone git://github.com/itandrew/Newznab-InnoDB-Dropin.git innodb` + + `php5 innodb/lib/innodb/convertToInnoDB.php` + + * Create a folder. + * Move to that folder. + * Clone my github + + `cd /var/www/newznab/misc/update_scripts/nix_scripts/` + + `git clone https://github.com/jonnyboy/newznab-tmux.git tmux` + + `cd tmux` + + `nano edit_these.sh` + + * Edit some permissions, run as root. + + `./set_perms.sh` + + * Run my script, as user. + + `./start.sh` + + * If you connect using putty, then under Window/Translation set Remote character set to UTF-8. + + * If something looks stalled, it probably isn't. If all 5 panes are still there, it is most likely, as it should be. + diff --git a/bin/cleanup_scripts.sh b/bin/cleanup_scripts.sh new file mode 100755 index 000000000..cb46ade77 --- /dev/null +++ b/bin/cleanup_scripts.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -e + +source ../edit_these.sh + +LASTOPTIMIZE1=`date +%s` +LASTOPTIMIZE2=`date +%s` +LASTOPTIMIZE3=`date +%s` +while : + + do + +#create releases from binaries +cd $NEWZNAB_PATH +[ -f $NEWZNAB_PATH/update_releases.php ] && $PHP $NEWZNAB_PATH/update_releases.php + +CURRTIME=`date +%s` +#every 2 hours and during first loop +DIFF=$(($CURRTIME-$LASTOPTIMIZE1)) +if [ "$DIFF" -gt 3600 ] || [ "$DIFF" -lt 1 ] +then + LASTOPTIMIZE1=`date +%s` + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/update_predb.php ] && $PHP $NEWZNAB_PATH/update_predb.php true + cd $TESTING_PATH + [ -f $TESTING_PATH/update_parsing.php ] && $PHP $TESTING_PATH/update_parsing.php + [ -f $TESTING_PATH/removespecial.php ] && $PHP $TESTING_PATH/removespecial.php + [ -f $TESTING_PATH/update_cleanup.php ] && $PHP $TESTING_PATH/update_cleanup.php +fi + +CURRTIME=`date +%s` +#every 12 hours +DIFF=$(($CURRTIME-$LASTOPTIMIZE2)) +if [ "$DIFF" -gt 43200 ] +then + LASTOPTIMIZE2=`date +%s` + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/optimise_db.php ] && $PHP $NEWZNAB_PATH/optimise_db.php +fi + +CURRTIME=`date +%s` +#every 12 hours and during 1st loop +DIFF=$(($CURRTIME-$LASTOPTIMIZE3)) +if [ "$DIFF" -gt 43200 ] || [ "$DIFF" -lt 1 ] +then + LASTOPTIMIZE3=`date +%s` + cd $NEWZNAB_PATH + #[ -f $NEWZNAB_PATH/optimise_db.php ] && $PHP $NEWZNAB_PATH/optimise_db.php true + [ -f $NEWZNAB_PATH/update_tvschedule.php ] && $PHP $NEWZNAB_PATH/update_tvschedule.php + [ -f $NEWZNAB_PATH/update_theaters.php ] && $PHP $NEWZNAB_PATH/update_theaters.php +fi + +echo "waiting $NEWZNAB_POST_SLEEP_TIME seconds..." +sleep $NEWZNAB_POST_SLEEP_TIME + +done + diff --git a/bin/config.php b/bin/config.php new file mode 100755 index 000000000..a79ff9112 --- /dev/null +++ b/bin/config.php @@ -0,0 +1,6 @@ + diff --git a/bin/innodb_import_full_threaded.sh b/bin/innodb_import_full_threaded.sh new file mode 100755 index 000000000..e0b5f544a --- /dev/null +++ b/bin/innodb_import_full_threaded.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +source ../edit_these.sh +eval $( sed -n "/^define/ { s/.*('\([^']*\)', '*\([^']*\)'*);/\1=\"\2\"/; p }" ../../../../../www/config.php ) + +export MYSQL_CMD="UPDATE groups set backfill_target=backfill_target+1 where active=1 and backfill_target<$MAXDAYS;" + +while : + do + + #import nzb's + cd $INNODB_PATH + [ -f $INNODB_PATH/nzb-import.php ] && $PHP $INNODB_PATH/nzb-import.php ${NZBS} & + + #make active groups current + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/update_binaries_threaded.php ] && $PHP $NEWZNAB_PATH/update_binaries_threaded.php & + + #get backfill for all active groups + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/backfill_threaded.php ] && $PHP $NEWZNAB_PATH/backfill_threaded.php & + + wait + + #increment backfill days + $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" + + echo "imports waiting $NEWZNAB_IMPORT_SLEEP_TIME seconds..." + sleep $NEWZNAB_IMPORT_SLEEP_TIME + +done + diff --git a/bin/innodb_import_threaded.sh b/bin/innodb_import_threaded.sh new file mode 100755 index 000000000..2cd8c859f --- /dev/null +++ b/bin/innodb_import_threaded.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +source ../edit_these.sh +eval $( sed -n "/^define/ { s/.*('\([^']*\)', '*\([^']*\)'*);/\1=\"\2\"/; p }" ../../../../../www/config.php ) + +export MYSQL_CMD="UPDATE groups set backfill_target=backfill_target+1 where active=1 and backfill_target<$MAXDAYS;" + +while : + do + + #import nzb's + cd $INNODB_PATH + [ -f $INNODB_PATH/nzb-import.php ] && $PHP $INNODB_PATH/nzb-import.php ${NZBS} & + + #make active groups current + cd $INNODB_PATH + [ -f $INNODB_PATH/update_binaries.php ] && $PHP $INNODB_PATH/update_binaries.php & + + #get backfill for all active groups + cd $INNODB_PATH + [ -f $INNODB_PATH/backfill.php ] && $PHP $INNODB_PATH/backfill.php & + + wait + + #increment backfill days + $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" + + echo "imports waiting $NEWZNAB_IMPORT_SLEEP_TIME seconds..." + sleep $NEWZNAB_IMPORT_SLEEP_TIME + +done + diff --git a/bin/monitor.php b/bin/monitor.php new file mode 100755 index 000000000..985ab7003 --- /dev/null +++ b/bin/monitor.php @@ -0,0 +1,63 @@ + + diff --git a/bin/myisam_import_full_threaded.sh b/bin/myisam_import_full_threaded.sh new file mode 100755 index 000000000..f7f1c4cd1 --- /dev/null +++ b/bin/myisam_import_full_threaded.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +source ../edit_these.sh +eval $( sed -n "/^define/ { s/.*('\([^']*\)', '*\([^']*\)'*);/\1=\"\2\"/; p }" ../../../../../www/config.php ) + +export MYSQL_CMD="UPDATE groups set backfill_target=backfill_target+1 where active=1 and backfill_target<$MAXDAYS;" + +while : + do + + #import nzb's + cd $ADMIN_PATH + [ -f $ADMIN_PATH/nzb-importmodified.php ] && $PHP $ADMIN_PATH/nzb-importmodified.php ${NZBS} & + + #make active groups current + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/update_binaries_threaded.php ] && $PHP $NEWZNAB_PATH/update_binaries_threaded.php & + + #get backfill for all active groups + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/backfill_threaded.php ] && $PHP $NEWZNAB_PATH/backfill_threaded.php & + + wait + + #increment backfill days + #$MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" + + echo "imports waiting $NEWZNAB_IMPORT_SLEEP_TIME seconds..." + sleep $NEWZNAB_IMPORT_SLEEP_TIME + +done + diff --git a/bin/myisam_import_threaded.sh b/bin/myisam_import_threaded.sh new file mode 100755 index 000000000..2ca5bf50d --- /dev/null +++ b/bin/myisam_import_threaded.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +source ../edit_these.sh +eval $( sed -n "/^define/ { s/.*('\([^']*\)', '*\([^']*\)'*);/\1=\"\2\"/; p }" ../../../../../www/config.php ) + +export MYSQL_CMD="UPDATE groups set backfill_target=backfill_target+1 where active=1 and backfill_target<$MAXDAYS;" + +while : + do + + #import nzb's + cd $ADMIN_PATH + [ -f $ADMIN_PATH/nzb-importmodified.php ] && $PHP $ADMIN_PATH/nzb-importmodified.php ${NZBS} & + + #make active groups current + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/update_binaries.php ] && $PHP $NEWZNAB_PATH/update_binaries.php & + + #get backfill for all active groups + cd $NEWZNAB_PATH + [ -f $NEWZNAB_PATH/backfill.php ] && $PHP $NEWZNAB_PATH/backfill.php & + + wait + + #increment backfill days + $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" + + echo "imports waiting $NEWZNAB_IMPORT_SLEEP_TIME seconds..." + sleep $NEWZNAB_IMPORT_SLEEP_TIME + +done + diff --git a/bin/postprocess_nfo.php b/bin/postprocess_nfo.php new file mode 100755 index 000000000..3b25241df --- /dev/null +++ b/bin/postprocess_nfo.php @@ -0,0 +1,36 @@ + 10) { + $postprocess = new PostProcess(true); + $postprocess->processNfos(); + } else { + echo "sleeping.....\n"; + sleep(10); + } +} + +mysql_free_result($result); + +?> + diff --git a/bin/postprocessing.php b/bin/postprocessing.php new file mode 100755 index 000000000..d67347769 --- /dev/null +++ b/bin/postprocessing.php @@ -0,0 +1,47 @@ +processTv(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processGames(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processMovies(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processMusic(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processBooks(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processUnwanted(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processMusicFromMediaInfo(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processOtherMiscCategory(); + } + { + $postprocess = new PostProcess(true); + $postprocess->processUnknownCategory(); + } +} + +?> diff --git a/bin/processAlternate.php b/bin/processAlternate.php new file mode 100755 index 000000000..912798a2d --- /dev/null +++ b/bin/processAlternate.php @@ -0,0 +1,12 @@ +processAdditional(); +} diff --git a/conf/.tmux.conf b/conf/.tmux.conf new file mode 100644 index 000000000..297f6613c --- /dev/null +++ b/conf/.tmux.conf @@ -0,0 +1,40 @@ +# C-b is not acceptable -- Vim uses it +set-option -g prefix C-a +bind-key C-a last-window + +# Start numbering at 1 +#set -g base-index 1 + +# Allows for faster key repetition +set -s escape-time 0 + +# Set status bar +set -g status-bg black +set -g status-fg white +set -g status-left "" +set -g status-right "#[fg=green]#H" + +# Rather than constraining window size to the maximum size of any client +# connected to the *session*, constrain window size to the maximum size of any +# client connected to *that window*. Much more reasonable. +setw -g aggressive-resize on + +# Allows us to use C-a a to send commands to a TMUX session inside +# another TMUX session +bind-key a send-prefix + +# Activity monitoring +setw -g monitor-activity on +set -g visual-activity on + +# # Refresh the status bar every 10 seconds. +#set-option -g status-interval 10000 + +# Example of using a shell command in the status line +#set -g status-right "#[fg=yellow]#(uptime | cut -d ',' -f 2-)" +set -g status-right "#[fg=red]#(ls -1 changeme | wc -l) NZB's left to process #[fg=yellow]#(uptime | cut -d ',' -f 2-)" +set-option -g status-right-length 200 +#set -g status-right '#[fg=green][#[fg=blue]%Y-%m-%d #[fg=white]%H:%M#[default] #($HOME/bin/battery)#[fg=green]]' + +# Highlight active window +set-window-option -g window-status-current-bg red diff --git a/edit_these.sh b/edit_these.sh new file mode 100755 index 000000000..35ca28054 --- /dev/null +++ b/edit_these.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +##EDIT THESE## + + +export NEWZNAB_PATH='/var/www/newznab/misc/update_scripts' +export INNODB_PATH='/var/www/newznab/misc/testing/innodb' +export TESTING_PATH='/var/www/newznab/misc/testing' +export ADMIN_PATH='/var/www/newznab/www/admin' +export NEWZPATH='/var/www/newznab' +export USERNAME='whats your name' # this is the user name that will run these scripts +export NEWZNAB_IMPORT_SLEEP_TIME='60' # in seconds - this includes import_nzb backfill and current fill +export NEWZNAB_POST_SLEEP_TIME='1' # in seconds - this is for post processing - sleep between loops +export MAXDAYS='200' #max days for backfill +export NZBS='/path/to/nzbs' #path to your nzb files to be imported + +#Choose to run the threaded or non-threaded newznab scripts true/false +export THREADS='true' + +#Choose your database, comment the one true/false +export INNODB='true' + +#By using this script you understand that the programmer is not responsible for any loss of data, users, or sanity. +#You also agree that you were smart enough to make a backup of your database and files. Do you agree? yes/no + +export AGREED='no' + +##END OF EDITS## + + +export MYSQL=`which mysql` +export SED=`which sed` +command -v php5 >/dev/null && export PHP=`which php5` || { export PHP=`which php`; } +command -v tmux >/dev/null && export TMUX=`which tmux` || { echo tmux command not found.\n apt-get install tmux\n or if on a real server yum install tmux; exit 1; } +#command -v bwm-ng >/dev/null && export BWMNG=`which bwm-ng` || { echo bwm-ng command not found.\n apt-get install bwm-ng; exit 1; } + diff --git a/set_perms.sh b/set_perms.sh new file mode 100755 index 000000000..053b3255b --- /dev/null +++ b/set_perms.sh @@ -0,0 +1,29 @@ +#!/bin/bash +source edit_these.sh + +if [ "$AGREED" == "no" ]; then + echo "Please edit the edit_these.sh file" + exit +fi + +cp $TESTING_PATH/nzb-importmodified.php $NEWZPATH/www/admin/ +cp conf/.tmux.conf /home/$USERNAME/.tmux.conf +$SED -i -e "s/'changeme',1/'$NZBPATH',0/" conf/.tmux.conf +$SED -i -e 's/$this->processAdditional();/\/\/$this->>processAdditional();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processNfos();/\/\/$this->processNfos();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processUnwanted();/\/\/$this->processUnwanted();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processMovies();/\/\/$this->processMovies();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processMusic();/\/\/$this->processMusic();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processBooks();/\/\/$this->processBooks();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processGames();/\/\/$this->processGames();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processTv();/\/\/$this->processTv();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processMusicFromMediaInfo();/\/\/$this->processMusicFromMediaInfo();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processOtherMiscCategory();/\/\/$this->processOtherMiscCategory();/' $NEWZPATH/www/lib/postprocess.php +$SED -i -e 's/$this->processUnknownCategory();/\/\/$this->processUnknownCategory();/' $NEWZPATH/www/lib/postprocess.php +chmod 777 $NEWZPATH/www/lib/smarty/templates_c +chmod -R 777 $NEWZPATH/www/covers +chmod 777 $NEWZPATH/www +chmod 777 $NEWZPATH/www/install +chmod -R 777 $NEWZPATH/nzbfiles +chown -R www-data:www-data /var/www/ +chmod 777 $NEWZPATH/nzbfiles/tmpunrar/ diff --git a/start.sh b/start.sh new file mode 100755 index 000000000..9c9a73424 --- /dev/null +++ b/start.sh @@ -0,0 +1,35 @@ +#!/bin/bash +source edit_these.sh + + +if [ "$AGREED" == "no" ]; then + echo "Please edit the edit_these.sh file" + exit +fi + +$TMUX new-session -d -s NewzNab -n NewzNab 'echo "processNfos Working......" && sleep 30 && $PHP bin/postprocess_nfo.php' +$TMUX selectp -t 0 +$TMUX splitw -v -p 82 'echo "monitor Working......" && $PHP bin/monitor.php' + + +if [ "$THREADS" == "true" -a "$INNODB" == "true" ]; then + $TMUX splitw -v -p 66 'cd bin && echo "imports Working......" && sleep 45 && ./innodb_import_full_threaded.sh' +elif [ "$THREADS" != "true" -a "$INNODB" == "true" ]; then + $TMUX splitw -v -p 66 'cd bin && echo "imports Working......" && sleep 45 && ./innodb_import_threaded.sh' +elif [ "$THREADS" == "true" -a "$INNODB" != "true" ]; then + $TMUX splitw -v -p 66 'cd bin && echo "imports Working......" && sleep 45 && ./myisam_import_full_threaded.sh' +else + $TMUX splitw -v -p 66 'cd bin && echo "imports Working......" && sleep 45 && ./myisam_import_threaded.sh' +fi + +$TMUX selectp -t 0 +$TMUX splitw -h -p 66 'echo "processAdditional Working......" && sleep 35 && $PHP bin/processAlternate.php' +$TMUX splitw -h -p 50 'echo "postProcessing Working......" && sleep 40 && $PHP bin/postprocessing.php' +$TMUX selectp -t 3 +$TMUX splitw -h -p 50 #'$BWMNG' +$TMUX selectp -t 5 +$TMUX splitw -h -p 50 'cd bin && echo "create Releases Working......" && ./cleanup_scripts.sh' + + +$TMUX select-window -tNewzNab:0 +$TMUX attach-session -d -tNewzNab