From c02bdf5ca4049839a6590b2396c1905ce5e4a174 Mon Sep 17 00:00:00 2001 From: jonnyboy Date: Sat, 30 Mar 2013 13:58:12 -0400 Subject: [PATCH] updated cj's scripts, fixed a few typos --- bin/monitor.php | 2 +- commit.sh | 6 +- conf/jonnyboys_defaults.sh | 534 ++++++++++++++++++ conf/jonnyboys_my.cnf | 138 +++++ config.sh | 2 +- nnscripts/config.php | 6 - nnscripts/lib/Getopt.php | 432 ++++++++++++++ nnscripts/lib/nnscripts.php | 591 ++++++++++++++++++++ nnscripts/nnscripts.php | 239 -------- nnscripts/remove_parts_without_releases.php | 256 ++++----- nnscripts/settings.ini | 35 ++ nnscripts/update_missing_movie_info.php | 136 ++--- 12 files changed, 1923 insertions(+), 454 deletions(-) create mode 100644 conf/jonnyboys_defaults.sh create mode 100644 conf/jonnyboys_my.cnf delete mode 100644 nnscripts/config.php create mode 100644 nnscripts/lib/Getopt.php create mode 100644 nnscripts/lib/nnscripts.php delete mode 100644 nnscripts/nnscripts.php create mode 100644 nnscripts/settings.ini diff --git a/bin/monitor.php b/bin/monitor.php index d907a45ac..843ab7a1c 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -2,7 +2,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); -$version="0.1r757"; +$version="0.1r758"; $db = new DB(); diff --git a/commit.sh b/commit.sh index e00d7883f..cd4d4ab52 100755 --- a/commit.sh +++ b/commit.sh @@ -4,9 +4,9 @@ commit=`git log | grep "^commit" | wc -l` commit=`expr $commit + 1` sed -i -e "s/\$version=.*$/\$version=\"0.1r$commit\";/" bin/monitor.php -cp -f /etc/mysql/my.cnf conf/my.cnf -cp -f defaults.sh conf/defaults.sh -sed -i -e 's/export SVN_PASSWORD=.*$/export SVN_PASSWORD="password"/' conf/defaults.sh +cp -f /etc/mysql/my.cnf conf/jonnyboys_my.cnf +cp -f defaults.sh conf/jonnyboys_defaults.sh +sed -i -e 's/export SVN_PASSWORD=.*$/export SVN_PASSWORD="password"/' conf/jonnyboys_defaults.sh git commit -a diff --git a/conf/jonnyboys_defaults.sh b/conf/jonnyboys_defaults.sh new file mode 100644 index 000000000..61814ff36 --- /dev/null +++ b/conf/jonnyboys_defaults.sh @@ -0,0 +1,534 @@ +#!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +#Copy this file to defaults.sh +#DO NOT EDIT THIS FILE + +#########################EDIT THESE######################### +############################################################ + +#This is the shutdown, true/false +#on, it runs, off and no scripts will be RESTARTED, when all panes are DEAD, killall tmux +#if this is set to false, the script will run 1 loop and terminate +export RUNNING="true" + +#these scripts set the 'nice'ness of each script, default is 19, the lowest, the highest is -20 +#anything between -1 and -20 require root/sudo to run +export NICENESS="10" + +#these scripts can add some serious load to your system, without proper monitoring it can be +#to much, you can set the max load that any pane will be started at +#for example, if you set load to 2, no pane will start when your system load exceeds 2 +#this does not mean the the desired load will not be exceeded, just that no panes will be be restarted +#this one is for all panes except update_releases +export MAX_LOAD="3.85" + +#this one is for update_releases +export MAX_LOAD_RELEASES="3.85" + +############################################################ + +#Set paths +export NEWZPATH="/var/www/newznab" + +#Should not need to change +export NEWZNAB_PATH=$NEWZPATH"/misc/update_scripts" +export TESTING_PATH=$NEWZPATH"/misc/testing" +export ADMIN_PATH=$NEWZPATH"/www/admin" + +############################################################ + +#Post Processing Additional is the post processing that downloads rar files and attempts to get info for your site +#you are able to set the number of process to be run from 1-32, remember that each process uses 1 of your nntp connections +#so, if you have 20, and you set this to 32, you will have errors, lots of errors, nfo lookup uses 1 -3 connections +#binaries and backfill threaded default up to 10 connections each and predb uses 1, so understand how many connections you are using when setting +#trial and error for this, set to 1 will run > 0, set to 2 will run > 200, 3 will run > 300 and so on. +#At some point, increasing this begins to slow things down. It will need to be adjusted for your system +#to get the desired performance, 0 will disable all post processing +export POST_TO_RUN="10" + +#or you can run them as they are run in stock, during update releases, this will disable all postprocessing windows and only run during update_releases +export ALMOST_STOCK="true" + +############################################################ + +#post processing per category, setting the above to 0 does not disable these +#run processNfos +export NFOS="true" + +#run processGames +export GAMES="true" + +#run processMovies +export MOVIES="true" + +#run processMusic +export MUSIC="true" + +#run processTV +export TVRAGE="true" + +#run processEbook +export EBOOK="true" + +#run processOther +export OTHERS="true" + +#run processUnwanted +export UNWANTED="true" + +############################################################ + +#Enter the session name to be used by tmux, no spaces allowed in the name, this can be changed after scripts start +#if you are running multiple servers, you could put your hostname here +export TMUX_SESSION="Ubuntu-dev" + +#Set, in seconds - how often the monitor.php (left top pane) script should update run the queries against the database +#the monitor script will update itself and each pane, once every 5 seconds plus the lagg time time on the loop the db is queried +#to press the point, the db is not queried any sooner than the time set below, but the script loops each iteration once every 5 seconds +#this makes it more responsive to stop/kill without slamming the db with needless queries, starts are still controlled by the queries +export MONITOR_UPDATE="30" + +#you may want to kill the update_binaries, backfill and import if no releases have been add in x minutes, set the timer to anything other than 0 to enable +#this will only run every 5 loops of monitor +export KILL_UPDATES="0" + +#do you want to just cause it to restart or keep off until a release is created, true for off until a release is created +#this will stop all downloads and imports +export KEEP_KILLED="false" + +############################################################ + +#You can have backfill loop constantly and interject binaries every so often +#by setting this next to true, if true, the normal backfill pane will be dead +#this works by setting the 2 start timers and which is run at the start of the loop is determined like this +#if at the start of the loop, the BINARIES_SEQ_TIMER has expired, then update_binaries will run and the BINARIES_SEQ_TIMER timer is reset +#if BINARIES_SEQ_TIMER has not expired, then if BACKFILL_SEQ_TIMER has expired, the backfill will run and BACKFILL_SEQ_TIMER timer is reset +export SEQUENTIAL="true" + +#time between loop start for update_binaries, in seconds, this is a countdown timer, not a sleep after it runs +#default is 30 minutes +#will run on first loop and then not again for at least 1800 seconds +export BINARIES_SEQ_TIMER="900" + +#this will not run on first loop, time between loop start for backfill, in seconds +#default is 10 seconds, this will run after time has expired, binaries will take precedence and run before this, if its time has expired +export BACKFILL_SEQ_TIMER="10" + +############################################################ + +#Choose to run update_binaries true/false +export BINARIES="true" + +#Choose to run the threaded or non-threaded newznab binaries scripts true/false +#update_binaries.php or update_binaries_threaded.php +export BINARIES_THREADS="true" + +#Set, in seconds - how long the update_binaries should sleep between runs +#top right pane +#sleep timers are not used when using SEQ +export BINARIES_SLEEP="900" + +#Set the max amount of binaries in the binaries table and still allow update_binaries to run +#set to 0 to disable +export BINARIES_MAX_BINS="5000" + +#Set the max amount of unprocessed releases and still allow update_binaries to run +#set to 0 to disable +export BINARIES_MAX_RELEASES="0" + +#Set the max amount of of rows in the parts table and still allow update_binaries to run +#set to 0 to disable +export BINARIES_MAX_ROWS="0" + +############################################################ + +#Choose to run backfill script true/false +export BACKFILL="true" + +#Choose to run the threaded or non-threaded newznab backfill scripts true/false +#backfill.php or backfill_threaded.php +export BACKFILL_THREADS="true" + +#Set, in seconds - how long the backfill should sleep between runs +#in pane below update_binaries +#sleep timers are not used when using SEQ +export BACKFILL_SLEEP="5" + +#Set the max amount of unprocessed releases and still allow backfill to run +#set to 0 to disable +export BACKFILL_MAX_RELEASES="0" + +#Set the max amount of binaries in the binaries table and still allow backfill to run +#set to 0 to disable +export BACKFILL_MAX_BINS="5000" + +#Set the max amount of of rows in the parts table and still allow backfill to run +#set to 0 to disable +export BACKFILL_MAX_ROWS="0" + +#Set the maximum days to backfill, you set the Newznab+ admin/edit backfill to 1 +#this will increment your database by 1 after each backfill loop +#once your backfill numbers reach $MAXDAYS, then it will no long increment the database +#backfill will continue to run, and do no work, at that point you should disable backfill, below +export MAXDAYS="1700" + +############################################################ + +#use kevin123's safer_backfill_parts.php instead of normal backfill or backfill threaded +#this is the script I use, it does 1 group at a time from z to a (wanted to start with tv groups first) 100k parts, +#then the script stops (once per loop), if your first_record_postdate on the group is 2012-06-24 +#it will be skipped (target reached). When that group is done, it will do another ( again from z to a). +#this does not use increment, it works by the date set below +#you also need to enable kevin's compression mod, those files are needed and you still need to enable BACKFILL +export KEVIN_SAFER="truef" + +#use kevin123's backfill_parts.php instead of normal backfill +export KEVIN_BACKFILL_PARTS="trued" + +#use kevin123's backfill_parts_threaded.php instead of normal +export KEVIN_THREADED="truek" + +#set the date to go back to, must be in the format of YYYY-MM-DD, like 2012-06-24 +export KEVIN_DATE="2008-08-15" + +#set the number of articles/headers to download at one time +export KEVIN_PARTS="1000000" + +############################################################ + +#Set the path to the nzb dump you downloaded from torrents, this is the path to bulk files folder of nzbs +#this does not recurse through subfolders, unless you set NZB_THREADS to true +export NZBS="/path/to/nzbs" + +#Choose to run import nzb script true/false +export IMPORT="false" + +#If, you have all of your nzbs in one folder select false +#If, you have all of you nzbs split into separate in with the root at $NZBS then select true +#and 10 nzbs will be imported from each subfolder per loop. +#Importing this way, allows all post processing scripts to run, such as book, music, movies +#Instead of doing all 1 type at once, spread the load +export NZB_THREADS="false" + +#Set max number of folders to process per loop. This includes empty folders. +export NZB_FOLDER_COUNT="20" + +#How many nzbs to import per loop, if using NZB_THREADS=true the per folder +export NZBCOUNT="10" + +#Set, in seconds - how long the nzb-import should sleep between runs +#below backfill +export IMPORT_SLEEP="40" + +#Set the max amount of unprocessed releases and still allow nzb-import to run +#set to 0 to disable +export IMPORT_MAX_RELEASES="0" + +#Set the max amount of of rows in the parts table and still allow nzb-import to run +#set to 0 to disable +export IMPORT_MAX_ROWS="0" + +#import nzbs using the filename as the release name true/false +export IMPORT_TRUE="false" + +############################################################ + +#MAX_RELEASES for each can be calculated on the total post processing or just the Misc category +#to calculate on just the Misc, enable this +export MISC_ONLY="true" + +############################################################ + +#Create releases, this is really only necessary to turn off when you only want to post process +export RELEASES="true" + +#Set, in seconds - how long the update_release should sleep between runs +#bottom right +export RELEASES_SLEEP="5" + +############################################################ + +#Choose to run optimize_innodb.php or optimize_mysiam.php script true/false +#set to false by default, you should test the optimize scripts in bin first +#optimize_myisam on small tables runs after every 5th loop of update_releases +export OPTIMIZE="truej" + +#optimize can wait, patiently while all other panes stop and then run +#or, forcefully terminate all panes while it runs, to kill all panes and run optimize, enable +export OPTIMIZE_KILL="true" + +#How often to run optimize_myisam on small tables seconds, default is 10 min +export MYISAM_SMALL="1800" + +#How often to run optimize_myisam on large tables seconds, default is 1 hr +export MYISAM_LARGE="3600" + +#How often to run optimize_innodb on small tables in seconds, default is 2 hr +export INNODB_SMALL="7200" + +#How often to run optimize_innodb on large tables in seconds, default is 48 hrs +export INNODB_LARGE="172800" + +############################################################ + +#Choose your database engine, comment the one true/false +#you should have already converted your database to InnoDB engine, if you select true here +export INNODB="false" + +############################################################ + +#Choose to run update_cleanup.php and removespecial.php true/false +#set to false by default, you will need to edit /misc/testing/update_cleanup.php and /misc/testing/update_parsing.php +#to actually do anything, directions are in the file +export CLEANUP="false" + +#edit update_cleanup.php and update_parsing.php and svn up, this will only mod files when you run scripts/update_svn.sh or scripts/fix_files.sh +export CLEANUP_EDIT="true" + +#How often do you want update_cleanup.php and removespecial.php to run, in seconds +export CLEANUP_TIMER="3600" + +############################################################ + +#Choose to run update_parsing.php true/false +#set to false by default, you will need to edit /misc/testing/update_parsing.php +#to actually do anything, directions are in the file +export PARSING="true" + +#choose to use kevin123's update_parsing script +#this also includes kevin123's categorymod.php, you must run either update_svn.sh or fix_files.sh to copy the file into place +export PARSING_MOD="true" + +#run update_parsing.php against the whole db or just the last 24 hours +export PAST_24_HOURS="false" + +#How often do you want update_parsing.php to run, in seconds. this takes alot of memory and processing time, default is every 12 hrs +export PARSING_TIMER="300" + +############################################################ + +#Choose to run update_predb.php +export PREDB="true" + +#How often to update the PreDB in seconds +export PREDB_TIMER="900" + +############################################################ + +#Choose to run processSpotnab.php +export SPOTNAB="true" + +#How often to update the PreDB in seconds +export SPOTNAB_TIMER="900" + +#automatically set ALL sources to active after retrieving sources +export SPOTNAB_ACTIVE="true" + +############################################################ + +#update the tv schedule and in theaters listings +export TV_SCHEDULE="true" + +#How often to update the TV Schedule and the In Theaters in seconds +export TVRAGE_TIMER="43200" + +############################################################ + +#Choose to run sphinx.php script true/false +#set to false by default, you should test the script first, php sphinx.php from the bin folder +export SPHINX="false" + +#How often to run sphinx in seconds +export SPHINX_TIMER="3600" + +############################################################ + +#mediainfo and ffmpeg can hang occasionally, set timer, in seconds, to anything other than 0 to enable +#it should not need to run longer that 120 seconds +export KILL_PROCESS="120" + +#look at man killall - if you have the -q option, enable this, otherwise leave it disabled +export KILL_QUIET="true" + +############################################################ + +#Delete parts and binaries older than retention days, which is set in edit - site +#this uses a script posted by cj https://github.com/NNScripts/nn-custom-scripts +export DELETE_PARTS="trued" + +#how often should this be run, default it 1 hr +export DELETE_TIMER="3600" + +#Releases may be added/edited with an imdb-id that does not exists in the movieinfo table. This script, update_missing_movie_info, +#will fetch all the missing imdb id's from the releases table. +export FETCH_MOVIE="true" + +#how often should this be run, default it 12 hr +export MOVIE_TIMER="43200" + +############################################################ + +#Specify your SED binary, if you are using freebsd or mac, you need to install gnu sed (gsed) and adjust the path +export SED="/bin/sed" +#export SED="/usr/local/bin/gsed" + +#freebsd, and maybe mac, does not contain SIGSTKFLT, SIGCLD, SIGPOLL, SIGPWR +#and powerprocess.php will error on one of those, but appears to work if they +#are commented out, enable to have the script comment them out while running update_svn +export FIX_POSIX="false" + +############################################################ + +#Select some monitoring script, if they are not installed, it will not affect the running of the scripts +#these are set to false by default, enable if you want them +export USE_HTOP="false" +export USE_BWMNG="false" +export USE_MYTOP="false" +export USE_VNSTAT="false" +export USE_ATOP="false" +export USE_NMON="false" +export USE_IOTOP="false" + +#freebsd does not have iotop, but can run top -m io -o total +export USE_TOP="false" + +export USE_IFTOP="false" +#select interface to listen, only 1 interface +export INTERFACE="eth0" + +#an additional window can be created manually with Ctrl-a c or it can be created at start of script +export USE_CONSOLE="false" + +############################################################ + +#Use powerline scripts to display the status bar +#To display properly, you need a modified font, download and install the font and then select that font in your terminal +#This is done on the terminal computer, not the newznab++ server +#download fonts from https://github.com/jonnyboy/powerline-fonts +#I recommend Consolas if you are using putty in Win7 +export POWERLINE="true" + +############################################################ + +#set your LANG to which ever you like, only effects these scripts +export LANG="en_US.UTF-8" + +#to help IMDB return only English titles, enable this, you will need to run update_svn.php or fix_files.sh +export EN_IMDB="true" + +############################################################ + +#newzdash is a web front end to show statistics of your Newznab+ install +#to use, you must first install from https://github.com/AlienXAXS/newzdash + +#this is YOUR shared key and allows these scripts to communicate with newzdash +export NEWZDASH_SHARED_SECRET="" + +#the url of your newzdash install, ensure it include HTTP:// or HTTPS:// or it will fail +#to disable leave blank ie. export NEWZDASH_URL="" +export NEWZDASH_URL="" + +########################################################### + +#Use tmpfs to run post processing on true/false +#to keep from running scripts as root, you can create your own ramdisk by adding the next line to /etc/fstab +#tmpfs /var/www/newznab/nzbfiles/tmpunrar1 tmpfs user,nodev,nodiratime,nosuid,size=256M,mode=777 0 0 +#edit the path, the path MUST be the path in site edit with a "1" appended to the end, like above +#you still need to set this to true or mount it manually as your user, not as root +export RAMDISK="true" + +#for freebsd, it is just a little different, you can either create the ramdisk and mount it by adding the next line to /etc/fstab +#tmpfs /var/www/newznab/nzbfiles/tmpunrar1 tmpfs rw,size=256M,mode=777 0 0 +#or, give users the permission to mount it by running sudo sysctl vfs.usermount=1 and then add the next line to /etc/fstab +#tmpfs /var/www/newznab/nzbfiles/tmpunrar1 tmpfs rw,noauto,size=256M,mode=777 0 0 + +############################################################ + +#set svn password, for use with scripts/update_svn.sh +#update_svn.sh is destructive, it update your version to match the svn version +export SVN_PASSWORD="password" + +#running update_svn as root will change file ownership of every file in the svn path +#to chown -R the path, enable and set user/group +#newznab/nzbfiles is not chown'd +export CHOWN_TRUE="true" + +#set CHOWN_TRUE="true" and WWW_USER="{youruser}:www-data" and run update_svn.sh or fix_files.sh +#and you will will not need root to run these scripts +export WWW_USER="jonnyboy:www-data" + +########################################################### + +#if you have a ramdisk and would like to monitor it's use, set path here +#this is not the same as RAMDISK above, I keep my parts table on a ramdisk +export RAMDISK_PATH="/var/ramdisk" + +########################################################### + +#logs can be written, per pane, to the logs folder +export WRITE_LOGS="truel" + +########################################################### + +#user defined scripts, you can define 3 scripts to run at specific times +#you must have them in the user_scripts, no other location +#This one will run before tmux creates the ui, so if you want to run something before the scripts add here +#it is assumed that the script is a bash script and has been chmod +x +export USER_DEF_ONE="" + +#this one will run before MyIsam Large +export USER_DEF_TWO="" + +#this one will run after MyIsam Large +export USER_DEF_THREE="" + +########################################################### + +#By using this script you understand that the programmer is not responsible for any loss of data, users, or sanity. +#You also agree that you were smart enough to make a backup of your database and files. Do you agree? yes/no +export AGREED="yes" + +############################################################ + +##END OF EDITS## + +command -v mysql >/dev/null 2>&1 || { echo >&2 "I require mysql but it's not installed. Aborting."; exit 1; } && export MYSQL=`command -v mysql` +command -v php5 >/dev/null 2>&1 && export PHP=`command -v php5` || { export PHP=`command -v php`; } +command -v tmux >/dev/null 2>&1 || { echo >&2 "I require tmux but it's not installed. Aborting."; exit 1; } && export TMUXCMD=`command -v tmux` +command -v nice >/dev/null 2>&1 || { echo >&2 "I require nice but it's not installed. Aborting."; exit 1; } && export NICE=`command -v nice` +command -v tee >/dev/null 2>&1 || { echo >&2 "I require tee but it's not installed. Aborting."; exit 1; } && export TEE=`command -v tee` + +if [[ $USE_HTOP == "true" ]]; then + command -v htop >/dev/null 2>&1|| { echo >&2 "I require htop but it's not installed. Aborting."; exit 1; } && export HTOP=`command -v htop` +fi +if [[ $USE_NMON == "true" ]]; then + command -v nmon >/dev/null 2>&1 || { echo >&2 "I require nmon but it's not installed. Aborting."; exit 1; } && export NMON=`command -v nmon` +fi +if [[ $USE_BWMNG == "true" ]]; then + command -v bwm-ng >/dev/null 2>&1|| { echo >&2 "I require bwm-ng but it's not installed. Aborting."; exit 1; } && export BWMNG=`command -v bwm-ng` +fi +if [[ $USE_IOTOP == "true" ]]; then + command -v iotop >/dev/null 2>&1|| { echo >&2 "I require iotop but it's not installed. Aborting."; exit 1; } && export IOTOP=`command -v iotop` +fi +if [[ $USE_TOP == "true" ]]; then + command -v top >/dev/null 2>&1|| { echo >&2 "I require top but it's not installed. Aborting."; exit 1; } && export TOP=`command -v top` +fi +if [[ $USE_MYTOP == "true" ]]; then + command -v mytop >/dev/null 2>&1|| { echo >&2 "I require mytop but it's not installed. Aborting."; exit 1; } && export MYTOP=`command -v mytop` +fi +if [[ $USE_VNSTAT == "true" ]]; then + command -v vnstat >/dev/null 2>&1|| { echo >&2 "I require vnstat but it's not installed. Aborting."; exit 1; } && export VNSTAT=`command -v vnstat` +fi +if [[ $USE_IFTOP == "true" ]]; then + command -v iftop >/dev/null 2>&1|| { echo >&2 "I require iftop but it's not installed. Aborting."; exit 1; } && export IFTOP=`command -v iftop` +fi +if [[ $USE_ATOP == "true" ]]; then + command -v atop >/dev/null 2>&1|| { echo >&2 "I require atop but it's not installed. Aborting."; exit 1; } && export ATOP=`command -v atop` +fi +if [[ $POWERLINE == "true" ]]; then + export TMUX_CONF="powerline/tmux.conf" +else + export TMUX_CONF="conf/tmux.conf" +fi + diff --git a/conf/jonnyboys_my.cnf b/conf/jonnyboys_my.cnf new file mode 100644 index 000000000..768ec8274 --- /dev/null +++ b/conf/jonnyboys_my.cnf @@ -0,0 +1,138 @@ +# +# The MySQL database server configuration file. +# +# You can copy this to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# +user = mysql +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /var/ramdisk +lc-messages-dir = /usr/share/mysql +skip-external-locking +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 127.0.0.1 +# +# * Fine Tuning +# +key_buffer = 16M +max_allowed_packet = 16M +thread_stack = 192K +thread_cache_size = 20 +group_concat_max_len = 8192 + +tmp_table_size = 512M +max_heap_table_size = 512M +table_open_cache = 2000 +low_priority_updates = 1 +concurrent_insert = ALWAYS +join_buffer_size = 2M +key_buffer_size = 128M +read_buffer_size = 8M + +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +myisam-recover = BACKUP +#max_connections = 100 +#table_cache = 64 +#thread_concurrency = 10 +# +# * Query Cache Configuration +# +query_cache_limit = 128M +query_cache_size = 1024M +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error log - should be very few entries. +# +log_error = /var/lib/mysql/error.log +# +# Here you can see queries with especially long duration +#log_slow_queries = /var/log/mysql/mysql-slow.log +#long_query_time = 2 +#log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#log_bin = /var/log/mysql/mysql-bin.log +expire_logs_days = 10 +max_binlog_size = 100M +#binlog_do_db = include_database_name +#binlog_ignore_db = include_database_name +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + + + +[mysqldump] +quick +quote-names +max_allowed_packet = 16M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completition + +[isamchk] +key_buffer = 16M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!includedir /etc/mysql/conf.d/ diff --git a/config.sh b/config.sh index b417220c7..592d6108c 100644 --- a/config.sh +++ b/config.sh @@ -318,7 +318,7 @@ export PREDB_TIMER="900" #Choose to run processSpotnab.php export SPOTNAB="false" -#How often to update the PreDB in seconds +#How often to update the SpotNab in seconds export SPOTNAB_TIMER="900" #automatically set ALL sources to active after retrieving sources diff --git a/nnscripts/config.php b/nnscripts/config.php deleted file mode 100644 index 9cc18abf4..000000000 --- a/nnscripts/config.php +++ /dev/null @@ -1,6 +0,0 @@ - - diff --git a/nnscripts/lib/Getopt.php b/nnscripts/lib/Getopt.php new file mode 100644 index 000000000..cd94dd7ad --- /dev/null +++ b/nnscripts/lib/Getopt.php @@ -0,0 +1,432 @@ + + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/** + * Getopt.PHP allows for easy processing of command-line arguments. + * It is a more powerful, object-oriented alternative to PHP's built-in getopt() function. + * + * @version 1.0 + * @link https://github.com/ulrichsg/getopt-php + */ +class Getopt { + + const NO_ARGUMENT = 0; + const REQUIRED_ARGUMENT = 1; + const OPTIONAL_ARGUMENT = 2; + + /** @var string */ + protected $scriptName; + /** @var array */ + protected $optionList = array(); + /** @var array */ + protected $options = array(); + /** @var array */ + protected $operands = array(); + + /** + * Create a new Getopt object. + * + * The argument $options can be either a string in the format accepted by the PHP library + * function getopt() or an array + * + * @param mixed $options Array of options, a String, or null + * + * @link https://www.gnu.org/s/hello/manual/libc/Getopt.html GNU Getopt manual + */ + public function __construct($options = null) { + if ($options !== null) { + $this->addOptions($options); + } + } + + /** + * Evaluate the given arguments. These can be passed either as a string or as an array. + * If nothing is passed, the running script's command line arguments are used. + * + * A {@link \UnexpectedValueException} or {@link \InvalidArgumentException} is thrown + * when the arguments are not well-formed or do not conform to the options passed by the user. + * + * @param mixed $arguments optional ARGV array or space separated string + * + * @return void + */ + public function parse($arguments = null) { + $this->options = array(); + if (!isset($arguments)) { + global $argv; + $arguments = $argv; + $this->scriptName = array_shift($arguments); // $argv[0] is the script's name + } elseif (is_string($arguments)) { + $this->scriptName = $_SERVER['PHP_SELF']; + $arguments = explode(' ', $arguments); + } + + $num_args = count($arguments); + for ($i = 0; $i < $num_args; ++$i) { + $arg = trim($arguments[$i]); + if (empty($arg)) { + continue; + } + if ($arg == '--' || mb_substr($arg, 0, 1) != '-') { + // no more options, treat the remaining arguments as operands + $first_operand_index = $arg == '--' ? $i + 1 : $i; + $this->operands = array_slice($arguments, $first_operand_index); + break; + } + if (mb_substr($arg, 0, 2) == '--') { + // long option + $option = mb_substr($arg, 2); + if (strpos($option, '=') === false) { + if ($i < $num_args - 1 + && mb_substr($arguments[$i + 1], 0, 1) != '-' + && $this->optionHasArgument($option, true)) { + $value = $arguments[$i + 1]; + ++$i; + } else { + $value = null; + } + } else { + list($option, $value) = explode('=', $option, 2); + } + $this->addOption($option, $value, true); + } else { + // short option + $option = mb_substr($arg, 1); + if (mb_strlen($option) > 1) { + // multiple options strung together + foreach ($this->mb_str_split($option, 1) as $ch) { + $this->addOption($ch, null, false); + } + } else { + if ($i < $num_args - 1 + && mb_substr($arguments[$i + 1], 0, 1) != '-' + && $this->optionHasArgument($option, false)) { + $value = $arguments[$i + 1]; + ++$i; + } else { + $value = null; + } + $this->addOption($option, $value, false); + } + } + } // endfor + + // remove '--' from operands array + $operands = array(); + foreach($this->operands as $operand) { + if ($operand !== '--') { + $operands[] = $operand; + } + } + $this->operands = $operands; + } + + /** + * Return the value of the given option. Must be invoked after parse(). + * + * The return value can be any of the following: + * + * + * @param string $name The (short or long) option. + * + * @return mixed + */ + public function getOption($name) { + return isset($this->options[$name]) ? $this->options[$name] : null; + } + + /** + * Return all the options. Must be invoked after parse(). + * + * Will return an empty array if called before parse() is called. + * + * @return array + */ + public function getOptions() { + return $this->options; + } + + /** + * Prints help message based on the available options. + * + * @param int $padding Number of characters to pad output of options to + */ + public function showHelp($padding = 25) { + echo $this->getHelpText($padding); + } + + /** + * @param int $padding Number of characters to pad output of options to + * + * @return string help message for given options. + */ + public function getHelpText($padding = 25) { + $help_text = sprintf("Usage: php %s [options] [operands]\n", $this->scriptName); + $help_text .= "Options:\n"; + foreach ($this->optionList as $name => $option) { + @list($short, $long, $arg, $description) = $option; + switch ($arg) { + case self::NO_ARGUMENT: $arg = ''; break; + case self::REQUIRED_ARGUMENT: $arg = ""; break; + case self::OPTIONAL_ARGUMENT: $arg = "[]"; break; + } + $short = ($short) ? '-'.$short : ''; + $long = ($long) ? '--'.$long : ''; + if ($short && $long) { + $options = $short.', '.$long; + } else if ($short) { + $options = $short; + } else { + $options = $long; + } + $padded = str_pad(sprintf(" %s %s", $options, $arg), $padding); + $help_text .= sprintf("%s %s\n", $padded, $description); + } + return $help_text; + } + + + /** + * Return the list of operands. Must be invoked after parse(). + * + * @return array + */ + public function getOperands() { + return $this->operands; + } + + /** + * Parse an option string. + * + * @param string $string the option string + * + * @throws \InvalidArgumentException + * @return array + * @internal + */ + protected function parseOptionString($string) { + if (!mb_strlen($string)) { + throw new \InvalidArgumentException('Option string must not be empty'); + } + $option_list = array(); + $eol = mb_strlen($string) - 1; + $next_can_be_colon = false; + for ($i = 0; $i <= $eol; ++$i) { + $ch = $string[$i]; + if (!preg_match('/^[A-Za-z]$/', $ch)) { + $colon = $next_can_be_colon ? " or ':'" : ''; + throw new \InvalidArgumentException("Option string is not well formed: " + . "expected a letter$colon, found '$ch' at position " . ($i + 1)); + } + if ($i == $eol || $string[$i + 1] != ':') { + $option_list[] = array($ch, null, self::NO_ARGUMENT); + $next_can_be_colon = true; + } elseif ($i < $eol - 1 && $string[$i + 2] == ':') { + $option_list[] = array($ch, null, self::OPTIONAL_ARGUMENT); + $i += 2; + $next_can_be_colon = false; + } else { + $option_list[] = array($ch, null, self::REQUIRED_ARGUMENT); + ++$i; + $next_can_be_colon = true; + } + } + return $option_list; + } + + /** + * Check that the argument conforms to Getopt.PHP's option array rules. + * Throws an exception on failure. + * + * @param array $options the option list + * + * @throws \InvalidArgumentException + * @return array the validated options array + * @internal + */ + protected function validateOptions(array $options) { + $valid_argument_specs = array( + self::NO_ARGUMENT, self::OPTIONAL_ARGUMENT, self::REQUIRED_ARGUMENT + ); + if (empty($options)) { + throw new \InvalidArgumentException('No options given'); + } + foreach ($options as $option) { + if (!is_array($option) || count($option) < 3) { + throw new \InvalidArgumentException("Too few fields in argument, must be 3 (short/long/type)"); + } + if (!(is_null($option[0]) || preg_match("/^[a-zA-Z]$/", $option[0]))) { + throw new \InvalidArgumentException("First component of option must be " + . "null or a letter, found '" . $option[0] . "'"); + } + if (!(is_null($option[1]) || preg_match("/^[a-zA-Z0-9_-]*$/", $option[1]))) { + throw new \InvalidArgumentException("Second component of option must be " + . "null or an alphanumeric string, found '" . $option[1] . "'"); + } + if (!mb_strlen($option[0]) && !mb_strlen($option[1])) { + throw new \InvalidArgumentException("The short and long name of an option must not both be empty"); + } + if (!in_array($option[2], $valid_argument_specs, true)) { + throw new \InvalidArgumentException("Third component of option must be one of " + . "Getopt::NO_ARGUMENT, Getopt::OPTIONAL_ARGUMENT and Getopt::REQUIRED_ARGUMENT"); + } + if (!isset($option[3])) { + $option[3] = ""; // description + } + } + return $options; + } + + /** + * Add an option to the list of known options. + * + * @param string $option the option's name + * @param string $value the option's value (or null) + * @param boolean $is_long whether the option name is long or short + * + * @throws \UnexpectedValueException + * @return void + * @internal + */ + protected function addOption($option, $value, $is_long) { + foreach ($this->optionList as $opt) { + if (($is_long && $opt[1] == $option) || (!$is_long && $opt[0] == $option)) { + if ($opt[2] == self::REQUIRED_ARGUMENT && !mb_strlen($value)) { + throw new \UnexpectedValueException("Option '$option' must have a value"); + } + // for no-argument options, check if they are duplicate + if ($opt[2] == self::NO_ARGUMENT) { + $old_value = $this->getOption($option); + $value = is_null($old_value) ? 1 : $old_value + 1; + } + // for optional-argument options, set value to 1 if none was given + if (!mb_strlen($value)) { + $value = 1; + } + // add both long and short names (if they exist) to the option array to facilitate lookup + if (mb_strlen($opt[0]) > 0) { + $this->options[$opt[0]] = $value; + } + if (mb_strlen($opt[1]) > 0) { + $this->options[$opt[1]] = $value; + } + return; + } + } + throw new \UnexpectedValueException("Option '$option' is unknown"); + } + + /** + * Return true if the given option can take an argument, false if it can't or is unknown. + * + * @param string $name the option's name + * @param boolean $is_long whether it is a long option + * + * @return boolean + * @internal + */ + protected function optionHasArgument($name, $is_long) { + foreach ($this->optionList as $option) { + if ((!$is_long && $option[0] == $name) + || ($is_long && $option[1] == $name)) { + return $option[2] != self::NO_ARGUMENT; + } + } + return false; + } + + /** + * Return the option list used by this Getopt. + * + * This function is used for testing. It is not meant for production use. + * + * @return array + * @internal + */ + public function getOptionList() { + return $this->optionList; + } + + /** + * Parses and Adds options + * The argument $options can be either a string in the format accepted by the PHP library + * function getopt() or an array + * + * @param mixed $options Array of options, a String, or null + * @throws \InvalidArgumentException + * @return array + */ + public function addOptions($options) + { + if (is_string($options)) { + return $this->addParsedOptions($this->parseOptionString($options)); + } + + if (is_array($options)) { + return $this->addParsedOptions($this->validateOptions($options)); + } + + throw new \InvalidArgumentException("Getopt(): argument must be string or array"); + } + + /** + * Merges new options with the ones already in the Getopt optionList. + * + * @param array $options The array from parsing from parseOptionString() or validateOptions() + * + * @return array + * @internal + */ + protected function addParsedOptions (array $options) + { + return $this->optionList = array_merge($this->optionList, $options); + } + + /** + * @param string $str string to split + * @param int $l + * + * @return string + * @internal + */ + protected function mb_str_split($str, $l = 0) + { + if ($l > 0) { + $ret = array(); + $len = mb_strlen($str, "UTF-8"); + for ($i = 0; $i < $len; $i += $l) { + $ret[] = mb_substr($str, $i, $l, "UTF-8"); + } + + return $ret; + } + + return preg_split("//u", $str, -1, PREG_SPLIT_NO_EMPTY); + } +} + diff --git a/nnscripts/lib/nnscripts.php b/nnscripts/lib/nnscripts.php new file mode 100644 index 000000000..66d048e76 --- /dev/null +++ b/nnscripts/lib/nnscripts.php @@ -0,0 +1,591 @@ +checkPHPVersion(); + + // Build the site variable + $site = new Sites(); + $this->site = $site->get(); + + // Initiate the database connection + $this->db = new DB(); + + // Read the settings from the ini file + $this->readSettings(); + + // Set the limit + $this->setLimit(); + + // Add the "now" date + $date = new DateTime(); + $this->settings['now'] = $date->format('Y-m-d H:i:s'); + } + + + /** + * Destructor + * + */ + public function __destruct() + { + if( isset( $_SERVER["HTTP_HOST"] ) && true === $this->settings['display'] ) + { + echo ''; + } + } + + + /** + * Check php version to be 5.3.10 or greater + * + * @throws Exception + * @return void + */ + protected function checkPHPVersion() + { + // Init + $current = ( false === strpos( phpversion(), '-' ) ? phpversion() : substr( phpversion(), 0, strpos( phpversion(), '-' ) ) ); + $required = '5.3.10'; + + if( 0 > strnatcmp( $current, $required ) ) + { + throw new Exception( sprintf( 'NNScripts'. PHP_EOL .'Error: PHP version %s is required but version %s is found.', $required, $current ) ); + } + } + + + /** + * Set the limit in the settings + * The limit is read from the site setting "rawretentiondays" + * + * @return void + */ + protected function setLimit() + { + $limit = ( isset( $this->settings['limit'] ) ? $this->settings['limit'] : null ); + if( null === $limit && in_array( 'limit', $this->allowedSettings ) ) + { + // Not set by settings.ini + if( isset( $this->site ) && isset( $this->site->rawretentiondays ) && ( !isset($this->nolimit) || true !== $this->nolimit ) ) + { + // Get the limit and convert to hours + $limit = (float)str_replace( ',', '.', $this->site->rawretentiondays ) * 24; + } + + // Set the new limit + $this->settings['limit'] = $limit; + } + } + + + /** + * Read the settings from the config.ini fille + * + * @return void + */ + protected function readSettings() + { + // The default settings + $settings = array( + 'display' => true, // Display output + 'remove' => null, // Remove or keep releases/data + 'limit' => null, // Limit + 'full' => null, // Full database + 'debug' => null, // Debug mode + 'querylimit' => 10000 // Query limit (used in remove_parts_without_releases) + ); + + // Try to read the ini file + $settingsFile = "settings.ini"; + $iniArray = $this->parseIniFile( $settingsFile, true ); + + // Parse the "global" settings + $this->parseIniSettings( $settings, $iniArray, 'global' ); + + // Parse the class settings + $this->parseIniSettings( $settings, $iniArray, get_class( $this ) ); + + // Remove all the not used settings + if( is_array( $this->allowedSettings ) && 0 < count( $this->allowedSettings ) ) + { + $allowed = $this->allowedSettings; + array_map( function( $key ) use ( &$settings, $allowed ) { + if( !in_array( $key, $allowed ) ) + { + unset( $settings[ $key ] ); + } + }, array_keys( $settings ) ); + } + + // Return + $this->settings = $settings; + } + + + /** + * Parse ini settings + * + * @param array $settings + * @param array $iniArray + * @param string $key + * @return void + */ + protected function parseIniSettings( array &$settings, array $iniArray, $key ) + { + // Parse the global settings + if( array_key_exists( $key, $iniArray ) && is_array( $iniArray[ $key ] ) ) + { + foreach( array_keys( $settings ) AS $settingKey ) + { + if( array_key_exists( $settingKey, $iniArray[ $key ] ) ) + { + $value = $iniArray[ $key ][ $settingKey ]; + if( is_numeric( $value ) ) + { + $value = ( preg_match('/^[0-9]+$/', $value ) ? (int)$value : (float)$value ); + } + $settings[ $settingKey ] = $value; + } + } + } + } + + + /** + * Parse the ini file + * + * @param string $settingsFile + * @return array + */ + protected function parseIniFile( $settingsFile ) + { + // Init + $ret = array(); + + if( file_exists( $settingsFile ) && is_readable( $settingsFile ) ) + { + $ini = file( $settingsFile ); + + if ( 0 === count( $ini ) ) + return $ret; + + $sections = $values = $globals = array(); + $i = 0; + + foreach( $ini as $line ) + { + $line = trim( $line ); + + // Comments + if ( '' === $line || ';' === $line{0} ) + continue; + + // Sections + if ( '[' === $line{0} ) + { + $sections[] = substr( $line, 1, -1 ); + $i++; + continue; + } + + // Key-value pair + list( $key, $value ) = explode( '=', $line, 2 ); + $key = trim( $key ); + $value = trim( $value ); + + // True / Fasle + if( "true" === $value ) + $value = true; + if( "false" === $value ) + $value = false; + + if ( 0 === $i ) + { + // Array values + if ( '[]' === substr( $line, -1, 2 ) ) + $globals[ $key ][] = $value; + else + $globals[ $key ] = $value; + } + else + { + // Array values + if ( '[]' === substr( $line, -1, 2 ) ) + $values[ $i - 1 ][ $key ][] = $value; + else + $values[ $i - 1 ][ $key ] = $value; + } + } + + for( $j=0; $j<$i; $j++ ) + { + if( array_key_exists( $j, $sections ) && array_key_exists( $j, $values ) ) + { + $ret[ $sections[ $j ] ] = $values[ $j ]; + } + } + + return $ret + $globals; + } + + // Return + return $ret; + } + + + /** + * Process the cli options + * + * @param array $options + * @param bool $addLimit + * @param bool $addRemove + * @param bool $addDebug + * @param bool $addHelp + * @internal param bool $addNoLimit + * @return void + */ + protected function setCliOptions( array $options, array $basics=array('limit', 'remove') ) + { + // Add a PHP_EOL to the last option + if( 0 < count( $options ) ) + { + $last = array_keys( $options ); + $last = end( $last ); + if( isset( $options[ $last ][3] ) ) + $options[ $last ][3] = $options[ $last ][3] . PHP_EOL; + } + + // Add the quiet option + if( in_array( 'display', $basics ) ) + { + $o1 = ( true === $this->settings['display'] ? ' (settings default)' : '' ); + $options[] = array( 'd', 'display', Ulrichsg\Getopt::NO_ARGUMENT, sprintf( 'Enable output%s', $o1 ) ); + + $o2 = ( false === $this->settings['display'] ? ' (settings default)' : '' ); + $options[] = array( 'q', 'quiet', Ulrichsg\Getopt::NO_ARGUMENT, sprintf( 'Disable output%s'. PHP_EOL, $o2 ) ); + } + + // Add the nolimit option + if( in_array( 'limit', $basics ) ) + { + $options[] = array( 'l', 'limit', Ulrichsg\Getopt::REQUIRED_ARGUMENT, sprintf('The limit in hours (default limit is %d hour%s)', $this->settings['limit'], (1 === $this->settings['limit'] ? '' : 's') ) ); + $options[] = array( 'n', 'nolimit', Ulrichsg\Getopt::NO_ARGUMENT, 'Disable the limit' . PHP_EOL ); + } + + // Remove options + if( in_array( 'remove', $basics ) ) + { + $o1 = ( true === $this->settings['remove'] ? ' (settings default)' : '' ); + $options[] = array( 'r', 'remove', Ulrichsg\Getopt::NO_ARGUMENT, sprintf( 'Enable the removal of releases%s', $o1 ) ); + + $o2 = ( false === $this->settings['remove'] ? ' (settings default)' : '' ); + $options[] = array( 'k', 'keep', Ulrichsg\Getopt::NO_ARGUMENT, sprintf( 'Disable the removal of releases%s'. PHP_EOL, $o2 ) ); + } + + // Add the debug option + if( in_array( 'debug', $basics ) ) + { + $o1 = ( true === $this->settings['debug'] ? ' (settings default)' : '' ); + $options[] = array( null, 'debug', Ulrichsg\Getopt::NO_ARGUMENT, sprintf( 'Enable debug mode%s'. PHP_EOL, $o1 ) ); + } + + // Add the help option + if( in_array( 'help', $basics ) ) + { + $options[] = array( 'h', 'help', Ulrichsg\Getopt::NO_ARGUMENT, 'Shows this help'. PHP_EOL ); + } + + // Set the command line options + $this->options = new Ulrichsg\Getopt( $options ); + $this->options->parse(); + + // Check for the help + if( $this->options->getOption('help') ) + { + $this->displayHeader(); + $this->options->showHelp(); + die(); + } + + // Merge the cli options with the settings + $this->mergeSettings(); + } + + + /** + * Merge the commandline options with the settings + * + * @return void + */ + protected function mergeSettings() + { + if( null !== $this->options && $this->options instanceof Ulrichsg\Getopt ) + { + // Check for limit settings + if( $this->options->getOption('nolimit') ) + { + $this->settings['limit'] = false; + } + if( $this->options->getOption('limit') ) + { + if( !is_numeric( $this->options->getOption('limit') ) ) + { + throw new Exception( 'Option \'limit\' must be numeric' ); + } + + $this->settings['limit'] = false; + if( 0 < (float)$this->options->getOption('limit') ) + { + $this->settings['limit'] = (float)$this->options->getOption('limit'); + } + } + + // Full database + if( $this->options->getOption('full') ) + $this->settings['full'] = true; + + // Check for remove setting + if( $this->options->getOption('remove') ) + $this->settings['remove'] = true; + if( $this->options->getOption('keep') ) + $this->settings['remove'] = false; + + // Check for display options + if( $this->options->getOption('quiet') ) + $this->settings['display'] = false; + if( $this->options->getOption('display') ) + $this->settings['display'] = true; + + // Debug? + if( $this->options->getOption('debug') ) + $this->settings['debug'] = true; + + // Query limit + if( $this->options->getOption('querylimit') ) + if( is_numeric( $this->options->getOption('querylimit') ) ) + $this->settings['querylimit'] = (int)$this->options->getOption('querylimit'); + } + } + + + /** + * Display a message + * + * The function will also try to detect a browser call and convert + * new-lines to
's + * + * @param string $message + * @return void + */ + protected function display( $message ) + { + if( true === $this->settings['display'] ) + { + // Convert for webbrowser? + if( isset( $_SERVER["HTTP_HOST"] ) ) + { + $message = nl2br( $message ); + } + + // Display + echo $message; + } + } + + + /** + * Display a page start + * Only usefull when viewing with a webbrowser + * + * @return void + */ + protected function displayPageStart() + { + if( isset( $_SERVER["HTTP_HOST"] ) && true === $this->display ) + { + echo '
';
+        }
+    }
+
+    /**
+     * Display the title
+     *
+     * @return void
+     */
+    protected function displayTitle()
+    {
+        // Display the script header line
+        $this->display( sprintf( "%s - version %s". PHP_EOL, $this->scriptName, $this->scriptVersion ) );
+    }
+
+
+    /**
+     * Display the header line and settings info
+     *
+     * @return void
+     */
+    protected function displayHeader()
+    {
+        // Display the page start in case of an html page
+        $this->displayPageStart();
+
+        // Show the title
+        $this->displayTitle();
+        $this->display( PHP_EOL );
+    }
+    
+    /**
+     * Display app settings
+     * @return void
+     */
+    protected function displaySettings()
+    {
+        $show = false;
+        // Display settings?
+        if( ( isset( $this->settings['remove'] ) && null !== $this->settings['remove'] ) ||
+            ( isset( $this->settings['debug'] ) && null !== $this->settings['debug'] ) ||
+            ( isset( $this->settings['limit'] ) && null !== $this->settings['limit'] ) ||
+            ( isset( $this->settings['querylimit'] ) && null !== $this->settings['querylimit'] ) )
+        {
+            $show = true;
+            $this->display( 'Settings:' );
+
+            // "debug" setting
+            if( isset( $this->settings['debug'] ) && is_bool( $this->settings['debug'] ) )
+            {
+                 $this->display( PHP_EOL . sprintf( "- debug : %sabled", ( true === $this->settings['debug'] ? 'En' : 'Dis' ) ) );
+            }
+
+            // "limit" setting
+            if( isset( $this->settings['limit'] ) && null !== $this->settings['limit'] )
+            {
+                $line = 'no limit';
+                if( is_float( $this->settings['limit'] ) )
+                {
+                    $line = sprintf( "%s hour%s", $this->settings['limit'], ( 1.0 !== $this->settings['limit'] ? 's' : '' ) );
+                }
+                $this->display( PHP_EOL . sprintf( "- limit : %s", $line ) );
+            }
+
+            // "remove" setting
+            if( isset( $this->settings['remove'] ) && null !== $this->settings['remove'] )
+            {
+                $line = "Data will be removed.";
+                if( true !== $this->settings['remove'] )
+                {
+                    $line = 'No data is removed from the database!'. PHP_EOL .'          Use the --remove option or change the settings.ini file if you want to remove releases or parts';
+                }
+                $this->display( PHP_EOL . sprintf( "- remove: %s", $line ) );
+            }
+
+            // "update" setting
+            if( isset( $this->settings['update'] ) && null !== $this->settings['update'] )
+            {
+                $line = "Data will be updated.";
+                if( true !== $this->settings['update'] )
+                {
+                    $line = 'No data is updated in the database!'. PHP_EOL .'          Change the "UPDATE" setting if you want to enable update';
+                }
+                $this->display( PHP_EOL . sprintf( "- update: %s", $line ) );
+            }
+            
+            // querylimit
+            if( isset( $this->settings['querylimit'] ) && null !== $this->settings['querylimit'] )
+            {
+                $this->display( PHP_EOL . sprintf( "- querylimit: %s", $this->settings['querylimit'] ) );
+            }
+
+            // Spacer
+            $this->display( PHP_EOL );
+        }
+
+        // End spacer
+        if( true === $show )
+            $this->display( PHP_EOL );
+    }
+}
diff --git a/nnscripts/nnscripts.php b/nnscripts/nnscripts.php
deleted file mode 100644
index b29f22312..000000000
--- a/nnscripts/nnscripts.php
+++ /dev/null
@@ -1,239 +0,0 @@
-checkPHPVersion();
-
-        // Set the script name and version
-        $this->scriptName = $scriptName;
-        $this->version = $version;
-        
-        // Set the display setting
-        $this->display = ( !defined('DISPLAY') || true === DISPLAY ? true : false );
-        
-        // Query Limit
-        if( defined('LIMIT') )
-        {
-            $this->limit = ( ( is_int(LIMIT) && 0 < LIMIT ) ? LIMIT : false );
-        }
-        
-        // Should data be removed?
-        if( defined('REMOVE') )
-        {
-            $this->remove = ( ( true === REMOVE ) ? true : false );
-        }
-        
-        // Should data be updated?
-        if( defined('UPDATE') )
-        {
-            $this->update = ( ( true === UPDATE ) ? true : false );
-        }
-
-        // Debug mode?
-        if( defined('DEBUG') )
-        {
-            $this->debug = ( ( true === DEBUG ) ? true : false );
-        }
-    }
-    
-    
-    /**
-     * Destructor
-     * 
-     */
-    public function __destruct()
-    {
-        if( isset( $_SERVER["HTTP_HOST"] ) && true === $this->display )
-        {
-            echo '
'; - } - } - - - /** - * Check php version to be 5.4 or greater - * - * @throws Exception - * @return void - */ - protected function checkPHPVersion() - { - // Init - $current = ( false === strpos( phpversion(), '-' ) ? phpversion() : substr( phpversion(), 0, strpos( phpversion(), '-' ) ) ); - $required = '5.3.10'; - - if( 0 > strnatcmp( $current, $required ) ) - { - throw new Exception( sprintf( 'NNScripts'. PHP_EOL .'Error: PHP version %s is required but version %s is found.', $required, $current ) ); - } - } - - - /** - * Display a message - * - * The function will also try to detect a browser call and convert - * new-lines to
's - * - * @param string $message - * @return void - */ - public function display( $message ) - { - if( true === $this->display ) - { - // Convert for webbrowser? - if( isset( $_SERVER["HTTP_HOST"] ) ) - { - $message = nl2br( $message ); - } - - // Display - echo $message; - } - } - - - /** - * Display a page start - * Only usefull when viewing with a webbrowser - * - * @return void - */ - public function displayPageStart() - { - if( isset( $_SERVER["HTTP_HOST"] ) && true === $this->display ) - { - echo '
';
-        }
-    }       
-
-
-    /**
-     * Display the header line and settings info
-     * 
-     * @return void
-     */
-    public function displayHeader()
-    {
-        // Display the page start in case of an html page
-        $this->displayPageStart();
-        
-        // Display the script header line
-        $this->display( sprintf( "%s - version %s". PHP_EOL, $this->scriptName, $this->version ) );
-        
-        // Display settings?
-        if( null !== $this->limit || null !== $this->remove || null !== $this->update )
-        {
-            $this->display( PHP_EOL .'Settings:' );
-
-            // Display the "debug" setting
-            if( true === $this->debug )
-            {
-                 $this->display( PHP_EOL . "- debug mode enabled" );
-            }
-            
-            // Display the "limit" setting
-            if( null !== $this->limit )
-            {
-                $line = 'no limit';
-                if( is_int($this->limit) )
-                {
-                    $line = sprintf( "%d hour%s", $this->limit, ( 1 < $this->limit ? 's' : '' ) );
-                }
-                $this->display( PHP_EOL . sprintf( "- limit : %s", $line ) );
-            }
-        
-            // Display the "remove" setting
-            if( null !== $this->remove )
-            {
-                $line = "Data will be removed.";
-                if( true !== $this->remove )
-                {
-                    $line = 'No data is removed from the database!'. PHP_EOL .'          Change the "REMOVE" setting if you want to remove releases or parts';
-                }
-                $this->display( PHP_EOL . sprintf( "- remove: %s", $line ) );
-            }
-            
-            // Display the "update" setting
-            if( null !== $this->update )
-            {
-                $line = "Data will be updated.";
-                if( true !== $this->update )
-                {
-                    $line = 'No data is updated in the database!'. PHP_EOL .'          Change the "UPDATE" setting if you want to enable update';
-                }
-                $this->display( PHP_EOL . sprintf( "- update: %s", $line ) );
-            }
-        
-            // Spacer
-            $this->display( PHP_EOL );
-        }
-        
-        // End spacer
-        $this->display( PHP_EOL );
-    }
-}
diff --git a/nnscripts/remove_parts_without_releases.php b/nnscripts/remove_parts_without_releases.php
index d4bd28561..8f0eed0fa 100644
--- a/nnscripts/remove_parts_without_releases.php
+++ b/nnscripts/remove_parts_without_releases.php
@@ -1,87 +1,76 @@
 nnscripts = $nnscripts;
+        // Call the parent constructor
+        parent::__construct();
         
-        // Set the database variable
-        $this->db = $db;
+        // Set the commandline options
+        $options = array(
+            array( null, 'querylimit',  Ulrichsg\Getopt::REQUIRED_ARGUMENT, 'The query limit (default 10000)' ),
+        );
+        $this->setCliOptions( $options, array('display', 'limit', 'help') );
+        
+        // Show the header
+        $this->displayHeader();
 
-        // Add the "now" date
-        $this->now = new DateTime();
-        if( defined('LIMIT') && is_int(LIMIT) && 0 < LIMIT )
-        {
-            // substract the limit hours
-            $this->now->sub( new DateInterval('PT'. LIMIT .'H') );
-        }
+        // Show the settings
+        $this->displaySettings();
     }
     
     
@@ -97,9 +86,12 @@ class removeParts
         
         // Remove parts where the binary is missing
         $this->removePartsWithMissingBinary();
+        
+        // Remove binaries without parts
+        $this->removeBinariesWithoutParts();
     }
-
-
+    
+    
     /**
      * Remove parts without a binary linked
      * 
@@ -107,48 +99,21 @@ class removeParts
      */
     private function removePartsWithoutBinary()
     {
-        $this->nnscripts->display( "Removing old part records which are not linked to a binary: " );
+        $this->display( "Removing old part records which are not linked to a binary: " );
         
         // Build the sql
-        $sql = sprintf(
-            "DELETE FROM parts
-             WHERE parts.binaryID = 0
-             %s",
-            ( ( defined('LIMIT') && is_int(LIMIT) && 0 < LIMIT ) ? sprintf("AND parts.dateadded < '%s'", $this->now->format('Y-m-d H:i:s')) : '' )
-        );
-        $result = $this->db->queryDirect( $sql );
-        $this->nnscripts->display( ( $result ? "Done" : "Error" ) . PHP_EOL );
-    }
-
-
-    /**
-     * Get the total count of parts to remove
-     * 
-     * @return int
-     */
-    protected function getTotalPartsToRemove()
-    {
-        // Init
-        $ret = 0;
-
-        // Get the total to remove
-        $sql = sprintf("SELECT count(*) AS total
-                        FROM parts AS p
-                        LEFT JOIN binaries AS b ON (b.ID = p.binaryID)
-                        WHERE b.ID IS NULL
-                        %s",
-                        ( ( defined('LIMIT') && is_int(LIMIT) && 0 < LIMIT ) ? sprintf("AND p.dateadded < '%s'", $this->now->format('Y-m-d H:i:s')) : '' ) );
-        $result = $this->db->query( $sql );
-        if( is_array($result) && 1 === count($result) )
+        $sql = "DELETE FROM parts
+                WHERE binaryID = 0";
+        if( is_numeric( $this->settings['limit'] ) && 0 < $this->settings['limit'] )
         {
-            $ret = $result[0]['total'];
+            $sql .= sprintf( ' AND dateadded < "%s" - INTERVAL %s HOUR', $this->settings['now'], $this->settings['limit'] );
         }
-
-        // Return
-        return $ret;
+        $result = $this->db->queryDirect( $sql );
+        
+        $this->display( ( $result ? "Done" : "Error" ) . PHP_EOL );
     }
     
-
+    
     /**
      * Remove parts where the binary is missing
      * 
@@ -156,18 +121,17 @@ class removeParts
      */
     private function removePartsWithMissingBinary()
     {
-        // The query limit
-        $queryLimit = ( defined('QUERYLIMIT') && is_int(QUERYLIMIT) && 0 < QUERYLIMIT ? QUERYLIMIT : 10000 );
-        
         // Get the total parts to remove
         $total = $this->getTotalPartsToRemove();
         if( 0 < $total )
         {
             $line = sprintf( "Removing %s parts with missing binaries", number_format($total, 0, '.', '') );
-            if( $total > $queryLimit )
-                $line .= sprintf( ' (in groups of %d parts)', $queryLimit );
+            if( $total > $this->settings['querylimit'] )
+            {
+                $line .= sprintf( ' (in groups of %d parts)', $this->settings['querylimit'] );
+            }
             $line .= ': ';
-            $this->nnscripts->display( $line );
+            $this->display( $line );
             
             // Set the lastID
             $lastId = null;
@@ -186,8 +150,12 @@ class removeParts
                                 ORDER BY p.ID ASC
                                 LIMIT %d",
                     ( null !== $lastId ? sprintf('AND p.ID > %d', $lastId) : '' ),
-                    ( ( defined('LIMIT') && is_int(LIMIT) && 0 < LIMIT ) ? sprintf("AND p.dateadded < '%s'", $this->now->format('Y-m-d H:i:s')) : '' ),
-                    $queryLimit
+                    (
+                        ( is_numeric( $this->settings['limit'] ) && 0 < $this->settings['limit'] )
+                        ? sprintf( ' AND p.dateadded < "%s" - INTERVAL %s HOUR', $this->settings['now'], $this->settings['limit'] )
+                        : ''
+                    ),
+                    $this->settings['querylimit']
                 );
                 $result = $this->db->query( $sql );
                 if( is_array( $result ) && 0 < count( $result ) )
@@ -199,9 +167,9 @@ class removeParts
                     {
                         $ids[] = $lastId = $row['ID'];
                         $counter++;
-                        if( $queryLimit === $counter )
+                        if( $this->settings['querylimit'] === $counter )
                         {
-                            $this->nnscripts->display( "." );
+                            $this->display( "." );
                             $sql = sprintf( "DELETE FROM parts WHERE ID IN (%s)", implode(',', $ids) );
                             $this->db->queryDirect( $sql );
                             $ids = array();
@@ -213,7 +181,7 @@ class removeParts
                     unset( $result );
                     
                     // Lower the total
-                    $total -= $queryLimit;
+                    $total -= $this->settings['querylimit'];
                 }
                 else
                 {
@@ -223,43 +191,81 @@ class removeParts
             }
             
             // And remove the rest
-            if( 0 < count($ids) )
+            if( is_array($ids) && 0 < count($ids) )
             {
-                $this->nnscripts->display( "." );
+                $this->display( "." );
                 $sql = sprintf( "DELETE FROM parts WHERE ID IN (%s)", implode(',', $ids) );
                 $this->db->queryDirect( $sql );
             }
             
-            $this->nnscripts->display( PHP_EOL );
+            $this->display( PHP_EOL );
         }
         else
         {
-            $this->nnscripts->display( 'No parts with missing binaries to remove.'. PHP_EOL );
+            $this->display( 'No parts with missing binaries to remove.'. PHP_EOL );
         }
     }
+    
+    
+    /**
+     * Get the total count of parts to remove
+     * 
+     * @return int
+     */
+    protected function getTotalPartsToRemove()
+    {
+        // Init
+        $ret = 0;
+
+        // Get the total to remove
+        $sql = "SELECT count(*) AS total
+                FROM parts AS p
+                LEFT JOIN binaries AS b ON (b.ID = p.binaryID)
+                WHERE b.ID IS NULL";
+        if( is_numeric( $this->settings['limit'] ) && 0 < $this->settings['limit'] )
+        {
+            $sql .= sprintf( ' AND p.dateadded < "%s" - INTERVAL %s HOUR', $this->settings['now'], $this->settings['limit'] );
+        }
+        
+        $result = $this->db->queryOneRow( $sql );
+        if( is_array($result) && array_key_exists('total', $result) )
+        {
+            $ret = (int)$result['total'];
+        }
+
+        // Return
+        return $ret;
+    }
+    
+    
+    /**
+     * Remove orhpan binary records
+     * 
+     * @return void
+     */
+    protected function removeBinariesWithoutParts()
+    {
+        $this->display( "Removing old binary records which do not have parts: " );
+        
+        $sql = "DELETE binaries FROM binaries
+                LEFT JOIN parts ON (parts.binaryID = binaries.ID)
+                WHERE parts.ID IS NULL";
+        if( is_numeric( $this->settings['limit'] ) && 0 < $this->settings['limit'] )
+        {
+            $sql .= sprintf( ' AND binaries.dateadded < "%s" - INTERVAL %s HOUR', $this->settings['now'], $this->settings['limit'] );
+        }
+        $result = $this->db->queryDirect( $sql );
+        
+        $this->display( ( $result ? "Done" : "Error" ) . PHP_EOL );
+    }
 }
 
+// Main application
 try
 {
-    // Init
-    $scriptName    = 'Remove parts which are not linked to a binary';
-    $scriptVersion = '0.2';
-    
-    // Load the NNscript class
-    $nnscripts = new NNScripts( $scriptName, $scriptVersion );
-    
-    // Display the header
-    $nnscripts->displayHeader();
-
-    // Load the application part
-    $db = new DB;
-    if( !$db )
-        throw new Exception("Error loading database library");
-        
-    // Load the categoryReleases class
-    $cr = new removeParts( $nnscripts, $db );
-    $cr->cleanup();
+    // Load the remove_parts_without_releases class
+    $rpwr = new remove_parts_without_releases();
+    $rpwr->cleanup();
 } catch( Exception $e ) {
     echo $e->getMessage() . PHP_EOL;
 }
-
diff --git a/nnscripts/settings.ini b/nnscripts/settings.ini
new file mode 100644
index 000000000..fe91baee5
--- /dev/null
+++ b/nnscripts/settings.ini
@@ -0,0 +1,35 @@
+[global]
+display = true
+
+[check_database]
+;display = false
+
+[fix_android_releases]
+;display = false
+;limit = 36
+
+[remove_blacklist_releases]
+;display = false
+;limit = 36
+;limit = false
+;full = true
+;debug = true
+remove = false
+
+[remove_category_releases]
+;display = false
+;limit = 36
+
+[remove_parts_without_releases]
+;display = false
+;limit = 36
+querylimit = 10000
+
+[remove_unwanted_releases]
+;display = false
+;limit = 36
+remove = false
+
+[update_missing_movie_info]
+;display = false
+
diff --git a/nnscripts/update_missing_movie_info.php b/nnscripts/update_missing_movie_info.php
index c7bd15dfb..a57c51922 100644
--- a/nnscripts/update_missing_movie_info.php
+++ b/nnscripts/update_missing_movie_info.php
@@ -4,52 +4,50 @@
  * 
  * Releases may be added/eddited with an imdb-id that does not exists
  * in the movieinfo table. This script will fetch all the missing imdb
- * id's from the releases table.
- * 
- * The location of the script needs to be "misc/custom" or the
- * "misc/testing" directory. if used from another location,
- * change lines 32 to 35 to require the correct files.
+ * id's from the releases table and updates the movie info.
  *
  * @author    NN Scripts
  * @license   http://opensource.org/licenses/MIT MIT License
  * @copyright (c) 2013 - NN Scripts
  *
  * Changelog:
- * 0.1  - Initial version
+ * 0.2 - Added settings.ini
+ *       Start using nnscript library
+ * 
+ * 0.1 - Initial version
  */
-
-//----------------------------------------------------------------------
-// Settings
-
-// Display settings
-define('DISPLAY', true);
-
-// Should the move info be updated?
-define('UPDATE', true);
-//----------------------------------------------------------------------
-
 // Load the application
-require(dirname(__FILE__)."/../bin/config.php");
-require_once(WWW_DIR."/lib/framework/db.php");
-require_once(WWW_DIR."lib/movie.php");
-require_once('nnscripts.php');
+require_once(dirname(__FILE__)."/../bin/config.php");
+
+// nnscripts includes
+require_once("lib/nnscripts.php");
+
+// newznab includes
+require_once(WWW_DIR."/lib/movie.php"); 
+
 
 /**
- * Update missing movie information
+ * Update missing movie info
  */
-class movieInfo
+class update_missing_movie_info extends NNScripts
 {
     /**
-     * NNScripts class
-     * @var NNScripts
+     * The script name
+     * @var string
      */
-    private $nnscripts;    
+    protected $scriptName = 'Update missing movie information';
     
     /**
-     * The database object
-     * @var DB
+     * The script version
+     * @var string
      */
-    private $db;
+    protected $scriptVersion = '0.2';
+    
+    /**
+     * Allowed settings
+     * @var array
+     */
+    protected $allowedSettings = array('display');
     
     /**
      * The movie object
@@ -57,25 +55,27 @@ class movieInfo
      */
     private $movie;
     
-
+    
+    
     /**
-     * Constructor
-     * 
-     * @param NNScripts $nnscripts
-     * @param DB $db
-     * @param Movie $movie
+     * The constructor
+     *
      */
-    public function __construct( NNScripts $nnscripts, DB $db, Movie $movie )
+    public function __construct()
     {
-        // Set the NNScripts variable
-        $this->nnscripts = $nnscripts;
+        // Call the parent constructor
+        parent::__construct();
         
-        // Set the database variable
-        $this->db = $db;
+        // Set the commandline options
+        $options = array();
+        $this->setCliOptions( $options, array('display', 'help') );
+        
+        // Show the header
+        $this->displayHeader();
         
         // Set the movie variable
-        $this->movie = $movie;
-    }
+        $this->movie = new Movie();
+    } 
     
     
     /**
@@ -105,53 +105,31 @@ class movieInfo
                     $name = sprintf("%s (%s)", str_replace(array('.', '_'), ' ', trim($matches[1])), $matches[2]);
                 }
 
-                $this->nnscripts->display( sprintf( "Updating: %s ", $name ) );
-                if( defined('UPDATE') && true === UPDATE )
-                {
-                    // Update the movie info
-                    $this->movie->updateMovieInfo( $row['imdbID'] );
+                $this->display( sprintf( "Updating: %s ", $name ) );
+
+                // Update the movie info
+                $this->movie->updateMovieInfo( $row['imdbID'] );
                         
-                    // Prevent overloading the remote servers
-                    sleep(2);
-                }
-                $this->nnscripts->display( PHP_EOL );
+                // Prevent overloading the remote servers
+                sleep(2);
+                
+                $this->display( PHP_EOL );
             }
         }
         else
         {
-            $this->nnscripts->display( 'No missing movie information found'. PHP_EOL );
+            $this->display( 'No missing movie information found'. PHP_EOL );
         }
-        
-        $this->nnscripts->display( PHP_EOL );
-    }
+    } 
 }
 
+
+// Main application
 try
 {
-    // Init
-    $scriptName    = 'Update missing movie information';
-    $scriptVersion = '0.1';
-    
-    // Load the NNscript class
-    $nnscripts = new NNScripts( $scriptName, $scriptVersion );
-    
-    // Display the header
-    $nnscripts->displayHeader();
-    
-    // Load the application part
-    $movie = new Movie;
-    if( !$movie )
-        throw new Exception("Error loading movie library");
-    
-    $db = new DB;
-    if( !$db )
-        throw new Exception("Error loading database library");
-        
-    // Load the blacklistReleases class
-    $mi = new movieInfo( $nnscripts, $db, $movie );
-    $mi->update();
-    
+    // Update missing movie information
+    $ummi = new update_missing_movie_info();
+    $ummi->update();
 } catch( Exception $e ) {
     echo $e->getMessage() . PHP_EOL;
 }
-