Updates to Sharing, IRCSraper and monitor

This commit is contained in:
Darko
2014-04-07 09:30:25 +02:00
parent f5d4cfcf0a
commit 2aa5841e1d
11 changed files with 65 additions and 37 deletions
+19 -15
View File
@@ -9,7 +9,7 @@ require_once(dirname(__FILE__)."/../lib/showsleep.php");
require_once(dirname(__FILE__)."/../lib/functions.php");
$version="0.3r1070";
$version="0.3r1072";
$db = new DB();
$functions = new Functions();
@@ -127,6 +127,7 @@ $proc_tmux = "SELECT "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'others') AS others, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'delete_parts') AS delete_parts, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'delete_timer') AS delete_timer, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'sharing_timer') AS sharing_timer, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'sphinx') AS sphinx, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'sphinx_timer') AS sphinx_timer, "
. "(SELECT VALUE FROM tmux WHERE SETTING = 'unwanted') AS unwanted, "
@@ -784,6 +785,9 @@ while( $i > 0 )
if ($proc_tmux_result[0]['show_query'] != NULL) {
$show_query = $proc_tmux_result[0]['show_query'];
}
if ($proc_tmux_result[0]['sharing_timer'] != NULL) {
$sharing_timer = $proc_tmux_result[0]['sharing_timer'];
}
//reset monitor paths before query
@@ -1155,7 +1159,7 @@ if ($running == 1){
if (( $maxload >= get_load()) && ($killed != "true") && ($binaries != 0)) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes0[2]);
shell_exec("tmux respawnp -t${tmux_session}:0.2 'echo \"\033[38;5;\"$color\"m\" && $_update_cmd 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $bins_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:0.2 'echo \"\033[38;5;\"$color\"m\" && $_update_cmd 2>&1 $log; $_sleep $bins_timer' 2>&1 1> /dev/null");
} else if (( $binaries != 0 ) && ( $maxload <= get_load())) {
$color = get_color($colors_start, $colors_end, $colors_exc);
shell_exec("tmux respawnp -t${tmux_session}:0.2 'echo \"\033[38;5;\"$color\"m\n$panes0[2] Disabled by Max Load\"' 2>&1 1> /dev/null");
@@ -1393,19 +1397,19 @@ if ($running == 1){
if (( $maxloadr >= get_load()) && ( $releases_run == 2 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes0[5]);
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/releases_threaded.py && 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $rel_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/releases_threaded.py && 2>&1 $log; $_sleep $rel_timer' 2>&1 1> /dev/null");
} elseif (( $maxloadr >= get_load()) && ( $releases_run == 1 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes0[5]);
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php update_releases.php && 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $rel_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php update_releases.php && 2>&1 $log; $_sleep $rel_timer' 2>&1 1> /dev/null");
} elseif (( $maxloadr >= get_load()) && ( $releases_run == 2 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes0[5]);
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/releases_threaded.py 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $rel_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/releases_threaded.py 2>&1 $log; $_sleep $rel_timer' 2>&1 1> /dev/null");
} elseif (( $maxloadr >= get_load()) && ( $releases_run == 1 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes0[5]);
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php update_releases.php && 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $rel_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php update_releases.php && 2>&1 $log; $_sleep $rel_timer' 2>&1 1> /dev/null");
} elseif (( $releases_run !== 0 ) && ( $maxloadr <= get_load())) {
$color = get_color($colors_start, $colors_end, $colors_exc);
shell_exec("tmux respawnp -t${tmux_session}:0.5 'echo \"\033[38;5;\"$color\"m\n$panes0[5] Disabled by Max Load Releases\"' 2>&1 1> /dev/null");
@@ -1451,7 +1455,7 @@ if ($running == 1){
if (( $maxload >= get_load()) && ( TIME() - $time9 >= $sphinx_timer ) && ( $sphinx == 1)) {
$color=get_color();
$log = writelog($panes1[2]);
shell_exec("tmux respawnp -t${tmux_session}:1.2 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php sphinx.php 2>&1 $log && echo \" \033[1;0;33m\"' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:1.2 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php sphinx.php 2>&1 $log' 2>&1 1> /dev/null");
$time9 = TIME();
} else if ($sphinx == 0) {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1466,10 +1470,10 @@ if ($running == 1){
}
//run comment sharing in in pane 1.3
if (( $maxload >= get_load()) && (( TIME() - $time16 ) >= 600 )) {
if (( $maxload >= get_load()) && (( TIME() - $time16 ) >= $sharing_timer )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes1[3]);
shell_exec("tmux respawnp -t${tmux_session}:1.3 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php postprocess_new.php sharing 2>&1 $log' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:1.3 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php postprocess_new.php sharing 2>&1 $log; $_sleep $sharing_timer' 2>&1 1> /dev/null");
$time16 = TIME();
} else if ( $maxload <= get_load()) {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1548,7 +1552,7 @@ if ($running == 1){
if (($maxload >= get_load()) && ($post_amazon == 1) && ($music_releases_proc > 0)) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes2[3]);
shell_exec("tmux respawnp -t${tmux_session}:2.3 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php postprocess_new.php music 2>&1 $log && $_sleep $post_timer_amazon && echo \" \033[1;0;33m\"' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:2.3 'echo \"\033[38;5;\"$color\"m\" && cd $_bin && $_php postprocess_new.php music 2>&1 $log && $_sleep $post_timer_amazon' 2>&1 1> /dev/null");
} else if (($maxload >= get_load()) && ($post_amazon == 1) && ($music_releases_proc == 0)) {
$color = get_color($colors_start, $colors_end, $colors_exc);
shell_exec("tmux respawnp -t${tmux_session}:2.3 'echo \"\033[38;5;${color}m\n${panes2[3]} has been disabled/terminated by No Music to process\"'");
@@ -1670,7 +1674,7 @@ if ($running == 1){
} elseif (( $maxload >= get_load()) && ( $fix_names == 1 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes3[0]);
shell_exec("tmux respawnp -t${tmux_session}:3.0 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php fixReleaseNames.php 3 true other yes show 2>&1 $log && $_php fixReleaseNames.php 5 true other yes show 2>&1 $log && $_php fixReleaseNames.php 7 true other yes show 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $fix_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:3.0 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php fixReleaseNames.php 3 true other yes show 2>&1 $log && $_php fixReleaseNames.php 5 true other yes show 2>&1 $log && $_php fixReleaseNames.php 7 true other yes show 2>&1 $log; $_sleep $fix_timer' 2>&1 1> /dev/null");
$time27 = TIME();
} elseif ( $fix_names == 0 ) {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1683,7 +1687,7 @@ if ($running == 1){
if (($maxload >= get_load()) && (($dehash == 1) || ($dehash == 3))) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes3[1]);
shell_exec("tmux respawnp -t${tmux_session}:3.1 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php predb_hash_decrypt.php 1000 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $dehash_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:3.1 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php predb_hash_decrypt.php 1000 2>&1 $log; $_sleep $dehash_timer' 2>&1 1> /dev/null");
$time28 = TIME();
} else if ( $maxload <= get_load()) {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1696,12 +1700,12 @@ if ($running == 1){
if (( $maxload >= get_load()) && ( $lookreqids == 2 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes3[2]);
shell_exec("tmux respawnp -t${tmux_session}:3.2 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/requestid_threaded.py 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $lookreqids_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:3.2 'echo \"\033[38;5;\"$color\"m\" && cd $_py && $_python ${DIR}/../python/requestid_threaded.py 2>&1 $log; $_sleep $lookreqids_timer' 2>&1 1> /dev/null");
$time30 = TIME();
} elseif (( $maxload >= get_load()) && ( $lookreqids == 1 )) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes3[2]);
shell_exec("tmux respawnp -t${tmux_session}:3.2 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php requestID.php 1000 true 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $lookreqids_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:3.2 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php requestID.php 1000 true 2>&1 $log; $_sleep $lookreqids_timer' 2>&1 1> /dev/null");
$time30 = TIME();
} else if ( $maxload <= get_load()) {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1766,7 +1770,7 @@ if ($running == 1){
if (($maxload >= get_load()) && (($dehash == 2) || ($dehash == 3))) {
$color = get_color($colors_start, $colors_end, $colors_exc);
$log = writelog($panes3[4]);
shell_exec("tmux respawnp -t${tmux_session}:3.4 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php postprocess_pre.php 2>&1 $log && echo \" \033[1;0;33m\" && $_sleep $dehash_timer' 2>&1 1> /dev/null");
shell_exec("tmux respawnp -t${tmux_session}:3.4 'echo \"\033[38;5;\"$color\"m\" && cd $_lib && $_php postprocess_pre.php 2>&1 $log; $_sleep $dehash_timer' 2>&1 1> /dev/null");
$time32 = TIME();
} else if ( $maxload <= get_load()) {
$color = get_color($colors_start, $colors_end, $colors_exc);
+5 -4
View File
@@ -183,7 +183,8 @@ INSERT INTO tmux (setting, value) values ('defrag_cache','900'),
('lastpretime', '0'),
('nntpretries', '10'),
('alternate_nntp', '0'),
('sqlpatch','16');
('sharing_timer', '60'),
('sqlpatch','17');
DROP TABLE IF EXISTS country;
CREATE TABLE country (
@@ -466,12 +467,12 @@ CREATE TABLE sharing (
enabled TINYINT(1) NOT NULL DEFAULT '0',
posting TINYINT(1) NOT NULL DEFAULT '0',
start_position TINYINT(1) NOT NULL DEFAULT '0',
fetching TINYINT(1) NOT NULL DEFAULT '0',
auto_enable TINYINT(1) NOT NULL DEFAULT '0',
fetching TINYINT(1) NOT NULL DEFAULT '1',
auto_enable TINYINT(1) NOT NULL DEFAULT '1',
hide_users TINYINT(1) NOT NULL DEFAULT '1',
last_article BIGINT UNSIGNED NOT NULL DEFAULT '0',
max_push MEDIUMINT UNSIGNED NOT NULL DEFAULT '40',
max_pull INT UNSIGNED NOT NULL DEFAULT '1000',
max_pull INT UNSIGNED NOT NULL DEFAULT '200',
PRIMARY KEY (site_guid)
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci ;
@@ -7,7 +7,7 @@ CREATE TABLE sharing_sites (
first_time DATETIME DEFAULT NULL,
enabled TINYINT(1) NOT NULL DEFAULT '0',
comments MEDIUMINT UNSIGNED NOT NULL DEFAULT '0',
PRIMARY KEY (id)
PRIMARY KEY (ID)
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
DROP TABLE IF EXISTS sharing;
@@ -17,12 +17,12 @@ CREATE TABLE sharing (
enabled TINYINT(1) NOT NULL DEFAULT '0',
posting TINYINT(1) NOT NULL DEFAULT '0',
start_position TINYINT(1) NOT NULL DEFAULT '0',
fetching TINYINT(1) NOT NULL DEFAULT '0',
auto_enable TINYINT(1) NOT NULL DEFAULT '0',
fetching TINYINT(1) NOT NULL DEFAULT '1',
auto_enable TINYINT(1) NOT NULL DEFAULT '1',
hide_users TINYINT(1) NOT NULL DEFAULT '1',
last_article BIGINT UNSIGNED NOT NULL DEFAULT '0',
max_push MEDIUMINT UNSIGNED NOT NULL DEFAULT '40',
max_pull INT UNSIGNED NOT NULL DEFAULT '1000',
max_pull INT UNSIGNED NOT NULL DEFAULT '200',
PRIMARY KEY (site_guid)
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci ;
@@ -0,0 +1,3 @@
INSERT IGNORE INTO tmux (setting, value) VALUE ('sharing_timer', '60');
UPDATE tmux SET value = '17' WHERE setting = 'sqlpatch';
+2 -2
View File
@@ -214,7 +214,7 @@ class IRCScraper
'A\s+NZB\s+is\s+available.*?To\s+Download' . // a.b.sony.psp
'|' .
'\s+NZB:\s+http:\/\/scnzb\.eu\/' . // scnzb
'|' .
//'|' .
//'^\[SBINDEX\]' . // tvnzb
'|' .
'^\[(MOD|OLD|RE|UN)?NUKE\]' . // Nukes. various channels
@@ -763,7 +763,7 @@ class IRCScraper
protected function scnzb(&$message)
{
//[Complete][512754] Formula1.2014.Malaysian.Grand.Prix.Team.Principals.Press.Conference.720p.HDTV.x264-W4F NZB: http://scnzb.eu/1pgOmwj
if (preg_match('/\[Complete\]\[(?P<reqid>\d+)\]\s+(?P<title>.+?)\s+NZB:/i', $message, $matches)) {
if (preg_match('/\[Complete\]\[(?P<reqid>\d+)\]\s*(?P<title>.+?)\s+NZB:/i', $message, $matches)) {
$this->CurPre['source'] = '#scnzb';
$this->CurPre['groupid'] = $this->getGroupID('alt.binaries.boneless');
$this->siftMatches($matches);
+14 -9
View File
@@ -155,7 +155,8 @@ Class Sharing
$siteName = uniqid('newznab_', true);
$this->db->exec(
sprintf('
INSERT INTO sharing (site_name, site_guid, max_push, max_pull, hide_users, start_position) VALUES (%s, %s, 40 , 1000, 1, 1)',
INSERT INTO sharing (site_name, site_guid, max_push, max_pull, hide_users, start_position, auto_enable, fetching)
VALUES (%s, %s, 40 , 200, 1, 1, 1, 1)',
$this->db->escapeString($siteName),
$this->db->escapeString(($siteGuid === '' ? sha1($siteName) : $siteGuid))
)
@@ -295,7 +296,7 @@ Class Sharing
$this->siteSettings['last_article'] = $ourOldest = (string)($group['last'] - 1000);
}
} else {
$ourOldest = $this->siteSettings['last_article'] + 1;
$ourOldest = (string)($this->siteSettings['last_article'] + 1);
}
// Set our newest to our oldest wanted + max pull setting.
@@ -391,7 +392,11 @@ Class Sharing
$this->db->escapeString($matches['guid'])
)
);
$found++;
// Update once in a while in case the user cancels the script.
if ($found++ % 10 == 0) {
$this->siteSettings['lastarticle'] = $header['Number'];
$this->db->exec(sprintf('UPDATE sharing SET last_article = %d',$header['Number']));
}
echo '.';
}
}
@@ -399,12 +404,7 @@ Class Sharing
}
// Update sharing's last article number.
$this->siteSettings['lastarticle'] = $newest;
$this->db->exec(
sprintf('
UPDATE sharing SET last_article = %d',
$newest
)
);
$this->db->exec(sprintf('UPDATE sharing SET last_article = %d',$newest));
if ($found > 0) {
echo PHP_EOL . '(Sharing) Fetched ' . $found . ' new comments.' . PHP_EOL;
} else {
@@ -447,6 +447,11 @@ Class Sharing
return false;
}
// Set sn names to anon.
if (substr($body['USER'], 0, 3) === 'sn-') {
$body['USER'] = 'ANON';
}
// Check if we have the user.
$user = $this->db->queryOneRow(
sprintf('SELECT ID FROM users WHERE username = %s',
@@ -180,6 +180,7 @@ function ajax_sharing_reset(id)
// fade.. mm
$('#message').fadeOut(5000);
setTimeout('history.go(0);',1500);
},
error: function(xhr,err,e) { alert( "Error in ajax_sharing_reset: " + err ); }
});
@@ -204,6 +205,7 @@ function ajax_sharing_site_purge(id)
// fade.. mm
$('#message').fadeOut(5000);
setTimeout('history.go(0);',1500);
},
error: function(xhr,err,e) { alert( "Error in ajax_sharing_site_purge: " + err ); }
});
@@ -22,7 +22,7 @@
<li><a href="{$smarty.const.WWW_TOP}/db-optimise.php">Optimise</a> Tables</li>
<li><a href="{$smarty.const.WWW_TOP}/comments-list.php">View Comments</a></li>
<li><a href="{$smarty.const.WWW_TOP}/spotnab-list.php">View</a> <a style="padding:0;" href="{$smarty.const.WWW_TOP}/spotnab-edit.php?action=add">Add</a> Spotnab Sources</li>
<li><a href="{$smarty.const.WWW_TOP}/sharing.php">Sharing Settings</a></li>
<li><a href="{$smarty.const.WWW_TOP}/sharing.php">Comment Sharing Settings</a></li>
<li><a href="{$smarty.const.WWW_TOP}/user-list.php">View</a> <a style="padding:0;" href="{$smarty.const.WWW_TOP}/user-edit.php?action=add">Add</a> Users</li>
<li><a href="{$smarty.const.WWW_TOP}/role-list.php">View</a> <a style="padding:0;" href="{$smarty.const.WWW_TOP}/role-edit.php?action=add">Add</a> Roles</li>
<li><a href="{$smarty.const.WWW_TOP}/site-stats.php">Site Stats</a> <a style="padding:0;" href="{$smarty.const.WWW_TOP}/site-debug.php">Debug</a></li>
@@ -7,7 +7,7 @@
<br />
<div id="message" style="width:717px;">msg</div>
{if $local}
<form action="{$SCRIPT_NAME}?action=submit" method="post">
<form action="{$SCRIPT_NAME}" method="post">
<fieldset style="width:717px;">
<legend>Local sharing settings.</legend>
<table class="input">
@@ -119,7 +119,8 @@
<a href="javascript:ajax_sharing_startposition(1, 1)" class="sharing_startposition_deactive">[ENABLE]</a>
{/if}
</strong>
When pulling the first time, or after resetting, start from the beginning of the group (takes more time).
When pulling the first time, or after resetting, start from the beginning of the group (takes more time).<br />
<strong>With the backfill setting, the group has many junk articles before you will start finding comments, you will want to raise Max Downloads setting(20000 or so) and run sharing manually a few times until you start finding comments, then cancel and drop it back to 200.</strong>
</div>
</td>
</tr>
@@ -888,6 +888,14 @@
</td>
</tr>
<tr>
<td style="width:180px;"><label for="sharing_timer">Comments Sharing Timer:</label></td>
<td>
<input class="short" id="sharing_timer" name="sharing_timer" type="text" value="{$ftmux->sharing_timer}" />
<div class="hint">Set the timer for updating comments shared between newznab and nZEDb servers</div>
</td>
</tr>
<tr>
<td style="width:180px;"><label for="debuginfo">Debug information:</label></td>
<td>
+4
View File
@@ -0,0 +1,4 @@
*.pyc
*.pyo
nntpproxy.conf
nntpproxy_a.conf