From 81e8ab99b8debdb14c96bf8a9fe135a60392807b Mon Sep 17 00:00:00 2001 From: jonnyboy Date: Sat, 30 Mar 2013 16:32:36 -0400 Subject: [PATCH] updated ugo's scripts, removed load check on threads, added misc_sorter, and disables all regexes if using ugo's assembler, added script to reactivate all regexes that were deactivated --- bin/monitor.php | 39 +- conf/.gitignore | 1 + conf/default_regexes.txt | 1711 ++++++++++++++++++++++ conf/jonnyboys_defaults.sh | 11 +- config.sh | 12 + scripts/reactivate_regexes.sh | 16 + ugo/automake.php | 1566 ++++++++++++++++++++ ugo/automake_threaded.php | 155 ++ ugo/finish_postproc.php | 39 + ugo/makenzb.php | 120 ++ ugo/makenzb_threaded.php | 210 +++ ugo/misc_sorter3.php | 2335 ++++++++++++++++++++++++++++++ ugo/misc_sorter_cleanup.php | 98 ++ ugo/misc_sorter_threaded.php | 140 ++ ugo/postproc.php | 355 +++++ ugo/postproc_threaded.php | 171 +++ ugo/update_binaries_threaded.php | 120 ++ 17 files changed, 7092 insertions(+), 7 deletions(-) create mode 100644 conf/default_regexes.txt create mode 100755 scripts/reactivate_regexes.sh create mode 100644 ugo/automake.php create mode 100644 ugo/automake_threaded.php create mode 100644 ugo/finish_postproc.php create mode 100644 ugo/makenzb.php create mode 100644 ugo/makenzb_threaded.php create mode 100644 ugo/misc_sorter3.php create mode 100644 ugo/misc_sorter_cleanup.php create mode 100644 ugo/misc_sorter_threaded.php create mode 100644 ugo/postproc.php create mode 100644 ugo/postproc_threaded.php create mode 100644 ugo/update_binaries_threaded.php diff --git a/bin/monitor.php b/bin/monitor.php index 8fa73cffb..be2ffce07 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -2,7 +2,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); -$version="0.1r760"; +$version="0.1r761"; $db = new DB(); @@ -49,6 +49,7 @@ $_alienx = dirname(__FILE__)."/../alienx"; $_conf = dirname(__FILE__)."/../conf"; $_powerline = dirname(__FILE__)."/../powerline"; $_cj = dirname(__FILE__)."/../nnscripts"; +$_ugo = dirname(__FILE__)."/../ugo"; $_user = dirname(__FILE__)."/../user_scripts"; $NNPATH="{$array['NEWZPATH']}{$array['NEWZNAB_PATH']}"; $TESTING="{$array['NEWZPATH']}{$array['TESTING_PATH']}"; @@ -61,6 +62,14 @@ $_backfill_increment = "UPDATE groups set backfill_target=backfill_target+1 wher $mysql_command_1 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_backfill_increment\""; $reset_bin = "UPDATE binaries SET procstat=0, procattempts=0, regexID=NULL, relpart=0, reltotalpart=0, relname=NULL;"; $mysql_command_2 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$reset_bin\""; +$_active_regex = "select ID from releaseregex where status=1;"; +$mysql_command_3 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_active_regex\" >> ../conf/active_regexes.txt"; +$_disable_regex = "update releaseregex set status=0 where status=1;"; +$mysql_command_4 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_disable_regex\""; + +if ( $array['UGO_THREADED'] == "true" ) { + shell_exec("$mysql_command_3 && $mysql_command_4"); +} //got microtime function microtime_float() @@ -159,6 +168,7 @@ $time17 = TIME(); $time18 = TIME(); $time19 = TIME(); $time20 = TIME(); +$time21 = TIME(); if ( $array['INNODB'] == "true" ) { $time5 = TIME(); @@ -351,7 +361,6 @@ while( $i > 0 ) printf("$a.."); sleep(1); } - $rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)"); } //defrag the query cache every 15 minutes @@ -543,6 +552,14 @@ while( $i > 0 ) $panes4 = str_replace("\n", '', explode(" ", $panes_win_5)); $panes5 = str_replace("\n", '', explode(" ", $panes_win_6)); + //reset binaries every 2 hours + if ((( TIME() - $time21 >= 7200 ) || ( $i == 1 )) && ($array['UGO_THREADED'] == "true" )) + { + $color = get_color(); + $rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)"); + shell_exec("$_tmux respawnp -k -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\n$panes0[5]\nKilled in to reset binaries\" && date +\"%D %T\" && echo \"This is color #$color\" && $ds1 $panes0[5] $ds4'"); + } + //kill update_binaries.php backfill.php and import-nzb if timer exceeded $killit=explode(" ", relativeTime("$newestdate")); $killed="false"; @@ -1057,10 +1074,10 @@ while( $i > 0 ) } //runs update_release and in 0.5 once if needed and exits - if (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) { + if (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $array['UGO_THREADED'] == "true" ) && ( $optimize_safe_to_run != "true" )) { $color = get_color(); $log = writelog($panes0[5]); - shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_bin && $_php update_releases.php && 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null"); + shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_ugo && $_php automake_threaded.php 2>&1 $log && cd $_bin && $_php update_releases.php 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null"); } elseif (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) { $color = get_color(); $log = writelog($panes0[5]); @@ -1092,12 +1109,22 @@ while( $i > 0 ) } //run $_php update_parsing.php in 1.1 - if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ( $optimize_safe_to_run != "true" )) { + if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ($array['MISC_SORTER'] == "true" ) && ( $optimize_safe_to_run != "true" )) { + $color = get_color(); + $log = writelog($panes1[1]); + shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_ugo && $_php misc_sorter_threaded.php 2>&1 $log && cd $TESTING && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null"); + $time3 = TIME(); + } elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ($array['MISC_SORTER'] != "true" ) && ( $optimize_safe_to_run != "true" )) { $color = get_color(); $log = writelog($panes1[1]); shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $TESTING && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null"); $time3 = TIME(); - } elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ( $optimize_safe_to_run != "true" )) { + } elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ($array['MISC_SORTER'] == "true" ) && ( $optimize_safe_to_run != "true" )) { + $color = get_color(); + $log = writelog($panes1[1]); + shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_ugo && $_php misc_sorter_threaded.php 2>&1 $log && cd $_bin && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null"); + $time3 = TIME(); + } elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ($array['MISC_SORTER'] != "true" ) && ( $optimize_safe_to_run != "true" )) { $color = get_color(); $log = writelog($panes1[1]); shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_bin && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null"); diff --git a/conf/.gitignore b/conf/.gitignore index 2d2fd07c1..10e4af8d6 100644 --- a/conf/.gitignore +++ b/conf/.gitignore @@ -1,2 +1,3 @@ tmux_user.conf my.cnf +active_regexes.txt diff --git a/conf/default_regexes.txt b/conf/default_regexes.txt new file mode 100644 index 000000000..9870bef37 --- /dev/null +++ b/conf/default_regexes.txt @@ -0,0 +1,1711 @@ +ID +12 +21 +31 +34 +40 +44 +47 +61 +94 +127 +147 +148 +160 +166 +170 +177 +178 +186 +188 +194 +208 +215 +221 +251 +283 +285 +288 +299 +300 +319 +344 +345 +353 +357 +363 +375 +376 +402 +408 +413 +416 +417 +418 +424 +430 +433 +437 +438 +439 +441 +442 +443 +444 +446 +447 +448 +457 +462 +474 +480 +481 +482 +485 +492 +495 +496 +497 +498 +500 +501 +502 +503 +515 +519 +521 +525 +528 +529 +530 +531 +532 +533 +535 +536 +537 +538 +539 +541 +543 +544 +546 +547 +548 +549 +550 +551 +552 +553 +555 +557 +558 +559 +560 +561 +562 +563 +566 +568 +569 +571 +572 +573 +574 +575 +578 +581 +583 +584 +586 +587 +589 +590 +591 +592 +593 +596 +601 +602 +603 +607 +610 +611 +612 +614 +615 +618 +619 +621 +623 +624 +625 +627 +633 +634 +635 +637 +640 +641 +642 +644 +646 +647 +652 +653 +654 +656 +657 +658 +659 +661 +666 +670 +671 +672 +673 +674 +675 +677 +678 +679 +680 +681 +682 +683 +684 +685 +687 +690 +698 +700 +702 +709 +710 +711 +712 +713 +714 +719 +720 +721 +722 +724 +725 +727 +729 +730 +733 +734 +735 +736 +738 +740 +741 +742 +743 +744 +745 +746 +747 +749 +750 +751 +752 +754 +755 +756 +757 +758 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +771 +772 +773 +775 +776 +777 +778 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +830 +831 +832 +833 +834 +835 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +863 +864 +865 +866 +867 +870 +871 +872 +873 +874 +876 +877 +879 +880 +881 +882 +883 +884 +885 +886 +888 +889 +890 +891 +892 +893 +894 +896 +897 +898 +899 +900 +902 +903 +904 +906 +907 +909 +910 +911 +912 +913 +914 +915 +916 +917 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 +1001 +1002 +1003 +1004 +1005 +1006 +1007 +1008 +1009 +1010 +1011 +1012 +1013 +1014 +1015 +1016 +1017 +1018 +1019 +1020 +1021 +1022 +1023 +1024 +1025 +1026 +1027 +1028 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1037 +1038 +1039 +1040 +1041 +1042 +1043 +1044 +1045 +1046 +1047 +1048 +1049 +1050 +1051 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1059 +1060 +1061 +1062 +1063 +1064 +1065 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1075 +1076 +1077 +1079 +1080 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1091 +1092 +1093 +1094 +1095 +1096 +1097 +1099 +1100 +1101 +1103 +1104 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1116 +1117 +1118 +1119 +1120 +1121 +1122 +1123 +1124 +1125 +1126 +1127 +1128 +1129 +1130 +1131 +1132 +1133 +1134 +1135 +1136 +1137 +1138 +1139 +1140 +1141 +1142 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1154 +1155 +1157 +1158 +1159 +1160 +1161 +1162 +1163 +1164 +1165 +1167 +1168 +1170 +1171 +1172 +1173 +1174 +1175 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1187 +1189 +1190 +1191 +1192 +1193 +1194 +1195 +1196 +1197 +1199 +1201 +1202 +1203 +1204 +1205 +1206 +1207 +1208 +1209 +1210 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1221 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1229 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1240 +1241 +1242 +1243 +1244 +1245 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1254 +1255 +1256 +1257 +1258 +1259 +1260 +1261 +1262 +1263 +1265 +1266 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1275 +1276 +1277 +1278 +1279 +1280 +1281 +1282 +1284 +1286 +1290 +1291 +1292 +1293 +1294 +1296 +1297 +1299 +1300 +1301 +1302 +1303 +1304 +1305 +1306 +1308 +1309 +1311 +1312 +1313 +1314 +1315 +1316 +1317 +1318 +1319 +1320 +1321 +1322 +1323 +1324 +1325 +1326 +1327 +1328 +1330 +1331 +1332 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 +1341 +1342 +1343 +1344 +1345 +1346 +1347 +1348 +1349 +1350 +1351 +1352 +1353 +1354 +1355 +1356 +1357 +1358 +1359 +1360 +1361 +1363 +1364 +1365 +1366 +1367 +1368 +1369 +1370 +1371 +1372 +1373 +1374 +1375 +1376 +1377 +1378 +1380 +1381 +1382 +1383 +1384 +1385 +1386 +1387 +1388 +1389 +1390 +1391 +1393 +1396 +1397 +1398 +1399 +1400 +1401 +1402 +1403 +1404 +1405 +1406 +1407 +1408 +1409 +1410 +1411 +1412 +1413 +1414 +1415 +1416 +1417 +1418 +1419 +1420 +1421 +1422 +1423 +1424 +1425 +1426 +1427 +1428 +1429 +1430 +1431 +1432 +1433 +1434 +1435 +1436 +1437 +1438 +1439 +1440 +1441 +1442 +1443 +1444 +1445 +1446 +1447 +1448 +1449 +1450 +1451 +1452 +1453 +1454 +1455 +1456 +1457 +1458 +1459 +1460 +1461 +1462 +1463 +1464 +1465 +1466 +1467 +1468 +1469 +1470 +1471 +1472 +1473 +1474 +1475 +1476 +1477 +1478 +1479 +1480 +1481 +1483 +1484 +1485 +1486 +1487 +1488 +1489 +1490 +1491 +1492 +1493 +1494 +1495 +1496 +1497 +1498 +1499 +1500 +1501 +1502 +1503 +1504 +1505 +1506 +1507 +1508 +1509 +1510 +1511 +1513 +1514 +1515 +1516 +1517 +1518 +1519 +1520 +1521 +1522 +1523 +1524 +1525 +1527 +1528 +1529 +1530 +1531 +1532 +1534 +1535 +1536 +1537 +1538 +1539 +1541 +1542 +1543 +1544 +1545 +1546 +1547 +1548 +1549 +1550 +1551 +1552 +1553 +1554 +1555 +1556 +1557 +1558 +1559 +1560 +1561 +1562 +1563 +1564 +1565 +1566 +1567 +1568 +1569 +1570 +1571 +1572 +1573 +1574 +1575 +1576 +1577 +1578 +1579 +1580 +1581 +1582 +1583 +1584 +1585 +1586 +1587 +1588 +1589 +1590 +1591 +1592 +1593 +1594 +1595 +1597 +1598 +1599 +1600 +1602 +1603 +1604 +1605 +1606 +1607 +1608 +1609 +1610 +1611 +1612 +1613 +1614 +1615 +1616 +1617 +1618 +1619 +1620 +1621 +1622 +1623 +1624 +1625 +1626 +1627 +1628 +1629 +1630 +1633 +1634 +1635 +1636 +1637 +1638 +1639 +1640 +1641 +1642 +1643 +1644 +1645 +1646 +1647 +1648 +1649 +1650 +1651 +1652 +1653 +1654 +1655 +1656 +1657 +1658 +1659 +1660 +1661 +1662 +1663 +1664 +1665 +1666 +1667 +1668 +1669 +1670 +1671 +1672 +1673 +1674 +1675 +1676 +1677 +1678 +1679 +1680 +1681 +1682 +1683 +1684 +1685 +1686 +1687 +1688 +1689 +1692 +1693 +1694 +1695 +1696 +1697 +1698 +1699 +1700 +1701 +1702 +1703 +1704 +1705 +1706 +1707 +1708 +1709 +1710 +1711 +1712 +1713 +1714 +1715 +1716 +1717 +1718 +1719 +1720 +1721 +1722 +1723 +1724 +1725 +1726 +1727 +1728 +1729 +1731 +1732 +1733 +1734 +1735 +1736 +1737 +1738 +1739 +1740 +1741 +1742 +1743 +1744 +1745 +1746 +1747 +1748 +1749 +1750 +1751 +1752 +1753 +1754 +1755 +1756 +1757 +1758 +1759 +1760 +1761 +1762 +1763 +1764 +1765 +1766 +1767 +1768 +1769 +1770 +1771 +1772 +1773 +1774 +1775 +1776 +1777 +1778 +1779 +1786 +1787 +1788 +1789 +1790 +1791 +1792 +1793 +1794 +1795 +1796 +1797 +1798 +1799 +1800 +1801 +1802 +1803 +1804 +1805 +1806 +1807 +1808 +1809 +1810 +1811 +1812 +1813 +1814 +1815 +1816 +1817 +1818 +1819 +1820 +1821 +1822 +1823 +1824 +1825 +1826 +1827 +1828 +1829 +1830 +1831 +1832 +1833 +1834 +1835 +1836 +1837 +1838 +1839 +1840 +1841 +1842 +1843 +1844 +1845 +1846 +1847 +1848 +1849 +1850 +1851 +1853 +1854 +1855 +1856 +1857 +1858 +1859 +1860 +1861 +1862 +1863 +1864 +1865 +1866 +1867 +1868 +1869 +1870 +1871 +1872 +1873 +1874 +1875 +1876 +1877 +1878 +1879 +1880 +1881 +1882 +1884 +1885 +1887 +1888 +1889 +1890 +1892 +1893 +1894 +1895 +1897 +1898 +1900 +1901 +1902 +1903 +1904 +1907 +1908 +1909 +1910 +1911 +1912 +1913 +1914 +1915 +1916 +1917 +1918 +1919 +1920 +1921 +1922 +1923 +1924 +1925 +1926 +1927 +1928 +1929 +1930 +1931 +1932 +1933 +1934 +1935 +1938 +1939 +1940 +1941 +1942 +1943 +1944 +1945 +1946 +1947 +1948 +1949 +1950 +1951 +1952 +1953 +1954 +1955 +1956 +1957 +1958 +1959 +1960 +1961 +1962 +1963 +1964 +1965 +1966 +1967 +1968 +1969 +1970 +1971 +1972 +1973 +1974 +1975 +1976 +1977 +1978 +1979 +1980 +1981 +1982 +1983 +1984 +1985 +1986 +1987 +1988 +1990 +1991 +1992 +1993 +1994 +1995 +1996 +1997 +1998 +1999 +2000 +2001 +2003 +2004 +2005 +2006 +2007 +2008 +2009 +2010 +2011 +2012 +2013 +2014 +2015 +2016 +2017 +2018 +2019 +2020 +2021 +2022 +2023 +2024 +2025 +2026 +2027 +2028 +2029 +2030 +2031 +2032 +2033 +2034 +2035 +2036 +2037 +2038 +2039 +2040 +2041 +2042 +2043 +2044 +2045 +2046 +2047 +2048 +2049 +2050 +2051 +2052 +2053 +2054 +2055 +2056 +2057 +2058 +2059 +2060 +2061 +2062 +2063 +2064 +2065 +2066 +2067 +2068 +2069 +2070 +2071 +2072 +2073 +2074 +2075 +2076 +2077 +2078 +2079 +2080 +2081 +2083 +2084 +2085 +2086 +2087 +2088 +2089 +2090 +2091 +2092 +2093 +2094 +2095 +2096 +2097 +2098 +2099 +2100 +2101 +2102 +2103 +2104 +2105 +2106 +2107 +2108 +2109 +2110 +2111 +2112 +2113 +2114 +2115 +2116 +2117 +2118 +2119 +2120 +2121 +2122 +2123 +2124 +2125 +2126 +2127 +2128 +2129 +2130 +2131 +2132 +2133 +2134 +2135 +2136 +2137 +2138 +2139 +2140 +2141 +2142 +2143 +2144 +2145 +2146 +2147 +2148 +2149 +2150 +2151 +2152 +2153 +2154 +2155 +2156 +2157 +2158 +2159 +2160 +2161 +2162 +2163 +2164 +2165 +2166 +2167 +2168 +2169 +2170 +2171 +2172 +2173 +2174 +2175 +2176 +2177 +2178 +2179 +2180 +2181 +2182 +2183 +2184 +2185 +2186 +2187 +2188 +2189 +2190 +2191 +2192 +2193 +2194 +2195 +2196 +2197 +2198 +2199 +2200 +2201 +2202 +2203 +2204 +2205 +2206 +2207 +2208 +2210 +2211 +2212 +2213 +2214 +2215 +2216 +2217 +2218 +2219 +2220 +2221 +2222 +2223 +2224 +2225 +2226 +2227 +2228 +2229 +2230 +2231 +2232 +2233 +2234 +2235 +2236 +2237 +2238 +2239 +2240 +2241 +2242 +2243 +2244 +2245 +2246 +2247 +2248 +2249 +2250 +2251 +2252 +2253 +2254 +2255 +2256 +2257 +2258 +2259 +2260 +2261 +2262 +2263 +2264 +2265 +2266 +2267 +2268 +2269 +2270 +2271 +2272 +2273 +2274 +2275 +2276 +2277 +2278 +2279 +2280 +2281 +2282 +2283 +2284 +2285 +2286 +2287 +2289 +2290 +2292 +2293 +2294 +2295 +2296 +2297 +2298 +2299 +2300 +2301 +2302 +2303 +2304 +2305 +2306 +2307 +2308 +2309 +2310 +2311 +2312 +2313 +2314 +2315 +2316 +2317 +2318 +2319 +2320 +2321 +2322 +2323 +100017 +100016 +100006 +100018 +100019 +100020 +100021 +100023 diff --git a/conf/jonnyboys_defaults.sh b/conf/jonnyboys_defaults.sh index 677815282..1efc72d23 100644 --- a/conf/jonnyboys_defaults.sh +++ b/conf/jonnyboys_defaults.sh @@ -240,6 +240,12 @@ export MISC_ONLY="true" ############################################################ +#run ugo's automake.php script to create releases, this does not use regexes and will run in a loop prior +#to update_releases.php +export UGO_THREADED="true" + +############################################################ + #Create releases, this is really only necessary to turn off when you only want to post process export RELEASES="true" @@ -300,6 +306,9 @@ export PARSING="true" #this also includes kevin123's categorymod.php, you must run either update_svn.sh or fix_files.sh to copy the file into place export PARSING_MOD="true" +#choose to use ugo's misc_sorter3.php script also, this will run before update_parsing, in the same loop +export MISC_SORTER="true" + #run update_parsing.php against the whole db or just the last 24 hours export PAST_24_HOURS="false" @@ -467,7 +476,7 @@ export RAMDISK_PATH="/var/ramdisk" ########################################################### #logs can be written, per pane, to the logs folder -export WRITE_LOGS="true" +export WRITE_LOGS="truel" ########################################################### diff --git a/config.sh b/config.sh index 968ee99b4..987c387c0 100644 --- a/config.sh +++ b/config.sh @@ -239,6 +239,15 @@ export MISC_ONLY="false" ############################################################ +#run ugo's automake.php script to create releases, this does not use regexes and will run in a loop prior +#to update_releases.php, this can be considerably slower, but may give you release that were being missed +#this might just overwhelm your db, so expect things to move slower, but you might get more in return +#this will deactivate all regexes, to reactive the regexes, run scripts/reactivate_regexes.sh +#still a wip +export UGO_THREADED="false" + +############################################################ + #Create releases, this is really only necessary to turn off when you only want to post process export RELEASES="false" @@ -299,6 +308,9 @@ export PARSING="false" #this also includes kevin123's categorymod.php, you must run either update_svn.sh or fix_files.sh to copy the file into place export PARSING_MOD="false" +#choose to use ugo's misc_sorter3.php script also, this will run before update_parsing, in the same loop +export MISC_SORTER="false" + #run update_parsing.php against the whole db or just the last 24 hours export PAST_24_HOURS="true" diff --git a/scripts/reactivate_regexes.sh b/scripts/reactivate_regexes.sh new file mode 100755 index 000000000..949ca3f9c --- /dev/null +++ b/scripts/reactivate_regexes.sh @@ -0,0 +1,16 @@ +SOURCE="${BASH_SOURCE[0]}" +DIR="$( dirname "$SOURCE" )" +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +cd $DIR"/../conf" +while read p; do + mysql --defaults-file=my.cnf -uroot newznab -e "update releaseregex set status=1 where status=0 and ID=$p" +done < active_regexes.txt + +rm active_regexes.txt diff --git a/ugo/automake.php b/ugo/automake.php new file mode 100644 index 000000000..fed77da5f --- /dev/null +++ b/ugo/automake.php @@ -0,0 +1,1566 @@ + 3) + { + $name = $relname; + } + + echo "Rname: ".$name."\n"; + + $name = cleanstr($name); + + $name = preg_replace('/\(?yenc\)?/iU','',$name); + + $name = preg_replace("/^(\[\d+\].\[.+\].\[#?a.b.[^\]]+\].(\[))/i", " ", $name); + + $name = preg_replace("/\[(?![^\[]+\])|\((?![^\(]+\))|\{(?![^\{]+\})/i", " ", $name); + + $name = preg_replace("/^\(?\?+\)/i", " ", $name); + +// $name = preg_replace('/ \b(rar|zip|avi|mp3|mp4|dmg|tar|nfo|nzb|par2)\b/iU','.$1',$name); + + $name = preg_replace('/\bmac[\. ]osx|mac[\. ]os[\. ]x\b|macos[\. ]x\b/iU', 'macosx', $name); + + $name = preg_replace('/ - - /iU',' - ',$name); + + $name = preg_replace('/^[\.\-\_\<\|]|[\.\-\_\|]$|\"/iU',' ',$name); + + $name = preg_replace('/^\s|\s$/iU','',$name); + + $name = preg_replace('/ +/iU',' ',$name); + + $name = cleanstr($name); + + $name = preg_replace('/(\d{1,4}? ?)?([o0]f ?\d{1,4}?)$/iU','',$name); + + $name = preg_replace('/ ?[o0]f ?$/iU','',$name); + + $name = preg_replace('/ +/iU',' ',$name); + + + if ($qual != '') + { + $name = $name." ".$qual; + } + + echo "release: ".$name." * * * ".$parent['name']."\n"; + + $i = 1; + $parts = count($results); + $notyet = true; +// echo $parts."\n"; + +// echo $results[0]['reltotalpart']."\n"; +// echo $results[0]['ID']."\n"; + + if (isset($results[0]) && ($results[0]['reltotalpart'] <= $parts) && $parts > 0 && is_string($name)) + { + $notyet = false; + echo "doing update: ".$parts." ".count($results)."\n"; + + $parts = count($results); + + $query = "Select * from `binaries` where `relname`= ".$db->escapeString($name)." and procstat = 5;"; + + $count = $db->query($query); + + if (count($count) > 0) + { + $nombre = $db->escapeString($name." ".$parent["ID"]); + $notyet = clearmatches ($name, $results, $code = -6); + return $notyet; + echo "duplicate $nombre\n"; + } else { + $nombre = $db->escapeString($name); + } + + foreach ($results as $r) + { + $query = sprintf("UPDATE `binaries` SET `relname`=%s, `relpart`=%d, `reltotalpart`=%d, `procstat`=%d, `regexID`=%d, `fromname`='%s' where procstat != 5 AND `ID` = %d;", utf8_encode($nombre), $i++, $parts, $PROCSTAT_TITLEMATCHED, -(ord($sect) - ord("A") + 1), $mail, $r["ID"]); + + echo "doing update: ".$r['name']."\n"; + echo $query."\n"; + + var_dump($db->query($query)); + + var_dump($db->getLastError()); + var_dump($db->getAffectedRows()); + + } + } else if ($parts == 1){ + //$notyet = clearmatches('', $results, $db); + } + + return $notyet; +} + +function cleanstr ($str = '', $full = false) +{ +// echo "CS1: ".$str."\n"; + +// $str = preg_replace('/\(\?\?\?\?\)/iU', '', $str); + $str = preg_replace('/\(??yenc\)??/iU', '', $str); + $str = preg_replace('/(?<=\.\b[a-z][a-z0-9]{2})(\.\b\d{1,4}?\b)(?!\.[0-9])/U', ' ', $str); +// $str = preg_replace('/^\s*(<*?(www\.)?[^\[\"]+?(\.\b(com|net|org|info)\b[^\[\(\"]*?))/iU','', $str); + $str1 = preg_replace('/\s*([<>]*?(www\.)?[^\[\<>"]+?(\.\b(com|net|org|info)\b(?!\.[a-zA-Z][a-zA-Z0-9]{2})[^\[\(\"<>]*?))/U',' ', $str); + + if (strlen($str1)>0) + $str = $str1; + + $str = preg_replace('/(\.\b[a-z][a-z0-9]{2}?\b|\.NFO|\.part[0-9]{1,4}?)+?/U',' ', $str); + $str = preg_replace('/\.vol\d{1,4}?\+\d{1,4}?|\binfo\b|\.par2\b|\bp\d{1,4}?\b/iU', ' ', $str); + +$str = preg_replace('/[\<\>]/iU', ' ', $str); + + //$str = preg_replace('/[\-\"\:]|(?![a-z0-9])\'(?![a-z0-9])/',' ',$str); + $str = preg_replace('/[\"\:]|(?![a-z0-9])\'(?![a-z0-9])/',' ',$str); + $str = preg_replace('/\-(?!\d\d)/',' ',$str); + $str = preg_replace('/[\.](?!\d|\b[a-z]\b)(? 0); + + +// echo "CS2: ".$str."\n"; + return $str; + +} + +function getname ($namemat = array()) +{ + $name = $namemat[0]; + +// echo "G1: ".$name."\n"; + +// $name = preg_replace('/[\(\{\[]\d{1,4}?(?!\d\)?)/iU','',$name); + $name = preg_replace('/^re:|Repost\b(:?)/iU','',$name); + $name = preg_replace('/^AutoRarPar\d{3,5} /iU','',$name); + $name = preg_replace('/[\x01-\x1f]/iU',' ',$name); + + if (strlen($name) <= 3) + { + $name = $namemat[count($namemat)-1]; + } +// $name = preg_replace('/\bmac[\. ]osx|mac[\. ]os[\. ]x\b|macos[\. ]x\b/iU', 'macosx', $name); +// $name = preg_replace('/(\.\b[a-z][a-z0-9]{2}\b|\.part[0-9]{1,4}?)(?1) + { + do + { + if (strcmp($arr[$i-1] ,$arr[$i]) == 0) + { + unset($arr[$i]); + $arr = array_values($arr); + } else { + $i++; + } + } while ($i < count($arr)); + } + +// var_dump($arr); +// echo "\n"; + return $arr; +} + +function checknames ($post, $matches, $fullname, $parts, $r, $db, $mail) +{ + global $relname, $suffix; + global $nzb, $nfo; + global $pars, $files; + global $qual; + + echo "cn: ".$post."\n"; + echo "fn: ".$fullname."\n"; + + $qualities = array('480', '720', '720p', '1080', '1080p', 'ac3', 'audio_ts', 'avi', 'bd25', 'bd50', 'bdmv', 'bdrip', 'bluray', 'BR-Disk', 'BR-Rip', 'brrip', 'cam', 'camrip', 'dc', 'directors cut', 'divx\d?', 'dts', 'DVD-R', 'DVD-Rip', 'dvd', 'dvdr', 'dvdrip', 'dvdscr', 'extended', 'h264', 'hd', 'hdcam', 'hdts', 'iso', 'm2ts', 'mkv', 'mpeg', 'mpg', 'ntsc', 'pal', 'proper', 'ppvrip', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'repack', 'rip', 'scr', 'screener', 'tc', 'telecine', 'telesync', 'ts', 'video_ts', 'unrated', 'x264', 'xvid'); + + $pattern = '/[\{\[\(]0*?1\/'.$r['totalParts'].'??[\]\}\)]$/iU'; + + $fullname = preg_replace($pattern, '', $fullname); + + $post = preg_replace($pattern, '', $post); + + $str = $post; + + if (preg_match('/^[\{\[\(]?(attn|att|attention)\:?/iU', $post)) + { + $str = preg_replace('/\batt[^ \:]*?\:?/iU','', $post); +// do { +// $str = preg_replace('/^[\[\{\(]|[\]\}\)]$|^ | $/iU','', $str, -1, $count); +// } while ($count > 0); +// if (preg_match('/ /iU', $str)) +// { +// $str = $fullname; +// } else { +// $str = $post; +// } + + echo "attn: ".$str."\n"; + } + + if (!preg_match('/\d{1,4}?\/\d{1,4}?/iU', $fullname) || preg_match('/(File|Post) \d{1,4}? ?[o0]f ?\d{1,4}?/iU', $fullname)) + { + //$str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $post); + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $post); + } else { + //$str = $post; + if (preg_match('/\d{1,4}?\/\d{1,4}?[\]\}\)]/iU', $fullname) && preg_match('/ ?[o0]f ?\d{1,4}?/iU', $fullname)) + { + $str = preg_replace('/[\{\{\(]\d{1,4}?\/(\d{1,4}?)[\]\}\)]/iU','', $str); + echo "cn1: ".$str."\n"; + } else { + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $str); + $str = preg_replace('/\d{1,4}?\/(\d{1,4}?)/iU','$1', $str); + echo "cn2: ".$str."\n"; + } + } + +// $str = preg_replace('/(?<=\.\b[a-z][a-z0-9]{2})(\.\b\d{1,4}?\b)/iU','', $str); +// $str = preg_replace('/(\.(par2|\b[a-z][a-z0-9]{2}?\b|vol\d{1,4}?\+\d{1,4}?))+?(\.\d{1,4}?)?/iU',' ',$str); +// $str = preg_replace('/(\binfo\b|\"|\-| +)/iU',' ',$str); +// $str = preg_replace('/[\(\[\{]??\d{1,4}?\/\d{1,4}?[\]\)\}]??/iU','', $str); + + $postwords = $namemat = splitarray($str); + + +// echo count($matches); +// print_r($postwords); +// echo "Name: ".cleanstr($str)."\n"; +// echo "\n"; + + $actual = array(); + + $nfo = 0; + $nzb = 0; + $segs = 0; + $segsarr = array(); + $pars = 0; + $files = 0; + $mode = array(); + $extras = 0; + + foreach ($matches as $m) + { + + $str = $m['name']; + + if ($suffix == '') + { + $tmp = preg_split('/\b(rar|zip|tar)\b/', $str, 0, PREG_SPLIT_DELIM_CAPTURE); + + if (count($tmp) > 1) + { + $suffix = ".".$tmp[1]; + } + } + + if (!preg_match('/\d{1,4}?\/\d{1,4}?/iU', $m['name'])) + { + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $m['name']); + } else { + if (preg_match('/\d{1,4}?\/\d{1,4}?[\]\}\)]/iU', $m['name']) && preg_match('/ ?[o0]f ?\d{1,4}?/iU', $m['name'])) + { + $str = preg_replace('/[\{\{\(]\d{1,4}?\/(\d{1,4}?)[\]\}\)]/iU','', $m['name']); + } else { + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $m['name']); + $str = preg_replace('/\d{1,4}?\/(\d{1,4}?)/iU','$1', $m['name']); + } + } + + +// $str = preg_replace('/(?<=\.\b[a-z][a-z0-9]{2})(\.\b\d{1,4}?\b)/iU','', $str); +// $str = preg_replace('/(\.(par2|\b[a-z][a-z0-9]{2}?\b|vol\d{1,4}?\+\d{1,4}?))+?(\.\d{1,4}?)?/iU',' ',$str); +// $str = preg_replace('/(\binfo\b|\"|\-| +)/iU',' ',$str); +// $str = preg_replace('/[\(\[\{]??\d{1,4}?\/\d{1,4}?[\]\)\}]??/iU','', $str); + +// $str = cleanstr($str); + + $keywords = splitarray($str); + +// print_r($keywords); +// echo "m ".$str."\n"; + + $i = 0; + foreach ($postwords as $p) + { + foreach ($keywords as $k) + { +// echo $p." ".$k."\n"; + + if (strcmp($p,$k) == 0) + { +// echo $p." ".$k."\n"; + $i++; + } + } + } + $mode[] = $i; + } + +$mode = array_count_values($mode); +ksort($mode); + +// echo "mode\n"; +// var_dump($mode); + +end($mode); +$index = 0; +$key = 0; + +do { + $index = $index + current($mode); +echo "a".current($mode)."\n"; + $key++; + +} while (($index < $parts) && (prev($mode) !== false)); + +echo "index raw".$index." ".$key."\n"; + +if (current($mode) !== false) +{ + if ($index > $parts) + { + $j = current($mode); + echo "c".$j."\n"; + next($mode); + $index = $index - $j; + if ($index < $parts * 0.9) + { + prev($mode); + } + echo "d".current($mode)."\n"; + } +} else { +echo "e".current($mode)."\n"; + end($mode); +echo "f".current($mode)."\n"; +} + +if (current($mode) === false) +{ + end($mode); +echo "g".current($mode)."\n"; +} + +// if (end($mode) > prev($mode) || current($mode) > 1000) +// { +// end($mode); +// } + +$index = key($mode); + +echo "index ".$index."\n"; + +// if ($index < 5) +// { +// $index = count($postwords); +// } + +foreach ($matches as $m) +{ + $str = $m['name']; + + if ($suffix == '') + { + $tmp = preg_split('/\b(rar|zip|tar)\b/', $str, 0, PREG_SPLIT_DELIM_CAPTURE); + + if (count($tmp) > 1) + { + $suffix = ".".$tmp[1]; + } + } + + if (!preg_match('/\d{1,4}?\/\d{1,4}?/iU', $m['name'])) + { + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $m['name']); + } else { + if (preg_match('/\d{1,4}?\/\d{1,4}?[\]\}\)]/iU', $m['name']) && preg_match('/ ?[o0]f ?\d{1,4}?/iU', $m['name'])) + { + $str = preg_replace('/[\{\{\(]\d{1,4}?\/(\d{1,4}?)[\]\}\)]/iU','', $m['name']); + } else { + $str = preg_replace('/\d{1,4}? ?([o0]f ?\d{1,4}?)/iU','$1', $m['name']); + $str = preg_replace('/\d{1,4}?\/(\d{1,4}?)/iU','$1', $m['name']); + } + } + + $keywords = splitarray($str); + + $i = 0; + foreach ($postwords as $p) + { + foreach ($keywords as $k) + { + + if (strcmp($p,$k) == 0) + { +// echo $p." ".$k."\n"; + $i++; + } + } + } + + +// echo $i." ".$index." ".$m['name']." ".$m['ID']."\n"; + if ($i >= $index && $index > 0) + { + +// echo $i." ".$index." ".$m['name']." ".$m['ID']."\n"; + + $actual[] =$m; + $str = preg_replace('/[\.](?!\d|\b[a-z]\b)(?= 1) + { + $pars = $pars + $m['totalParts']; + $extras++; +// echo "pars: ".$m['name']." ".$n."\n"; + + } else if (preg_match('/sfv|\bnfo\b|\bnzb\b|par2/Ui', $str) >= 1) + { + $extras++; +// echo "extra: ".$m['name']." ".$n."\n"; + } else + { + $segsarr[] = $m['totalParts']; +// echo "segs: ".$m['name']." ".$n."\n"; + } + + $str = cleanstr($str); + +//echo $str."\n"; + + $str = preg_split("/[\s_\"]+/", $str, 0, PREG_SPLIT_DELIM_CAPTURE); + foreach ($str as $n) + { + $namemat[] = $n; + if (strcasecmp($n, 'nfo') == 0) + { + $nfo = $m['ID']; + } else if (strcasecmp($n, 'nzb') == 0) + { + $nzb = $m['ID']; + } +// echo $n."\n"; + } + } +// print_r($actual); +// echo "\n"; + +// echo "suffix".$suffix."\n"; +} + +echo "p2: ".$pars." files: ".array_sum($segsarr)." e: ".$extras."\n"; + +$files = count($segsarr); + +if (array_sum($segsarr) > 0) +{ + $segs = $pars /array_sum($segsarr) * ($files + $extras); +} + +$pars = round($segs); + +echo "p2: ".$pars." files: ".$files."\n"; + +$namemat = array_count_values($namemat); +asort($namemat); +//var_dump($namemat); + +if (count($namemat) > 1) +{ + reset($namemat); + $i = 1; + $max = count($actual)-1; + + while (next($namemat) < $max && current($namemat) !== false) + { +//echo current($namemat)." ".$i."\n"; + $i++; + } + + $namemat = array_slice($namemat, $i,count($namemat), true); + +// var_dump($namemat); + + $j = 0; + $name = ''; + $imat = array(); + foreach ($actual as $m) + { + $i = 0; + + foreach ($namemat as $key => $value) + { + $str = $m['name']; + $str = preg_replace('/(\.(vol\d{1,4}?\+\d{1,4}?|sfv|part\d{1,4}?|nfo|\bnzb\b|par2))/iU','', $str); + //$str = preg_replace('/[\[\{\(]??\d{1,4}?\/\d{1,4}?[\]\}\)]??/iU','', $str); + //$str = preg_replace('/[\[\{\(]??\d{1,4}?(\/\d{1,4}?| [o0]f \d{1,4}?)[\]\}\)]??/iU','$1', $str); + if(strripos($str, $key) !== false) + { + $i++; + } + } +// echo $i." ".$j." ".$str."\n"; + $imat[] = $i; + if ($i > $j) + { + $name = $str; + $j = $i; + } + } + + $name = cleanstr($name); + + echo 'name = '.$name."\n"; + + $name2 = ''; + + $name = preg_replace('/[\.](?!\d|\b[a-z]\b)(? $value) + { + +// echo $n." ".$key."\n"; + if (strcmp($key,$n) == 0) + { +// echo $n."\n"; + $name2 = $name2.' '.$n; + unset($namemat[$key]); + } + } + } + + do { + $name2 = preg_replace('/^[\[\{\(](?!\d+\])|[\]\}\)]$|^ | $/iU','', $name2, -1, $count); + } while ($count > 0); + + echo "name2 = ".$name2."\n"; + $relname = cleanstr($name2); + echo "relname = ".$relname."\n"; + +//var_dump($namemat); + + $qual = ''; + + foreach ($namemat as $key => $value) + { + foreach($qualities as $q) + { +// echo $key." ".$q."\n"; + if (stripos ($key , $q) !== false) + { + $qual = $qual.' '.$q; + continue(2); + } + } + } + + + +// var_dump($namemat); + + if ($nfo == 0) + { + $tmp = addnfo ($mail, $r, $db, $relname); + + echo "addnfo: ".count($actual)."\n"; + + if (isset($tmp)) + { +// $actual[] = $tmp; + + // var_dump($actual); + } + } +} +return $actual; +} + +function clearmatches ($name, $results, $code = -1) +{ + global $db; + + echo "clearing ".count($results)."\n"; + + global $relname; + + foreach ($results as $r) + { + if ($name == '') + { + $query = sprintf("update binaries set procstat=%d, `procattempts`=0, `regexID`=NULL, `relpart`=0, `reltotalpart`=0 where (ID = %d OR binaryhash ='%s') and procstat != 5" , $code, $r["ID"], $r['binaryhash'] ); + } else { + + $query = sprintf("update binaries set procstat=%d, `procattempts`=0, `regexID`=NULL, `relpart`=0, `reltotalpart`=0, `relname`=%s where (ID = %d OR binaryhash ='%s') and procstat != 5" , $code, $db->escapeString($relname), $r["ID"], $r['binaryhash']); + } + +// echo $query."\n"; + + $db->query($query); + } + return false; +} + +function checkmatches ($query, $r, $short = false) +{ + +global $relname, $nfo; +global $db; + + $query = $query." AND groupID=".$r['groupID']; + + if ($short) + { + $query = $query." AND `date` <= DATE_ADD('".$r['date']."', INTERVAL 6 HOUR) AND `date` >= DATE_SUB('".$r['date']."', INTERVAL 6 HOUR) "; + } else { + $query = $query." AND `date` <= DATE_ADD('".$r['date']."', INTERVAL 2 DAY) AND `date` >= DATE_SUB('".$r['date']."', INTERVAL 2 DAY) "; + } + + if (preg_match('/^(\[\d+\])/', $r['name'], $matches)) + { + $query = $query." AND name like '%".$matches[1]."%' "; + } + + + $query = $query." AND `procstat` != 5"; + + $query = $query." GROUP BY binaryhash"; + + $query = $query." ORDER BY name;"; + + $time = microtime(true); + + $matches = $db->query($query); + + $time = microtime(true) - $time; + + //if ($time > 1) + //trigger_error("\ntime = $time \tmatches = ".count($matches)."\nquery = ".$query."\n"); + + echo count($matches)." ".$query."\n"; + + if (count($matches) == 0) + { + $query = str_ireplace("match (name) against ('"," `name` like '%", $query); + $query = str_ireplace("' IN NATURAL LANGUAGE MODE)","%'", $query); + + $matches = $db->query($query); + + $matches = $db->query($query); + } + + + echo $query."\n"; + return $matches; +} + +function makenzb ($nzb, $nzb1, $nfo, $nfo1) +{ + global $nntpconnected, $nntp; + global $relname; + + +//echo $nzb." ".$nfo."\n"; + + if (!$nntpconnected) + { + //$nntp->doConnect(); + //$nntpconnected = true; + } + if ($nzb == 0) + { + $nzb = $nzb1; + } + + if ($nzb != 0) + { +echo "getting nzb\n"; + + $relname = preg_replace('/[\[\{\(]?\d{1,4}?\/\d{1,4}?[\]\}\)]?/iU','', $relname); + $relname = preg_replace('/\/|\:|^\.|^ | $|^[\[\]\{\}\(\)](?!\d+\])/iU','', $relname); + $relname = preg_replace('/^[\`\'\"\<\/]/iU','', $relname); + $relname = preg_replace('/^[\-\+\_]/iU','', $relname); + $relname = preg_replace('/( \d{1,4}?)? ?[o0]f ?\d{1,4}?$/iU','', $relname); + $relname = cleanstr($relname); + + //$bin = $nntp->getBinary($nzb); + //file_put_contents(FS_ROOT."/nzbs/".$relname.".nzb", $bin); + } + if ($nfo == 0) + { + $nfo = $nfo1; + } + + if ($nfo != 0) + { +echo "getting nfo\n"; + + // $bin = $nntp->getBinary($nfo); + + // file_put_contents(FS_ROOT."/nzbs/".$relname.".nfo", $bin); + } +} + +function addnfo ($mail, $r, $db, $relname) +{ + $query = "SELECT * FROM `binaries` where match (name) against (".$db->escapeString($relname)." IN NATURAL LANGUAGE MODE)"; + + $query = $query." AND `name` like '%.nfo%'"; + + $query = $query." AND `fromname` like '%".$mail."%'"; + + $query = $query." AND groupID=".$r['groupID']; + + $query = $query." AND `date` <= DATE_ADD('".$r['date']."', INTERVAL 2 DAY) AND `date` >= DATE_SUB('".$r['date']."', INTERVAL 2 DAY) "; + + $query = $query." AND `procstat` != 5"; + + $query = $query." GROUP BY binaryhash"; + + $query = $query." limit 1;"; + + $matches = $db->query($query); + + if (isset($matches[0])) + { + return $matches[0]; + } else { + return null; + } +} + +function removemail($name, $mail) +{ + $poster = preg_split('/@/', $mail, 0, PREG_SPLIT_DELIM_CAPTURE); + +// var_dump($poster); + + $poster = preg_replace('/[\.\_\-\']/','.?', $poster[0]); + + $poster = str_replace('/', ' ', $poster); + + $poster = str_replace('\\', ' ', $poster); + + $poster = preg_quote($poster); + + echo "mail ".$mail."\n"; + + echo "poster $poster\n"; + + return preg_replace('/\b'.$poster.'\b/', '', $name); +} + +function okresults (&$results, &$cuenta, $parts, $i, $patern, $patern2, $r, $mail, $dogn = true) +{ + global $pars, $relname; + global $db; + + echo "OKr CR: ".count($results)." P2: $pars P: $parts\n"; + + if (count($results) + $pars < $parts || count($results) > 2 * $parts) + { + $cuenta1 = preg_split($patern, $r['name'], 0, PREG_SPLIT_DELIM_CAPTURE); + +echo "cuenta1 ".$i." ".$dogn; +//var_dump($cuenta1); + + if (isset($cuenta1) && $dogn) + { + $cuenta1[0] = getname($cuenta1); +//var_dump($cuenta1); + } + + if (!isset($cuenta1[$i]) || strlen($cuenta1[$i]) < 3) + { + $cuenta1[$i] = $cuenta1[0]; + } + + $cuenta1[0] = preg_replace($patern2 ,'', $cuenta1[$i]); + +echo "ok: ".$cuenta1[0]."\n"; +// var_dump($cuenta1); + + + $query = "SELECT * FROM `binaries` where name like '%".preg_replace('/\'/','\\\'', removemail($cuenta1[0], $mail))."%'"; + + $query = $query." AND `fromname` like '%".$mail."%'"; + + $matches1 = checkmatches($query, $r); + + $relname1 = $relname; + + $pars1 = $pars; + + $results1 = checknames($cuenta1[0], $matches1, $r['name'], $parts, $r, $db, $mail); + +echo "OKr1 CR1: ".count($results1)." P2.1: $pars1 P: $parts\n"; + + if (abs(count($results1) + $pars - $parts) < abs(count($results) + $pars1 -$parts) && count($results1) < 2 * $parts) + { +echo "using OKr1\n"; + $matches = $matches1; + $cuenta[0] = $cuenta1[0]; + $results = $results1; + $relname1 = $relname; + $pars1 = $pars; + } + + $pars = $pars1; + $relname = $relname1; + if (count($results) > $parts) + { + $matches1 = checkmatches($query, $r, true); + + $relname1 = $relname; + + $pars1 = $pars; + + $results1 = checknames($cuenta1[0], $matches1, $r['name'], $parts, $r, $db, $mail); + + echo "OKr2 CR2: ".count($results1)." P2.2: $pars1 P: $parts\n"; + + if (abs(count($results1) + $pars - $parts) < abs(count($results) + $pars1 -$parts) && count($results1) < 2 * $parts) + { + echo "using OKr2\n"; + $matches = $matches1; + $cuenta[0] = $cuenta1[0]; + $results = $results1; + $relname1 = $relname; + $pars1 = $pars; + } + + $pars = $pars1; + $relname = $relname1; + + } + } +} + +function domatching ($pattern, $db, $r, $oldname, $sect) +{ + global $notyet, $jump; + + global $ptr; + + global $nzb, $nfo; + + global $pars, $files; + + global $relname; + + $mail = preg_split('/(\b[a-z0-9\.\_\-\' ]+?\b@\b[a-z0-9\.\_\-\' ]+?\b)/Ui', $r['fromname'], 0, PREG_SPLIT_DELIM_CAPTURE); + + if (!isset($mail[1])) + { + $mail[1] = $r['fromname']; + } + + $mail[1] = preg_replace('/^\'|\'$/','', $db->escapeString($mail[1])); + + $cuenta = preg_split($pattern, removemail($r['name'], $mail[1]), 0, PREG_SPLIT_DELIM_CAPTURE); + + if (isset($cuenta[1]) && $notyet) + { + + echo "doing ".$sect."\n"; +// var_dump($cuenta); +// echo "\n"; + + echo "1: ".$r['name']."\n"; + echo "2: ".$cuenta[0]."\n"; + + $cuenta[0] = getname($cuenta); + + if (strlen($cuenta[0]) <= 3) + { + //$cuenta[0] = $cuenta[count($cuenta)-1]; + } + + echo "3: ".preg_quote($cuenta[0])."\n"; + + $query = "SELECT * FROM `binaries` where match (name) against (".$db->escapeString(removemail($cuenta[0], $mail[1]))." IN NATURAL LANGUAGE MODE)"; +// $query = "SELECT * FROM `binaries` where match (name) against (".$db->escapeString($r['name'])." IN NATURAL LANGUAGE MODE)"; + + $query = $query." AND `fromname` like '%".$mail[1]."%'"; + + $query = $query." AND name like '%".preg_replace('/\'/','\\\'', $cuenta[1])."%'"; + + if (is_null($r['importname'])) + $query = $query." AND importname is null"; + else + $query = $query." AND importname like '".$r['importname']."'"; + + $matches = checkmatches($query, $r); + + $parts = intval(preg_replace('/[^\d]/','', $cuenta[1])); + +// echo $query."\n"; + +// echo count($matches)."\n"; + echo $cuenta[1]."\n"; + + $results = checknames($cuenta[0] , $matches, $r['name'], $parts, $r, $db, $mail[1]); + + $patern = $pattern; + $patern2 = '/(\.(\b[a-z][a-z0-9]{2}?\b|\bNFO|\bnfo\b|p\d{1,4}?|par2|part\d{1,4}?|vol\d{1,4}?\+\d{1,4}?)(?!\]).+?)+?/iU'; + okresults ($results, $cuenta, $parts, 2, $patern, $patern2, $r, $mail[1], false); + + $patern = $pattern; + $patern2 = '/(\.(\b[a-z][a-z0-9]{2}?\b|\bNFO|\bnfo\b|p\d{1,4}?|par2|part\d{1,4}?|vol\d{1,4}?\+\d{1,4}?)(?!\]).+?)+?/U'; + okresults ($results, $cuenta, $parts, 0, $patern, $patern2, $r, $mail[1], false); + + $patern = '/([^\"\)\]]+?)(\.(\b[a-z][a-z0-9]{2}?\b|\bNFO|\bnfo\b|info\b|p\d{1,4}?|par2|part\d{1,4}?|vol\d{1,4}?\+\d{1,4}?))+?/U'; + $patern2 = '/(\.(\b[a-z][a-z0-9]{2}?\b|\bNFO|\bnfo\b|p\d{1,4}?|par2|part\d{1,4}?|vol\d{1,4}?\+\d{1,4}?))+?(?!\])/U'; + okresults ($results, $cuenta, $parts, 1, $patern, $patern2, $r, $mail[1], true); + + $nfo1 = $nfo; + $nzb1 = $nzb; + $pars1 =$pars; + $cuenta1 = $cuenta; + $relname1 = $relname; + +echo "OK end CR: ".count($results)." P: ". $parts." P2: ".$pars."\n"; + + $results1 = checknames($r['name'], $matches, $r['name'], $parts, $r, $db, $mail[1]); + + if (abs(count($results1) + $pars - $parts) < abs(count($results) + $pars1 -$parts) && count($results1) < 2 * $parts) + { + $results = $results1; + $relname1 = $relname; + $pars1 = $pars; + } + + $pars = $pars1; + $relname = $relname1; + +echo "OK end CR: ".count($results)." P: ". $parts." P2: ".$pars."\n"; + +// echo "6: ".$cuenta[0]."\n"; + $name = getname($cuenta); + +// echo "7: ".$name."\n"; + + //$results = checknames($name, $matches, $r['name'], $parts, $r, $db, $mail[1]); + + + echo $ptr." N:".$name." CM:".count($matches)." P:".$parts." CR:".count($results)." ".$r['age']." days ".$sect." ".$r['ID']."\n"; +// echo $query."\n"; + + + echo "stats: P2:".$pars." CR:".count($results)." P:".$parts." F:".$files." N:".$nzb."\n"; + + + //if ((count($results) +1 >= $parts || ($nzb && $nfo && $parts < 4)) && $oldname != $name) + if (($nzb != 0 || $nzb1 != 0) && $files == 0) + { +echo "D1\n"; + makenzb($nzb, $nzb1, $nfo, $nfo1); + $jump = $jump && clearmatches($name, $results, -3); + } else if (((((count($results) + $pars >= $parts) && ($files > 0 || count($results) == $parts)) || (count($results) >= $parts && ($files > 0 || count($results) == $parts))) && ($files <= 2 * $parts)) && $oldname != $name) + { +// echo "makerelease ".$sect."\n"; +echo "D2\n"; + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if ((((count($results) + $pars >= $parts) && $files > 2) || (count($results) >= $parts && $files > 2)) && ($files < 2 * $parts)) + { +echo "D3\n"; + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if (($nzb != 0 || $nzb1 != 0) && ($r['age'] >= 3 || $files == 0)) + { +echo "D4\n"; + makenzb($nzb, $nzb1, $nfo, $nfo1); + $jump = $jump && clearmatches($name, $results, -3); + } else if (count($results) >= $parts && $parts == 2 && $files == 2) { +echo "doing a 2 part release\n"; + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if (count($results) == 1 && $parts == 1 && $r['totalParts'] > 10 && $nzb == 0 && $nzb1 == 0) { +echo "doing a 1 part release\n"; + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if (count($results) <= 2 * $parts || count($results) > 0) { +echo "clearmatches ".$sect."\n"; +echo "stats: ".$pars." ".count($results)." ".$parts." ".$files." ".$nzb."\n"; + $jump = $jump && clearmatches('', $results); + } else { +echo "skipping\n"; +echo "stats: ".$pars." ".count($results)." ".$parts." ".$files." ".$nzb."\n"; + } + $notyet = false; + } +} + +function domatching2 ($pattern, $db, $r, $oldname, $sect, $parts = -1) +{ + global $notyet, $jump; + + global $relname; + + $mail = preg_split('/(\b[a-z0-9\.\_\-\' ]+?\b@\b[a-z0-9\.\_\-\' ]+?\b)/Ui', $r['fromname'], 0, PREG_SPLIT_DELIM_CAPTURE); + + if (!isset($mail[1])) + { + $mail[1] = $r['fromname']; + } + + $mail[1] = preg_replace('/^\'|\'$/','', $db->escapeString($mail[1])); + + $cuenta = preg_split($pattern, removemail($r['name'], $mail[1]), 0, PREG_SPLIT_DELIM_CAPTURE); + + + + if (isset($cuenta[1]) && $notyet) + { + //var_dump($cuenta); + echo "d2 1: ".$r['name']."\n"; + echo "d2 2: ".$cuenta[0]."\n"; + if (strlen($cuenta[0]) <= 3) + { + $cuenta[0] = $cuenta[1]; + } + echo "doing ".$sect."\n"; + $oldname = $cuenta[0]; +// $cuenta[0] = getname($cuenta); + $query = "SELECT * FROM `binaries` where match (name) against (".$db->escapeString($cuenta[0])." IN NATURAL LANGUAGE MODE)"; + + if ($pattern == '/^(\[\d+\])/') + { + $query = "SELECT * FROM `binaries` where name regexp '^".preg_quote($cuenta[0])."' "; + $query = str_replace('\\', "\\\\", $query); + } + + + $query = $query." AND `fromname` like '%".$mail[1]."%'"; + + $matches = checkmatches($query, $r); + +echo "CM ".count($matches)."\n"; + + if (count($matches) == 0) + { + $cuenta = preg_split($pattern, $r['name'], 0, PREG_SPLIT_DELIM_CAPTURE); + + if (strlen($cuenta[0]) <= 3) + { + $cuenta[0] = $cuenta[1]; + } + + $oldname = $cuenta[0]; + + $query = "SELECT * FROM `binaries` where match (name) against (".$db->escapeString($cuenta[0])." IN NATURAL LANGUAGE MODE)"; + + $query = $query." AND `fromname` like '%".$mail[1]."%'"; + + $matches = checkmatches($query, $r); + } + + $arr = array(); + + foreach ($matches as $value) + $arr[] = $value['name']; + natcasesort($arr); + $arr = array_values($arr); + + $files = array(); + + foreach ($matches as $value) + { + $key = array_search($value['name'], $arr); + + $files[$key] = $value; + } + + ksort($files); + + $matches = $files; + + $i = 0; + $k = 0; + $q = 0; + $par = 0; + $arr = array(); + $files = array(); + + foreach ($matches as $value) + { + + $tmp = str_replace('/', ' ', $oldname); + $tmp = str_replace('\\', ' ', $tmp); + + $tmp = preg_replace('/(\.[a-z][a-z0-9]{2,3})+$/', '', $tmp); + + echo "1100 ".preg_quote($tmp)."\n"; + echo "1100 ".$value['name']."\n"; + + if (preg_match('/('.preg_quote($tmp).')/iU', str_replace('/', ' ', $value['name']), $tmp)) + { + echo "1101"; + //var_dump($tmp); + preg_match('/^(.+)(?|(?:\.(?:part|r)(\d{1,4}?))|\.(\d{1,4}?)[\]"]|\.(\d{1,4}?)$)/iU', $value['name'], $tmp); + echo "1102"; + //var_dump($tmp); + if (isset($tmp[2]) && !preg_match('/\.(par2|vol\d{1,4}?\+\d{1,4}?|nzb|nfo)/iU', $value['name'])) + { + $j = similar_text($cuenta[0], $tmp[1], $p); + + $p = floor($p); + + $a = strval($tmp[2]) + 0; + +// echo "j = $j, p = $p, a= $a\n"; + + if ($j > $k || $p > $q) + { +// echo "reset\n"; + $i = $a - 1; + $k = $j; + $q = $p; + $arr = array(); + } + + + if ($a > $i && $j >= $k && $p >= $q) + { + $i = $a; + $k = $j; + $q = $p; + $arr[] = $value; + } + } else { + $par = $par + $value['totalParts']; + $files[] = $value; + } + } + } + + if (count($arr) > 0 && preg_match('/^(.+)(?|(?:\.(?:part|r)(\d{1,4}?))|\.(\d{1,4}?)[\]"]|\.(\d{1,4}?)$)/iU', $arr[0]['name'], $tmp)) + { + $arr1 = array(); + + foreach ($arr as $value) + $arr1[] = $value['name']; + natcasesort($arr1); + $arr1 = array_values($arr1); + + $files1 = array(); + + foreach ($arr as $value) + { + $key = array_search($value['name'], $arr1); + + $files1[$key] = $value; + } + + ksort($files1); + + $arr = $files1; + + $j = $tmp[2] - 1; + $missing = $j; + + $l = end($arr); + + $l = $l['totalParts']; + + $s = prev($arr); + + $s = $s['totalParts']; + + $tmp= prev($arr); + + $tmp = $tmp['totalParts']; + + if ($tmp > $s) + $s = $tmp; + + if ($s == 0) + $s = 1; + + $missing = $missing - ($par / $s); + + foreach ($arr as $a) + { + preg_match('/^(.+)(?|(?:\.(?:part|r)(\d{1,4}?))|\.(\d{1,4}?)[\]"]|\.(\d{1,4}?)$)/iU', $a['name'], $tmp); + { + if ($tmp[2] -$j != 1) + $missing = $missing - ($tmp[2] -$j) + 1; + if ($a['totalParts'] != $s) + $missing = $missing + 1; + $l = $a['totalParts']; + } + $j = $tmp[2] + 0; + } + + if ($l < $s) + $missing = $missing - 1; + else + echo "last not smaller\n"; + + echo "$i\n"; + if ($missing) + echo "missing $missing\n"; + + $arr = array_merge($arr, $files); + + $name = getname($cuenta); + + $results = checknames($oldname , $arr, $r['name'], count($arr), $r, $db, $mail[1]); +//trigger_error("CR= ".count($results)." AR= ".count($arr)." missing= $missing age= ".$r['age']." l= $l s= $s P=$parts ".$r['ID']); + + echo "CR= ".count($results)." AR= ".count($arr)." missing= $missing age= ".$r['age']." l= $l s= $s\n"; + + if (substr_count($r['xref'], ":") > 5) { + // spam if in more than five groups + echo "cleaning spam ".$r['ID']."\n"; + $jump = $jump && clearmatches($name, $results, -2); + } else if ($missing <= 0 && $l < $s && $r['age'] > .25 && count($results) == count($arr) && count($results) > 3 && $parts == -1) + { + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if ($missing <= 0 && $r['age'] > 2.5 && count($results) == count($arr) && count($results) > 3 && $parts == -1) { + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if ($r['age'] > .5 && count($results) == count($arr) && count($results) == 1 && $parts == 1) { + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else if ($r['age'] > .5 && count($results) == count($arr) && $parts == 0) { + $jump = $jump && makerelease($results, $r, $name, $db, $sect, $mail[1]); + } else + $jump = $jump && clearmatches('', $results); + } + +// echo "relname $relname * * * name $name\n"; + +// var_dump($results); + +// var_dump($files); + $notyet = false; + } +} + +$nzb = 0; +$nfo = 0; +$pars = 0; +$files = 0; +$qual = ''; + +$nntpconnected = false; + +$db = new Db; +$nntp = new Nntp; + +//$db->query("UPDATE `binaries` SET `procstat`=0 WHERE `procstat`=-1"); + +echo "starting auto matcher\n"; +$time = microtime(true); + +$ptr = 0; +$i = 0; +$j = 0; +$oldname = ''; +$name = "*"; +$oldid = 0; + +if (isset($argv[1])) + { + echo "doing ID ".$argv[1]."\n"; + } + +do { + +$time2 = microtime(true); + +echo "\nptr $ptr\n"; + +if (isset($argv[1])) + { +// $query = "SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age from binaries b where b.`ID` in (select a.ID from binaries a where a.`procstat`=0 AND a.groupID = ".$argv[1]." AND a.`name` REGEXP '^\\\\[[0-9]+\\\\]|\\\\.[0-9]{3}\"|\\\\.[0-9]{3}$|\\\\.r[0-9]+\\\\b|\\\\.part[0-9]+|([0-9] ?([o0]f|\\\\/) ?[0-9])|([\\\\(\\\\{\\\\[]?[0-9]+ *(segs|parts))' group by a.binaryhash order by null) limit ".$ptr.", 2"; + $query = "SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age from binaries where `ID` = ".$argv[1]; + $argv[1] = -1; + + $rel = $db->query($query); + + } else { + + $rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6"); + +// $query = "SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age from binaries b where b.`ID` in (select a.ID from binaries a where a.`procstat`=0 AND a.`name` REGEXP '^\\\\[[0-9]+\\\\]|\\\\.[0-9]{3}\"|\\\\.[0-9]{3}$|\\\\.r[0-9]+\\\\b|\\\\.part[0-9]+|([0-9] ?([o0]f|\\\\/) ?[0-9])|([\\\\(\\\\{\\\\[]?[0-9]+ *(segs|parts))' order by a.`groupID`, a.`name`) group by binaryhash order by null limit ".$ptr.", 2"; + $query = "SELECT *, HOUR (TIMEDIFF(now(), `date`)) / 24 AS age FROM binaries b where b.`ID` in ( SELECT a.ID FROM binaries a WHERE a.`procstat` = 0 GROUP BY a.binaryhash order by a.ID) limit ".$ptr.", 2"; +// $query = "SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age from binaries b where b.ID in (select a.ID from binaries a where a.`procstat`=0 AND a.name REGEXP '([0-9] ?([o0]f|\\/) ?[0-9])|([\\(\\{\\[]?[0-9]+ *(segs|parts))' order by a.`groupID`, a.`name`) order by null limit ".$ptr.", 2"; +// $query = "SELECT *, HOUR (TIMEDIFF(now(), `date`)) / 24 AS age FROM binaries b where b.`ID` = 4724491"; + + $rel = $db->query($query); + } + + //$rel = $db->query("SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age FROM `binaries` where `ID` = 2060224 order by `groupID`, `name` limit ".$ptr.", 5"); + //$rel = $db->query("SELECT *, hour(TIMEDIFF(now(), `date`))/24 as age FROM `binaries` where `name` LIKE '%Red Dwarf S10E05 %' order by `groupID`, `name` limit ".$ptr.", 5"); + // + + $patern = '/^\[[0-9]+\]|\.[0-9]{3}\"|\.[0-9]{3}$|\.r[0-9]+\b|\.part[0-9]+|([0-9] ?([o0]f|\/) ?[0-9])|([\(\{\[]?[0-9]+ *(segs|parts))/'; + + +$tim = microtime(true) - $time2; +echo "time1 = ".$tim."\n"; + +echo " * *\n"; +echo " * * * \n"; +echo " * * * *\n"; + + if (isset($rel[0])) + { + if ($oldid == $rel[0]['ID']) + { + $ptr++; + //$rel = $db->query("SELECT * FROM `binaries` WHERE `procstat`=0 and `name` REGEXP '(File|Post) [0-9]+ [o0]f [0-9]+' order by `groupID`, `name` limit ".$ptr.", 5"); + continue; + } + $oldid = $rel[0]['ID']; + } + + $relnum = count($rel); + + //echo "doing select 1 ".$relnum."\n"; + + if ($relnum > 0) + { +//echo "ptr = ".$ptr."\n"; +echo "relnum is $relnum\n"; + foreach ($rel as $r) + { + + echo "spam count ".substr_count($r['xref'], ":")."\n"; + +// trigger_error($r['procstat']); + if ($r['procstat'] != 0) + break; + +// trigger_error($r['name']); + + $relnum = $relnum - 1; + + $j++; + + $suffix = ''; + $notyet = true; + $jump = true; + + + // $r['name'] = preg_replace('/\'/','',$r['name']); + + + + + echo "name: ".$r['name']." ".$r['ID']."\n"; + + $relname = ''; + + $pattern = '/[\{\[\(]0*?1\/'.$r['totalParts'].'??[\]\}\)]$/iU'; + + $r['name'] = preg_replace($pattern, '', $r['name']); + +// echo "name: ".$r['name']."\n"; + +//WHERE `name` REGEXP CONCAT( '[\{\[\(]0*1\/', `totalParts` , '[\]\}\)]$' ) AND `totalParts` >10 + + if (preg_match('/(Usenet Index Post)/i', $r['name'])) + $jump = $jump && clearmatches('', array(array('ID' => $r['ID'], 'binaryhash' => $r['binaryhash'])), -3); + + + $pattern = '/(?:File|Post) \d{1,4}? ?[o0]f ?(\d{1,4}?)/iU'; + + domatching($pattern, $db, $r, $oldname, "A"); + + + // $pattern = '/\d{1,3}? ?[o0]f ?(\d{1,3}?) ?.*[\[\{\(]\d{1,3}?\/\d{1,3}?[\}\]\)]/iU'; + + // domatching($pattern, $db, $r, $oldname, "B"); + + + $pattern = '/(? 0 && $ptr <100000); +echo "finnis ". $i." ".$j."\n"; +if ($nntpconnected) +{ + $nntp->doQuit(); + $nntpconnected = false; +} + +//unset($db); +unset($nntp); +$time = microtime(true) - $time; +echo "time = ".$time."\n"; + +//sql code to reset binaries (nothing processed) +//UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE 1 = 1 + +//sql code to see the releases +//SELECT distinct `relname`, `name`, `reltotalpart`, count(*), `groupID`, `ID`, `procstat`, `regexID` FROM `binaries` where reltotalpart > 0 and procstat = 5 group by `relname` ORDER BY count(*) DESC +//SELECT relname, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, CASE WHEN regexID < 0 THEN 1 ELSE coalesce(g.minfilestoformrelease, s.minfilestoformrelease) END as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, regexID, COUNT(ID) AS num FROM binaries WHERE procstat = 5 GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL +//SELECT nombre, bID groupID, reqID, SUM(num) AS num FROM ( SELECT name as nombre, relname, `binaries`.`ID` AS bID, `regexID`, reltotalpart, groupID, reqID, fromname, COUNT(ID) AS num FROM binaries WHERE procstat = -1 GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x LEFT OUTER JOIN groups g ON g.ID = x.groupID INNER JOIN ( SELECT VALUE AS minfilestoformrelease FROM site WHERE setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname ORDER BY `num` desc + +//sql code to see what didn't get released +//SELECT nombre, SUM(num) AS num, bid, groupID, CASE WHEN regexID < 0 THEN 1 ELSE coalesce(g.minfilestoformrelease, s.minfilestoformrelease) END as minfilestoformrelease FROM ( SELECT name as nombre, ID as bid, relname, date, reltotalpart, groupID, reqID, fromname, regexID, COUNT(ID) AS num FROM binaries WHERE procstat = -1 GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY `num` desc + +//sql code to see the state of the binaries (-3 nzb to be dled, -2 spam, -1 can't processed, 0 unprocessed, 5 ready for release +//SELECT `procstat` , COUNT( * ), sum(totalParts) FROM `binaries` GROUP BY `procstat` + +//sql code to see how many potential records can be processed by the assmembler +//SELECT count(*) FROM `binaries` where `procstat`=0 AND name REGEXP '([0-9] ?(of|\\/) ?[0-9])|([\\(\\{\\[]?[0-9]+ *(segs|parts))' order by `groupID`, `name` + +//SELECT * FROM `binaries` WHERE `date` < DATE_SUB(now(), INTERVAL 4 DAY) AND `name` LIKE '%.nzb%' and `procstat` != 5 ORDER BY `binaries`.`name` ASC + +//sql code to clean orphaned binaries's records +//select * FROM `binaries` WHERE `ID` NOT IN (SELECT p.`binaryID` FROM `parts` p) +//delete FROM parts WHERE `binaryID` NOT IN (SELECT b.id FROM binaries b) + +//select * FROM `parts` WHERE NOT EXISTS(SELECT ID FROM `binaries`); + +?> +//ALTER TABLE `newznab`.`binaries` ADD INDEX ( `binaryhash` ) diff --git a/ugo/automake_threaded.php b/ugo/automake_threaded.php new file mode 100644 index 000000000..97b8aed09 --- /dev/null +++ b/ugo/automake_threaded.php @@ -0,0 +1,155 @@ +query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)"); + echo "binaries have been reset\n"; + } else if ($argv[1] == "cont") { + $cont = false; + } +} + +echo "cleaning spam..... can take a few moments\n"; + +//$db->disableAutoCommit(); +//$db->disableForeignKeyChecks(); + +$rel = $db->query("SELECT `ID` from `binaries` WHERE `xref` REGEXP '(.+:[0-9]+ ){5,}' AND `procstat` = 0"); + +foreach($rel as $r) +{ + $db->query("UPDATE `binaries` set `procstat` = -2 WHERE `ID` = ".$r['ID']); + if ($db->getLastError() != '') + var_dump($db->getLastError()); +} + +echo "spam count ".count($rel)."\n"; + +//$db->commit(); + +//$db->enableAutoCommit(); +//$db->enableForeignKeyChecks(); + + +if ($cont) + $rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6"); + +echo "Getting work to be done\n"; +$query = "SELECT ID FROM binaries WHERE `procstat` = 0 GROUP BY binaryhash order by RAND()"; + +$theList = $db->query($query); +unset($db); + +$theCount = count($theList); + +$ps = new PowerProcess(); +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxChildren = 8; +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->threadTimeLimit = 0; // Disable child timeout + +$tim = microtime(true) - $time; +echo "time = ".$tim."\n"; + +echo "Starting threaded assembly process\n"; +echo "\nToday will be doing $theCount binaries\n"; +echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t"; + +while ($ps->RunControlCode()) +{ + // Start the parent loop + $listcount = count($theList); + if ($listcount) + { + // We still have groups to process + if ($ps->SpawnReady()) + { + // Spawn another thread + $ps->threadData = array_pop($theList); +// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." post(s) to update after this\n"; + $ps->spawnThread(); + } + } + else + { + // No more groups to process + echo "\nNo more posts to process - Initiating shutdown\n"; + $ps->threadTimeLimit = 200; + do { + $ps->tick(); + } while ($ps->myThreads); + $ps->Shutdown(); + echo "\nShutdown complete\n"; + $tim = microtime(true) - $time; + echo "final time = ".$tim."\n"; + } +} +$tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +unset($theList); + +if ($ps->RunThreadCode()) +{ + $post = $ps->threadData; + + $thread = sprintf("%05d",$ps->GetPID()); + +// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n"; + + $param = $post['ID']; + + $dir = dirname(__FILE__); + $file = 'automake.php'; + + $output = shell_exec("php {$dir}/{$file} {$param}"); + + $count = $theCount - $listcount; + $countpct = $count / $theCount; + $cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT); + + if ($listcount % 250 == 0 && $listcount > 0) + { + $tim = microtime(true) - $time; + echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); + $tim = ($listcount * $tim) / $count + time(); +// $tim = number_format(($listcount * $tim) / 60 / $count, 1); + if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0)) + echo "\t".date('M jS Y H:i:s', $tim); +// echo "\t".$tim."m to go" ; + echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t"; + + } elseif ($listcount % 25 == 0) + echo " ."; + + +// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n"; + $tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +} + +// Exit to call back to parent - Let know that child has completed +exit(0); + +// Create callback function +function psUpdateComplete() +{ + echo "[Thread-MASTER] Threaded assembly process complete\n"; +} + +?> + diff --git a/ugo/finish_postproc.php b/ugo/finish_postproc.php new file mode 100644 index 000000000..3dc98922f --- /dev/null +++ b/ugo/finish_postproc.php @@ -0,0 +1,39 @@ +query($thequery); + if ($result[0]["thecount"] < 10 ) + { + $postprocess->processTv(); + $postprocess->processAll(); + $postprocess->processSpotNab(); + echo "done...waiting"; + sleep(600); + } else { + for ($i = 0; $i < $result[0]["thecount"]; $i=$i+20) + { + echo "\n"; + echo $result[0]["thecount"] - $i. " more to go: \n"; +// $postprocess->processAdditional(); + $postprocess->processNfos(); + $postprocess->processUnwanted(); + $postprocess->processMovies(); + $postprocess->processMusic(); + $postprocess->processBooks(); + $postprocess->processGames(); + $postprocess->processTv(); + $postprocess->processMusicFromMediaInfo(); + $postprocess->processOtherMiscCategory(); + $postprocess->processUnknownCategory(); + + } + } +} diff --git a/ugo/makenzb.php b/ugo/makenzb.php new file mode 100644 index 000000000..feedad596 --- /dev/null +++ b/ugo/makenzb.php @@ -0,0 +1,120 @@ +queryOneRow($query); + + echo $row['name']." is ".($row['size']/1024/1024)." MB\n"; + + if (!isset($name)) + { + $name = $row['relname']; + } + + if (preg_match('/\.nzb\b/i', $row['name']) && $row['size'] < 10000000) + { +echo "doing nzb\n"; + $nntp = new Nntp; + if ($nntp->doConnect()) + { + $bin = $nntp->getBinary($id); + + echo strlen($bin)."\n"; + + $name = preg_replace('/ /','_',$name); + + if (file_exists($path.$name.".nzb")) + { + $name = $name." ".$id; + } + + echo $name." ".strlen($bin)."\n"; + + + if (strlen($bin) == 0) + $ok = false; + elseif (file_put_contents($path.$name.".nzb", $bin) == 0) + $ok = false; + + + echo $ok."\n"; + } + $nntp->doQuit(); + + unset($nntp); + unset($bin); + } + + if ($ok) + { + echo "clearing \n"; + $query = "UPDATE `binaries` SET `procstat` = 6 WHERE ID = ".$id; + + echo $query."\n"; + $db->query($query); + + } + +} + + +mb_internal_encoding("UTF-8"); + +$db = new Db; + +if (isset($argv[1])) +{ + if (isset($argv[2])) + { + makenzb($argv[1], $argv[2]); + } else { + makenzb($argv[1]); + } +} else { + + $db->query("UPDATE `binaries` SET `binaries`.`procstat` = 6 WHERE `binaries`.`procstat` = - 3 AND binaries.ID NOT IN ( SELECT `parts`.`binaryID` FROM `parts` )"); + + $rel = $db->query("SELECT * FROM `binaries` WHERE `procstat` = -3"); + + foreach($rel as $bin) + { + + $name = $bin['relname']; + + $name = preg_replace('/yenc/iU',' ',$name); + + $name = preg_replace('/\d+?\/\d+?/iU',' ',$name); + + $name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name); + + $name = preg_replace('/[\`\'\"\:\/\-\<\>]/iU',' ',$name); + + $name = preg_replace('/\.(?!nzb)/iU',' ',$name); + + $name = preg_replace('/ +?/iU',' ',$name); + + $name = preg_replace('/^ | $/iU','',$name); + + $name = preg_replace('/ /','_',$name); + + $id = $bin['ID']; + + echo "doing $name\n"; + + makenzb($id, $name); + + } +} + + +unset($db); +?> diff --git a/ugo/makenzb_threaded.php b/ugo/makenzb_threaded.php new file mode 100644 index 000000000..8ec618827 --- /dev/null +++ b/ugo/makenzb_threaded.php @@ -0,0 +1,210 @@ +query($query); +unset($db); + +$theCount = count($theList); + +$ps = new PowerProcess(); +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxThreads = 20; +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->threadTimeLimit = 0; // Disable child timeout + +$tim = microtime(true) - $time; +echo "time = ".$tim."\n"; + +echo "Starting threaded nzb exporter process\n"; +echo "\nToday will be doing $theCount binaries\n"; +echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t"; + +while ($ps->RunControlCode()) +{ + // Start the parent loop + $listcount = count($theList); + if ($listcount) + { + // We still have groups to process + if ($ps->SpawnReady()) + { + // Spawn another thread + $ps->threadData = array_pop($theList); + //echo "[Thread-MASTER] Spawning new thread. Still have " . count($rel) ." nzb(s) to update after this\n"; + $ps->spawnThread(); + } + else + { + // There are no more slots available to run + //$ps->tick(); + //echo ".\n"; + } + } + else + { + // No more groups to process +// $query = "UPDATE `binaries` SET `procstat` = 6 WHERE `procstat` = -3 OR `procstat` = -2"; +// $db = new Db(); +// var_dump($db->query($query)); + echo "\nNo more nzbs to process - Initiating shutdown\n"; + $ps->threadTimeLimit = 200; + do { + $ps->tick(); + } while ($ps->myThreads); + $ps->Shutdown(); + echo "\nShutdown complete\n"; + + + $path = FS_ROOT."/nzbs/"; + + $filestoprocess = glob($path."*.nzb"); + + foreach($filestoprocess as $nzbFile) + { + + $xml = file_get_contents( $nzbFile ); + + $matches = preg_split('/\A(\<\?xml[^\r\n]+?\?\>)\s*\R+/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE); + + $name = preg_split('/\.nzb$/i', $nzbFile, 0, PREG_SPLIT_DELIM_CAPTURE); + + $name = $name[0]; + +// $marker = $matches[0]; +// +// if (strlen($marker) == 0 || strlen($marker) >100) +// { +// $marker = $matches[1]; +// } + + $marker = ''; + + foreach ($matches as $m) + { + if (preg_match('/^(\<\?xml[^\r\n]+?\?\>)/iU', $m)) + { + echo "$m\n"; + $marker = $m; + continue; + } + } + + +echo "marker "."/(".preg_quote($marker)."\s*\R+)/iU\n"; + + if (!empty($marker)) + { + $matches = preg_split('/('.preg_quote($marker).'\s*\R+)/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE); + + if (strlen($matches[0]) == 0) + { + unset($matches[0]); + $matches = array_values($matches); + } + + echo count($matches)."\n"; + + if (count($matches) > 2) + { + for ($i = 0; $i < count($matches); $i = $i + 2) + { + echo $i."\n"; + $outxml = $matches[$i] . $matches[$i + 1]; + $fileout = $name . '.' . $i . '.nzb'; + file_put_contents($fileout, $outxml); + echo $fileout."\n"; + } + } + } + } + $tim = microtime(true) - $time; + echo "final time = ".$tim."\n"; + } +} +$tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +unset($theList); +unset($db); + +if ($ps->RunThreadCode()) +{ + $bin = $ps->threadData; + + $thread = sprintf("%05d",$ps->GetPID()); + +// echo "[Thread-{$thread}] Begining processing for nzb {$bin['ID']}\n"; + + $name = $bin['relname']; + + $name = preg_replace('/yenc/iU',' ',$name); + + $name = preg_replace('/\d+?\/\d+?/iU',' ',$name); + + $name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name); + + $name = preg_replace('/[\`\'\"\:\/\-\<\>\|]/iU',' ',$name); + + $name = preg_replace('/\.(?!nzb)/iU',' ',$name); + + $name = preg_replace('/ +?/iU',' ',$name); + + $name = preg_replace('/^ | $/iU','',$name); + + $name = preg_replace('/ /','_',$name); + + $id = $bin['ID']; + + if ($name == '') + { + $name = $id; + } + +// echo "\nname: ".$name." \n"; + + $dir = dirname(__FILE__); + $file = 'makenzb.php'; + +// echo "php {$dir}/{$file} {$id} {$name}\n"; + + $output = shell_exec("php {$dir}/{$file} {$id} {$name}"); + + $count = $theCount - $listcount; + $countpct = $count / $theCount; + + if ($listcount % 100 == 0 && $listcount > 0) + { + $tim = microtime(true) - $time; + echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); + $tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time(); +// $tim = number_format(($listcount * $tim) / 60 / $count, 1); + if ($countpct > 0.1 || $count > 1500) + echo "\t".date('H:i:s', $tim); +// echo "\t".$tim."m to go" ; + echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t"; + + } + if ($listcount % 10 == 0) + echo " ."; + +// echo "[Thread-{$thread}] Completed update for {$bin['name']}\n"; +} + +// Exit to call back to parent - Let know that child has completed +exit(0); + +// Create callback function +function psUpdateComplete() +{ + echo "Threaded export process complete\n"; +} + +?> diff --git a/ugo/misc_sorter3.php b/ugo/misc_sorter3.php new file mode 100644 index 000000000..0578f8a72 --- /dev/null +++ b/ugo/misc_sorter3.php @@ -0,0 +1,2335 @@ + 1) + $author = trim($tmp[1]." ".$tmp[0]); + + $rel = $book->getBookInfoByName($author, $title); + + echo "book1 "; + //var_dump($rel); + + if ($rel === false) + { + $query = "Select * from `bookinfo` WHERE `title` LIKE '%".$title."%' AND ID in (SELECT ID FROM `bookinfo` WHERE `author` sounds LIKE '".$author."')"; + $rel = $db->query($query); + + echo "book2 "; + //var_dump($rel); + + if (isset($rel[0]["ID"])) + { + $bookID = $rel[0]["ID"]; + } else { + + $bookID = $book->updateBookInfo($author, $title); + } + } else { + $bookID = $rel['ID']; + } + unset($book); + echo "book ".$bookID."\n"; + return $bookID; +} + + +function findmusicname($row, $cat = 3010) +{ + global $db; + $ok = false; + + $query = "SELECT * FROM releases WHERE ( releases.passwordstatus < 0 AND releases.haspreview < 0 ) AND ID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel !== false) + return false; + + $query = "SELECT releasevideo.releaseID FROM releasevideo WHERE releasevideo.releaseID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel !== false) + return false; + + $query = "SELECT releasefiles.releaseID, releasefiles.`name` FROM releasefiles INNER JOIN releases ON releasefiles.releaseID = releases.ID WHERE releases.ID = ".$row['ID']." order by releasefiles.`name`"; + $rel = $db->query($query); + + $name = ''; + $ok = false; + + $m3u = ''; + $alt = ''; + $mp3 = false; + $mp3name = ''; + $files = 0; + $extras = 0; + $folder = ''; + $bookId = 0; + $musicId = 0; + + if (count($rel) > 0) + { + echo "findmusicname ".$row['ID']." ".count($rel)."\n"; + foreach($rel as $r) + { + $sub = $r['name']."\n"; + + $sub = preg_replace('/[\x01-\x1f]/', '', $sub); + + $sub = preg_replace('/(? 0.7) && $mp3)." ".$mp3." ".$m3u."\n"; + + if (($m3u != '' || (($files + $extras) / count($rel) > 0.7)) && $mp3) + { + $name = $m3u; + + if ($files == 1) + $name = $mp3name; + + if (empty($name)) + { + $name = matchnames($row['name'], $alt); + } + + echo $row['guid']." ".$name.' '.$mp3.' '.($files + $extras) / count($rel)."\n"; + } + + + // if (preg_match('/\.mp3|\.m3u|\.sfv|\.nfo|\.flac/|\.asf/', $r['name'])) + // echo "release file ".$r['name']."\n"; + + $name = preg_replace("/^\d+\-/i", '', $name); + $name = trim(preg_replace("/\\\\|^ +| +$/i", ' ', $name)); + +echo "file name $name\n"; + + } else { + $name = domusicfiles ($row); + } + +echo "found name $name\n"; + + if ($name != '') + { + $music = new Music(); + $result = $music->parseArtist($name); + $artist = $result['artist']; + $album = $result['album']; + + if (empty($album) && empty($artist)) + $album = $name; + +echo "1artist = $artist album = $album\n"; + + $album = stripname($album); + $artist = stripname($artist); + + $artist = preg_replace('/\d+/', '', $artist); + + $tmp = explode(",", $artist); + + if (count($tmp) > 1) + $artist = trim($tmp[1]." ".$tmp[0]); + + +echo "2artist = $artist album = $album\n"; + + $rel = $music->getMusicInfoByName($artist, $album); +// var_dump($rel); + $musicId = $rel['ID']; + + if ($musicId <= 1) + { + echo "amazon music1 "; + $musicId = $music->updateMusicInfo($artist, $album, 0); + + if ($musicId === false) + { +// $book = new Book(true); +// $bookId = $book->updateBookInfo($artist, $album); +// unset($book); + $bookId = findbookid ($artist, $album); + } +//var_dump($musicId); + } + +if (preg_match('/\.flac/', $mp3name)) + $cat = 3040; + +echo "music name: $name $musicId $bookId\n"; + if ($musicId > 1) + { + $ok = dodbupdate($row['ID'], $cat, $name, $musicId, 'music'); + } else if ($bookId > 0) { + $ok = dodbupdate($row['ID'], 3030, $name, $bookId, 'book'); + } else { + if (preg_match('/book/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 3030, $name); + else + $ok = dodbupdate($row['ID'], $cat, $name); + } + + unset($music); + } + return $ok; +} + +function stripname($name) +{ + if (is_array($name)) + return $name; + + global $qualities; + + $quality = preg_replace('/(.+?)/iU', "/\b\\1\b.*?$/iU", $qualities); + +// var_dump($quality); + + $name1 = $name; + + do { + $original = $name1; +// echo "fm1: $name1\n"; + $name1 = preg_replace($quality, " ", $name1); + $name1 = preg_replace('/(as )?(requested|request|req):?/i', " ", $name1); + $name1 = preg_replace('/Thanks To Original Poster/i', " ", $name1); + $name1 = preg_replace('/\b[a-z]{1,2}[0-9]{1,2} /i', " ", $name1); + $name1 = preg_replace('/^\d+|\d+$/', '', $name1); + $name1 = preg_replace('/\battn:? [^ ]+\b/i', " ", $name1); + $name1 = preg_replace('/^National.?Geographic|^nat.?geo|^bbc/i', " ", $name1); + $name1 = preg_replace("/\.?(s\d+?e\d+?)\.?/iU", " \\1 ", $name1); + $name1 = preg_replace("/^(\[\d+\].\[.+\].(\[#?a.b.[^\]]+\])*?.)/iU", "", $name1); + $name1 = preg_replace("/\[(?![^\[]+\])|\((?![^\(]+\))|\{(?![^\{]+\})/iU", " ", $name1); + $name1 = preg_replace("/^\(?\?+?\)/iU", " ", $name1); + $name1 = preg_replace("/[\x01-\x1f\_\!\?\[\{\}\]\/\:\|]/iU", " ", $name1); + $name1 = preg_replace("/(?parseNameEpSeason($name); + + if (empty($showinfo['cleanname'])) + { + $showinfo['name'] = $name; + $showinfo['cleanname'] = $name; + } + +echo "clean ".$showinfo['cleanname']."\n"; + + $rageID = $tvrage->getByTitle($showinfo['cleanname']); + + } + +// echo "1 $name\n"; + + if ($rageID == false) + { + $rageID = $tvrage->parseRageIdFromNfo($nfo); + + $name1 = $tvrage->getRageInfoFromService($rageID); +// var_dump($name1); + if (isset($name1['name']) && $name1['name'] != '') + $name = $name1['name']; + else + { + if (!isset($showinfo['cleanname'])) + $showinfo = $tvrage->parseNameEpSeason($name); + $showInfo1 = array( + 'name' => '', + 'season' => '', + 'episode' => '', + 'seriesfull' => '', + 'airdate' => '', + 'country' => '', + 'year' => '', + 'cleanname' => '' + ); + foreach ($showInfo1 as $k => $v) + if (isset($showinfo[$k])) + $showInfo1[$k] = $showinfo[$k]; + $showinfo = $showInfo1; + echo "calling rage $name\n"; +// var_dump($showinfo); + try { + $tvrShow = $tvrage->getRageMatch($showinfo); +// var_dump($tvrShow); + if (isset($tvrShow["showid"])) + $rageID = $tvrShow["showid"]; +// echo "$rageID \n"; + } catch (Exception $e) { + echo 'Caught exception: ', $e->getMessage(), "\n"; + } + } + if (isset($tvrShow) && $rageID > 0) + $tvrage->updateRageInfo($rageID, $showinfo, $tvrShow, null); + } + +// echo "2 $name\n"; + + if ($rageID == false) + { + $query = "SELECT * FROM releases WHERE ( releases.passwordstatus < 0 AND releases.haspreview < 0 ) AND ID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel !== false) + return false; + + $query = "SELECT releasevideo.releaseID FROM releasevideo WHERE releasevideo.releaseID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel === false) + return false; + + $query = "SELECT releasefiles.releaseID, releasefiles.`name` FROM releasefiles INNER JOIN releases ON releasefiles.releaseID = releases.ID WHERE releases.ID = ".$row['ID']; + $rel = $db->query($query); + + foreach($rel as $r) + { + $name1 = preg_replace('/(?parseNameEpSeason($name1); + +// var_dump($parsed); + + $rageID = $tvrage->getByTitle($parsed['cleanname']); + + if ($rageID == false) + { + if ($parsed) + $name1 = $parsed['cleanname']; + $sql = sprintf("SELECT rageID from tvrage where (releasetitle = %s or releasetitle = %s)", $db->escapeString($name1), $db->escapeString(str_replace(' and ', ' & ', $name1))); + + $sql = str_replace(" = '", " like '%", $sql); + $sql = str_replace("' or", "%' or", $sql); + $sql = str_replace("')", "%')", $sql); + +// echo "$sql\n"; + + $res = $db->queryOneRow($sql); + if ($res) + $rageID = $res["rageID"]; + } + } + if ($rageID != 0 && $name1 != '') + $name = $name1; + } + + $tmp = $tvrage->getByRageID($rageID); + +// echo "ltvrageid = $rageID\n"; +// var_dump($tmp); + + if (!isset($tmp[0])) + { + + $name = preg_replace('/[\_\.]|bbc|nat.?geo|national.?geographic/i', " ", $name1); +// $name = preg_replace('/(19|20)\d{2}.*$/i', " ", $name); + $name = preg_replace('/\d{1,3} ?of ?\d{1,3}.*$/i', " ", $name); + $name = preg_replace('/^[\]\}\)]|[\[\{\(]/i', "", $name); + $name = trim($name); + +echo "name == $name\n"; + $showinfo = $tvrage->parseNameEpSeason($name); + if (!isset($showinfo['cleanname'])) + $showinfo = array( + 'name' => $name, + 'season' => '', + 'episode' => '', + 'seriesfull' => '', + 'airdate' => '', + 'country' => '', + 'year' => '', + 'cleanname' => $name + ); + + + + try { + $tvrShow = $tvrage->getRageMatch($showinfo); + if (isset($tvrShow["showid"])) + $rageID = $tvrShow["showid"]; + } catch (Exception $e) { + echo 'Caught exception: ', $e->getMessage(), "\n"; + } +// var_dump($tvrShow); + if ($tvrShow === false) + { + $name1 = preg_replace('/(\D+?)( s?\d{1,3}?(?!\d).*?)/iU', '\1', $name); + $showinfo = $tvrage->parseNameEpSeason($name1); + if (!isset($showinfo['cleanname'])) + $showinfo = array( + 'name' => $name1, + 'season' => '', + 'episode' => '', + 'seriesfull' => '', + 'airdate' => '', + 'country' => '', + 'year' => '', + 'cleanname' => $name1 + ); +echo "$name1\n"; + $tvrShow = $tvrage->getRageMatch($showinfo); +// var_dump($tvrShow); + if ($tvrShow !== false) + { + if (isset($tvrShow["showid"])) + $rageID = $tvrShow["showid"]; + } + } + + if (($tvrShow !== false && is_array($tvrShow) && $tvrShow['title'] == $name) || $rageID) + { + $showinfo['name'] = $tvrShow["title"]; + $showinfo['cleanname'] = $tvrShow["title"]; + if ($rageID != 0) + $tvrage->updateRageInfo($rageID, $showinfo, $tvrShow, NULL); + } + $tmp = $tvrage->getByRageID($rageID); + } + + if (isset($tmp[0])) + { +// echo "rageID = $rageID\n"; +// echo "title = ".$tmp[0]['releasetitle']."\n"; +// var_dump($tmp); + + $show = $tvrage->parseNameEpSeason($tmp[0]["releasetitle"]." ".$row['searchname']); +// var_dump($tvrage->getEpisodeInfo($rageID, $show['season'], $show['episode'])); + + $name = matchnames($row['name'], $tmp[0]['releasetitle']." ".$show['seriesfull']); + + $name = findquality ($nfo, $name); + + if (dodbupdate($row['ID'], $cat, $name, $rageID, 'tv')) + { + if (is_array($show) && $show['name'] != '') + { + // $tvrage->updateEpInfo($show, $row['ID']); + } + $ok = $rageID; + } + } + + unset($tvrage); + return $ok; +} + +function cleanname($name) +{ + global $qualities; + + $quality = preg_replace('/(.+?)/iU', "/\.*?(\b\\1\b)\.*?/iU", $qualities); + + do { + $original = $name; +// echo "CN: $name\n"; + $name = preg_replace($quality, " \\1 ", $name); + $name = preg_replace("/\.?(s\d+?e\d+?)\.?/iU", " \\1 ", $name); + $name = preg_replace("/yenc/iU", " ", $name); + $name = preg_replace("/^(\[\d+\].\[.+\].(\[#?a.b.[^\]]+\])*?.)/iU", "", $name); + $name = preg_replace("/\[(?![^\[]+\])|\((?![^\(]+\))|\{(?![^\{]+\})/iU", " ", $name); + $name = preg_replace("/^\(?\?+?\)/iU", " ", $name); + $name = preg_replace("/[\x01-\x1f\_\!\?\[\{\}\]\/\:\|]/iU", " ", $name); + $name = preg_replace("/(?getByRageID($rageID); + unset($tvrage); + + $name1 = $name1[0]["releasetitle"]; + + $name = matchnames($row['name'], $name1); + + echo "doing tv1 ".$name." ".$row['guid']."\n"; + + $ok = dodbupdate($id, $cat, $name, $rageID, 'tv'); + + } else { + +//echo "$name $name1\n"; + + $name = matchnames($row['name'], $name1); + + echo "doing tv2 ".$name." ".$row['guid']."\n"; + +// $matches = preg_split('/(\btitle\:|\bname=)[ \.\_]*?(.*)\r/iU', $nfo, 0, PREG_SPLIT_DELIM_CAPTURE); + +// echo $matches[1]."\n"; + + if (empty($name)) + { + $ok = dodbupdate($id, $cat, ''); + } else { + $ok = dodbupdate($id, $cat, $name); + } + } + return $ok; +} + +function doAmazon ($name, $id, $nfo = "", $q, $region = 'com', $case = false, $nfo ='', $row = '') +{ + global $db; + $s = new Sites(); + $site = $s->get(); + $amazon = new AmazonProductAPI($site->amazonpubkey, $site->amazonprivkey, $site->amazonassociatetag); + $ok = false; + + + try { + switch ($case) + { + case 'upc': + $amaz = $amazon->getItemByUpc(trim($q), $region); + break; + case 'asin': + $amaz = $amazon->getItemByAsin(trim($q), $region); + break; + case 'isbn': + $amaz = $amazon->searchProducts(trim($q), '', "ISBN"); + break; + } + } catch (Exception $e) { + echo 'Caught exception: ', $e->getMessage(), "\n"; + unset($s); + unset($amaz); + unset($amazon); + return $ok; + } + + if (!isset($amaz->Items->Item)) + return $ok; + + $type = $amaz->Items->Item->ItemAttributes->ProductGroup; + + switch ($type) + { + case 'Book': + case 'eBooks': + $new = (string) $amaz->Items->Item->ItemAttributes->Author; + $new = $new . " - ". (string) $amaz->Items->Item->ItemAttributes->Title; + $name = matchnames($name, $new); + + $query = "SELECT ID FROM `bookinfo` WHERE `asin` = '".(string) $amaz->Items->Item->ASIN."'"; + $rel = $db->query($query); + if (count($rel) == 0) + { + $book = new Book(); + $item = array(); + $item["asin"] = (string) $amaz->Items->Item->ASIN; + $item["url"] = (string) $amaz->Items->Item->DetailPageURL; + $item["coverurl"] = (string) $amaz->Items->Item->LargeImage->URL; + if ($item['coverurl'] != "") + $item['cover'] = 1; + else + $item['cover'] = 0; + $item["author"] = (string) $amaz->Items->Item->ItemAttributes->Author; + $item["dewey"] = (string) $amaz->Items->Item->ItemAttributes->DeweyDecimalNumber; + $item["ean"] = (string) $amaz->Items->Item->ItemAttributes->EAN; + $item["isbn"] = (string) $amaz->Items->Item->ItemAttributes->ISBN; + $item["publisher"] = (string) $amaz->Items->Item->ItemAttributes->Publisher; + $item["publishdate"] = (string) $amaz->Items->Item->ItemAttributes->PublicationDate; + $item["pages"] = (string) $amaz->Items->Item->ItemAttributes->NumberOfPages; + $item["title"] = (string) $amaz->Items->Item->ItemAttributes->Title; + $item["review"] = ""; + if (isset($amaz->Items->Item->EditorialReviews)) + $item["review"] = trim(strip_tags((string) $amaz->Items->Item->EditorialReviews->EditorialReview->Content)); + + $bookId = $book->addUpdateBookInfo($item['title'], $item['asin'], $item['url'], + $item['author'], $item['publisher'], $item['publishdate'], $item['review'], + $item['cover'], $item['dewey'], $item['ean'], $item['isbn'], $item['pages'] ); + unset($book); + } else { + $bookId = $rel[0]['ID']; + } + + + $query = "SELECT * FROM releases INNER JOIN releaseaudio ON releases.ID = releaseaudio.releaseID WHERE releases.ID = $id"; + $rel = $db->query($query); + if (count($rel) == 0) + { + $ok = dodbupdate($id, 7020, $name, $bookId, 'book'); + } else { + $ok = dodbupdate($id, 3030, $name, $bookId, 'book'); + } + unset($rel); + break; + + case 'Digital Music Track': + case 'Digital Music Album': + case 'Music': + $new = (string) $amaz->Items->Item->ItemAttributes->Artist; + if ($new != '') + $new = $new . " - "; + $new = $new .(string) $amaz->Items->Item->ItemAttributes->Title; + $name = matchnames($name, $new); + + $query = "SELECT * FROM `bookinfo` WHERE `asin` = '".(string) $amaz->Items->Item->ASIN."'"; + +// echo "$query\n"; + $rel = $db->query($query); + if (count($rel) == 0) + { + $mus = array(); + $mus['coverurl'] = (string) $amaz->Items->Item->LargeImage->URL; + if ($mus['coverurl'] != "") + $mus['cover'] = 1; + else + $mus['cover'] = 0; + + $mus['title'] = (string) $amaz->Items->Item->ItemAttributes->Title; + + $mus['asin'] = (string) $amaz->Items->Item->ASIN; + + $mus['url'] = (string) $amaz->Items->Item->DetailPageURL; + + $mus['salesrank'] = (string) $amaz->Items->Item->SalesRank; + if (empty($mus['salesrank'])) + $mus['salesrank'] = 'null'; + + $mus['artist'] = (string) $amaz->Items->Item->ItemAttributes->Artist; + + $mus['publisher'] = (string) $amaz->Items->Item->ItemAttributes->Publisher; + + $mus['releasedate'] = $db->escapeString((string) $amaz->Items->Item->ItemAttributes->ReleaseDate); + if ($mus['releasedate'] == "''") + $mus['releasedate'] = 'null'; + + $mus['review'] = ''; + if (isset($amaz->Items->Item->EditorialReviews)) + $mus['review'] = trim(strip_tags((string) $amaz->Items->Item->EditorialReviews->EditorialReview->Content)); + + $mus['year'] = ($mus['releasedate'] != 'null' ? substr($mus['releasedate'], 1, 4) : date("Y")); + + $mus['tracks'] = ''; + if (isset($amaz->Items->Item->Tracks)) + { + $tmpTracks = (array) $amaz->Items->Item->Tracks->Disc; + $tracks = $tmpTracks['Track']; + $mus['tracks'] = (is_array($tracks) && !empty($tracks)) ? implode('|', $tracks) : ''; + } + + $music = new Music(); + $musicId = $music->addUpdateMusicInfo($mus['title'], $mus['asin'], $mus['url'], + $mus['salesrank'], $mus['artist'], $mus['publisher'], $mus['releasedate'], $mus['review'], + $mus['year'],'', $mus['tracks'], $mus['cover'] ); + + unset($music); + } else { + $musicId = $rel[0]['ID']; + } + + $ok = dodbupdate($id, 3010, $name, $musicId, 'music'); + break; + + case 'Movies': + case 'DVD': + $new = (string) $amaz->Items->Item->ItemAttributes->Title; + $new = $new . " (" . substr((string) $amaz->Items->Item->ItemAttributes->ReleaseDate, 0, 4) . ")"; + $name = matchnames($name, $new); + + $ok = matchimdb ($nfo, $row); + + if (!$ok) + { + $name = findquality ($nfo, $name); + $ok = dodbupdate($id, 2000, $name); + } + break; + + default: + echo "* * * * * * uncatched amazon category $type ".$name; + break; + } +//echo $query."\n"; + + unset($s); + unset($amaz); + unset($amazon); + return $ok; +} + +function doarray($matches) +{ + $r = array(); + + foreach ($matches as $m) + { + if (strlen($m) < 50) + { + $str = preg_replace("/\s/iU", "", $m); + + $m = strtolower($str); + + if ($m == 'audiobook') { + $r[-1] = $m; + } else if ($m == 'anidb.net') { + $r[-2] = $m; + } else if ($m == 'upc') { + $r[-3] = $m; + } else if ($m == 'amazon.') { + $r[-4] = $m; + } else if ($m == 'asin') { + $r[-5] = $m; + } else if ($m == 'imdb') { + $r[-6] = $m; + } else if ($m == 'os') { + $r[-7] = $m; + } else if ($m == 'mac' || $m == 'macintosh' || $m == 'dmg' || $m == 'macos' || $m == 'macosx' || $m == 'osx') { + $r[-8] = $m; + } else if ($m == 'itunes.apple.com/') { + $r[-9] = $m; + } else if (preg_match('/sports|deportes|nhl|nfl|\bnba/i', $m)) { + $r[1000] = $m; + } else if (preg_match('/avi|xvid|divx|mkv/i', $m)) { + $r[1001] = $m; + } else if (preg_match('/\.(?:rar|001)/i', $m)) { + $r[1002] = $m; + } else { + $r[] = $m; + } + } + } + + ksort($r); + return $r; +} + +function doOS ($nfo, $name, $cat, $id) +{ + global $db; + + $ok = false; + + $pattern = '/(.*v(?:er)?[\.\s]*?\d+\.\d(?:\.\d+)??[^\v]*?)|(?getNZBPath($row['guid']); + $m3u = ''; + $alt = ''; + $mp3 = false; + $mp3name = ''; + $files = 0; + $extras = 0; + + echo "doing music ".$row['guid']."\n"; + + if ($nzbinfo->loadFromFile($file)) + { + $name = $row['name']; + + $name = preg_replace("/\//",' ', $name); + +// echo "$name\n"; + + $name = preg_quote($name); + + + echo "$name\n"; + + + foreach($nzbinfo->nzb as $nzbsubject) + { + + $sub = $nzbsubject['subject']."\n"; + + $sub = preg_replace("/$name/i",'', $sub); + +// echo "doing music file = $sub\n"; + + if (preg_match('/\.(vol\d{1,3}?\+\d{1,3}?|par2|nfo\b|sfv|par\b|p\d{1,3}?|sv\b)/iU', $sub)) + { + $extras++; + $alt = preg_replace('/(\.vol\d{1,3}?\+\d{1,3}?|\.par2|\.[a-z][a-z0-9]{2})+?".+?/iU', '', $sub); + } + + if (preg_match('/\.mp3|\.flac/', $sub, $matches)) + { + $mp3name = preg_replace('/(\.mp3".+?)/iU', '.mp3', $sub); + $mp3name = preg_replace('/(\.flac".+?)/iU', '.flac', $sub); + $mp3name = preg_replace('/(?iU)^[^\"]+\"(0\d+?-?(00)??)??/iU', '', $mp3name); + + $mp3 = true; + $files++; + } + + if (preg_match('/\.m3u|\"00+[ \-\_\.]+?|\.nfo\b|\.sfv/iU', $sub, $matches)) + { + if (preg_match('/\.url|playlist/iU', $sub)) + { + continue; + } + $sub = preg_replace('/(\.vol\d{1,3}?\+\d{1,3}?|\.par2|\.[a-z][a-z0-9]{2})+?".+?/iU', '', $sub); + + $m3u = preg_replace('/(?iU)^[^\"]+\"(0\d+?-?(00)??)??/iU', '', $sub); + } + + } + } + $name = ''; + + if ( count($nzbinfo->nzb) > 0) + { + + echo (($m3u != '' || ($files + $extras) / count($nzbinfo->nzb) > 0.7) && $mp3)." ".$mp3." ".$m3u."\n"; + + if (($m3u != '' || (($files + $extras) / count($nzbinfo->nzb) > 0.7)) && $mp3) + { + $name = $m3u; + + if ($files == 1) + $name = $mp3name; + + if (empty($name)) + { + $name = matchnames($row['name'], $alt); + } + + echo $row['guid']." ".$name.' '.$mp3.' '.($files + $extras) / count($nzbinfo->nzb)."\n"; + + } + + echo (($m3u != '' || ($files + $extras) / count($nzbinfo->nzb) > 0.7) && $mp3)." ".$mp3." ".$m3u."\n"; + + } + +echo "music files $name\n"; + $name = cleanname($name); +echo "cleaned name $name\n"; + unset($file); + + unset($nzbinfo); + unset($nzb); + + return $name; +} + +function findquality ($nfo, $name) +{ + global $qualities; + $qual = array(); +// echo "$name\n"; + foreach ($qualities as $quality) + { + if (preg_match("/(?$quality) + { + if (preg_match("/$quality/iU", $name)) + { + unset($qual[$key]); + } + } + + $n = ''; + if (count($qual) > 0) + { + foreach ($qual as $quality) + { + $n = $n ." ".$quality; + } + } + + return trim($name .' '.$n); +} + +function matchanime ($nfo, $id, $name) +{ + $ok = false; + $anime = ''; + + if (preg_match('/anidb.net.*aid=(\d+?)/iU', $nfo, $set)) + { + $anidb = new AniDB(); + if ($anidb->getAnimeInfo($set[1]) === false) + { + $anidb->addTitle($anidb->AniDBAPI($set[1])); + } + $anime = $anidb->getAnimeInfo($set[1]); + + if ($anime !== false) + { + + $anime = matchnames($name, $anime['title']); + + $ok = dodbupdate($id, 5070, $anime, $set[1], 'anime'); + } + unset($anidb); + + } + return $ok; +} + +function matchimdb ($nfo, $row) +{ + $ok = false; + + preg_match('/imdb\.[a-z0-9\.\_\-\/]+?(?:tt|\?)(\d+?)\/?/iU', $nfo, $set); + +// echo "imdb name= ".$row['name']."\n"; + + if (isset($set[1])) + { + $imdb = $set[1] + 0; + + $movie = new Movie(); + + $title = $movie->getMovieInfo($imdb); + + if ($title === false) + { + $movie->fetchImdbProperties($imdb); + $movie->updateMovieInfo($imdb); + $title = $movie->getMovieInfo($imdb); + } + + $name = $title['title']." (".$title['year'].")"; + +// $name = matchnames($row['name'], $name); + + // echo "quality ".findquality ($nfo, $name)."\n"; + $name = findquality ($nfo, $name); + + $name = matchnames($row['name'], $name); + + echo "imdb $name\n"; + +// echo (strval($imdb) + 0) . " * * * " . (strval($title['imdbID']) + 0) ."\n"; + + if ((strval($imdb) + 0) == (strval($title['imdbID']) + 0)) + { + if (preg_match('/sport/iU', $title['genre'])) { + $ok = dodbupdate($row['ID'], 5060, $name, $imdb, 'imdb'); + } else if (preg_match('/docu/iU', $title['genre'])) { + $ok = dodbupdate($row['ID'], 5080, $name, $imdb, 'imdb'); + } else if (preg_match('/talk\-show/iU', $title['genre'])) { + $ok = dodbupdate($row['ID'], 5000, $name, $imdb, 'imdb'); + } else if (preg_match('/tv/iU', $title['genre']) || preg_match('/episode/iU', $title['genre']) || preg_match('/reality/iU', $title['genre'])) { + $ok = updateTV ($title['title']." (".$title['year'].")", $row['ID'], $row, 5000, $nfo); + } else { + $ok = dodbupdate($row['ID'], 2000, $name, $imdb, 'imdb'); + } + } + unset($movie); + } +// echo (bool) $ok."\n"; + return $ok; +} + +function matchaudiobook ($nfo, $id, $oldname, $row) +{ + global $db; + $ok = false; + + if (preg_match('/Sample Rate|MP3|speech|Encoded|ID3|(un)?abridged|Copyright|Ripped|khz|stero|mono/i', $nfo)) + { + if (preg_match('/mpeg|pixels|fps/i', $nfo)) + { + echo "not audio\n"; + return $ok; + } + $query = "SELECT * FROM releases INNER JOIN releasevideo ON releases.ID = releasevideo.releaseID where ( releases.passwordstatus >= 0 OR releases.haspreview >= 0 ) AND releases.ID = ".$id; + $rel = $db->query($query); + if (count($rel) > 0) + { + echo "not audio\n"; + return $ok; + } + unset ($rel); + + echo "audiobooks $id\n"; + $name = $oldname; + $author = ''; + $title = ''; + $bookID = 0; + $set = preg_split('/\btitle\:?(\.\.+?|\s\s+?|__+?)([^\r]*)\r/Ui', $nfo, 0, PREG_SPLIT_DELIM_CAPTURE); + + + if (isset($set[2])) + { + + $set[2] = trim(preg_replace('/[\|\:\-]| +/', ' ', $set[2])); + +// echo $set[2]."\n"; + $name = $set[2]; + $title = $name; + + $set = preg_split('/author\:?(\.\.+?|\s\s+?|__+?)([^\r]*)\r/Ui', $nfo, 0, PREG_SPLIT_DELIM_CAPTURE); + + if (isset($set[2])) + { + $set[2] = trim(preg_replace('/[\|\:\-]| +/', ' ', $set[2])); + +// echo $set[2]."\n"; + $name = $set[2]." - ".$name; + $author = $set[2]; + } + + $bookID = findbookid ($author, $title); +// +// $book = new Book(); +// $rel = $book->getBookInfoByName($author, $title); +// +// if ($rel === false) +// { +// $query = "Select * from `bookinfo` WHERE `title` LIKE '%".$title."%' AND ID in (SELECT ID FROM `bookinfo` WHERE `author` sounds LIKE '".$author."')"; +// $rel = $db->query($query); +// // echo "$query\n"; +// +// if (isset($rel[0]["ID"])) +// { +// // var_dump($rel); +// $bookID = $rel[0]["ID"]; +// } else { +// // echo "not found\n"; +// $bookID = $book->updateBookInfo($author, $title); +// // echo "$bookID\n"; +// } +// } else { +// $bookID = $rel['ID']; +// } + + $name = matchnames($oldname, $name); + if ($bookID > 0) + { + $ok = dodbupdate($id, 3030, $name, $bookID, 'book'); + } else { + $ok = dodbupdate($id, 3030, $name); + } + } else { + $ok = findmusicname($row, 3030); +// $ok = dodbupdate($id, 3030); + } + } +// unset($book); + return $ok; +} + +function matchmusic($nfo, $row) +{ + $artist = $album = $title = ''; + + $artist = preg_split('/(?:a\s?r\s?t\s?i\s?s\s?t\b)+? *?(?!(?:[^ \.\:\}\]\*] ?){2,}?\b)(?:[\*\?\-\=\|\;\:\.\[\}\]\( \xb0-\x{3000}]+?)[ \.\>\:]([a-z0-9]?(?!:).+?(?\:]([a-z0-9]?(?!:).+?(?1 && $title[1] <50 && isset($artist[1]) && strlen($artist[1]) >1 && $artist[1] <50) + { + +// echo "\n * * * * * title= ".$title[1]."\t ".$row['name']."\n"; + if (isset($artist[1])) + { + $new = trim($artist[1]); + $artist1 = trim($artist[1]); + } + + + if (isset($title[1])) + { + if (strlen($new) > 1) + { + $new = $new . " -*- "; + } + $new = $new . trim($title[1]); + $album = trim($title[1]); + } + + $name = matchnames($row['name'], $new); + + $artist = matchnames($artist1, ''); + $album = matchnames($album, ''); + +// echo "$artist * * $album\n"; + } else { + + $name = domusicfiles ($row); + if ($name != "") + { + $result = $music->parseArtist($name); + $artist = $result['artist']; + $album = $result['album']; + } + } + +echo "3artist = $artist album = $album\n"; + + $album = stripname($album); + $artist = stripname($artist); + +if (is_array($artist)) + //var_dump($artist); + + $tmp = explode(",", $artist); + + if (count($tmp) > 1) + $artist = trim($tmp[1]." ".$tmp[0]); + +echo "4artist = $artist album = $album\n"; + + if ($artist != '' && $album != '') + $musicId = 0; + else { + $rel = $music->getMusicInfoByName($artist, $album); +// var_dump($rel); + $musicId = $rel['ID']; + } + + if ($musicId <= 1) + { + echo "amazon music2 "; + $musicId = $music->updateMusicInfo($artist, $album, 0); + + if ($musicId === false) + { +// var_dump($artist); +// var_dump($album); + $book = new Book(true); + $bookId = $book->updateBookInfo($artist, $album); + unset($book); + } +// var_dump($musicId); + + } + + if ($musicId > 0) + { + $ok = dodbupdate($row['ID'], 3010, $name, $musicId, 'music'); + } else if ($bookId > 0) { + $ok = dodbupdate($row['ID'], 3030, $name, $bookId, 'book'); + } + + if (!$ok) + { + $ok = findmusicname($row, 3010); + +// $name = matchnames($row['name'], domusicfiles($row)); +// $ok = dodbupdate($row['ID'], 3010, ''); + } + if (!$ok) + { + if (preg_match('/book/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 3030, $name); + else + $ok = dodbupdate($row['ID'], 3010, $name); + } + unset($music); + return $ok; +} + +function matchmag($nfo, $row) +{ + $ok = false; + $set = preg_split('/(?:\btitle|\bname|release)\b(?:[\-\:\.\}\[\s\xb0-\x{3000}]+?)\B([a-z0-9\.\- \(\)]+?)/Uui', $nfo, 0, PREG_SPLIT_DELIM_CAPTURE|PREG_SPLIT_NO_EMPTY); + if (isset($set[1])) + { + $name = trim($set[1]); + if (preg_match('/(?:issue)\b(?:[\-\:\.\}\[\s\xb0-\x{3000}]+?)\B([a-z0-9\.\- \(\)\/]+?)/iuU', $nfo, $set)) + { + if (isset($set[1])) + { + $name = $name." ".trim($set[1]); + } + $name = matchnames($row['name'], $name); + $ok = dodbupdate($row['ID'], 7010, $name ); + } + } + return $ok; +} + +function dodbupdate($id, $cat, $name = '', $typeid = 0, $type='none', $debug = DEBUGGING) +{ + global $db; + + $query = "UPDATE `releases` SET `categoryID` = $cat"; + + if ($name != '') + $query = $query.", `searchname` = ".$db->escapeString($name); + switch ($type) { + case 'imdb': + if ($typeid != 0) + $query = $query.", `imdbID` = $typeid"; + break; + case 'book': + if ($typeid != 0) + $query = $query.", `bookinfoID` = $typeid"; + break; + case 'music': + if ($typeid != 0) + $query = $query.", `musicinfoID` = $typeid"; + break; + case 'anime': + if ($typeid != 0) + $query = $query.", `anidbID` = $typeid"; + break; + case 'tv': + if ($typeid != 0) + $query = $query.", `rageID` = $typeid"; + break; + default: + break; + } + + $query = $query." where `ID` = $id"; +echo $debug." ".$query."\n"; + if (!$debug) + { +echo "updating"; + if ($db->query($query) !== false) + return true; + } else { + return true; + } + return false; +} + +function findmovie($name) +{ + $imdb = 0; + + $j = 0; + + $name = stripname($name); + + $url = "http://www.omdbapi.com/?r=xml&s="; + $name1 = $name; + if (preg_match('/([^\[\r\]]+)[\[\{\(\_\. ]+?((?:19|20)\d{2})/U', $name, $title)) + { + $url = $url.urlencode($title[1])."&y=".$title[2]; + $name1 = $title[1]." ".$title[2]; + } + else + $url = $url.urlencode($name); + +echo "omdbapi: $url\n"; +echo cleanname($name1)."\n"; + + $data = getUrl($url); + +//echo "$data\n"; + $xmlObj = @simplexml_load_string(html_entity_decode($data)); + $arrXml = objectsIntoArray($xmlObj); + +// var_dump($arrXml); + + if (isset($arrXml["Movie"])) + { + + $tvrage = new TVRage(true); + + $name1 = ''; + + foreach ($arrXml["Movie"] as $ax) + { +// var_dump($ax); + + if (isset($ax["@attributes"])) + $a = $ax["@attributes"]; + else + $a = $ax; + if ((isset($title[2]) && $a["Year"] == $title[2]) || !isset($title[2])) + { + $i = 0; + $a["Title"] = preg_replace('/&/', ' and ', $a["Title"]); + $name = preg_replace('/&/', ' and ', $name); + +echo "name $name\n"; + + $arr = preg_split('/[\:\:\.\, \-\_\(\)\[\]\{\}]/', $a["Title"], 0, PREG_SPLIT_NO_EMPTY); + foreach ($arr as $r) + { + $r = str_replace("/", " ", $r); + $r = str_replace("\\", " ", $r); + $r = preg_replace("/(\b\w+\b)[^ ]/", "\\1", $r); + if (preg_match('/'.preg_quote($r).'/i', $name)) + $i++; + } +echo count($arr)." $i $j \n"; + if (count($arr) == $i && $i > $j) + { + $name1 = $name; + $imdb = $a["imdbID"]; + $j = $i; + } +// echo "comp result ".count($arr)." ".$i." ".$name."\n"; +// echo $a["Title"]."\n"; + } + } + echo "$name1 $imdb\n"; + + unset($tvrage); + } + echo "findmovie = $imdb\n"; + + return array(strval($imdb), $j); +} + +function matchvideo($row, $nfo) +{ + global $db; + + $ok = false; + + $imdb = ''; + + $j = 0; + + $query = "SELECT releasevideo.releaseID FROM releasevideo WHERE releasevideo.releaseID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel === false) + { + $query = "SELECT releasefiles.`name`, releasefiles.releaseID, releasefiles.size FROM releasefiles WHERE releasefiles.releaseID = ".$row['ID']; + $res = $db->queryDirect($query); + + $cont = false; + while ($rel = $db->getAssocArray($res)) + { + if(preg_match('/\.(iso|mkv|mp4|iso|divx)$|\.([^\.]+)$/', $rel['name'], $ext)) + { + echo "ext = ".$ext[1]." ".$row['guid']."\n"; + //var_dump($ext); + $cont = true; + } + } + if (!$cont) + return false; + } + $thename = $name = $row['name']; +echo "name1: $name\n"; + $rageID = lookupTV ($name, $nfo, $row, 5000); + + if ($rageID == 0) + { + $imdb = findmovie($name); + $imdb = $imdb[0]; + } + + if ($rageID == 0 && empty($imdb) ) + { + $set = preg_split('/(?:\btitle|\bname).+[\.\s]+?[^a-z0-9]*?([a-z0-9\-\.\_ ]+?)/iU', $nfo, 0, PREG_SPLIT_DELIM_CAPTURE|PREG_SPLIT_NO_EMPTY); +//var_dump($set); + if (isset($set[1])) + { +echo "name2: ".$set[1]."\n"; + $rageID = lookupTV ($set[1], $nfo, $row, 5000); + if ($rageID) + { + $thename = $set[1]; + } else + { + $imdb = findmovie($set[1]); + $imdb = $imdb[0]; + } + + } + + if ($rageID == 0 && empty($imdb) ) + { + $query = "SELECT releasefiles.releaseID, releasefiles.`name` FROM releasefiles INNER JOIN releases ON releasefiles.releaseID = releases.ID WHERE releases.ID = ".$row['ID']; + $rel = $db->query($query); + + $ok = false; + $j = 0; + + + if (count($rel) > 0) + { + foreach($rel as $r) + { + $name1 = preg_replace('/video[\_\. ]ts/iU', ' ', $r['name']); + + $name1 = preg_replace('/\\\\/', ' ', $name1); + + $name1 = preg_replace('/\.[a-z][a-z0-9]{2}$/iU', '', $name1); + + $name1 = preg_replace('/ +/', ' ', $name1); + + $name1 = preg_replace('/^ | $/', '', $name1); + + echo "file ".$name1."\n"; +echo "name3: $name1\n"; + $rageID = lookupTV ($name1, $nfo, $row, 5000); + + if ($rageID && $j == 0) + { + $thename = $name1; + break; + } else { + $tmp = findmovie($name1); +// var_dump($tmp); + if ($tmp[1] > $j) + { + echo "$rageID\n"; + $imdb = $tmp[0]; + $j = $tmp[1]; + $thename = $name1; + } + } + + } + } + } + } + + if (preg_match('/^[a-f0-9]+$/', $thename)) + { + if (isset($set[1])) + $thename = $set[1]; + else if (preg_match('/[a-f0-9][ .]/', $name1)) + $thename = $name1; + $thename = preg_replace('/\.\_/', ' ', $thename); + + } + +echo "$j $rageID $imdb\n"; + + if ($imdb != '') + { + $imdb = preg_replace('/^tt/', '', $imdb); + $movie = new Movie(); + $tmp = $movie->getMovieInfo($imdb); + + if ($tmp === false) + { + $movie->fetchImdbProperties($imdb); + $movie->updateMovieInfo($imdb); + $tmp = $movie->getMovieInfo($imdb); + } + + + $thename = matchnames($row['name'], $thename); + $thename = matchnames($thename, $tmp['title']); + $thename = findquality ($nfo, $thename); + + unset($movie); + +echo "$thename\t"; +// var_dump($tmp); + + if (isset($tmp['ID'])) + { + if (preg_match('/sport/iU', $tmp['genre'])) { + $ok = dodbupdate($row['ID'], 5060, $thename, $imdb, 'imdb'); + } else if (preg_match('/docu/iU', $tmp['genre'])) { + $ok = dodbupdate($row['ID'], 5080, $thename, $imdb, 'imdb'); + } else if (preg_match('/talk\-show/iU', $tmp['genre'])) { + $ok = dodbupdate($row['ID'], 5000, $thename, $imdb, 'imdb'); + } else if (preg_match('/tv/iU', $tmp['genre']) || preg_match('/episode/iU', $tmp['genre']) || preg_match('/reality/iU', $tmp['genre'])) { + $ok = updateTV ($tmp['title']." (".$tmp['year'].")", $row['ID'], $row, 5000, $nfo); + } else { + $ok = dodbupdate($row['ID'], 2000, $thename, $imdb, 'imdb'); + } + } + + + } + + if ($rageID) + { + $tvrage = new TVRage(true); + $tmp = $tvrage->getByRageID($rageID); + $thename = matchnames($row['name'], $thename); + $thename = matchnames($thename, $tmp[0]['releasetitle']); + $thename = findquality ($nfo, $thename); + + $name = $tmp[0]['releasetitle']; + + // $tmp = $tvrage->getByRageID($rageID); + + unset($tvrage); +// var_dump($tmp); +echo "doing updates"; + if (isset($tmp[0])) + { + if (preg_match('/sports/i', $tmp[0]['genre'])) + $ok = dodbupdate($row['ID'], 5060, $thename, $rageID, 'tv'); + elseif (preg_match('/documen/i', $tmp[0]['genre'])) + $ok = dodbupdate($row['ID'], 5080, $thename, $rageID, 'tv'); + else + $ok = dodbupdate($row['ID'], 5000, $thename, $rageID, 'tv'); + } + } + + if ($rageID == 0 && empty($imdb) ) + { + $thename = matchnames($row['name'], $thename); + $thename = findquality ($nfo, $thename); +echo "the name = $thename\n"; + if (preg_match('/sport/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 5060, $thename); + elseif (preg_match('/docu/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 5080, $thename); + elseif (preg_match('/tv/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 5000, $thename); + elseif (preg_match('/mov/i', $row['gname'])) + $ok = dodbupdate($row['ID'], 2000, $thename); + else + $ok = dodbupdate($row['ID'], 2020, $thename); + } + + return $ok; +} + + +echo "starting\n"; +$db = new DB(); + +$res = $db->query("SET NAMES 'utf8'"); +$res = $db->query("SET CHARACTER SET 'utf8'"); + +//mysql_query("SET NAMES 'utf8'"); +//mysql_query("SET CHARACTER SET 'utf8'"); +mb_internal_encoding("UTF-8"); +mb_regex_encoding("UTF-8"); +mb_http_output("UTF-8"); +mb_language("uni"); + +$query = "SELECT ID FROM category WHERE parentID = 8000"; // OR parentID = 2000 OR parentID = 5000 + +$res = $db->query($query); + +$thecategory = $res[0]['ID']; + +unset($res[0]); + +foreach($res as $r) + $thecategory = $thecategory.", ".$r['ID']; + +if (isset($argv[1])) +{ + $query = "SELECT uncompress(releasenfo.nfo) AS nfo, releases.ID, releases.guid, releases.`name`, releases.searchname, groups.`name` AS gname FROM releasenfo INNER JOIN releases ON releasenfo.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID WHERE releases.ID = ".$argv[1]; +} else { + $query = "SELECT uncompress(releasenfo.nfo) AS nfo, releases.ID, releases.guid, releases.`name`, releases.searchname, groups.`name` AS gname FROM releasenfo INNER JOIN releases ON releasenfo.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID WHERE releases.categoryID IN ( $thecategory )"; +} +//$query = "SELECT uncompress(releasenfo.nfo) AS nfo, releases.ID, releases.guid, releases.`name`, releases.searchname, groups.`name` AS gname FROM releasenfo INNER JOIN releases ON releasenfo.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID"; +//$query = "SELECT uncompress(releasenfo.nfo) AS nfo, releases.ID, releases.guid, releases.`name`, releases.searchname, groups.`name` AS gname FROM releasenfo INNER JOIN releases ON releasenfo.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID WHERE releases.guid = '6dc77dc7fb48745ee76ec345c4fe789b'"; +// +//echo $query."\n"; + +$res = $db->queryDirect($query); + +echo "got matches\n"; + +while ($row = $db->getAssocArray($res)) +{ + $nfo = utf8_decode($row['nfo']); + + if (strlen($nfo) > 0) + { +// trigger_error("doing nfo ".$row['ID']." ".$row['name']); + + $pattern = '/.+(\.rar|\.001) [0-9a-f]{6,10}?|(imdb)\.[a-z0-9\.\_\-\/]+?(tt|\?)(\d+?)\/?|(tvrage)|(\bASIN)|(isbn)|(UPC\b)|(comic book)|(comix)|(tv series)|(\bos\b)|(documentaries)|(documentary)|(doku)|(macintosh)|(dmg)|(mac[ _\.\-]??os[ _\.\-]??x??)|(\bos\b\s??x??)|(\bosx\b)'; + $pattern = $pattern . '|(\bios\b)|(iphone)|(ipad)|(ipod)|(pdtv)|(hdtv)|(video streams)|(movie)|(audiobook)|(audible)|(recorded books)|(spoken book)|(speech)|(read by)\:?|(narrator)\:?|(narrated by)'; + $pattern = $pattern . '|(dvd)|(ntsc)|(m4v)|(mov\b)|(avi\b)|(xvid)|(divx)|(mkv)|(amazon\.)[a-z]{2,3}.*\/dp\/|(anidb.net).*aid=|(\blame\b)|(\btrack)|(t r a c k)|(music)|(44.1kHz)|type:(game)|(game) Type|(platform)|\b(win(?:dows|all|xp)\b)|(\bwin\b)'; + $pattern = $pattern . '|(m3u)|(flac\b)|(application)|(plugin)|(\bcrack)|(install)|(setup)|(magazin)|(x264)|(itunes\.apple\.com\/)|(sports)|(deportes)|(nhl)|(nfl)|(\bnba)|(ncaa)|(album)|(epub)|(mobi)|format\W+(pdf)/iU'; + + $matches = preg_split($pattern, $nfo, -1, PREG_SPLIT_NO_EMPTY|PREG_SPLIT_DELIM_CAPTURE); + + $matches = doarray($matches); + + if (count($matches) > 0) + { + // var_dump($matches); + } + + foreach($matches as $m) + { + if (isset($m)) { + $case = preg_replace('/ /', '', $m); + } else { + $case = ''; + } + if ($row['name'] == '4d5bdbf0da623fb9cf849b8b745c60e9') + print_r($matches); + + if (($m == 'os' || $m == 'platform') && preg_match('/(?:\bos\b(?: type)??|platform)[ \.\:\}]+(\w+?).??(\w*?)/iU', $nfo, $set)) + { + + if (isset($set[1])) + { + // var_dump($set); + $case = strtolower($set[1]); + } + if (strlen($set[2]) && (stripos($set[2], 'mac') !== false || stripos($set[2], 'osx') !== false)) + { + $case = strtolower($set[2]); + } + } + + + echo "\n".$case." ".$row['guid']."\n"; + // echo $row['name']."\n"; + // echo $row['searchname']."\n"; + echo "$case ".$row['guid']."\n"; + + switch (strtolower($case)) + { + case 'itunes.apple.com/': + echo $row['guid']."\n"; + preg_match('/itunes\.apple\.com\/(.*)/iU', $nfo, $set); + echo 'itunes.apple.com'.$set[1]."\n"; + break; + + case 'imdb': + if (matchimdb($nfo, $row)) + break(2); + break; + + + case "asin": + case "isbn": + case "amazon.": + case "upc": + if ($case == 'asin' || $case == 'isbn') + { + preg_match('/(?:isbn|asin)[ \:\.]*? *?([a-zA-Z0-9\-\.]{8,20}?)/iU', $nfo, $set); + //var_dump($set); + if (isset($set[1])) + { + $set[1] = preg_replace('/[\-\.]/', '', $set[1]); + echo "asin ".$set[1]."\n"; + if (strlen($set[1])>13) + break; + if (isset($set[1])) + { + $set[2] = $set[1]; + $set[1] = "com"; + } + } + } else if ($case == 'amazon.') { + preg_match('/amazon\.([a-z]*?\.?[a-z]{2,3}?)\/.*\/dp\/([a-zA-Z0-9]{8,10}?)/iU', $nfo, $set); + } else if ($case == 'upc') { + preg_match('/UPC\:?? *?([a-zA-Z0-9]*?)/iU', $nfo, $set); + if (isset($set[1])) + { + $set[2] = $set[1]; + $set[1] = "All"; + } + } else { + echo "* * * * * error in amazon"; + break; + } + + if (count($set) > 1) + { + if (doAmazon ($row['name'], $row['ID'], $nfo, $set[2], $set[1], $case, $nfo, $row)) + { + break(2); + } + } + break; + case 'comicbook': + case 'comix': + if (dodbupdate($row['ID'], 7030)) + break(2); + break; + case 'epub': + case 'mobi': + case 'pdf': + echo "ebooks\n"; + $name = ''; + if (preg_match('/(?:\btitle\b)+?(?! *[^ \.\:\}\]]{2,}?\b)(?:[\|\;\:\.\[\}\]\( \xb0-\x{3000}]+?)[ \.]([a-z0-9](?!:)(?:.(?!\s\s+))+?[^\s]+?)/Uuim', $nfo, $title)) + $name = $title[1]; + if (dodbupdate($row['ID'], 7020, $name)) + break(2); + + break; + case 'audiobook': + case 'audible': + case 'recordedbooks': + case 'spokenbook': + case 'readby': + case 'narratedby': + case 'narrator': + case 'speech': + if (matchaudiobook ($nfo, $row['ID'], $row['name'], $row)) + break(2); + break; + + case 't r a c k': + case 'track': + case 'lame': + case 'album': + case 'music': + case '44.1kHz': + case 'm3u': + case 'flac': + if (preg_match('/(a\s?r\s?t\s?i\s?s\s?t|l\s?a\s?b\s?e\s?l|mp3|e\s?n\s?c\s?o\s?d\s?e\s?r|rip|stereo|mono)/i', $nfo)) + { + if (!preg_match('/(\bavi\b|x\.?264|divx|mvk|xvid|install|Setup\.exe|unzip|unrar)/i', $nfo)) + { + if (matchmusic($nfo, $row)) + { + // echo "breaking 2\n"; + break(2); + } + } else { + // echo "$case ".$row['guid']."\n"; + if (findmusicname($row, 3010)) + break (2); + } + } + break; + + case 'magazin': + // echo "magazin\n"; + if (matchmag($nfo, $row)) + break(2); + break; + + case 'dmg': + case 'mac': + case 'macintosh': + case 'macos': + case 'macosx': + case 'osx': + if (doOS($nfo, $row['name'], 4030, $row['ID'])) + break(2); + break; + + case 'ios': + case 'iphone': + case 'ipad': + case 'ipod': + + if (doOS($nfo, $row['name'], 4060, $row['ID'])) + break(2); + break; + + case 'application': + case 'install': + case 'setup': + if (!preg_match('/(Setup(\.exe)?|unzip|unrar)/i', $nfo)) + continue; + if (preg_match('/(\bavi\b|x\.?264|divx|mvk|xvid|install)/i', $nfo)) + continue; + + case 'windows': + case 'win': + case 'winall': + case 'winxp': + case 'plugin': + case 'crack': + case 'linux': + if (doOS($nfo, $row['name'], 4020, $row['ID'])) + break(2); + break; + + case 'android': + if (doOS($nfo, $row['name'], 4070, $row['ID'])) + break(2); + break; + + case 'game': + $set = preg_split('/\>(.*)\queryDirect($query); + +echo "doing part 2\n"; + +while ($row = $db->getAssocArray($res)) +{ +// trigger_error("doing part 2".$row['ID']); + $query = "SELECT releasevideo.releaseID FROM releasevideo WHERE releasevideo.releaseID = ".$row['ID']; + $rel = $db->queryOneRow($query); + + if ($rel !== false) + continue; + + echo "\n".$row['guid']."\n"; + + $query = 'SELECT releasenfo.releaseID, uncompress(releasenfo.nfo) AS nfo FROM releasenfo WHERE releasenfo.releaseID = '.$row['ID']; + $rel = $db->queryOneRow($query); + + $nfo = ''; + if ($rel !== false) + if ($rel['releaseID'] == $row['ID']) + $nfo = $rel['nfo']; + + matchmusic($nfo, $row); + //findmusicname($row, 3010); +} + +if (isset($argv[1])) +{ + $query = "SELECT releasefiles.`name` AS rname, releasefiles.size, releases.*, groups.`name` AS gname FROM releasefiles INNER JOIN releases ON releasefiles.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID WHERE releases.ID = ".$argv[1]." ORDER BY releasefiles.releaseID, releasefiles.`name`"; +} else { + $query = "SELECT releasefiles.`name` AS rname, releasefiles.size, releases.*, groups.`name` AS gname FROM releasefiles INNER JOIN releases ON releasefiles.releaseID = releases.ID INNER JOIN groups ON releases.groupID = groups.ID WHERE releases.categoryID IN ( $thecategory ) ORDER BY releasefiles.releaseID, releasefiles.`name`"; +} + +$res = $db->queryDirect($query); + +echo "doing part 3\n"; + +while ($row = $db->getAssocArray($res)) +{ +// trigger_error("doing part 3".$row['ID']); + if(preg_match('/\.([^\.]{3,4})$/', $row['rname'], $ext)) + { + $ext[1] = strtolower($ext[1]); + + echo "ext = ".$ext[1]." ".$row['guid']."\n"; +// var_dump($ext); + switch ($ext[1]) + { + case 'epub': + case 'mobi': + if (preg_match('/ \- /', $row['rname'])) + { + $title = preg_split("/ \- /", $row['rname']); + $author = preg_replace('/(\.[^\.]+)/', '', $title[1]); + + $tmp = preg_replace('/'.$row['name'].'/', '', $title[0]); + $tmp = preg_split('/\\\\/', $tmp, 0, PREG_SPLIT_NO_EMPTY); + + + + echo "author: ".$author." title: ".$tmp[count($tmp)-1]."\n"; + + $bookId = findbookid ($author, $tmp[count($tmp)-1]); + + $title1 = $tmp[count($tmp)-1]; + + echo $bookId."\n"; + + if ($bookId == 0) + { + $bookId = findbookid ($tmp[count($tmp)-1], $author); + echo "inverse ".$bookId."\n"; + + $title1 = $author; + $author = $tmp[count($tmp)-1]; + + } + + $title1 = matchnames($row['name'], $author.' - '.$title1); + + if ($bookId != 0) + $ok = dodbupdate($row['ID'], 7020, $title1, $bookId, 'book'); + else + $ok = dodbupdate($row['ID'], 7020, $title1); + } + break; + + + case 'avi': + case 'mkv': + case 'mp4': + case 'iso': + case 'wmv': + case 'mpg': + case 'mov': + case 'm4v': + $tmp = str_replace('/', ' ', $row['rname']); + $tmp = preg_replace("/([^\\\\]+?)\\\\(\\1)/", "\\2", $tmp ); + $tmp = preg_replace("/".preg_quote(str_replace('/', ' ', $row['name']))."/", "", $tmp); + $tmp = preg_replace("/\\\\/", " ", $tmp); + $tmp = stripname($tmp); + + $tvrage = new TVRage(true); + $parsed = $tvrage->parseNameEpSeason($tmp); + + unset($tvrage); + echo "name = ".$tmp."\n"; + + $nfo = $tmp."\n"; + + //var_dump($parsed); + + matchvideo($row, $nfo); + + break; + + + default: + break; + } + + } + +} + +echo "all done\n"; + +unset($res); +unset($row); +unset($matches); + +if (!isset($argv[1])) +{ + $movie = new Movie(); + $movie->processMovieReleases(); + $movie->updateUpcoming(); + + $music = new Music(true); + $music->processMusicReleases(); + + $book = new Book(true); + $book->processBookReleases(); + + $anidb = new AniDB(true); + $anidb->animetitlesUpdate(); + $anidb->processAnimeReleases(); + + $tvrage = new TVRage(true); + $tvrage->processTvReleases(true); + + $thetvdb = new TheTVDB(true); + $thetvdb->processReleases(); + + echo "cleaning 5000\n"; + + $query = "update releases set releases.categoryID= 5060 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE movieinfo.genre REGEXP 'sport' AND NOT movieinfo.genre REGEXP 'comedy' AND NOT movieinfo.genre REGEXP 'romance' AND NOT movieinfo.genre REGEXP 'drama' )"; + + $db->query($query); + + $query = "update releases set releases.categoryID= 5000 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'tv|talk\-show|reality|episode' ) AND categoryID NOT IN ( SELECT ID FROM category WHERE parentID = 5000 )"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 10500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 2500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2020 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000)"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2040 WHERE `categoryID` = 2050 AND size <10000000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=2030 WHERE `categoryID` = 2040 AND size < 2500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=5040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 1500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=5030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 10000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=5050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000)"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=5030 WHERE `categoryID` = 5040 AND size < 1500000000"; + + $db->query($query); + + $query = "UPDATE `releases` SET `categoryID`=5050 WHERE `categoryID` = 5030 AND size < 1000000"; + + $db->query($query); + + $query = "UPDATE releases SET categoryID = 5070 WHERE rageID IN ( SELECT tvrage.rageID FROM tvrage WHERE tvrage.genre REGEXP 'animat' )"; + + $db->query($query); + + $query = "update releases set releases.categoryID= 5080 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'docum' )"; + + $db->query($query); + +} +//ALTER TABLE `movieinfo` CHANGE `genre` `genre` VARCHAR( 256 ) CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL + +?> diff --git a/ugo/misc_sorter_cleanup.php b/ugo/misc_sorter_cleanup.php new file mode 100644 index 000000000..d87dc3b11 --- /dev/null +++ b/ugo/misc_sorter_cleanup.php @@ -0,0 +1,98 @@ +processMovieReleases(); +$movie->updateUpcoming(); + +$music = new Music(true); +$music->processMusicReleases(); + +$book = new Book(true); +$book->processBookReleases(); + +$anidb = new AniDB(true); +$anidb->animetitlesUpdate(); +$anidb->processAnimeReleases(); + +$tvrage = new TVRage(true); +$tvrage->processTvReleases(true); + +$thetvdb = new TheTVDB(true); +$thetvdb->processReleases(); + +echo "cleaning 5000\n"; + +$db = new DB(); + +$query = "update releases set releases.categoryID= 5060 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE movieinfo.genre REGEXP 'sport' AND NOT movieinfo.genre REGEXP 'comedy' AND NOT movieinfo.genre REGEXP 'romance' AND NOT movieinfo.genre REGEXP 'drama' )"; + +$db->query($query); + +$query = "update releases set releases.categoryID= 5000 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'tv|talk\-show|reality|episode' ) AND categoryID NOT IN ( SELECT ID FROM category WHERE parentID = 5000 )"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 10500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 2500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2020 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000)"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2040 WHERE `categoryID` = 2050 AND size <10000000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=2030 WHERE `categoryID` = 2040 AND size < 2500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=5040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 1500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=5030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 100000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=5050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000)"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=5030 WHERE `categoryID` = 5040 AND size < 1500000000"; + +$db->query($query); + +$query = "UPDATE `releases` SET `categoryID`=5050 WHERE `categoryID` = 5030 AND size < 100000000"; + +$db->query($query); + +$query = "UPDATE releases SET categoryID = 5070 WHERE rageID IN ( SELECT tvrage.rageID FROM tvrage WHERE tvrage.genre REGEXP 'animat' )"; + +$db->query($query); + +$query = "update releases set releases.categoryID= 5080 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'docum' )"; + +$db->query($query); + +?> diff --git a/ugo/misc_sorter_threaded.php b/ugo/misc_sorter_threaded.php new file mode 100644 index 000000000..cde1b90a3 --- /dev/null +++ b/ugo/misc_sorter_threaded.php @@ -0,0 +1,140 @@ +query($query); + +$thecategory = $res[0]['ID']; + +unset($res[0]); + +foreach($res as $r) + $thecategory = $thecategory.", ".$r['ID']; + +$query = "SELECT releases.ID, releases.`name` FROM releases WHERE releases.categoryID IN ( $thecategory )"; + +$theList = $db->query($query); +unset($db); + +$theCount = count($theList); + +$ps = new PowerProcess(); +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxChildren = 24; +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->threadTimeLimit = 0; // Disable child timeout + +$tim = microtime(true) - $time; +echo "time = ".$tim."\n"; + + +echo "\nToday will be going over $theCount releases\n"; +echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t"; + +while ($ps->RunControlCode()) +{ + // Start the parent loop + $listcount = count($theList); + if ($listcount) + { + // We still have groups to process + if ($ps->SpawnReady()) + { + // Spawn another thread + $ps->threadData = array_pop($theList); +// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." posts to sort after this\n"; + $ps->spawnThread(); + } + else + { + // There are no more slots available to run + //$ps->tick(); + //echo ".\n"; + } + } + else + { + // No more groups to process + echo "\nNo more to process - Initiating shutdown\n"; + $ps->threadTimeLimit = 200; + do { + $ps->tick(); + } while ($ps->myThreads); + $ps->Shutdown(); + echo "\nShutdown complete\n"; + $tim = microtime(true) - $time; + echo "final processing time = ".$tim."\n"; + + } +} +$tim = microtime(true) - $time; +//echo "time = ".$tim."\n"; +unset($theList); + +if ($ps->RunThreadCode()) +{ + $post = $ps->threadData; + + $thread = sprintf("%05d",$ps->GetPID()); + +// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n"; + + $param = $post['ID']; + + $dir = dirname(__FILE__); + $file = 'misc_sorter3.php'; + + $output = shell_exec("php {$dir}/{$file} {$param}"); + + $count = $theCount - $listcount; + $countpct = $count / $theCount; + $cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT); + + if ($listcount % 250 == 0 && $listcount > 0) + { + $tim = microtime(true) - $time; + echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); + $tim = ($listcount * $tim) / $count + time(); +// $tim = number_format(($listcount * $tim) / 60 / $count, 1); + if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0)) + echo "\t".date('M jS Y H:i:s', $tim); +// echo "\t".$tim."m to go" ; + echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t"; + + } elseif ($listcount % 25 == 0) + echo " ."; + + +// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n"; + $tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +} + +// Exit to call back to parent - Let know that child has completed +exit(0); + +// Create callback function +function psUpdateComplete() +{ + echo "[Thread-MASTER] Threaded sorting process complete\n"; +} + +?> + diff --git a/ugo/postproc.php b/ugo/postproc.php new file mode 100644 index 000000000..789296561 --- /dev/null +++ b/ugo/postproc.php @@ -0,0 +1,355 @@ +myThreads[$pid])) + if ($ps->myThreads[$pid]['time'] != time()) + { + $ps->myThreads[$pid]['time'] = time(); + $lastto = time(); + shm_put_var($buffer, 1, $ps); + // trigger_error($pid." ".$ps->myThreads[$pid]['time']); + } + } else { + trigger_error("$pid is orphaned"); + } +} + +$postprocess = new PostProcess(true); + +$s = new Sites(); +$site = $s->get(); +$nntp = new Nntp; + +$db = new DB; +$nzb = new Nzb; + +$maxattemptstocheckpassworded = 5; +$numtoProcess = 5; +$processVideoSample = ($site->ffmpegpath != '') ? true : false; +$processMediainfo = ($site->mediainfopath != '') ? true : false; +$processPasswords = ($site->unrarpath != '') ? true : false; +$processAudioSample = ($site->saveaudiopreview == 1) ? true : false; + +$nntpconnected = false; +$lastto = time()+1000; + +var_dump($argv); + +if (is_numeric($argv[1])) +{ + if (isset($argv[3])) + { + $buffer = shm_attach($argv[2]); + $pid = $argv[3]; + } + + $rel = $db->query(sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r + left join category c on c.ID = r.categoryID + where r.ID = %d ", $argv[1])); + + $rel = $rel[0]; + + if ($rel !== false) + { + $tmpPath = $site->tmpunrarpath; + + if (substr($tmpPath, -strlen( '/' ) ) != '/') + { + $tmpPath = $tmpPath.'/'; + } + + $tmpPath = $tmpPath.$rel['guid'].'/'; + + if (!file_exists($tmpPath)) + mkdir($tmpPath, 0766, true); + + + // Per release defaults + $passStatus = array(Releases::PASSWD_NONE); + $blnTookMediainfo = false; + $blnTookSample = ($rel['disablepreview'] == 1) ? true : false; //only attempt sample if not disabled + + if ($blnTookSample) + $db->query(sprintf("update releases set haspreview = 0 where id = %d", $rel['ID'])); + + // + // Go through the binaries for this release looking for a rar, a sample, and a mediafile + // + $nzbInfo = new nzbInfo; + $norar = 0; + + // only load nzbs and check for rar files if we are doing something with them. + if ($processVideoSample || $processMediainfo || $processPasswords || $processAudioSample) + { + $nzbfile = $nzb->getNZBPath($rel['guid'], $site ->nzbpath); + if (!$nzbInfo->loadFromFile($nzbfile)) + { + goto end; + } + + foreach($nzbInfo->nzb as $nzbsubject) + { + if (preg_match("/\w\.r00/i", $nzbsubject['subject'])) + $norar= 1; + } + } + + // attempt to process video sample file + if(!empty($nzbInfo->samplefiles) && $processVideoSample && $blnTookSample === false) + { + $sampleFile = $nzbInfo->samplefiles[0]; //first detected sample + $sampleMsgids = array_slice($sampleFile['segments'], 0, 1); //get first segment, increase to get more of the sample + $sampleGroup = $sampleFile['groups'][0]; + + echo "PostPrc : Fetching ".implode($sampleMsgids, ', ')." from {$sampleGroup}\n"; + if (!$nntpconnected) + $nntpconnected = $nntp->doConnect(); + + if ($nntpconnected) { + $sampleBinary = $nntp->getMessages($sampleGroup, $sampleMsgids); + watchdog (); + if ($sampleBinary === false) + echo "\nPostPrc : Couldnt fetch sample\n"; + else + { + $samplefile = $tmpPath.'sample.avi'; + + file_put_contents($samplefile, $sampleBinary); + + $blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']); + if ($blnTookSample) + $postprocess->updateReleaseHasPreview($rel['guid']); + + unlink($samplefile); + } + unset($sampleBinary); + } else { + echo "\nFailed to get NNTP connection\n"; + } + } + + // attempt to process loose media file + if(!empty($nzbInfo->mediafiles) && (($processVideoSample && $blnTookSample === false) || $processMediainfo)) + { + $mediaFile = $nzbInfo->mediafiles[0]; //first detected media file + $mediaMsgids = array_slice($mediaFile['segments'], 0, 2); //get first two segments + $mediaGroup = $mediaFile['groups'][0]; + + echo "PostPrc : Fetching ".implode($mediaMsgids, ', ')." from {$mediaGroup}\n"; + if (!$nntpconnected) + $nntpconnected = $nntp->doConnect(); + + if ($nntpconnected) { + $mediaBinary = $nntp->getMessages($mediaGroup, $mediaMsgids); + watchdog (); + if ($mediaBinary === false) + echo "\nPostPrc : Couldnt fetch media file\n"; + else + { + $mediafile = $tmpPath.'sample.avi'; + + file_put_contents($mediafile, $mediaBinary); + + if ($processVideoSample && $blnTookSample === false) + { + $blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']); + if ($blnTookSample) + $postprocess->updateReleaseHasPreview($rel['guid']); + } + + if ($processMediainfo) + $blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']); + + unlink($mediafile); + } + unset($mediaBinary); + } else { + echo "\nFailed to get NNTP connection\n"; + } + } + + // attempt to process audio sample file + if(!empty($nzbInfo->audiofiles) && $processAudioSample && $blnTookSample === false) + { + $audioFile = $nzbInfo->audiofiles[0]; //first detected audio file + $audioMsgids = array_slice($audioFile['segments'], 0, 1); //get first segment + $audioGroup = $audioFile['groups'][0]; + + echo "PostPrc : Fetching ".implode($audioMsgids, ', ')." from {$audioGroup}\n"; + if (!$nntpconnected) + $nntpconnected = $nntp->doConnect(); + + if ($nntpconnected) { + $audioBinary = $nntp->getMessages($audioGroup, $audioMsgids); + watchdog (); + if ($audioBinary === false) + echo "\nPostPrc : Couldnt fetch audio sample\n"; + else + { + $audiofile = $tmpPath.'sample.mp3'; + + file_put_contents($audiofile, $audioBinary); + + $blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']); + if ($blnTookSample !== false) + $postprocess->updateReleaseHasPreview($rel['guid'], 2); + + if ($processMediainfo) + $blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']); + + if ($site ->lamepath != "") + $postprocess->lameAudioSample($site ->lamepath, $rel['guid']); + + unlink($audiofile); + } + unset($audioBinary); + } else { + echo "\nFailed to get NNTP connection\n"; + } + } + + if (!empty($nzbInfo->rarfiles) && ($site ->checkpasswordedrar > 0 || (($processVideoSample || $processAudioSample) && $blnTookSample === false) || $processMediainfo)) + { + $mysqlkeepalive = 0; + foreach($nzbInfo->rarfiles as $rarFile) + { + //dont process any more rars if a passworded rar has been detected and the site is set to automatically delete them + if ($site ->deletepasswordedrelease == 1 && max($passStatus) == Releases::PASSWD_RAR) + { + echo "-Skipping processing of rar {$rarFile['subject']} as this release has already been marked as passworded.\n"; + goto end; + } + + $rarMsgids = array_slice($rarFile['segments'], 0, 1); //get first segment + $rarGroup = $rarFile['groups'][0]; + + echo "PostPrc : Fetching ".implode($rarMsgids, ', ')." from {$rarGroup} (".++$mysqlkeepalive.")\n"; + if (!$nntpconnected) + $nntpconnected = $nntp->doConnect(); + + if ($nntpconnected) { + $fetchedBinary = $nntp->getMessages($rarGroup, $rarMsgids); + watchdog (); + if ($fetchedBinary === false) + { + echo "\nPostPrc : Failed fetching rar file\n"; + $db->query(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $rel['ID'])); + goto end; + } + else + { + $relFiles = $postprocess->processReleaseFiles($fetchedBinary, $rel['ID']); + + if ($site ->checkpasswordedrar > 0 && $processPasswords) + { + $passStatus[] = $postprocess->processReleasePasswords($fetchedBinary, $tmpPath, $site ->unrarpath, $site ->checkpasswordedrar); + } + + // we need to unrar the fetched binary if checkpasswordedrar wasnt 2 + if ($site ->checkpasswordedrar < 2 && $processPasswords) + { + $rarfile = $tmpPath.'rarfile.rar'; + file_put_contents($rarfile, $fetchedBinary); + $execstring = '"'.$site ->unrarpath.'" e -ai -ep -c- -id -r -kb -p- -y -inul "'.$rarfile.'" "'.$tmpPath.'"'; + $output = runCmd($execstring, false, true); + unlink($rarfile); + } + + if ($processVideoSample && $blnTookSample === false) + { + $blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']); + if ($blnTookSample) + $postprocess->updateReleaseHasPreview($rel['guid']); + } + + $blnTookAudioSample = false; + if ($processAudioSample && $blnTookSample === false) + { + $blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']); + if ($blnTookSample) + { + $blnTookAudioSample = true; + $postprocess->updateReleaseHasPreview($rel['guid'], 2); + } + } + + if ($processMediainfo && $blnTookMediainfo === false) + { + $blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']); + } + + // + // Has to be done after mediainfo + // + if ($blnTookAudioSample && $site ->lamepath != "") + $postprocess->lameAudioSample($site ->lamepath, $rel['guid']); + + if ($mysqlkeepalive % 25 == 0) + { + $db->query("select 1"); + watchdog (); + } + } + + //clean up all files + foreach(glob($tmpPath.'*') as $v) + { + unlink($v); + } + } else { + echo "\nFailed to get NNTP connection\n"; + } + + } //end foreach msgid + } + elseif(empty($nzbInfo->rarfiles) && $norar == 1) + { + $passStatus[] = Releases::PASSWD_POTENTIAL; + } + + $hpsql = ''; + if (!$blnTookSample) + $hpsql = ', haspreview = 0'; + + $sql = sprintf("update releases set passwordstatus = %d %s where ID = %d", max($passStatus), $hpsql, $rel["ID"]); + $db->query($sql); + + } +end: + foreach(glob($tmpPath.'*') as $v) + { + unlink($v); + } + foreach(glob($tmpPath.'.*') as $v) + { + @unlink($v); + } + rmdir($tmpPath); + + if ($nntpconnected) + { + $nntp->doQuit(); + } + unset($postprocess); + unset($s); + unset($site); + unset($nntp); + unset($db); + unset($nzb); + unset($nzbInfo); + +} +?> diff --git a/ugo/postproc_threaded.php b/ugo/postproc_threaded.php new file mode 100644 index 000000000..047c31158 --- /dev/null +++ b/ugo/postproc_threaded.php @@ -0,0 +1,171 @@ +myThreads) < 1 || !isset($buffer)) + return false; + + $ps1 = shm_get_var($buffer, 1); + $change = false; + + $ps1c = count($ps1->myThreads); + $psc = count($ps->myThreads); + + foreach($ps->myThreads as $k => $v) + { + if (isset($ps1->myThreads[$k])) + { + if ($v['time'] != $ps1->myThreads[$k]['time']) + { + $q = (strval($ps1->myThreads[$k]['time']) - strval($ps->myThreads[$k]['time'])); +// echo "time changed for $q\n"; + $ps->myThreads[$k]['time'] = $ps1->myThreads[$k]['time']; + $change = true; + } + } else { +// echo "ps not current $psc $ps1c\n"; + $change = true; + } + } + + if ($change) + shm_put_var($buffer, 1, $ps); + +// echo "pause "; +// var_dump($ps->myThreads); +} + +$time = microtime(true); +$db = new DB(); + +$maxattemptstocheckpassworded = 10; +$numtoProcess = 1000; + +$query = sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r + left join category c on c.ID = r.categoryID + where (r.passwordstatus between %d and -1) + or (r.haspreview = -1 and c.disablepreview = 0) order by RAND() limit %d ", ($maxattemptstocheckpassworded + 1) * -1, $numtoProcess); + +//$query = "SELECT releases.ID, releases.rarinnerfilecount, releases.haspreview, releases.passwordstatus, releases.releasenfoID, releases.guid FROM releases WHERE releases.ID NOT IN ( SELECT releasefiles.releaseID FROM releasefiles ) AND releases.rageID < 1 AND releases.imdbID < 1 OR releases.rarinnerfilecount < 0"; + +$theList = $db->query($query); +unset($db); + +$theCount = count($theList); + +$ps = new PowerProcess(); +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxThreads = 16; +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->threadTimeLimit = 350; // Disable child timeout + + +$handle = ftok(__FILE__, chr(1)); +$buffer = shm_attach($handle, 10240); + +$tim = microtime(true) - $time; +echo "time = ".$tim."\n"; + +echo "Starting threaded rar inspection process\n"; +echo "\nToday will be be inspecting $theCount binaries\n"; +echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t"; + +while ($ps->RunControlCode()) +{ + // Start the parent loop + $listcount = count($theList); + if ($listcount) + { + // We still have groups to process + if ($ps->SpawnReady()) + { + // Spawn another thread + $ps->threadData = array_pop($theList); +// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n"; + $ps->spawnThread(); + processchanges(); + } + else + { + //There are no more slots available to run + $ps->tick(); + //echo ".\n"; + processchanges(); + } + } + else + { + // No more groups to process + echo "\nNo more release to inspect - Initiating shutdown\n"; + do { + $ps->tick(); + processchanges(); + } while ($ps->myThreads); + $ps->Shutdown(); + echo "\nShutdown complete\n"; +// var_dump($ps->myThreads); + $tim = microtime(true) - $time; + echo "final time = ".$tim."\n"; + } +} +$tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +unset($theList); + +if ($ps->RunThreadCode()) +{ + $release = $ps->threadData; + + $thread = sprintf("%05d",$ps->GetPID()); + +// echo "[Thread-{$thread}] Begining rar inspection for {$release['name']} {$handle} id= {$param}\n"; + + $param = $release['ID']; + + $dir = dirname(__FILE__); + $file = 'postproc.php'; + +//echo "start "; +//var_dump($ps->myThreads); + + $output = shell_exec("php {$dir}/{$file} {$param} {$handle} {$thread}"); + + $count = $theCount - $listcount; + $countpct = $count / $theCount; + + if ($listcount % 50 == 0 && $count > 0) + { + $tim = microtime(true) - $time; + echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); + $tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time(); +// $tim = number_format(($listcount * $tim) / 60 / $count, 1); + if ($countpct > 0.1 || $count > 1500) + echo "\t".date('H:i:s', $tim); +// echo "\t".$tim."m to go" ; + echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t"; + + } + if ($listcount % 2 == 0) + echo " ."; + +// echo "[Thread-{$thread}] Completed update for binary {$release['name']}\n"; + $tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +} + +// Exit to call back to parent - Let know that child has completed +exit(0); + +// Create callback function +function psUpdateComplete() +{ + echo "[Thread-MASTER] Threaded inspection process complete\n"; +} + +?> + diff --git a/ugo/update_binaries_threaded.php b/ugo/update_binaries_threaded.php new file mode 100644 index 000000000..57f2c83b7 --- /dev/null +++ b/ugo/update_binaries_threaded.php @@ -0,0 +1,120 @@ +getActive(); + +$query = "SELECT * FROM groups WHERE groups.active = 1 ORDER BY ( groups.last_record - groups.first_record ) ASC"; + +$time = microtime(true); +$db = new DB(); +$theList = $db->query($query); + +//unset($groups); +unset($db); + +$theCount = count($theList); + +$ps = new PowerProcess(); +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxThreads = 15; +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->threadTimeLimit = 1500; // Disable child timeout + +$tim = microtime(true) - $time; +echo "time = ".$tim."\n"; + +echo "Starting threaded binary update process\n"; +echo "\nToday will be be inspecting $theCount binaries\n"; +echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t"; + +while ($ps->RunControlCode()) +{ + // Start the parent loop + $listcount = count($theList); + if ($listcount) + { + // We still have groups to process + if ($ps->SpawnReady()) + { + // Spawn another thread + $ps->threadData = array_pop($theList); +// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n"; + $ps->spawnThread(); + } + else + { + // There are no more slots available to run + $ps->tick(); + //echo "- \n"; + } + } + else + { + // No more groups to process + echo "\nNo more groups to process - Initiating shutdown\n"; + do { + $ps->tick(); + } while ($ps->myThreads); + $ps->Shutdown(); + echo "\nShutdown complete\n"; + $tim = microtime(true) - $time; + echo "final time = ".$tim."\n"; + } +} +$tim = microtime(true) - $time; +// echo "time = ".$tim."\n"; +unset($theList); + +if ($ps->RunThreadCode()) +{ + $group = $ps->threadData; + + $thread = sprintf("%05d",$ps->GetPID()); + +// echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n"; + + $param = $group['name']; + + $dir = dirname(__FILE__); + $file = 'update_binaries.php'; + + $output = shell_exec("php {$dir}/{$file} {$param}"); + //$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}"); + + $count = $theCount - $listcount; + $countpct = $count / $theCount; + + if ($listcount % 12 == 0 && $count > 0) + { + $tim = microtime(true) - $time; + echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); + $tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time(); +// $tim = number_format(($listcount * $tim) / 60 / $count, 1); + if ($countpct > 0.1 || $count > 1500) + echo "\t".date('H:i:s', $tim); +// echo "\t".$tim."m to go" ; + echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t"; + + } + if ($listcount % 1 == 0) + echo " ."; + + +// echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; +} + +// Exit to call back to parent - Let know that child has completed +exit(0); + +// Create callback function +function psUpdateComplete() +{ + echo "Threaded update process complete\n"; +} + +?> +