From 1b80dbd89ba37021b019fa8618dd928f5caa9cd6 Mon Sep 17 00:00:00 2001 From: Darko Date: Mon, 29 Sep 2014 14:17:37 +0200 Subject: [PATCH] Add nntpproxy. --- lib/DB/patches/0088~site.sql | 3 + .../misc/update_scripts/nntpproxy.php | 68 + .../templates/nntmux/views/admin/sharing.tpl | 5 +- .../nntmux/views/admin/site-edit.tpl | 20 + python/lib/nntpproxy.conf.sample | 16 + python/lib/pynntp/.gitignore | 12 + python/lib/pynntp/COPYING | 674 +++++++ python/lib/pynntp/NOTE | 1 + python/lib/pynntp/README.md | 59 + python/lib/pynntp/__init__.py | 0 python/lib/pynntp/nntp/__init__.py | 10 + python/lib/pynntp/nntp/date.py | 527 ++++++ python/lib/pynntp/nntp/fifo.py | 150 ++ python/lib/pynntp/nntp/iodict.py | 103 ++ python/lib/pynntp/nntp/nntp.py | 1615 +++++++++++++++++ python/lib/pynntp/nntp/utils.py | 229 +++ python/lib/pynntp/nntp/yenc.py | 61 + python/lib/pynntp/setup.py | 13 + python/nntpproxy.py | 246 +++ start.php | 70 +- 20 files changed, 3873 insertions(+), 9 deletions(-) create mode 100644 lib/DB/patches/0088~site.sql create mode 100644 lib/copy_this/misc/update_scripts/nntpproxy.php create mode 100644 python/lib/nntpproxy.conf.sample create mode 100644 python/lib/pynntp/.gitignore create mode 100644 python/lib/pynntp/COPYING create mode 100644 python/lib/pynntp/NOTE create mode 100644 python/lib/pynntp/README.md create mode 100644 python/lib/pynntp/__init__.py create mode 100644 python/lib/pynntp/nntp/__init__.py create mode 100644 python/lib/pynntp/nntp/date.py create mode 100644 python/lib/pynntp/nntp/fifo.py create mode 100644 python/lib/pynntp/nntp/iodict.py create mode 100644 python/lib/pynntp/nntp/nntp.py create mode 100644 python/lib/pynntp/nntp/utils.py create mode 100644 python/lib/pynntp/nntp/yenc.py create mode 100644 python/lib/pynntp/setup.py create mode 100644 python/nntpproxy.py diff --git a/lib/DB/patches/0088~site.sql b/lib/DB/patches/0088~site.sql new file mode 100644 index 000000000..690620f04 --- /dev/null +++ b/lib/DB/patches/0088~site.sql @@ -0,0 +1,3 @@ +INSERT IGNORE INTO `site` (`setting`, `value`) VALUE + ('nntpproxyr','0'); +UPDATE `tmux` SET `value` = '88' WHERE `setting` = 'sqlpatch'; \ No newline at end of file diff --git a/lib/copy_this/misc/update_scripts/nntpproxy.php b/lib/copy_this/misc/update_scripts/nntpproxy.php new file mode 100644 index 000000000..f487dd95b --- /dev/null +++ b/lib/copy_this/misc/update_scripts/nntpproxy.php @@ -0,0 +1,68 @@ +get(); +$powerline = (isset($tmux->powerline)) ? $tmux->powerline : 0; +$s= new Sites(); +$site = $s->get(); + +$tmux_session = 'NNTPProxy'; + +function python_module_exist($module) +{ + exec("python -c \"import $module\"", $output, $returnCode); + return ($returnCode == 0 ? true : false); +} + +$nntpproxy = $site->nntpproxy; +if ($nntpproxy === '0') { + exit(); +} else { + $modules = array("socketpool"); + foreach ($modules as &$value) { + if (!python_module_exist($value)) { + exit($pdo->log->error("NNTP Proxy requires " . $value . " python module but it's not installed. Aborting.")); + } + } +} + +function window_proxy($tmux_session, $powerline) +{ + global $pdo; + global $site; + + $DIR = NN_MISC; + if ($powerline === '1') { + $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf"; + } else { + $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf"; + } + + $nntpproxy = $pdo->getSetting('nntpproxy'); + if ($nntpproxy === '1') { + $DIR = NN_MISC; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/_tmux/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/tmux//python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + shell_exec("cd ${DIR}/update_scripts/nix_scripts/tmux; tmux -f $tmuxconfig attach-session -t $tmux_session || tmux -f $tmuxconfig new-session -d -s $tmux_session -n NNTPProxy 'printf \"\033]2;\"NNTPProxy\"\033\" && python $nntpproxypy $nntpproxyconf'"); + } + } + + if ($nntpproxy == '1' && ($site->alternate_nntp == '1')) { + $DIR = NN_MISC; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + shell_exec("tmux selectp -t 0; tmux splitw -t $tmux_session:0 -h -p 50 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); + } + } +} + +window_proxy($tmux_session, $powerline); diff --git a/lib/copy_this/www/templates/nntmux/views/admin/sharing.tpl b/lib/copy_this/www/templates/nntmux/views/admin/sharing.tpl index bbfea111d..a4a98ee94 100644 --- a/lib/copy_this/www/templates/nntmux/views/admin/sharing.tpl +++ b/lib/copy_this/www/templates/nntmux/views/admin/sharing.tpl @@ -1,7 +1,10 @@

{$page->title}

- If you turn on or off the Alternate NNTP provider (not functional yet) you will need to click the reset button to reset sharing settings. + + Note: If you are running nntpproxy you will not be able to upload comments, turn off the post option if you are using it.
+ If you turn on or off the Alternate NNTP provider you will need to click the reset button to reset sharing settings. +
msg
diff --git a/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl b/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl index 7f06c6105..ae0d16de1 100644 --- a/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl +++ b/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl @@ -1292,6 +1292,26 @@
+ + + + + {html_radios id="nntpproxy" name='nntpproxy' values=$yesno_ids output=$yesno_names selected=$site->nntpproxy separator='
'} +
+ nntpproxy does not support the POST usenet command which is required for uploading usenet comments when using sharing. + Using the NNTP Proxy for newznab can improve performance of newznab. It uses + connection pooling which not only give more control over the number of connections to use but also + reduces time for connection setup/teardown. The proxy also takes care of compressed headers for you. + To use this featrure you will need to install socketpool (sudo pip install socketpool or sudo easy_install socketpool) + (ensure python2 is default) and edit the configuration file (nntpproxy.conf and nntpproxy_a.conf) in the + update_scripts/nix_scripts/tmux/python/lib (copy sample) directory and finally edit your www/config.php file + to use the proxy (username and password are ignored by the proxy - make then anything you like - the + proxy doesn't use ssl either). Make sure you turn off the use compressed headers option here in site + preferences (the proxy uses compressed headers by default and passes on decompressed data). +
+ + + diff --git a/python/lib/nntpproxy.conf.sample b/python/lib/nntpproxy.conf.sample new file mode 100644 index 000000000..177f39d53 --- /dev/null +++ b/python/lib/nntpproxy.conf.sample @@ -0,0 +1,16 @@ +{ + "usenet": { + "host": "your.usenet.host.com", + "port": 443, + "username": "your_username", + "password": "your_password", + "use_ssl": true + }, + "proxy": { + "host": "localhost", + "port": 9991 + }, + "pool": { + "size": 5 + } +} diff --git a/python/lib/pynntp/.gitignore b/python/lib/pynntp/.gitignore new file mode 100644 index 000000000..17ccbe7b5 --- /dev/null +++ b/python/lib/pynntp/.gitignore @@ -0,0 +1,12 @@ +# temp files +~* +.*.swp + +# python compiled +*.pyc + +# setuptools stuff +build/ +dist/ +MANIFEST +pynntp.egg-info/ diff --git a/python/lib/pynntp/COPYING b/python/lib/pynntp/COPYING new file mode 100644 index 000000000..94a9ed024 --- /dev/null +++ b/python/lib/pynntp/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/python/lib/pynntp/NOTE b/python/lib/pynntp/NOTE new file mode 100644 index 000000000..4230235ec --- /dev/null +++ b/python/lib/pynntp/NOTE @@ -0,0 +1 @@ +Cloned from 991a76331cdf5d8f9dbf5b18f6e29adc80749a2f https://github.com/greenbender/pynntp/commit/991a76331cdf5d8f9dbf5b18f6e29adc80749a2f \ No newline at end of file diff --git a/python/lib/pynntp/README.md b/python/lib/pynntp/README.md new file mode 100644 index 000000000..a16b8d7db --- /dev/null +++ b/python/lib/pynntp/README.md @@ -0,0 +1,59 @@ +pynntp +====== + +Python NNTP library. + +This package includes advanced NNTP features, including, compressed headers. + +The most important (useful) feature of this package over other nntp libaries is +the ablity to use generators to produce data. This allows for streaming download +of large responses to say an XOVER command (which can produce gigabytes of data) +and allows you to process the data at the same time is is being received. +Meaning that memory use is minimal (even for the largest responses) and that +cycles aren't being wasted waiting on a blocking read (even in a single threaded +application) + + +Example +------- + + >>> import nntp + >>> nntp_client = nntp.NNTPClient("usenet-host.com", 443, "user", "password", use_ssl=True) + >>> nntp_client.date() + datetime.datetime(2013, 10, 19, 6, 11, 41, tzinfo=_tzgmt()) + >>> nntp_client.xfeature_compress_gzip() + True + >>> nntp_client.date() + datetime.datetime(2013, 10, 19, 6, 13, 3, tzinfo=_tzgmt()) + + +Supported Commands +------------------ + +NNTP commands that are currently supported include: +* CAPABILITIES +* MODE READER +* QUIT +* DATE +* HELP +* NEWGROUPS (generator) +* NEWNEWS (generator) +* LIST ACTIVE (generator) +* LIST ACTIVE.TIMES (generator) +* LIST NEWSGROUPS (generator) +* LIST OVERVIEW.FMT (generator) +* LIST EXTENSIONS (generator) +* GROUP +* NEXT +* LAST +* ARTICLE +* HEAD +* BODY +* POST +* XGTITLE +* XHDR +* XZHDR +* XOVER (generator) +* XZVER (generator) +* XPAT (generator) +* XFEATURE COMPRESS GZIP diff --git a/python/lib/pynntp/__init__.py b/python/lib/pynntp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/lib/pynntp/nntp/__init__.py b/python/lib/pynntp/nntp/__init__.py new file mode 100644 index 000000000..fb9ed67ca --- /dev/null +++ b/python/lib/pynntp/nntp/__init__.py @@ -0,0 +1,10 @@ +from nntp import ( + NNTPError, + NNTPReplyError, + NNTPTemporaryError, + NNTPPermanentError, + NNTPProtocolError, + NNTPDataError, + BaseNNTPClient, + NNTPClient, +) diff --git a/python/lib/pynntp/nntp/date.py b/python/lib/pynntp/nntp/date.py new file mode 100644 index 000000000..37768464b --- /dev/null +++ b/python/lib/pynntp/nntp/date.py @@ -0,0 +1,527 @@ +#!/usr/bin/python +""" +Date utilities to do fast datetime parsing. +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +# TODO: At the moment this has been targeted toward the datetime formats used by +# NNTP as it was developed for use in a NNTP reader. There is, however, no +# reason why this module could not be extended to include other formats. + +import calendar +import datetime +import dateutil.parser +import dateutil.tz + +class _tzgmt(dateutil.tz.tzutc): + """GMT timezone. + """ + + def tzname(self, dt): + return "GMT" + +TZ_LOCAL = dateutil.tz.tzlocal() +"""Local timezone (at the time the module was loaded)""" + +TZ_UTC = dateutil.tz.tzutc() +"""UTC timezone.""" + +TZ_GMT = _tzgmt() +"""GMT timezone.""" + +_months = dict( + jan=1, feb=2, mar=3, apr=4, may=5, jun=6, + jul=7, aug=8, sep=9, oct=10,nov=11,dec=12 +) +"""Conversion dictionary for english abbreviated month to integer.""" + +def _offset(value): + """Parse timezone to offset in seconds. + + Args: + value: A timezone in the '+0000' format. An integer would also work. + + Returns: + The timezone offset from GMT in seconds as an integer. + """ + o = int(value) + if o == 0: + return 0 + a = abs(o) + s = a*36+(a%100)*24 + return (o//a)*s + +def timestamp_d_b_Y_H_M_S(value): + """Convert timestamp string to time in seconds since epoch. + + Timestamps strings like '18 Jun 2013 12:00:00 GMT' are able to be converted + by this function. + + Args: + value: A timestamp string in the format '%d %b %Y %H:%M:%S GMT'. + + Returns: + The time in seconds since epoch as an integer. + + Raises: + ValueError: If timestamp is invalid. + KeyError: If the abbrieviated month is invalid. + + Note: The timezone is ignored it is simply assumed to be UTC/GMT. + """ + d, b, Y, t, Z = value.split() + H, M, S = t.split(":") + return int(calendar.timegm(( + int(Y), _months[b.lower()], int(d), int(H), int(M), int(S), 0, 0, 0 + ))) + +def datetimeobj_d_b_Y_H_M_S(value): + """Convert timestamp string to a datetime object. + + Timestamps strings like '18 Jun 2013 12:00:00 GMT' are able to be converted + by this function. + + Args: + value: A timestamp string in the format '%d %b %Y %H:%M:%S GMT'. + + Returns: + A datetime object. + + Raises: + ValueError: If timestamp is invalid. + KeyError: If the abbrieviated month is invalid. + + Note: The timezone is ignored it is simply assumed to be UTC/GMT. + """ + d, b, Y, t, Z = value.split() + H, M, S = t.split(":") + return datetime.datetime( + int(Y), _months[b.lower()], int(d), int(H), int(M), int(S), tzinfo=TZ_GMT + ) + +def timestamp_a__d_b_Y_H_M_S_z(value): + """Convert timestamp string to time in seconds since epoch. + + Timestamps strings like 'Tue, 18 Jun 2013 22:00:00 +1000' are able to be + converted by this function. + + Args: + value: A timestamp string in the format '%a, %d %b %Y %H:%M:%S %z'. + + Returns: + The time in seconds since epoch as an integer. + + Raises: + ValueError: If timestamp is invalid. + KeyError: If the abbrieviated month is invalid. + """ + a, d, b, Y, t, z = value.split() + H, M, S = t.split(":") + return int(calendar.timegm(( + int(Y), _months[b.lower()], int(d), int(H), int(M), int(S), 0, 0, 0 + ))) - _offset(z) + +def datetimeobj_a__d_b_Y_H_M_S_z(value): + """Convert timestamp string to a datetime object. + + Timestamps strings like 'Tue, 18 Jun 2013 22:00:00 +1000' are able to be + converted by this function. + + Args: + value: A timestamp string in the format '%a, %d %b %Y %H:%M:%S %z'. + + Returns: + A datetime object. + + Raises: + ValueError: If timestamp is invalid. + KeyError: If the abbrieviated month is invalid. + """ + a, d, b, Y, t, z = value.split() + H, M, S = t.split(":") + return datetime.datetime( + int(Y), _months[b.lower()], int(d), int(H), int(M), int(S), + tzinfo=dateutil.tz.tzoffset(None, _offset(z)) + ) + +def timestamp_YmdHMS(value): + """Convert timestamp string to time in seconds since epoch. + + Timestamps strings like '20130618120000' are able to be converted by this + function. + + Args: + value: A timestamp string in the format '%Y%m%d%H%M%S'. + + Returns: + The time in seconds since epoch as an integer. + + Raises: + ValueError: If timestamp is invalid. + + Note: The timezone is assumed to be UTC/GMT. + """ + i = int(value) + S = i + M = S//100 + H = M//100 + d = H//100 + m = d//100 + Y = m//100 + return int(calendar.timegm(( + Y % 10000, m % 100, d % 100, H % 100, M % 100, S % 100, 0, 0, 0) + )) + +def datetimeobj_YmdHMS(value): + """Convert timestamp string to a datetime object. + + Timestamps strings like '20130618120000' are able to be converted by this + function. + + Args: + value: A timestamp string in the format '%Y%m%d%H%M%S'. + + Returns: + A datetime object. + + Raises: + ValueError: If timestamp is invalid. + + Note: The timezone is assumed to be UTC/GMT. + """ + i = int(value) + S = i + M = S//100 + H = M//100 + d = H//100 + m = d//100 + Y = m//100 + return datetime.datetime( + Y % 10000, m % 100, d % 100, H % 100, M % 100, S % 100, tzinfo=TZ_GMT + ) + +def timestamp_epoch(value): + """Convert timestamp string to a datetime object. + + Timestamps strings like '1383470155' are able to be converted by this + function. + + Args: + value: A timestamp string as seconds since epoch. + + Returns: + The time in seconds since epoch as an integer. + """ + return int(value) + +def datetimeobj_epoch(value): + """Convert timestamp string to a datetime object. + + Timestamps strings like '1383470155' are able to be converted by this + function. + + Args: + value: A timestamp string as seconds since epoch. + + Returns: + A datetime object. + + Raises: + ValueError: If timestamp is invalid. + """ + return datetime.datetime.utcfromtimestamp(int(value)).replace(tzinfo=TZ_GMT) + +def timestamp_fmt(value, fmt): + """Convert timestamp string to time in seconds since epoch. + + Wraps the datetime.datetime.strptime(). This is slow use the other + timestamp_*() functions if possible. + + Args: + value: A timestamp string. + fmt: A timestamp format string. + + Returns: + The time in seconds since epoch as an integer. + """ + return int(calendar.timegm( + datetime.datetime.strptime(value, fmt).utctimetuple() + )) + +def datetimeobj_fmt(value, fmt): + """Convert timestamp string to a datetime object. + + Wrapper for datetime.datetime.strptime(). This is slow use the other + timestamp_*() functions if possible. + + Args: + value: A timestamp string. + fmt: A timestamp format string. + + Returns: + A datetime object. + """ + return datetime.datetime.strptime(value, fmt) + +def timestamp_any(value): + """Convert timestamp string to time in seconds since epoch. + + Most timestamps strings are supported in fact this wraps the + dateutil.parser.parse() method. This is SLOW use the other timestamp_*() + functions if possible. + + Args: + value: A timestamp string. + + Returns: + The time in seconds since epoch as an integer. + """ + return int(calendar.timegm(dateutil.parser.parse(value).utctimetuple())) + +def datetimeobj_any(value): + """Convert timestamp string to a datetime object. + + Most timestamps strings are supported in fact this is a wrapper for the + dateutil.parser.parse() method. This is SLOW use the other datetimeobj_*() + functions if possible. + + Args: + value: A timestamp string. + + Returns: + A datetime object. + """ + return dateutil.parser.parse(value) + +_timestamp_formats = { + "%d %b %Y %H:%M:%S" : timestamp_d_b_Y_H_M_S, + "%a, %d %b %Y %H:%M:%S %z": timestamp_a__d_b_Y_H_M_S_z, + "%Y%m%d%H%M%S" : timestamp_YmdHMS, + "epoch" : timestamp_epoch, +} + +def timestamp(value, fmt=None): + """Parse a datetime to a unix timestamp. + + Uses fast custom parsing for common datetime formats or the slow dateutil + parser for other formats. This is a trade off between ease of use and speed + and is very useful for fast parsing of timestamp strings whose format may + standard but varied or unknown prior to parsing. + + Common formats include: + 1 Feb 2010 12:00:00 GMT + Mon, 1 Feb 2010 22:00:00 +1000 + 20100201120000 + 1383470155 (seconds since epoch) + + See the other timestamp_*() functions for more details. + + Args: + value: A string representing a datetime. + fmt: A timestamp format string like for time.strptime(). + + Returns: + The time in seconds since epoch as and integer for the value specified. + """ + if fmt: + return _timestamp_formats.get(fmt, + lambda v: timestamp_fmt(v, fmt) + )(value) + + l = len(value) + + if 19 <= l <= 24 and value[3] == " ": + # '%d %b %Y %H:%M:%Sxxxx' + try: + return timestamp_d_b_Y_H_M_S(value) + except (KeyError, ValueError, OverflowError): + pass + + if 30 <= l <= 31: + # '%a, %d %b %Y %H:%M:%S %z' + try: + return timestamp_a__d_b_Y_H_M_S_z(value) + except (KeyError, ValueError, OverflowError): + pass + + if l == 14: + # '%Y%m%d%H%M%S' + try: + return timestamp_YmdHMS(value) + except (ValueError, OverflowError): + pass + + # epoch timestamp + try: + return timestamp_epoch(value) + except ValueError: + pass + + # slow version + return timestamp_any(value) + +_datetimeobj_formats = { + "%d %b %Y %H:%M:%S" : datetimeobj_d_b_Y_H_M_S, + "%a, %d %b %Y %H:%M:%S %z": datetimeobj_a__d_b_Y_H_M_S_z, + "%Y%m%d%H%M%S" : datetimeobj_YmdHMS, + "epoch" : datetimeobj_epoch, +} + +def datetimeobj(value, fmt=None): + """Parse a datetime to a datetime object. + + Uses fast custom parsing for common datetime formats or the slow dateutil + parser for other formats. This is a trade off between ease of use and speed + and is very useful for fast parsing of timestamp strings whose format may + standard but varied or unknown prior to parsing. + + Common formats include: + 1 Feb 2010 12:00:00 GMT + Mon, 1 Feb 2010 22:00:00 +1000 + 20100201120000 + 1383470155 (seconds since epoch) + + See the other datetimeobj_*() functions for more details. + + Args: + value: A string representing a datetime. + + Returns: + A datetime object. + """ + if fmt: + return _datetimeobj_formats.get(fmt, + lambda v: datetimeobj_fmt(v, fmt) + )(value) + + l = len(value) + + if 19 <= l <= 24 and value[3] == " ": + # '%d %b %Y %H:%M:%Sxxxx' + try: + return datetimeobj_d_b_Y_H_M_S(value) + except (KeyError, ValueError): + pass + + if 30 <= l <= 31: + # '%a, %d %b %Y %H:%M:%S %z' + try: + return datetimeobj_a__d_b_Y_H_M_S_z(value) + except (KeyError, ValueError): + pass + + if l == 14: + # '%Y%m%d%H%M%S' + try: + return datetimeobj_YmdHMS(value) + except ValueError: + pass + + # epoch timestamp + try: + return datetimeobj_epoch(value) + except ValueError: + pass + + # slow version + return datetimeobj_any(value) + + +# testing +if __name__ == "__main__": + + import sys + import timeit + + log = sys.stdout.write + + times = ( + datetime.datetime.now(TZ_UTC), + datetime.datetime.now(TZ_GMT), + datetime.datetime.now(TZ_LOCAL), + datetime.datetime.now(), + ) + + # check timezones + for t in times: + log("%s\n" % t.strftime("%Y-%m-%d %H:%M:%S %Z")) + + # TODO validate values (properly) + + # check speed + values = ( + { + "name": "Implemented Format", + "time": "20130624201912", + "fmt" : "%Y%m%d%H%M%S" + }, + { + "name": "Unimplemented Format", + "time": "2013-06-24 20:19:12", + "fmt" : "%Y-%m-%d %H:%M:%S" + } + ) + tests = ( + { + "name" : "GMT timestamp (strptime version)", + "test" : "int(calendar.timegm(datetime.datetime.strptime('%(time)s', '%(fmt)s').utctimetuple()))", + "setup": "import calendar, datetime", + }, + { + "name" : "GMT timestamp (dateutil version)", + "test" : "int(calendar.timegm(dateutil.parser.parse('%(time)s').utctimetuple()))", + "setup": "import calendar, dateutil.parser", + }, + { + "name" : "GMT timestamp (fast version)", + "test" : "timestamp('%(time)s')", + "setup": "from __main__ import timestamp", + }, + { + "name" : "GMT timestamp (fast version with format hint)", + "test" : "timestamp('%(time)s', '%(fmt)s')", + "setup": "from __main__ import timestamp", + }, + { + "name" : "GMT datetime object (strptime version)", + "test" : "datetime.datetime.strptime('%(time)s', '%(fmt)s').replace(tzinfo=TZ_GMT)", + "setup": "import datetime; from __main__ import TZ_GMT", + }, + { + "name" : "GMT datetime object (dateutil version)", + "test" : "dateutil.parser.parse('%(time)s').replace(tzinfo=TZ_GMT)", + "setup": "import dateutil.parser; from __main__ import TZ_GMT", + }, + { + "name" : "GMT datetime object (fast version)", + "test" : "datetimeobj('%(time)s')", + "setup": "from __main__ import datetimeobj", + }, + { + "name" : "GMT datetime object (fast version with format hint)", + "test" : "datetimeobj('%(time)s', '%(fmt)s')", + "setup": "from __main__ import datetimeobj", + } + ) + iters = 100000 + for v in values: + log("%(name)s (%(fmt)s)\n" % v) + for t in tests: + log(" %(name)-52s" % t) + elapsed = timeit.timeit(t["test"] % v, t["setup"], number=iters) + log("%0.3f sec (%d loops @ %0.3f usec)\n" % ( + elapsed, iters, (elapsed/iters)*1000000 + )) diff --git a/python/lib/pynntp/nntp/fifo.py b/python/lib/pynntp/nntp/fifo.py new file mode 100644 index 000000000..61a0e106c --- /dev/null +++ b/python/lib/pynntp/nntp/fifo.py @@ -0,0 +1,150 @@ +#!/usr/bin/python +""" +A reasonably efficient FIFO buffer. +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#NOTE: docstrings beed to be added + +_DISCARD_SIZE = 0xffff + +class Fifo(object): + + def __init__(self, data="", eol="\r\n"): + + self.buf = data + self.eol = eol + self.buflist = [] + self.pos = 0 + + def __len__(self): + + return len(self.buf) - self.pos + + def __iter__(self): + + return self + + def __discard(self): + + if self.pos > _DISCARD_SIZE: + self.buf = self.buf[self.pos:] + self.pos = 0 + + def __append(self): + + self.buf += "".join(self.buflist) + self.buflist = [] + + def clear(self): + + self.buf = "" + self.buflist = [] + self.pos = 0 + + def write(self, data): + + self.buflist.append(data) + + def read(self, length=0): + + self.__append() + + if 0 < length < len(self): + newpos = self.pos + length + data = self.buf[self.pos:newpos] + self.pos = newpos + self.__discard() + return data + + data = self.buf[self.pos:] + self.clear() + return data + + def readline(self): + + self.__append() + + i = self.buf.find(self.eol, self.pos) + if i < 0: + return "" + + newpos = i + len(self.eol) + data = self.buf[self.pos:newpos] + self.pos = newpos + self.__discard() + return data + + def readuntil(self, token, size=0): + + self.__append() + + i = self.buf.find(token, self.pos) + if i < 0: + index = max(len(token) - 1, size) + newpos = max(len(self.buf) - index, self.pos) + data = self.buf[self.pos:newpos] + self.pos = newpos + self.__discard() + return False, data + + newpos = i + len(token) + data = self.buf[self.pos:newpos] + self.pos = newpos + self.__discard() + return True, data + + def peek(self, length=0): + + self.__append() + + if 0 < length < len(self): + newpos = self.pos + length + return self.buf[self.pos:newpos] + + return self.buf[self.pos:] + + def peekline(self): + + self.__append() + + i = self.buf.find(self.eol, self.pos) + if i < 0: + return "" + + newpos = i + len(self.eol) + return self.buf[self.pos:newpos] + + def peekuntil(self, token, size=0): + + self.__append() + + i = self.buf.find(token, self.pos) + if i < 0: + index = max(len(token) - 1, size) + newpos = max(len(self.buf) - index, self.pos) + return False, self.buf[self.pos:newpos] + + newpos = i + len(token) + return True, self.buf[self.pos:newpos] + + def next(self): + + line = self.readline() + if not line: + raise StopIteration() + + return line diff --git a/python/lib/pynntp/nntp/iodict.py b/python/lib/pynntp/nntp/iodict.py new file mode 100644 index 000000000..f8b3a08b9 --- /dev/null +++ b/python/lib/pynntp/nntp/iodict.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +""" +Case-insentitive ordered dictionary (useful for headers). +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#NOTE: docstrings beed to be added + +from collections import OrderedDict + +def _lower(v): + """assumes that classes that inherit list, tuple or dict have a constructor + that is compatible with those base classes. If you are using classes that + don't satisfy this requirement you can subclass them and add a lower() + method for the class""" + if hasattr(v, "lower"): + return v.lower() + if isinstance(v, (list, tuple)): + return v.__class__(_lower(x) for x in v) + if isinstance(v, dict): + return v.__class__(_lower(v.items())) + return v + + +# NOTE: This class makes assumptions about the OrderedDict class that it +# inherits from -- this a a bad idea. If The OrderedDict Class were to change +# behaviour it could break this class. +class IODict(OrderedDict): + """Case in-sensitive ordered dictionary. + >>> iod = IODict([('ABC', 1), ('DeF', 'A'), (('gHi', 'jkl', 20), 'b')]) + >>> iod + IODict([('ABC', 1), ('DeF', 'A'), (('gHi', 'jkl', 20), 'b')]) + >>> iod['ABC'], iod['abc'], iod['aBc'] + (1, 1, 1) + >>> iod['DeF'], iod['def'], iod['dEf'] + ('A', 'A', 'A') + >>> iod[('gHi', 'jkl', 20)], iod[('ghi', 'jKL', 20)] + ('b', 'b') + >>> iod == {"aBc": 1, "deF": 'A', ('Ghi', 'JKL', 20): 'b'} + True + >>> iod.popitem() + (('gHi', 'jkl', 20), 'b') + """ + + def __init__(self, *args, **kwds): + self.__map = {} + OrderedDict.__init__(self, *args, **kwds) + + def __setitem__(self, key, *args, **kwds): + l = _lower(key) + OrderedDict.__setitem__(self, l, *args, **kwds) + self.__map[l] = key + + def __getitem__(self, key, *args, **kwds): + l = _lower(key) + return OrderedDict.__getitem__(self, l, *args, **kwds) + + def __delitem__(self, key, *args, **kwds): + l = _lower(key) + OrderedDict.__delitem__(self, l, *args, **kwds) + del self.__map[l] + + def __contains__(self, key): + l = _lower(key) + return OrderedDict.__contains__(self, l) + + def __iter__(self): + for k in OrderedDict.__iter__(self): + yield self.__map[k] + + def __reversed__(self): + for k in OrderedDict.__reversed__(self): + yield self.__map[k] + + def clear(self): + OrderedDict.clear(self) + self.__map.clear() + + def __eq__(self, other): + """assumes that classes that inherit dict have a constructor that is + compatible with the dict class.""" + if len(self) != len(other) or not isinstance(other, dict): + return False + so = OrderedDict(zip(_lower(self.keys()), self.values())) + oo = other.__class__(zip(_lower(other.keys()), other.values())) + return so == oo + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/python/lib/pynntp/nntp/nntp.py b/python/lib/pynntp/nntp/nntp.py new file mode 100644 index 000000000..0bb327b38 --- /dev/null +++ b/python/lib/pynntp/nntp/nntp.py @@ -0,0 +1,1615 @@ +#!/usr/bin/python +""" +An NNTP library - a bit more useful than the nntplib one (hopefully). +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import ssl +import zlib +import socket +import datetime +import cStringIO +import utils +import iodict +import fifo +import yenc +import date + + +class NNTPError(Exception): + """Base class for all NNTP errors. + """ + pass + +class NNTPSyncError(NNTPError): + """NNTP sync errors. + + Generally raised when a command is issued while another command it still + active. + """ + pass + +class NNTPReplyError(NNTPError): + """NNTP response status errors. + """ + def __init__(self, code, message): + NNTPError.__init__(self, code, message) + + def code(self): + """The response status code. + """ + return self.args[0] + + def message(self): + """The response message. + """ + return self.args[1] + + def __str__(self): + return "%d %s" % self.args + +class NNTPTemporaryError(NNTPReplyError): + """NNTP temporary errors. + + Temporary errors have response codes from 400 to 499. + """ + pass + +class NNTPPermanentError(NNTPReplyError): + """NNTP permanent errors. + + Permanent errors have response codes from 500 to 599. + """ + pass + +# TODO: Add the status line as a parameter ? +class NNTPProtocolError(NNTPError): + """NNTP protocol error. + + Protcol errors are raised when the response status is invalid. + """ + pass + +class NNTPDataError(NNTPError): + """NNTP data error. + + Data errors are raised when the content of a response cannot be parsed. + """ + pass + + +class BaseNNTPClient(object): + """NNTP BaseNNTPClient. + + Base class for NNTP clients implements the basic command interface and + transparently handles compressed replies. + """ + def __init__(self, host, port=119, username="", password="", timeout=30, use_ssl=False): + """Constructor for BasicNNTPClient. + + Connects to usenet server and enters reader mode. + + Args: + host: Hostname for usenet server. + port: Port for usenet server. + username: Username for usenet account (default "anonymous") + password: Password for usenet account (default "anonymous") + timeout: Connection timeout (default 30 seconds) + use_ssl: Should we use ssl (default False) + + Raises: + IOError (socket.error): On error in underlying socket and/or ssl + wrapper. See socket and ssl modules for further details. + NNTPReplyError: On bad response code from server. + """ + self.socket = socket.socket() + if use_ssl: + self.socket = ssl.wrap_socket(self.socket) + self.socket.settimeout(timeout) + + self.__buffer = fifo.Fifo() + self.__generating = False + + self.username = username + self.password = password + + # connect + self.socket.connect((host, port)) + code, message = self.status() + if not code in [200, 201]: + raise NNTPReplyError(code, message) + + def __recv(self, size=4096): + """Reads data from the socket. + + Raises: + NNTPError: When connection times out or read from socket fails. + """ + data = self.socket.recv(size) + if not data: + raise NNTPError("Failed to read from socket") + self.__buffer.write(data) + + def __line_gen(self): + """Generator that reads a line of data from the server. + + It first attempts to read from the internal buffer. If there is not + enough data to read a line it then requests more data from the server + and adds it to the buffer. This process repeats until a line of data + can be read from the internal buffer. + + Yields: + A line of data when it becomes available. + """ + while True: + line = self.__buffer.readline() + if not line: + self.__recv() + continue + yield line + + def __buf_gen(self, length=0): + """Generator that reads a block of data from the server. + + It first attempts to read from the internal buffer. If there is not + enough data in the internal buffer it then requests more data from the + server and adds it to the buffer. + + Args: + length: An optional amount of data to retrieve. A length of 0 (the + default) will retrieve a least one buffer of data. + + Yields: + A block of data when enough data becomes available. + + Note: + If a length of 0 is supplied then the size of the yielded buffer can + vary. If there is data in the internal buffer it will yield all of + that data otherwise it will yield the the data returned by a recv + on the socket. + """ + while True: + buf = self.__buffer.read(length) + if not buf: + self.__recv() + continue + yield buf + + def status(self): + """Reads a command response status. + + If there is no response message then the returned status message will + be an empty string. + + Raises: + NNTPError: If data is required to be read from the socket and fails. + NNTPProtocolError: If the status line can't be parsed. + NNTPTemporaryError: For status code 400-499 + NNTPPermanentError: For status code 500-599 + + Returns: + A tuple of status code (as an integer) and status message. + """ + line = next(self.__line_gen()).rstrip() + parts = line.split(None, 1) + + try: + code, message = int(parts[0]), "" + except ValueError: + raise NNTPProtocolError(line) + + if code < 100 or code >= 600: + raise NNTPProtocolError(line) + + if len(parts) > 1: + message = parts[1] + + if 400 <= code <= 499: + raise NNTPTemporaryError(code, message) + + if 500 <= code <= 599: + raise NNTPPermanentError(code, message) + + return code, message + + def __info_plain_gen(self): + """Generator for the lines of an info (textual) response. + + When a terminating line (line containing single period) is received the + generator exits. + + If there is a line begining with an 'escaped' period then the extra + period is trimmed. + + Yields: + A line of the info response. + + Raises: + NNTPError: If data is required to be read from the socket and fails. + """ + self.__generating = True + + for line in self.__line_gen(): + if line == ".\r\n": + break + if line.startswith("."): + yield line[1:] + yield line + + self.__generating = False + + def __info_gzip_gen(self): + """Generator for the lines of a compressed info (textual) response. + + Compressed responses are an extension to the NNTP protocol supported by + some usenet servers to reduce the bandwidth of heavily used range style + commands that can return large amounts of textual data. + + This function handles gzip compressed responses that have the + terminating line inside or outside the compressed data. From experience + if the 'XFEATURE COMPRESS GZIP' command causes the terminating '.\\r\\n' + to follow the compressed data and 'XFEATURE COMPRESS GZIP TERMINATOR' + causes the terminator to be the last part of the compressed data (i.e + the reply the gzipped version of the original reply - terminating line + included) + + This function will produce that same output as the __info_plain_gen() + function. In other words it takes care of decompression. + + Yields: + A line of the info response. + + Raises: + NNTPError: If data is required to be read from the socket and fails. + NNTPDataError: If decompression fails. + """ + self.__generating = True + + inflate = zlib.decompressobj(15+32) + + done, buf = False, fifo.Fifo() + while not done: + try: + data = inflate.decompress(next(self.__buf_gen())) + except zlib.error: + raise NNTPDataError("Decompression failed") + if data: + buf.write(data) + if inflate.unused_data: + buf.write(inflate.unused_data) + for line in buf: + if line == ".\r\n": + done = True + break + if line.startswith("."): + yield line[1:] + yield line + + self.__generating = False + + def __info_yenczlib_gen(self): + """Generator for the lines of a compressed info (textual) response. + + Compressed responses are an extension to the NNTP protocol supported by + some usenet servers to reduce the bandwidth of heavily used range style + commands that can return large amounts of textual data. The server + returns that same data as it would for the uncompressed versions of the + command the difference being that the data is zlib deflated and then + yEnc encoded. + + This function will produce that same output as the info_gen() + function. In other words it takes care of decoding and decompression. + + Yields: + A line of the info response. + + Raises: + NNTPError: If data is required to be read from the socket and fails. + NNTPDataError: When there is an error parsing the yEnc header or + trailer, if the CRC check fails or decompressing data fails. + """ + + escape = 0 + dcrc32 = 0 + inflate = zlib.decompressobj(-15) + + # header + header = next(self.__info_plain_gen()) + if not header.startswith("=ybegin"): + raise NNTPDataError("Bad yEnc header") + + # data + buf, trailer = fifo.Fifo(), "" + for line in self.__info_plain_gen(): + if line.startswith("=yend"): + trailer = line + continue + data, escape, dcrc32 = yenc.decode(line, escape, dcrc32) + try: + data = inflate.decompress(data) + except zlib.error: + raise NNTPDataError("Decompression failed") + if not data: + continue + buf.write(data) + for l in buf: + yield l + + # trailer + if not trailer: + raise NNTPDataError("Missing yEnc trailer") + + # expected crc32 + ecrc32 = yenc.crc32(trailer) + if ecrc32 is None: + raise NNTPDataError("Bad yEnc trailer") + + # check crc32 + if ecrc32 != dcrc32 & 0xffffffff: + raise NNTPDataError("Bad yEnc CRC") + + def info_gen(self, code, message, compressed=False): + """Dispatcher for the info generators. + + Determines which __info_*_gen() should be used based on the supplied + parameters. + + Args: + code: The status code for the command response. + message: The status message for the command reponse. + compressed: Force decompression. Useful for xz* commands. + + Returns: + An info generator. + """ + if "COMPRESS=GZIP" in message: + return self.__info_gzip_gen() + if compressed: + return self.__info_yenczlib_gen() + return self.__info_plain_gen() + + def info(self, code, message, compressed=False): + """The complete content of an info response. + + This should only used for commands that return small or known amounts of + data. + + Returns: + A the complete content of a textual response. + """ + return "".join([x for x in self.info_gen(code, message, compressed)]) + + def command(self, verb, args=None): + """Call a command on the server. + + If the user has not authenticated then authentication will be done + as part of calling the command on the server. + + For commands that don't return a status message the status message + will default to an empty string. + + Args: + verb: The verb of the command to call. + args: The arguments of the command as a string (default None). + + Returns: + A tuple of status code (as an integer) and status message. + + Note: + You can run raw commands by supplying the full command (including + args) in the verb. + + Note: Although it is possible you shouldn't issue more than one command + at a time by adding newlines to the verb as it will most likely lead + to undesirable results. + """ + if self.__generating: + raise NNTPSyncError("Command issued while a generator is active") + + cmd = verb + if args: + cmd += " " + args + cmd += "\r\n" + + self.socket.sendall(cmd) + + try: + code, message = self.status() + except NNTPTemporaryError as e: + if e.code() != 480: + raise e + code, message = self.command("AUTHINFO USER", self.username) + if code == 381: + code, message = self.command("AUTHINFO PASS", self.password) + if code != 281: + raise NNTPReplyError(code, message) + code, message = self.command(verb, args) + + return code, message + + def close(self): + """Closes the connection at the client. + + Once this method has been called, no other methods of the NNTPClient object + should be called. + """ + self.socket.close() + + +class NNTPClient(BaseNNTPClient): + """NNTP NNTPClient. + + Implements many of the commands that are commonly used by current usenet + servers. Including handling commands that use compressed responses. + + Implements generators for commands for which generators are likely to + yield (bad pun warning) perfomance gains. These gains will be in the form + of lower memory consumption and the added ability to process and receive + data in parallel. If you are using commands that can take a range as an + argument or can return large amounts of data there should be a _gen() + version of the command and it should be used in preference to the standard + version. + + Note: All commands can raise the following exceptions: + NNTPError + NNTPProtocolError + NNTPPermanentError + NNTPReplyError + IOError (socket.error) + + Note: All commands that use compressed responses can also raise an + NNTPDataError. + """ + + def __init__(self, host, port=119, username="", password="", timeout=30, use_ssl=False, reader=True): + """Constructor for NNTP NNTPClient. + + Connects to usenet server.. + + Args: + host: Hostname for usenet server. + port: Port for usenet server. + username: Username for usenet account (default "") + password: Password for usenet account (default "") + timeout: Connection timeout (default 30 seconds) + use_ssl: Should we use ssl (default False) + reader: Use reader mode + + Raises: + socket.error: On error in underlying socket and/or ssl wrapper. See + socket and ssl modules for further details. + NNTPReplyError: On bad response code from server. + """ + super(NNTPClient, self).__init__(host, port, username, password, timeout, use_ssl) + + # reader + if reader: + self.mode_reader() + + + # session administration commands + + def capabilities(self, keyword=None): + """CAPABILITIES command. + + Determines the capabilities of the server. + + Although RFC3977 states that this is a required command for servers to + implement not all servers do, so expect that NNTPPermanentError may be + raised when this command is issued. + + See + + Args: + keyword: Passed directly to the server, however, this is unused by + the server according to RFC3977. + + Returns: + A list of capabilities supported by the server. The VERSION + capability is the first capability in the list. + """ + args = keyword + + code, message = self.command("CAPABILITIES", args) + if code != 101: + raise NNTPReplyError(code, message) + + return [x.strip() for x in self.info_gen(code, message)] + + def mode_reader(self): + """MODE READER command. + + Instructs a mode-switching server to switch modes. + + See + + Returns: + Boolean value indicating whether posting is allowed or not. + """ + code, message = self.command("MODE READER") + if not code in [200, 201]: + raise NNTPReplyError(code, message) + + return code == 200 + + def quit(self): + """QUIT command. + + Tells the server to close the connection. After the server acknowledges + the request to quit the connection is closed both at the server and + client. Only useful for graceful shutdown. If you are in a generator + use close() instead. + + Once this method has been called, no other methods of the NNTPClient + object should be called. + + See + """ + code, message = self.command("QUIT") + if code != 205: + raise NNTPReplyError(code, message) + + self.socket.close() + + + # information commands + + def date(self): + """DATE command. + + Coordinated Universal time from the perspective of the usenet server. + It can be used to provide information that might be useful when using + the NEWNEWS command. + + See + + Returns: + The UTC time according to the server as a datetime object. + + Raises: + NNTPDataError: If the timestamp can't be parsed. + """ + code, message = self.command("DATE") + if code != 111: + raise NNTPReplyError(code, message) + + ts = date.datetimeobj(message, fmt="%Y%m%d%H%M%S") + + return ts + + def help(self): + """HELP command. + + Provides a short summary of commands that are understood by the usenet + server. + + See + + Returns: + The help text from the server. + """ + code, message = self.command("HELP") + if code != 100: + raise NNTPReplyError(code, message) + + return self.info(code, message) + + def newgroups_gen(self, timestamp): + """Generator for the NEWGROUPS command. + + Generates a list of newsgroups created on the server since the specified + timestamp. + + See + + Args: + timestamp: Datetime object giving 'created since' datetime. + + Yields: + A tuple containing the name, low water mark, high water mark, + and status for the newsgroup. + + Note: If the datetime object supplied as the timestamp is naive (tzinfo + is None) then it is assumed to be given as GMT. + """ + if timestamp.tzinfo: + ts = timestamp.asttimezone(date.TZ_GMT) + else: + ts = timestamp.replace(tzinfo=date.TZ_GMT) + + args = ts.strftime("%Y%m%d %H%M%S %Z") + + code, message = self.command("NEWGROUPS", args) + if code != 231: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield utils.parse_newsgroup(line) + + def newgroups(self, timestamp): + """NEWGROUPS command. + + Retreives a list of newsgroups created on the server since the specified + timestamp. See newgroups_gen() for more details. + + See + + Args: + timestamp: Datetime object giving 'created since' datetime. + + Returns: + A list of tuples in the format given by newgroups_gen() + """ + return [x for x in self.newgroups_gen(timestamp)] + + def newnews_gen(self, pattern, timestamp): + """Generator for the NEWNEWS command. + + Generates a list of message-ids for articles created since the specified + timestamp for newsgroups with names that match the given pattern. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + timestamp: Datetime object giving 'created since' datetime. + + Yields: + A message-id as string. + + Note: If the datetime object supplied as the timestamp is naive (tzinfo + is None) then it is assumed to be given as GMT. If tzinfo is set + then it will be converted to GMT by this function. + """ + if timestamp.tzinfo: + ts = timestamp.asttimezone(date.TZ_GMT) + else: + ts = timestamp.replace(tzinfo=date.TZ_GMT) + + args = pattern + args += " " + ts.strftime("%Y%m%d %H%M%S %Z") + + code, message = self.command("NEWNEWS", args) + if code != 230: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield line.strip() + + def newnews(self, pattern, timestamp): + """NEWNEWS command. + + Retrieves a list of message-ids for articles created since the specified + timestamp for newsgroups with names that match the given pattern. See + newnews_gen() for more details. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + timestamp: Datetime object giving 'created since' datetime. + + Returns: + A list of message-ids as given by newnews_gen() + """ + return [x for x in self.newnews_gen(pattern, timestamp)] + + + # list commands + + def list_active_gen(self, pattern=None): + """Generator for the LIST ACTIVE command. + + Generates a list of active newsgroups that match the specified pattern. + If no pattern is specfied then all active groups are generated. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + + Yields: + A tuple containing the name, low water mark, high water mark, + and status for the newsgroup. + """ + args = pattern + + if args is None: + cmd = "LIST" + else: + cmd = "LIST ACTIVE" + + code, message = self.command(cmd, args) + if code != 215: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield utils.parse_newsgroup(line) + + def list_active(self, pattern=None): + """LIST ACTIVE command. + + Retreives a list of active newsgroups that match the specified pattern. + See list_active_gen() for more details. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + + Returns: + A list of tuples in the format given by list_active_gen() + """ + return [x for x in self.list_active_gen(pattern)] + + def list_active_times_gen(self): + """Generator for the LIST ACTIVE.TIMES command. + + Generates a list of newsgroups including the creation time and who + created them. + + See + + Yields: + A tuple containing the name, creation date as a datetime object and + creator as a string for the newsgroup. + """ + code, message = self.command("LIST ACTIVE.TIMES") + if code != 215: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + parts = line.split() + try: + name = parts[0] + timestamp = date.datetimeobj_epoch(parts[1]) + creator = parts[2] + except (IndexError, ValueError): + raise NNTPDataError("Invalid LIST ACTIVE.TIMES") + yield name, timestamp, creator + + def list_active_times(self): + """LIST ACTIVE TIMES command. + + Retrieves a list of newsgroups including the creation time and who + created them. See list_active_times_gen() for more details. + + See + + Returns: + A list of tuples in the format given by list_active_times_gen() + """ + return [x for x in self.list_active_times_gen()] + + def list_distrib_pats_gen(self): + """Generator for the LIST DISTRIB.PATS command. + """ + raise NotImplementedError() + + def list_distrib_pats(self): + """LIST DISTRIB.PATS command. + """ + return [x for x in self.list_distrib_pats_gen()] + + def list_headers_gen(self, arg=None): + """Generator for the LIST HEADERS command. + """ + raise NotImplementedError() + + def list_headers(self, arg=None): + """LIST HEADERS command. + """ + return [x for x in self.list_headers_gen(arg)] + + def list_newsgroups_gen(self, pattern=None): + """Generator for the LIST NEWSGROUPS command. + + Generates a list of newsgroups including the name and a short + description. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + + Yields: + A tuple containing the name, and description for the newsgroup. + """ + args = pattern + + code, message = self.command("LIST NEWSGROUPS", args) + if code != 215: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + parts = line.strip().split() + name, description = parts[0], "" + if len(parts) > 1: + description = parts[1] + yield name, description + + def list_newsgroups(self, pattern=None): + """LIST NEWSGROUPS command. + + Retrieves a list of newsgroups including the name and a short + description. See list_newsgroups_gen() for more details. + + See + + Args: + pattern: Glob matching newsgroups of intrest. + + Returns: + A list of tuples in the format given by list_newsgroups_gen() + """ + return [x for x in self.list_newsgroups_gen(pattern)] + + def list_overview_fmt_gen(self): + """Generator for the LIST OVERVIEW.FMT + + See list_overview_fmt() for more information. + + Yields: + An element in the list returned by list_overview_fmt(). + """ + code, message = self.command("LIST OVERVIEW.FMT") + if code != 215: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + try: + name, suffix = line.rstrip().split(":") + except ValueError: + raise NNTPDataError("Invalid LIST OVERVIEW.FMT") + if suffix and not name: + name, suffix = suffix, name + if suffix and suffix != "full": + raise NNTPDataError("Invalid LIST OVERVIEW.FMT") + yield (name, suffix == "full") + + def list_overview_fmt(self): + """LIST OVERVIEW.FMT command. + """ + return [x for x in self.list_overview_fmt_gen()] + + def list_extensions_gen(self): + """Generator for the LIST EXTENSIONS command. + """ + code, message = self.command("LIST EXTENSIONS") + if code != 202: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield line.strip() + + def list_extensions(self): + """LIST EXTENSIONS command. + """ + return [x for x in self.list_extensions_gen()] + + def list_gen(self, keyword=None, arg=None): + """Generator for LIST command. + + See list() for more information. + + Yields: + An element in the list returned by list(). + """ + if keyword: + keyword = keyword.upper() + + if keyword is None or keyword == "ACTIVE": + return self.list_active_gen(arg) + if keyword == "ACTIVE.TIMES": + return self.list_active_times_gen() + if keyword == "DISTRIB.PATS": + return self.list_distrib_pats_gen() + if keyword == "HEADERS": + return self.list_headers_gen(arg) + if keyword == "NEWSGROUPS": + return self.list_newsgroups_gen(arg) + if keyword == "OVERVIEW.FMT": + return self.list_overview_fmt_gen() + if keyword == "EXTENSIONS": + return self.list_extensions_gen() + + raise NotImplementedError() + + def list(self, keyword=None, arg=None): + """LIST command. + + A wrapper for all of the other list commands. The output of this command + depends on the keyword specified. The output format for each keyword can + be found in the list function that corresponds to the keyword. + + Args: + keyword: Information requested. + arg: Pattern or keyword specific argument. + + Note: Keywords supported by this function are include ACTIVE, + ACTIVE.TIMES, DISTRIB.PATS, HEADERS, NEWSGROUPS, OVERVIEW.FMT and + EXTENSIONS. + + Raises: + NotImplementedError: For unsupported keywords. + """ + return [x for x in self.list_gen(keyword, arg)] + + def group(self, name): + """GROUP command. + """ + args = name + + code, message = self.command("GROUP", args) + if code != 211: + raise NNTPReplyError(code, message) + + parts = message.split(None, 4) + try: + total = int(parts[0]) + first = int(parts[1]) + last = int(parts[2]) + group = parts[3] + except (IndexError, ValueError): + raise NNTPDataError("Invalid GROUP status '%s'" % message) + + return total, first, last, group + + def next(self): + """NEXT command. + """ + code, message = self.command("NEXT") + if code != 223: + raise NNTPReplyError(code, message) + + parts = message.split(None, 3) + try: + article = int(parts[0]) + ident = parts[1] + except (IndexError, ValueError): + raise NNTPDataError("Invalid NEXT status") + + return article, ident + + def last(self): + """LAST command. + """ + code, message = self.command("LAST") + if code != 223: + raise NNTPReplyError(code, message) + + parts = message.split(None, 3) + try: + article = int(parts[0]) + ident = parts[1] + except (IndexError, ValueError): + raise NNTPDataError("Invalid LAST status") + + return article, ident + + # TODO: Validate yEnc body + def article(self, msgid_article=None, decode=None): + """ARTICLE command. + """ + args = None + if msgid_article is not None: + args = utils.unparse_msgid_article(msgid_article) + + code, message = self.command("ARTICLE", args) + if code != 220: + raise NNTPReplyError(code, message) + + parts = message.split(None, 1) + + try: + articleno = int(parts[0]) + except ValueError: + raise NNTPProtocolError(message) + + # headers + headers = utils.parse_headers(self.info_gen(code, message)) + + # decoding setup + decode = "yEnc" in headers.get("subject", "") + escape = 0 + crc32 = 0 + + # body + body = [] + for line in self.info_gen(code, message): + + # decode body if required + if decode: + if line.startswith("=y"): + continue + line, escape, crc32 = yenc.decode(line, escape, crc32) + + body.append(line) + + return articleno, headers, "".join(body) + + def head(self, msgid_article=None): + """HEAD command. + """ + args = None + if msgid_article is not None: + args = utils.unparse_msgid_article(msgid_article) + + code, message = self.command("HEAD", args) + if code != 221: + raise NNTPReplyError(code, message) + + return utils.parse_headers(self.info_gen(code, message)) + + # TODO: Support yEnc article body validation + def body(self, msgid_article=None, decode=False): + """BODY command. + """ + args = None + if msgid_article is not None: + args = utils.unparse_msgid_article(msgid_article) + + code, message = self.command("BODY", args) + if code != 222: + raise NNTPReplyError(code, message) + + escape = 0 + crc32 = 0 + + body = [] + for line in self.info_gen(code, message): + + # decode body if required + if decode: + if line.startswith("=y"): + continue + line, escape, crc32 = yenc.decode(line, escape, crc32) + + # body + body.append(line) + + return "".join(body) + + def xgtitle(self, pattern=None): + """XGTITLE command. + """ + args = pattern + + code, message = self.command("XGTITLE", args) + if code != 282: + raise NNTPReplyError(code, message) + + return self.info(code, message) + + def xhdr(self, header, msgid_range=None): + """XHDR command. + """ + args = header + if range is not None: + args += " " + utils.unparse_msgid_range(msgid_range) + + code, message = self.command("XHDR", args) + if code != 221: + raise NNTPReplyError(code, message) + + return self.info(code, message) + + def xzhdr(self, header, msgid_range=None): + """XZHDR command. + + Args: + msgid_range: A message-id as a string, or an article number as an + integer, or a tuple of specifying a range of article numbers in + the form (first, [last]) - if last is omitted then all articles + after first are included. A msgid_range of None (the default) + uses the current article. + """ + args = header + if msgid_range is not None: + args += " " + utils.unparse_msgid_range(msgid_range) + + code, message = self.command("XZHDR", args) + if code != 221: + raise NNTPReplyError(code, message) + + return self.info(code, message, compressed=True) + + def xover_gen(self, range=None): + """Generator for the XOVER command. + + The XOVER command returns information from the overview database for + the article(s) specified. + + + + Args: + range: An article number as an integer, or a tuple of specifying a + range of article numbers in the form (first, [last]). If last is + omitted then all articles after first are included. A range of + None (the default) uses the current article. + + Returns: + A list of fields as given by the overview database for each + available article in the specified range. The fields that are + returned can be determined using the LIST OVERVIEW.FMT command if + the server supports it. + + Raises: + NNTPReplyError: If no such article exists or the currently selected + newsgroup is invalid. + """ + args = None + if range is not None: + args = utils.unparse_range(range) + + code, message = self.command("XOVER", args) + if code != 224: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield line.rstrip().split("\t") + + def xover(self, range=None): + """The XOVER command. + + The XOVER command returns information from the overview database for + the article(s) specified. + + + + Args: + range: An article number as an integer, or a tuple of specifying a + range of article numbers in the form (first, [last]). If last is + omitted then all articles after first are included. A range of + None (the default) uses the current article. + + Returns: + A table (list of lists) of articles and their fields as given by the + overview database for each available article in the specified range. + The fields that are given can be determined using the LIST + OVERVIEW.FMT command if the server supports it. + + Raises: + NNTPReplyError: If no such article exists or the currently selected + newsgroup is invalid. + """ + return [x for x in self.xover_gen(range)] + + def xzver_gen(self, range=None): + """Generator for the XZVER command. + + The XZVER command returns information from the overview database for + the article(s) specified. It is part of the compressed headers + extensions that are supported by some usenet servers. It is the + compressed version of the XOVER command. + + + + Args: + range: An article number as an integer, or a tuple of specifying a + range of article numbers in the form (first, [last]). If last is + omitted then all articles after first are included. A range of + None (the default) uses the current article. + + Returns: + A list of fields as given by the overview database for each + available article in the specified range. The fields that are + returned can be determined using the LIST OVERVIEW.FMT command if + the server supports it. + + Raises: + NNTPTemporaryError: If no such article exists or the currently + selected newsgroup is invalid. + NNTPDataError: If the compressed response cannot be decoded. + """ + args = None + if range is not None: + args = utils.unparse_range(range) + + code, message = self.command("XZVER", args) + if code != 224: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message, True): + yield line.rstrip().split("\t") + + def xzver(self, range=None): + """XZVER command. + + The XZVER command returns information from the overview database for + the article(s) specified. It is part of the compressed headers + extensions that are supported by some usenet servers. It is the + compressed version of the XOVER command. + + + + Args: + range: An article number as an integer, or a tuple of specifying a + range of article numbers in the form (first, [last]). If last is + omitted then all articles after first are included. A range of + None (the default) uses the current article. + + Returns: + A list of fields as given by the overview database for each + available article in the specified range. The fields that are + returned can be determined using the LIST OVERVIEW.FMT command if + the server supports it. + + Raises: + NNTPTemporaryError: If no such article exists or the currently + selected newsgroup is invalid. + NNTPDataError: If the compressed response cannot be decoded. + """ + return [x for x in self.xzver_gen(range)] + + def xpat_gen(self, header, msgid_range, *pattern): + """Generator for the XPAT command. + """ + args = " ".join( + [header, utils.unparse_msgid_range(msgid_range)] + list(pattern) + ) + + code, message = self.command("XPAT", args) + if code != 221: + raise NNTPReplyError(code, message) + + for line in self.info_gen(code, message): + yield line.strip() + + def xpat(self, header, id_range, *pattern): + """XPAT command. + """ + return [x for x in self.xpat_gen(header, id_range, *pattern)] + + def xfeature_compress_gzip(self, terminator=False): + """XFEATURE COMPRESS GZIP command. + """ + args = "TERMINATOR" if terminator else None + + code, message = self.command("XFEATURE COMPRESS GZIP", args) + if code != 290: + raise NNTPReplyError(code, message) + + return True + + def post(self, headers={}, body=""): + """POST command. + + Args: + headers: A dictionary of headers. + body: A string or file like object containing the post content. + + Raises: + NNTPDataError: If binary characters are detected in the message + body. + + Returns: + A value that evaluates to true if posting the message succeeded. + (See note for further details) + + Note: + '\\n' line terminators are converted to '\\r\\n' + + Note: + Though not part of any specification it is common for usenet servers + to return the message-id for a successfully posted message. If a + message-id is identified in the response from the server then that + message-id will be returned by the function, otherwise True will be + returned. + + Note: + Due to protocol issues if illegal characters are found in the body + the message will still be posted but will be truncated as soon as + an illegal character is detected. No illegal characters will be sent + to the server. For information illegal characters include embedded + carriage returns '\\r' and null characters '\\0' (because this + function converts line feeds to CRLF, embedded line feeds are not an + issue) + """ + code, message = self.command("POST") + if code != 340: + raise NNTPReplyError(code, message) + + # send headers + hdrs = utils.unparse_headers(headers) + self.socket.sendall(hdrs) + + if isinstance(body, basestring): + body = cStringIO.StringIO(body) + + # send body + illegal = False + for line in body: + if line.startswith("."): + line = "." + line + if line.endswith("\r\n"): + line = line[:-2] + elif line.endswith("\n"): + line = line[:-1] + if any(c in line for c in "\0\r"): + illegal = True + break + self.socket.sendall(line + "\r\n") + self.socket.sendall(".\r\n") + + # get status + code, message = self.status() + + # check if illegal characters were detected + if illegal: + raise NNTPDataError("Illegal characters found") + + # check status + if code != 240: + raise NNTPReplyError(code, message) + + # return message-id possible + message_id = message.split(None, 1)[0] + if message_id.startswith("<") and message_id.endswith(">"): + return message_id + + return True + +# testing +if __name__ == "__main__": + + import sys + import hashlib + + log = sys.stdout.write + + try: + host = sys.argv[1] + port = int(sys.argv[2]) + username = sys.argv[3] + password = sys.argv[4] + use_ssl = int(sys.argv[5]) + except: + log("%s \n" % sys.argv[0]) + sys.exit(1) + + nntp_client = NNTPClient(host, port, username, password, use_ssl=use_ssl, reader=False) + + try: + log("HELP\n") + try: + log("%s\n" % nntp_client.help()) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("DATE\n") + try: + log("%s\n" % nntp_client.date()) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("NEWGROUPS\n") + try: + log("%s\n" % nntp_client.newgroups(datetime.datetime.utcnow() - datetime.timedelta(days=50))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("NEWNEWS\n") + try: + log("%s\n" % nntp_client.newnews("alt.binaries.*", datetime.datetime.utcnow() - datetime.timedelta(minutes=1))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("CAPABILITIES\n") + try: + log("%s\n" % nntp_client.capabilities()) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("GROUP misc.test\n") + try: + total, first, last, name = nntp_client.group("misc.test") + log("%d %d %d %s\n" % (total, first, last, name)) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("HEAD\n") + try: + log("%r\n" % nntp_client.head(last)) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("BODY\n") + try: + log("%r\n" % nntp_client.body(last)) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("ARTICLE\n") + try: + result = nntp_client.article(last, False) + log("%d\n%s\n%r\n" % (result[0], result[1], result[2])) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("ARTICLE (auto yEnc decode)\n") + try: + result = nntp_client.article(last) + log("%d\n%s\n%r\n" % (result[0], result[1], result[2])) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XHDR Date %d-%d\n" % (last-10, last)) + try: + log("%s\n" % nntp_client.xhdr("Date", (last-10, last))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XZHDR Date %d-%d\n" % (last-10, last)) + try: + log("%s\n" % nntp_client.xzhdr("Date", (last-10, last))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XOVER %d-%d\n" % (last-10, last)) + try: + result = nntp_client.xover((last-10, last)) + log("Entries %d Hash %s\n" % (len(result), hashlib.md5("".join(["".join(x) for x in result])).hexdigest())) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XZVER %d-%d\n" % (last-10, last)) + try: + result = nntp_client.xzver((last-10, last)) + log("Entries %d Hash %s\n" % (len(result), hashlib.md5("".join(["".join(x) for x in result])).hexdigest())) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XFEATURE COMPRESS GZIP\n") + try: + log("%s\n" % nntp_client.xfeature_compress_gzip()) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XOVER %d-%d\n" % (last-10, last)) + try: + result = nntp_client.xover((last-10, last)) + log("Entries %d Hash %s\n" % (len(result), hashlib.md5("".join(["".join(x) for x in result])).hexdigest())) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XFEATURE COMPRESS GZIP TERMINATOR\n") + try: + log("%s\n" % nntp_client.xfeature_compress_gzip()) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("XOVER %d-%d\n" % (last-10, last)) + try: + result = nntp_client.xover((last-10, last)) + log("Entries %d Hash %s\n" % (len(result), hashlib.md5("".join(["".join(x) for x in result])).hexdigest())) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST\n") + try: + log("Entries %d\n" % len(nntp_client.list())) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST ACTIVE\n") + try: + log("Entries %d\n" % len(nntp_client.list("ACTIVE"))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST ACTIVE alt.binaries.*\n") + try: + log("Entries %d\n" % len(nntp_client.list("ACTIVE", "alt.binaries.*"))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST ACTIVE.TIMES\n") + try: + log("Entries %d\n" % len(nntp_client.list("ACTIVE.TIMES"))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST NEWSGROUPS\n") + try: + log("Entries %d\n" % len(nntp_client.list("NEWSGROUPS"))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST NEWSGROUPS alt.binaries.*\n") + try: + log("Entries %d\n" % len(nntp_client.list("NEWSGROUPS", "alt.binaries.*"))) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST OVERVIEW.FMT\n") + try: + log("%s\n" % nntp_client.list("OVERVIEW.FMT")) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("LIST EXTENSIONS\n") + try: + log("%s\n" % nntp_client.list("EXTENSIONS")) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("POST (with illegal characters)\n") + try: + log("%s\n" % nntp_client.post( + iodict.IODict({ + "From": "\"pynntp\" ", + "Newsgroups": "misc.test", + "Subject": "pynntp test article", + "Organization": "pynntp", + }), + "pip install pynntp\r\nthis\0contains\rillegal\ncharacters" + )) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("POST\n") + try: + log("%s\n" % nntp_client.post( + iodict.IODict({ + "From": "\"pynntp\" ", + "Newsgroups": "misc.test", + "Subject": "pynntp test article", + "Organization": "pynntp", + }), + "pip install pynntp" + )) + except NNTPError as e: + log("%s\n" % e) + log("\n") + + log("QUIT\n") + try: + nntp_client.quit() + except NNTPError as e: + log("%s\n" % e) + log("\n") + + finally: + log("CLOSING CONNECTION\n") + nntp_client.close() diff --git a/python/lib/pynntp/nntp/utils.py b/python/lib/pynntp/nntp/utils.py new file mode 100644 index 000000000..2952f5b8c --- /dev/null +++ b/python/lib/pynntp/nntp/utils.py @@ -0,0 +1,229 @@ +#!/usr/bin/python +""" +An NNTP library - a bit more useful than the nntplib one (hopefully). +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import cStringIO +import iodict + +def unparse_msgid_article(obj): + """Unparse a message-id or article number argument. + + Args: + obj: A messsage id or an article number. + + Returns: + The message id or article number as a string. + """ + return str(obj) + +def parse_msgid_article(obj): + """Parse a message-id or article number argument. + + Args: + str: Message id or article as a string. + + Returns: + The message id as a string or article number as an integer. + """ + try: + return int(obj) + except ValueError: + pass + return obj + +def unparse_range(obj): + """Unparse a range argument. + + Args: + obj: An article range. There are a number of valid formats; an integer + specifying a single article or a tuple specifying an article range. + If the range doesn't give a start article then all articles up to + the specified last article are included. If the range doesn't + specify a last article then all articles from the first specified + article up to the current last article for the group are included. + + Returns: + The range as a string that can be used by an NNTP command. + + Note: Sample valid formats. + 4678 + (,5234) + (4245,) + (4245, 5234) + """ + if isinstance(obj, (int, long)): + return str(obj) + + if isinstance(obj, tuple): + arg = str(obj[0]) + "-" + if len(obj) > 1: + arg += str(obj[1]) + return arg + + raise ValueError("Must be an integer or tuple") + +def parse_range(obj): + """Parse a range argument. + + Args: + obj: An article range as a string. + + Returns: + The range as a string that can be used by an NNTP command. + + Raises: + ValueError: If obj is not a valid range format. + + Note: Sample valid formats. + 4678 + (,5234) + (4245,) + (4245, 5234) + """ + if isinstance(obj, (int, long)): + return str(obj) + + if isinstance(obj, tuple): + arg = str(obj[0]) + "-" + if len(obj) > 1: + arg += str(obj[1]) + return arg + + raise ValueError("Must be an integer or tuple") + +def unparse_msgid_range(obj): + """Unparse a message-id or range argument. + + Args: + obj: A message id as a string or a range as specified by + unparse_range(). + + Raises: + ValueError: If obj is not a valid message id or range format. See + unparse_range() for valid range formats. + + Returns: + A message id or range as a string that can be used by an NNTP command. + """ + if isinstance(obj, basestring): + return obj + + return unparse_range(obj) + +def parse_newsgroup(line): + """Parse a newsgroup info line to python types. + + Args: + line: An info response line containing newsgroup info. + + Returns: + A tuple of group name, low-water as integer, high-water as integer and + posting status. + + Raises: + ValueError: If the newsgroup info cannot be parsed. + + Note: + Posting status is a character is one of (but not limited to): + "y" posting allowed + "n" posting not allowed + "m" posting is moderated + """ + parts = line.split() + try: + group = parts[0] + low = int(parts[1]) + high = int(parts[2]) + status = parts[3] + except (IndexError, ValueError): + raise ValueError("Invalid newsgroup info") + return group, low, high, status + +def parse_header(line): + """Parse a header line. + + Args: + line: A header line as a string. + + Returns: + None if end of headers is found. A string giving the continuation line + if a continuation is found. A tuple of name, value when a header line is + found. + + Raises: + ValueError: If the line cannot be parsed as a header. + """ + if not line or line == "\r\n": + return None + if line[0] in " \t": + return line[1:].rstrip() + name, value = line.split(":", 1) + return (name.strip(), value.strip()) + +def parse_headers(obj): + """Parse a string a iterable object (including file like objects) to a + python dictionary. + + Args: + obj: An iterable object including file-like objects. + + Returns: + An dictionary of headers. If a header is repeated then the last value + for that header is given. + + Raises: + ValueError: If the first line is a continuation line or the headers + cannot be parsed. + """ + if isinstance(obj, basestring): + obj = cStringIO.StringIO(obj) + hdrs = [] + for line in obj: + hdr = parse_header(line) + if not hdr: + break + if isinstance(hdr, basestring): + if not hdrs: + raise ValueError("First header is a continuation") + hdrs[-1] = (hdrs[-1][0], hdrs[-1][1] + hdr) + continue + hdrs.append(hdr) + return iodict.IODict(hdrs) + +def unparse_header(name, value): + """Parse a name value tuple to a header string. + + Args: + name: The header name. + value: the header value. + + Returns: + The header as a string. + """ + return ": ".join([name, value]) + "\r\n" + +def unparse_headers(hdrs): + """Parse a dictionary of headers to a string. + + Args: + hdrs: A dictionary of headers. + + Returns: + The headers as a string that can be used in an NNTP POST. + """ + return "".join([unparse_header(n, v) for n, v in hdrs.items()]) + "\r\n" diff --git a/python/lib/pynntp/nntp/yenc.py b/python/lib/pynntp/nntp/yenc.py new file mode 100644 index 000000000..5d020ef7b --- /dev/null +++ b/python/lib/pynntp/nntp/yenc.py @@ -0,0 +1,61 @@ +#!/usr/bin/python +""" +Basic yEnc decoder. +Copyright (C) 2013 Byron Platt + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#NOTE: docstrings beed to be added + +import re +import zlib +import struct +import binascii + + +_crc32_re = re.compile(r"\s+crc(?:32)?=([0-9a-fA-F]{8})") + +def crc32(trailer): + + match = _crc32_re.search(trailer) + if not match: + return None + + buf = binascii.unhexlify(match.group(1)) + + return struct.unpack(">I", buf)[0] + + +def decode(buff, escape=0, crc32=0): + + decoded = "" + + for c in buff: + byte = ord(c) + if escape: + byte = (byte - 106) & 0xff + escape = 0 + elif byte == 0x3d: + escape = 1 + continue + elif byte == 0x0d or byte == 0x0a: + continue + else: + byte = (byte - 42) & 0xff + decoded += chr(byte) + + crc32 = zlib.crc32(decoded, crc32) + + return decoded, escape, crc32 diff --git a/python/lib/pynntp/setup.py b/python/lib/pynntp/setup.py new file mode 100644 index 000000000..d95d9b0ee --- /dev/null +++ b/python/lib/pynntp/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name='pynntp', + version='0.8.4', + description='NNTP Library (including compressed headers)', + author='Byron Platt', + author_email='byron.platt@gmail.com', + license='GPL3', + url='https://github.com/greenbender/pynntp', + packages=['nntp'], + install_requires=['dateutils'], +) diff --git a/python/nntpproxy.py b/python/nntpproxy.py new file mode 100644 index 000000000..09c2a4e3a --- /dev/null +++ b/python/nntpproxy.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +import time +import random +import socket +import SocketServer +import socketpool +from lib.pynntp.nntp import nntp +from lib.info import bcolors +from socketpool import util + + +class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): + + def __init__(self, host, port, backend_mod, pool=None, + username="anonymous", password="anonymous", timeout=60, use_ssl=False): + self.host = host + self.port = port + self.backend_mod = backend_mod + self._pool = pool + self._connected = False + self._life = time.time() - random.randint(0, 10) + self._start_time = time.time() + if backend_mod.Socket != socket.socket: + raise ValueError("Bad backend") + nntp.NNTPClient.__init__(self, self.host, self.port, username, password, timeout=timeout, use_ssl=use_ssl) + self.id = self.socket.getsockname()[1] + print(bcolors.PRIMARY + "New NNTP connection to %s established with ID #%5d" % + (self.host, self.id) + bcolors.ENDC) + self._connected = True + self.xfeature_compress_gzip() + + def __del__(self): + self.release() + + def matches(self, **match_options): + target_host = match_options.get('host') + target_port = match_options.get('port') + return target_host == self.host and target_port == self.port + + def is_connected(self): + if self._connected: + return socketpool.util.is_connected(self.socket) + return False + + def handle_exception(self, exception): + print(bcolors.ERROR + str(exception) + bcolors.ENDC) + self.release() + self.invalidate() + + def get_lifetime(self): + return self._life + + def invalidate(self): + print(bcolors.PRIMARY + "Disconnecting from NNTP connection ID #%5d after %d seconds." % + (self.id, (time.time() - self._start_time)) + bcolors.ENDC) + self.close() + self._connected = False + self._life = -1 + + def release(self): + if self._pool is not None: + if self._connected: + self._pool.release_connection(self) + else: + self._pool = None + + +# NNTP proxy request handler for newznab +class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): + + def handle(self): + with self.server.nntp_client_pool.connection() as nntp_client: + + self.wfile.write("200 localhost NNRP Service Ready.\r\n") + + for line in self.rfile: + data = line.strip() + + if not data.startswith("POST"): + if data.startswith("GROUP"): + print(bcolors.ALTERNATE + "%5d %s" % (nntp_client.id, data)) + else: + print(bcolors.HEADER + "%5d " % nntp_client.id) + (bcolors.PRIMARY + "%s" % data) + + if data.startswith("XOVER"): + try: + rng = data.split(None, 1)[1] + rng = tuple(map(int, rng.split("-"))) + xover_gen = nntp_client.xover_gen(rng) + self.wfile.write("224 data follows\r\n") + for entry in xover_gen: + self.wfile.write("\t".join(entry) + "\r\n") + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("503 internal server error\r\n") + + elif data.startswith("BODY"): + msgid = data.split(None, 1)[1] + try: + body = nntp_client.body(msgid) + self.wfile.write("222 %s\r\n" % msgid) + self.wfile.write(body.replace("\r\n.", "\r\n..")) + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("430 no such article\r\n") + + elif data.startswith("GROUP"): + try: + total, first, last, group = nntp_client.group(data.split(None, 1)[1]) + self.wfile.write("211 %d %d %d %s\r\n" % (total, first, last, group)) + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("411 no such news group\r\n") + + elif data.startswith("LIST OVERVIEW.FMT"): + try: + fmt = nntp_client.list_overview_fmt() + self.wfile.write("215 Order of fields in overview database.\r\n") + fmt = "\r\n".join(["%s:%s" % (f[0], "full" if f[1] else "") for f in fmt]) + "\r\n" + self.wfile.write(fmt) + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("503 internal server error\r\n") + + elif data.startswith("HEAD"): + msgid = data.split(None, 1)[1] + try: + head = nntp_client.head(msgid) + self.wfile.write("221 %s\r\n" % msgid) + head = "\r\n".join([": ".join(item) for item in head.items()]) + "\r\n\r\n" + self.wfile.write(head) + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("430 no such article\r\n") + + elif data.startswith("ARTICLE"): + msgid = data.split(None, 1)[1] + try: + article = nntp_client.article(msgid, False) + # check no of return values for compatibility with pynntp<=0.8.3 + if len(article) == 2: + articleno, head, body = 0, article[0], article[1] + else: + articleno, head, body = article + self.wfile.write("220 %d %s\r\n" % (articleno, msgid)) + head = "\r\n".join([": ".join(item) for item in head.items()]) + "\r\n\r\n" + self.wfile.write(head) + self.wfile.write(body.replace("\r\n.", "\r\n..")) + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("430 no such article\r\n") + + elif data == "LIST": + try: + list_gen = nntp_client.list_gen() + self.wfile.write("215 list of newsgroups follows\r\n") + for entry in list_gen: + self.wfile.write("%s %d %d %s\r\n" % entry) + self.wfile.write(".\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("503 internal server error\r\n") + + elif data.startswith("LIST ACTIVE") and not data.startswith("LIST ACTIVE.TIMES"): + try: + pattern = data[11:].strip() or None + active_gen = nntp_client.list_active_gen(pattern) + self.wfile.write("215 list of newsgroups follows\r\n") + for entry in active_gen: + self.wfile.write("%s %d %d %s\r\n" % entry) + self.wfile.write(".\r\n") + self.wfile.write(str(e) + "\r\n") + except Exception as ex: + print(bcolors.ERROR + str(ex.message) + bcolors.ENDC) + self.wfile.write("503 internal server error\r\n") + + elif data.startswith("AUTHINFO user") or data.startswith("AUTHINFO pass"): + self.wfile.write("281 Ok\r\n") + + elif data.startswith("XFEATURE"): + self.wfile.write("290 feature enabled\r\n") + + elif data.startswith("QUIT"): + self.wfile.write("205 Connection closing\r\n") + break + + else: + self.wfile.write("500 What?\r\n") + + +# NNTP proxy server for newznab +class NNTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + + allow_reuse_address = True + + def __init__(self, server_address, request_handler, nntp_client_pool_obj, bind_and_activate=True): + SocketServer.TCPServer.__init__(self, server_address, request_handler, bind_and_activate=bind_and_activate) + self.nntp_client_pool = nntp_client_pool_obj + +if __name__ == "__main__": + + import sys + try: + if len(sys.argv) == 1: + import os + pathname = os.path.abspath(os.path.dirname(sys.argv[0])) + with open(pathname+"/lib/nntpproxy.conf", "rb") as fd: + config = json.load(fd) + else: + with open(sys.argv[1], "rb") as fd: + config = json.load(fd) + except IndexError: + sys.stderr.write("Usage: %s configfile\n" % sys.argv[0]) + sys.exit(1) + except IOError as e: + sys.stderr.write("Failed to open config file (%s)\n" % e) + sys.exit(1) + except ValueError as e: + sys.stderr.write("Failed to parse config file (%s)\n" % e) + sys.exit(1) + + nntp_client_pool = socketpool.ConnectionPool( + NNTPClientConnector, + retry_max=3, + retry_delay=1, + timeout=-1, + max_lifetime=30000., + max_size=int(config["pool"]["size"]), + options=config["usenet"] + ) + + proxy = NNTPProxyServer((config["proxy"]["host"], config["proxy"]["port"]), + NNTPProxyRequestHandler, nntp_client_pool) + remote = (config["usenet"]["host"], config["usenet"]["port"]) + print(bcolors.PRIMARY + + "NNTP proxy server started on: %s:%d, using a maximum pool size of %d." % + (config["proxy"]["host"], config["proxy"]["port"], config["pool"]["size"])) + proxy.serve_forever() diff --git a/start.php b/start.php index be94fc9aa..3d9c82df2 100644 --- a/start.php +++ b/start.php @@ -13,6 +13,19 @@ $site = $s->get(); $t = new Tmux(); $tmux = $t->get(); $patch = (isset($tmux->sqlpatch)) ? $tmux->sqlpatch : 0; +$nntpproxy = $site->nntpproxy; + +// Search for NNTPProxy session that might be running froma userthreaded.php run. Setup a clean environment to run in. +exec("tmux list-session | grep NNTPProxy", $nntpkill); +if (count($nntpkill) === 0) { + exec("tmux list-session", $sessions); + if (count($sessions) === 0) { + echo $db->log->notice("The above is a notice generated by tmux. \nWhen starting this script, we first check that nntpproxy is not already running in a tmux session. If it is not, the above notice is generated by tmux.\nIt is not an error."); + } +} else { + echo $db->log->notice("Found NNTPProxy tmux session and killing it."); + exec("tmux kill-session -t NNTPProxy"); +} // Check database patch version if ($patch < 87) { @@ -75,6 +88,16 @@ function python_module_exist($module) return ($returnCode == 0 ? true : false); } +$nntpproxy = $site->nntpproxy; +if ($nntpproxy == '1') { + $modules = array("nntp", "socketpool"); + foreach ($modules as &$value) { + if (!python_module_exist($value)) { + exit($db->log->error("\nNNTP Proxy requires " . $value . " python module but it's not installed. Aborting.\n")); + } + } +} + function start_apps($tmux_session) { $t = new Tmux(); @@ -125,6 +148,29 @@ function start_apps($tmux_session) } } +function window_proxy($tmux_session, $window) +{ + global $site; + $nntpproxy = $site->nntpproxy; + if ($nntpproxy === '1') { + $DIR = NN_MISC; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + exec("tmux new-window -t $tmux_session -n nntpproxy 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); + } + } + + if ($nntpproxy === '1' && ($site->alternate_nntp == '1')) { + $DIR = NN_MISC; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + exec("tmux selectp -t $tmux_session:$window.0; tmux splitw -t $tmux_session:$window -h -p 50 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); + } + } +} + function window_utilities($tmux_session) { exec("tmux new-window -t $tmux_session -n Utils 'printf \"\033]2;update_predb\033\"'"); @@ -206,10 +252,14 @@ if ($seq == 1) { window_utilities($tmux_session); window_post($tmux_session); - window_fixnames($tmux_session); - window_ircscraper($tmux_session); - window_sharing($tmux_session); - + if ($nntpproxy == 1) { + window_ircscraper($tmux_session); + window_proxy($tmux_session, 4); + window_sharing($tmux_session); + } else { + window_ircscraper($tmux_session); + window_sharing($tmux_session); + } start_apps($tmux_session); attach($DIR, $tmux_session); } else { @@ -222,10 +272,14 @@ if ($seq == 1) { window_utilities($tmux_session); window_post($tmux_session); - window_fixnames($tmux_session); - window_ircscraper($tmux_session); - window_sharing($tmux_session); - + if ($nntpproxy == 1) { + window_ircscraper($tmux_session); + window_proxy($tmux_session, 4); + window_sharing($tmux_session); + } else { + window_ircscraper($tmux_session); + window_sharing($tmux_session); + } start_apps($tmux_session); attach($DIR, $tmux_session); } \ No newline at end of file