...
 
Commits (10)
......@@ -5,7 +5,7 @@ stages:
- release
default:
image: namboy94/ci-docker-environment:0.5.0
image: namboy94/ci-docker-environment:0.8.0
before_script:
- echo "$SERVER_ACCESS_KEY" > ~/.ssh/id_rsa
- chmod 0600 ~/.ssh/id_rsa
......@@ -27,6 +27,12 @@ stylecheck:
script:
- python-codestyle-check
type_check:
stage: test
tags: [docker]
script:
- python-static-type-check
unittest:
stage: test
tags: [docker]
......
V 3.2.0:
- Added notice that python 2 is no longer supported
- Fixed program hanging when no channels are joined
- Added timeout to xdcc-dl
- Improved xdcc-search to include server configuration
V 3.1.2:
- Integrated sentry
- Integrated mypy static type checking
V 3.1.1:
- Use docker in CI
V 3.1.0:
......
......@@ -12,6 +12,8 @@ An XDCC File downloader based on the [irclib](https://github.com/jaraco/irc) fra
Either install the program using `pip install xdcc-dl` or `python setup.py install`
Please not that python 2 is no longer supported, the project requires python 3 to run.
## Usage
### Message-based CLI
......
......@@ -20,6 +20,8 @@ LICENSE"""
import argparse
from requests.exceptions import ConnectionError
from puffotter.init import cli_start, argparse_add_verbosity
from xdcc_dl import sentry_dsn
from xdcc_dl.xdcc.exceptions import DownloadIncomplete
from xdcc_dl.helper import set_throttle_value, set_logging_level, prepare_packs
from xdcc_dl.entities import XDCCPack
......@@ -28,39 +30,14 @@ from xdcc_dl.xdcc import download_packs
from xdcc_dl.pack_search.SearchEngine import SearchEngineType
def main():
def main(args: argparse.Namespace):
"""
Conducts a XDCC pack search
Conducts a XDCC pack search with the option to immediately download any
found packs
:param args: The command line arguments
:return: None
"""
parser = argparse.ArgumentParser()
parser.add_argument("search_term", help="The term to search for")
parser.add_argument("--search_engine",
default=SearchEngineType.HORRIBLESUBS.name.lower(),
choices=SearchEngineType.choices(True),
help="The Search Engine to use")
parser.add_argument("-s", "--server",
default="irc.rizon.net",
help="Specifies the IRC Server. "
"Defaults to irc.rizon.net")
parser.add_argument("-o", "--out",
help="Specifies the target file. "
"Defaults to the pack's file name. "
"When downloading multiple packs, index "
"numbers will be appended to the filename")
parser.add_argument("-v", "--verbose", action="store_true",
help="Sets the verbosity of the program to INFO")
parser.add_argument("-d", "--debug", action="store_true",
help="Sets the verbosity of the program to DEBUG")
parser.add_argument("-q", "--quiet", action="store_true",
help="Disables all output")
parser.add_argument("-t", "--throttle",
help="Limits the download speed of xdcc-dl. "
"Append K,M or G for more convenient units")
args = parser.parse_args()
try:
set_logging_level(args.quiet, args.verbose, args.debug)
set_throttle_value(args.throttle)
......@@ -83,16 +60,40 @@ def main():
for pack in packs:
Logger().info("Downloading pack {}".format(pack))
download_packs(packs)
download_packs(packs, timeout=args.timeout)
except ConnectionError:
print("Connection Error, could not conduct search")
except KeyboardInterrupt:
Logger().print("Thanks for using xdcc-dl!")
except DownloadIncomplete:
Logger().warning("Download incomplete.")
Logger().print("Thanks for using xdcc-dl!")
raise KeyboardInterrupt()
if __name__ == "__main__":
main()
parser = argparse.ArgumentParser()
parser.add_argument("search_term", help="The term to search for")
parser.add_argument("--search-engine",
default=SearchEngineType.HORRIBLESUBS.name.lower(),
choices=SearchEngineType.choices(True),
help="The Search Engine to use")
parser.add_argument("-s", "--server",
default="irc.rizon.net",
help="Specifies the IRC Server. "
"Defaults to irc.rizon.net")
parser.add_argument("-o", "--out",
help="Specifies the target file. "
"Defaults to the pack's file name. "
"When downloading multiple packs, index "
"numbers will be appended to the filename")
parser.add_argument("-t", "--throttle",
help="Limits the download speed of xdcc-dl. "
"Append K,M or G for more convenient units")
parser.add_argument("--timeout", default=120, type=int,
help="Sets a timeout for starting the download")
argparse_add_verbosity(parser)
cli_start(
main, parser,
sentry_dsn=sentry_dsn,
package_name="xdcc-dl",
exit_msg="Thanks for using xdcc-dl!"
)
......@@ -18,9 +18,10 @@ You should have received a copy of the GNU General Public License
along with xdcc-dl. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
# imports
import os
import argparse
from puffotter.init import cli_start, argparse_add_verbosity
from xdcc_dl import sentry_dsn
from xdcc_dl.xdcc import download_packs
from xdcc_dl.helper import set_throttle_value, set_logging_level, prepare_packs
from xdcc_dl.entities import XDCCPack
......@@ -28,12 +29,29 @@ from xdcc_dl.logging import Logger
from xdcc_dl.xdcc.exceptions import DownloadIncomplete
def main():
def main(args: argparse.Namespace):
"""
Starts the main method of the program
:param args: The command line arguments
:return: None
"""
try:
set_throttle_value(args.throttle)
set_logging_level(args.quiet, args.verbose, args.debug)
packs = XDCCPack.from_xdcc_message(
args.message, os.getcwd(), args.server
)
prepare_packs(packs, args.out)
download_packs(packs, timeout=args.timeout)
except DownloadIncomplete:
Logger().warning("Download incomplete.")
raise KeyboardInterrupt()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("message",
help="An XDCC Message. Supports ranges (1-100), "
......@@ -48,34 +66,15 @@ def main():
"Defaults to the pack's file name. "
"When downloading multiple packs, index "
"numbers will be appended to the filename")
parser.add_argument("-v", "--verbose", action="store_true",
help="Sets the verbosity of the program to INFO")
parser.add_argument("-d", "--debug", action="store_true",
help="Sets the verbosity of the program to DEBUG")
parser.add_argument("-q", "--quiet", action="store_true",
help="Disables all output")
parser.add_argument("-t", "--throttle",
help="Limits the download speed of xdcc-dl. "
"Append K,M or G for more convenient units")
args = parser.parse_args()
try:
set_logging_level(args.quiet, args.verbose, args.debug)
set_throttle_value(args.throttle)
packs = XDCCPack.from_xdcc_message(
args.message, os.getcwd(), args.server
)
prepare_packs(packs, args.out)
download_packs(packs)
except KeyboardInterrupt:
Logger().print("Thanks for using xdcc-dl!")
except DownloadIncomplete:
Logger().warning("Download incomplete.")
Logger().print("Thanks for using xdcc-dl!")
if __name__ == "__main__":
main()
parser.add_argument("--timeout", default=120, type=int,
help="Sets a timeout for starting the download")
argparse_add_verbosity(parser)
cli_start(
main, parser,
sentry_dsn=sentry_dsn,
package_name="xdcc-dl",
exit_msg="Thanks for using xdcc-dl!"
)
......@@ -19,32 +19,40 @@ along with xdcc-dl. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
import argparse
from puffotter.init import cli_start
from requests.exceptions import ConnectionError
from xdcc_dl import sentry_dsn
from xdcc_dl.pack_search.SearchEngine import SearchEngineType
def main():
def main(args: argparse.Namespace):
"""
Conducts a XDCC pack search
:param args: The command line arguments
:return: None
"""
parser = argparse.ArgumentParser()
parser.add_argument("search_term", help="The term to search for")
parser.add_argument("search_engine",
choices=SearchEngineType.choices(True),
help="The Search Engine to use")
args = parser.parse_args()
try:
search_engine = SearchEngineType.resolve(args.search_engine)
results = search_engine.search(args.search_term)
for result in results:
print(result)
message = "{} (xdcc-dl \"{}\")".format(result.filename, result.get_request_message(True))
if result.server.address != "irc.rizon.net":
message = message[0:-1] + " --server " + result.server.address + ")"
print(message)
except ConnectionError:
print("Connection Error, could not conduct search")
except KeyboardInterrupt:
print("Thanks for using xdcc-dl!")
raise KeyboardInterrupt()
if __name__ == "__main__":
main()
parser = argparse.ArgumentParser()
parser.add_argument("search_term", help="The term to search for")
parser.add_argument("search_engine",
choices=SearchEngineType.choices(True),
help="The Search Engine to use")
cli_start(
main, parser,
sentry_dsn=sentry_dsn,
package_name="xdcc-dl",
exit_msg="Thanks for using xdcc-dl!"
)
......@@ -46,7 +46,9 @@ if __name__ == "__main__":
"cfscrape",
"typing",
"colorama",
"irc"
"irc",
"puffotter",
"sentry-sdk"
],
include_package_data=True,
zip_safe=False
......
3.1.1
\ No newline at end of file
3.2.0
\ No newline at end of file
......@@ -16,3 +16,9 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with xdcc-dl. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
sentry_dsn = "https://f20c5998b8fc46109e71fd9ddeebd64b@sentry.namibsun.net/7"
"""
The sentry DSN used for logging exceptions
"""
......@@ -221,8 +221,7 @@ class XDCCPack(object):
return packs
except ValueError:
packnumbers = xdcc_message.rsplit("#", 1)[1]
start, end = packnumbers.split("-")
start, end = xdcc_message.rsplit("#", 1)[1].split("-")
try:
step = int(end.split(";")[1])
......
......@@ -22,7 +22,8 @@ import logging
from typing import List, Optional
from xdcc_dl.logging import Logger
from xdcc_dl.entities.XDCCPack import XDCCPack
from xdcc_dl.xdcc.XDCCClient import XDCCCLient
from xdcc_dl.xdcc.XDCCClient import XDCCClient
from puffotter.units import byte_string_to_byte_count
def set_throttle_value(throttle_string: str):
......@@ -34,20 +35,9 @@ def set_throttle_value(throttle_string: str):
"""
try:
if throttle_string is not None:
multiplier = 1
units = {"k": 1000, "m": 1000000, "g": 1000000000}
throttle_num = ""
for i, char in enumerate(throttle_string):
if char.isdigit():
throttle_num += char
else:
if len(throttle_string) - 1 != i:
raise KeyError
else:
multiplier = units[char.lower()]
limit = multiplier * int(throttle_num)
XDCCCLient.download_limit = limit
except KeyError:
limit = byte_string_to_byte_count(throttle_string)
XDCCClient.download_limit = limit
except ValueError:
print("Invalid throttle value")
sys.exit(1)
......
......@@ -18,7 +18,7 @@ along with xdcc-dl. If not, see <http://www.gnu.org/licenses/>.
LICENSE"""
from enum import Enum
from typing import List, Set
from typing import List, Set, Callable, Optional
from xdcc_dl.entities.XDCCPack import XDCCPack
from xdcc_dl.pack_search.procedures.nibl import find_nibl_packs
from xdcc_dl.pack_search.procedures.ixirc import find_ixirc_packs
......@@ -30,7 +30,7 @@ class SearchEngine:
An XDCC Pack Search Engine
"""
def __init__(self, name: str, procedure: callable):
def __init__(self, name: str, procedure: Callable):
"""
Initializes the Search Engine
:param name: The name of the search engine
......@@ -72,7 +72,7 @@ class SearchEngineType(Enum):
return set(choices)
@classmethod
def resolve(cls, name: str) -> SearchEngine or None:
def resolve(cls, name: str) -> Optional[SearchEngine]:
"""
Resolves a string identifier of a search engine and provides
the correct search engine
......
......@@ -94,6 +94,6 @@ def parse_result(result: str) -> Dict[str, str]:
else:
# Segment is a string
data[current_key] = segment
data[str(current_key)] = segment
return data
......@@ -23,6 +23,7 @@ from typing import List
from bs4 import BeautifulSoup
from xdcc_dl.entities.XDCCPack import XDCCPack
from xdcc_dl.entities.IrcServer import IrcServer
from puffotter.units import byte_string_to_byte_count
def find_ixirc_packs(search_phrase: str) -> List[XDCCPack]:
......@@ -79,7 +80,7 @@ def find_ixirc_packs(search_phrase: str) -> List[XDCCPack]:
analysing = False
# Establish search results
results = []
results = [] # type: List[XDCCPack]
for content in page_contents:
results += get_page_results(content)
......@@ -171,8 +172,7 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]:
elif column_count == 5:
pass # This is the 'gets' section, we don't need that
elif column_count == 6:
# File Size
size = line_part.text
size = line_part.text.replace("\xa0", " ").replace(" ", "")
# Resets state after a pack was successfully parsed,
# and adds xdcc pack to results
......@@ -184,7 +184,7 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]:
# Generate XDCCPack and append it to the list
result = XDCCPack(IrcServer(server), bot, pack_number)
result.set_filename(file_name)
result.set_size(size)
result.set_size(byte_string_to_byte_count(size))
results.append(result)
# Resets state after invalid pack
......
......@@ -24,17 +24,20 @@ import shlex
import socket
import irc.events
import irc.client
from threading import Thread
from irc.client import DCCConnection
from colorama import Fore, Back
from typing import Optional, IO, Any, List
from xdcc_dl.entities import User, XDCCPack
from xdcc_dl.logging import Logger
from xdcc_dl.xdcc.exceptions import InvalidCTCPException, \
AlreadyDownloadedException, DownloadCompleted, DownloadIncomplete, \
PackAlreadyRequested, UnrecoverableError
PackAlreadyRequested, UnrecoverableError, Timeout, BotDoesNotExist
from irc.client import SimpleIRCClient, ServerConnection, Event, \
ip_numstr_to_quad
class XDCCCLient(SimpleIRCClient):
class XDCCClient(SimpleIRCClient):
"""
IRC Client that can download an XDCC pack
"""
......@@ -51,17 +54,40 @@ class XDCCCLient(SimpleIRCClient):
for event in irc.events.all:
exec(
"def on_{}(self, c, e):\n"
" self.logger.debug("
" \"{}:\" + str(e.source) + \" \" + str(e.arguments),"
" back=Back.BLUE"
")".format(event, event)
" self.handle_generic_event(\"{}\", c, e)"
"".format(event, event)
)
def __init__(self, pack: XDCCPack, retry: bool = False):
def handle_generic_event(
self,
event_type: str,
_: ServerConnection,
event: Event
):
"""
Handles a generic event that isn't handled explicitly
:param event_type: The event type to handle
:param _: The connection to use
:param event: The received event
:return: None
"""
self.logger.debug("{}:{} {}".format(
event_type,
event.source,
event.arguments
), back=Back.BLUE)
def __init__(
self,
pack: XDCCPack,
retry: bool = False,
timeout: int = 120
):
"""
Initializes the XDCC IRC client
:param pack: The pack to downloadX
:param retry: Set to true for retried downloads.
:param timeout: Sets the timeout time for starting downloads
"""
self.logger = Logger()
......@@ -72,18 +98,21 @@ class XDCCCLient(SimpleIRCClient):
self.pack = pack
self.server = pack.server
self.downloading = False
self.xdcc_timestamp = 0
self.channels = None # change to list if channel joins are required
self.xdcc_timestamp = 0.0
self.channels = None # type: Optional[List[str]]
self.message_sent = False
self.connect_start_time = 0
self.connect_start_time = 0.0
self.timeout = timeout
self.timed_out = False
self.disconnected = False
# XDCC state variables
self.peer_address = ""
self.peer_port = -1
self.filesize = -1
self.progress = 0
self.xdcc_file = None
self.xdcc_connection = None
self.xdcc_file = None # type: Optional[IO[Any]]
self.xdcc_connection = None # type: Optional[DCCConnection]
self.retry = retry
if not self.retry:
......@@ -95,6 +124,23 @@ class XDCCCLient(SimpleIRCClient):
super().__init__()
def timeout_watcher():
"""
Monitors when the XDCC message is sent. If it is not sent by the
timeout time, a ping will be sent and handled by the on_ping method
:return: None
"""
self.logger.info("Timeout watcher started")
while not self.message_sent and not self.disconnected:
time.sleep(1)
self.logger.debug("Iterating timeout thread")
if self.timeout < (time.time() - self.connect_start_time):
self.logger.info("Timeout detected")
self.connection.ping(self.server.address)
time.sleep(2)
Thread(target=timeout_watcher).start()
def download(self) -> str:
"""
Downloads the pack
......@@ -133,6 +179,7 @@ class XDCCCLient(SimpleIRCClient):
finally:
self.logger.info("Disconnecting")
try:
self.disconnected = True
self._disconnect()
except (DownloadCompleted, ):
pass
......@@ -146,7 +193,7 @@ class XDCCCLient(SimpleIRCClient):
if not completed:
self.logger.error("Download Incomplete. Retrying.")
retry_client = XDCCCLient(self.pack, True)
retry_client = XDCCClient(self.pack, True, self.timeout)
retry_client.download_limit = self.download_limit
retry_client.download()
......@@ -156,6 +203,32 @@ class XDCCCLient(SimpleIRCClient):
return self.pack.get_filepath()
def on_ping(self, _: ServerConnection, __: Event):
"""
Handles a ping event.
Used for timeout checks
:param _: The IRC connection
:param __: The received event
:return: None
"""
self.logger.debug("PING")
if not self.message_sent \
and self.timeout < (time.time() - self.connect_start_time) \
and not self.timed_out:
self.logger.error("Timeout")
self.timed_out = True
raise Timeout()
def on_nosuchnick(self, _: ServerConnection, __: Event):
"""
When a bot does not exist or is not online right now, aborts.
:param _: The IRC connection
:param __: The received event
:return: None
"""
self.logger.error("This bot does not exist on this server")
raise BotDoesNotExist()
def on_welcome(self, conn: ServerConnection, _: Event):
"""
The 'welcome' event indicates a successful connection to the server
......@@ -182,7 +255,7 @@ class XDCCCLient(SimpleIRCClient):
channels = list(map(lambda x: "#" + x.split(" ")[0], channels))
self.channels = channels
for channel in self.channels:
for channel in channels:
# Join all channels to avoid only joining a members-only channel
conn.join(channel)
......@@ -197,21 +270,33 @@ class XDCCCLient(SimpleIRCClient):
"""
self.logger.info("WHOIS End")
if self.channels is None:
self.on_join(conn, _)
def on_join(self, conn: ServerConnection, event: Event):
self.on_join(conn, _, True)
def on_join(
self,
conn: ServerConnection,
event: Event,
force: bool = False
):
"""
The 'join' event indicates that a channel was successfully joined.
The first on_join call will send a message to the bot that requests
the initialization of the XDCC file transfer.
:param conn: The connection
:param event: The 'join' event
:param force: If set to True, will force sending an XDCC message
:return: None
"""
# Make sure we were the ones joining
if not event.source.startswith(self.user.get_name()):
if not event.source.startswith(self.user.get_name()) and not force:
return
self.logger.info("Joined Channel: " + event.target)
if force:
self.logger.info(
"Didn't find a channel using WHOIS, "
"trying to send message anyways"
)
else:
self.logger.info("Joined Channel: " + event.target)
if not self.message_sent:
self._send_xdcc_request_message(conn)
......@@ -289,6 +374,9 @@ class XDCCCLient(SimpleIRCClient):
:param event: The 'dccmsg' event
:return: None
"""
if self.xdcc_file is None:
return
data = event.arguments[0]
chunk_size = len(data)
......@@ -353,13 +441,13 @@ class XDCCCLient(SimpleIRCClient):
)
# TODO Handle queues
def on_error(self, _: ServerConnection, event: Event):
def on_error(self, _: ServerConnection, __: Event):
"""
Sometimes, the connection gives an error which may prove fatal for
the download process. A possible cause of error events is a banned
IP address.
:param _: The connection
:param event: The error event
:param __: The error event
:return: None
"""
self.logger.error("Unrecoverable Error: Is this IP banned?")
......
......@@ -19,16 +19,17 @@ LICENSE"""
from typing import List
from xdcc_dl.entities.XDCCPack import XDCCPack
from xdcc_dl.xdcc.XDCCClient import XDCCCLient
from xdcc_dl.xdcc.XDCCClient import XDCCClient
def download_packs(packs: List[XDCCPack]):
def download_packs(packs: List[XDCCPack], timeout: int = 120):
"""
Downloads a list of XDCC Packs
:param packs: The packs to download
:param timeout: Specifies timeout time
:return: None
"""
for pack in packs:
client = XDCCCLient(pack)
client = XDCCClient(pack, timeout=timeout)
client.download()
......@@ -56,3 +56,15 @@ class UnrecoverableError(Exception):
"""
Exception raised when an unrecoverable error occurs
"""
class Timeout(UnrecoverableError):
"""
Exception raised when a timeout occurs
"""
class BotDoesNotExist(UnrecoverableError):
"""
Exception raised if a bot does not exist
"""