#!/usr/bin/env python import feedparser import re import time import subprocess import urllib2 import tempfile import json class ConfigParticipant(object): def _import_attribute(self, config, key, cast, default): if key in config: try: v = cast(config[key]) except StandardError as e: print "Failed to cast value of %s '%s' to %s: %s" % (key, config[key], cast, e) return default return v else: return default def _import_list(self, config, key, cast): va = self._import_attribute(config, key, list, []) va_objs = [] for v in va: try: v = cast(v) except StandardError as e: print "Failed to cast value of %s '%s' to %s: %s" % (key, config[key], cast, e) continue va_objs.append(v) return va_objs def _import_data_for_list(self, data, key, obj_list): if key not in data: return try: l = list(data[key]) except TypeError: print "Failed to load list for key %s" % key return for i, v in enumerate(l): if i < len(obj_list): obj_list[i].import_data(v) else: print "Too many elements in data - data may be out-of-date" class App(ConfigParticipant): def __init__(self): self.feeds = [] self.config_filename = None self.data_filename = None def load_config(self, json_filename): self.config_filename = json_filename try: fd = file(self.config_filename) except IOError: print "No config file" return self.import_config(json.load(fd)) fd.close() def save_config(self): fd = file(self.config_filename, "w") json.dump(self.export_data(), fd) fd.close() def load_data(self, json_filename): self.data_filename = json_filename try: fd = file(json_filename) except IOError: print "No data file" return self.import_data(json.load(fd)) fd.close() def save_data(self): fd = file(self.data_filename, "w") json.dump(self.export_data(), fd) fd.close() def import_config(self, config): self.feeds = self._import_list(config, "feeds", Feed.import_config) def export_config(self): return { "feeds": [f.export_config() for f in self.feeds], } def import_data(self, data): self._import_data_for_list(data, "feeds", self.feeds) def export_data(self): return { "feeds": [f.export_data() for f in self.feeds], } def setup_env(self): opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) urllib2.install_opener(opener) def get_next_feed(self): """Returns a (delay, feed) tuple of time to delay before next feed, and next feed to run """ next_expiration = None next_feed = None for feed in self.feeds: expir = feed.poll_delay + feed.last_load if next_expiration < expir: next_expiration = expir next_feed = feed delay = next_expiration - int(time.time()) if delay < 0: delay = 0 return (delay, next_feed) def _run_relavent(self, feed): found = feed.find_relavant() for i in found: # download file here instead of relying on transmission since we may # have cookies that transmission does not f = urllib2.urlopen(i) temp = tempfile.NamedTemporaryFile() temp.write(f.read()) f.close() temp.flush() ret = subprocess.call(["transmission-remote", "localhost", "-a", temp.name]) temp.close() if ret: print "Error adding torrent" self.save_data() def main(self): try: for feed in self.feeds: self._run_relavent(feed) while True: (delay, feed) = self.get_next_feed() time.sleep(delay) try: self._run_relavent(feed) except urllib2.URLError, e: print e except KeyboardInterrupt: pass class Feed(ConfigParticipant): def __init__(self): self.url = None # in seconds self.poll_delay = 60 * 60 # list of Matcher objects self.matches = None # initialize to Epoch. Should always be integer number of seconds since # Epoch self.last_load = 0 # true when the default action (when no matcher matches) should be to # count the item as relavant self.default_white = False self.seen = [] def _load_feed(self): self.last_load = int(time.time()) f = urllib2.urlopen(self.url) d = feedparser.parse(f.read()) f.close() return d def find_relavant(self): d = self._load_feed() found = [] print "New RSS Items:" for i in d['items']: if i.link in self.seen: continue self.seen.append(i.link) print " ", i.title white = self.default_white for m in self.matches: if not m.matches(i): continue print " Matched. White:", m.white m.matched_count += 1 white = m.white if white: found.append(i.link) self.seen = self.seen[-len(d['items']):] return found @staticmethod def import_config(config): f = Feed() f.url = f._import_attribute(config, "url", str, f.url) f.poll_delay = f._import_attribute(config, "poll_delay", int, f.poll_delay) f.matches = f._import_list(config, "matches", Matcher.import_config) f.default_white = f._import_attribute(config, "default_white", bool, f.default_white) return f def export_config(self): return { "url": self.url, "poll_delay": self.poll_delay, "matches": [m.export_config() for m in self.matches], "default_white": self.white, } def import_data(self, data): self.seen = self._import_list(data, "seen", str) self._import_data_for_list(data, "matches", self.matches) def export_data(self): return { "seen": self.seen, "matches": [m.export_data() for m in self.matches], } class Matcher(ConfigParticipant): def __init__(self): self.str = None self.re = None # white marks item to be downloaded, black means don't download # similar to white and black lists self.white = True self.matched_count = 0 @staticmethod def import_config(config): m = Matcher() m.white = m._import_attribute(config, "white", bool, m.white) m.str = m._import_attribute(config, "value", str, m.str) m.matched_count = m._import_attribute(config, "matched_count", int, m.matched_count) m.re = re.compile(m.str) return m def export_config(self): return { "white": self.white, "str": self.str, "matched_count": self.matched_count, } def import_data(self, data): self.matched_count = self._import_attribute(data, "matched_count", int, self.matched_count) def export_data(self): return { "matched_count": self.matched_count, } def matches(self, item): return self.re.match(item.title) if __name__ == "__main__": app = App() app.load_config('.feed-transmission.json') app.load_data(".feed-transmission.data") app.setup_env() app.main()