1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
#!/usr/bin/env python
import feedparser
import re
import time
import subprocess
import urllib2
import tempfile
import shelve
import ConfigParser
class App(object):
def __init__(self):
self.seen = []
self.downloaded = []
self.feeds = []
def load_config(self):
config = ConfigParser.RawConfigParser({})
config.readfp(file('.feed-transmission.ini'))
feed_names = config.get("config", "feeds")
feed_names = feed_names.split(",")
for i in feed_names:
i = i.strip()
f = {}
url = config.get(i, "url")
poll_delay = config.getint(i, "poll_delay")
match = []
for m in config.items(i):
if not m[0].startswith("match"):
continue
try:
re.compile(m[1])
except re.error:
print "Invalid regular expression at %s, %s: %s" % (i, m[0], m[1])
match.append(m[1])
match = "|".join(match)
match = re.compile(match)
self.add_feed(Feed(url, poll_delay, match))
def load_stor(self):
self.stor = shelve.open(".feed-transmission.data")
if not "downloaded" in self.stor:
self.stor["downloaded"] = []
if not "seen" in self.stor:
self.stor["seen"] = []
self.downloaded = self.stor["downloaded"]
self.seen = self.stor["seen"]
def setup_env(self):
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
urllib2.install_opener(opener)
def add_feed(self, feed):
self.feeds.append(feed)
def get_next_feed(self):
"""Returns a (delay, feed) tuple of time to delay before next feed, and
next feed to run
"""
# TODO
return (self.feeds[0].poll_delay, self.feeds[0])
def run_relavent(self, feed):
found = feed.find_relavant(self.seen)
self.seen = self.seen[-500:]
self.stor["seen"] = self.seen
for i in found:
if i in self.downloaded:
continue
self.downloaded.append(i)
f = urllib2.urlopen(i)
temp = tempfile.NamedTemporaryFile()
temp.write(f.read())
f.close()
temp.flush()
ret = subprocess.call(["transmission-remote", "localhost", "-a", temp.name])
temp.close()
if ret:
print "Error adding torrent"
self.downloaded = self.downloaded[-500:]
self.stor["downloaded"] = self.downloaded
self.stor.sync()
def main(self):
try:
for feed in self.feeds:
self.run_relavent(feed)
while True:
(delay, feed) = self.get_next_feed()
time.sleep(delay)
try:
self.run_relavent(feed)
except urllib2.URLError, e:
print e
except KeyboardInterrupt:
pass
class Feed(object):
def __init__(self, url, poll_delay, match):
self.url = url
self.poll_delay = poll_delay
self.match = match
def load_feed(self):
f = urllib2.urlopen(self.url)
d = feedparser.parse(f.read())
f.close()
return d
def find_relavant(self, seen):
d = self.load_feed()
found = []
print "New RSS Items:"
for i in d['items']:
if i.title in seen:
continue
seen.append(i.title)
print " ", i.title
if not self.match.match(i.title):
continue
print " Matched"
found.append(i.link)
return found
if __name__ == "__main__":
app = App()
app.load_config()
app.load_stor()
app.setup_env()
app.main()
|
