commit de38800ce7a8ad243771b74cf84e36a84aebc0d1
parent 4d0d209d7b746599be83f63aad1c61682de36a1e
Author: Christoph Lohmann <20h@r-36.net>
Date: Mon, 17 Mar 2014 18:08:20 +0100
Add retry handling on error.
Diffstat:
| feeddb.py | | | 23 | +++++++++++++++++------ |
| zs.py | | | 26 | +++++++++++++++++++++++--- |
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/feeddb.py b/feeddb.py
@@ -106,18 +106,29 @@ class feeddb(object):
feed[key] = value
self.writefeed(uri, feed)
- def pause(self, uri):
- self.setfeedval(uri, "pause", True)
-
- def unpause(self, uri):
- self.setfeedval(uri, "pause", False)
-
def getfeedval(self, uri, key):
feed = self.readfeed(uri)
if feed == None:
return None
return feed[key]
+ def setretry(self, uri, retries):
+ self.setfeedval(uri, "retry", retries)
+
+ def incretry(self, uri):
+ retries = self.getfeedval(uri, "retry")
+ if retries == None:
+ self.setfeedval(uri, "retry", 1)
+ else:
+ self.setfeedval(uri, "retry", retries+1)
+
+ def pause(self, uri):
+ self.setfeedval(uri, "pause", True)
+
+ def unpause(self, uri):
+ self.setfeedval(uri, "pause", False)
+ self.setretry(uri, 0)
+
def ispaused(self, uri):
return self.getfeedval(uri, "pause")
diff --git a/zs.py b/zs.py
@@ -25,18 +25,33 @@ def run(db, selfeed=None, dryrun=False):
print("pause %s" % (feeduri))
continue
+ retries = db.getretry(feeduri)
+ estr = None
print("fetch %s" % (feeduri))
try:
curfeed = feed.fetch(feeduri)
except urllib.error.HTTPError as err:
if err.code == 404:
- print("404 -> pause %s" % (feeduri))
- db.pause(feeduri)
- continue
+ estr = "404"
+ retries += 1
except socket.gaierror:
continue
except TimeoutError:
continue
+ except ConnectionResetError:
+ estr = "connreset"
+ retries += 1
+
+ # retry handling
+ if estr != None:
+ if retries > 2:
+ sys.stderr.write("pause %s %s\n" % \
+ (estr, feeduri))
+ db.pause(feeduri)
+ db.setretry(feeduri, retries)
+ continue
+ elif retries > 0:
+ db.setretry(feeduri, 0)
clen = len(curfeed["articles"])
if clen == 0:
@@ -114,6 +129,11 @@ def main(args):
usage(args[0])
db.resetarticles(args[1])
+ elif args[1] == "retry":
+ if len(args) < 4:
+ usage(args[0])
+ db.setretry(args[1], int(args[2]))
+
elif args[1] == "pause":
if len(args) < 3:
usage(args[0])
