zs (6320B)
1 #!/usr/bin/env python 2 # coding=utf-8 3 # 4 # Copy me if you can. 5 # by 20h 6 # 7 8 import sys 9 import os 10 import zeitungsschau.feed as feed 11 import zeitungsschau.feeddb as feeddb 12 import zeitungsschau.opml as opml 13 import zeitungsschau.feedemail as feedemail 14 import socket 15 import http.client 16 import ssl 17 import getopt 18 import pprint 19 import requests.exceptions 20 import requests 21 import smtplib 22 import lxml 23 24 dodebug = False 25 26 def debug(msg): 27 global dodebug 28 if dodebug == True: 29 print("debug: %s" % (msg)) 30 31 def sendfeed(db, ufeed): 32 feedemail.send(ufeed, db.cfg["email"], db.cfg["smtphost"], \ 33 db.cfg["smtpport"], db.cfg["smtpssl"], \ 34 db.cfg["smtpstarttls"], db.cfg["smtpuser"], \ 35 db.cfg["smtppassword"], db.cfg["smtpcmd"], \ 36 db.cfg["smtpuselocal"]) 37 38 def run(db, selfeed=None, dryrun=False, onlychanges=False): 39 feeduris = db.listfeeds() 40 41 if feeduris != None and selfeed in feeduris: 42 feeduris = [selfeed] 43 44 for feeduri in feeduris: 45 if db.ispaused(feeduri): 46 print("pause %s" % (feeduri)) 47 continue 48 49 retries = db.getretry(feeduri) 50 estr = None 51 if onlychanges != True: 52 print("fetch %s" % (feeduri)) 53 curfeed = None 54 rcode = 0 55 56 """ 57 # All errors. 58 (rcode, curfeed) = feed.fetch(feeduri) 59 """ 60 try: 61 (rcode, curfeed) = feed.fetch(feeduri) 62 except socket.gaierror: 63 continue 64 except socket.timeout: 65 continue 66 except TimeoutError: 67 continue 68 except ConnectionResetError: 69 estr = "connreset" 70 retries += 1 71 except requests.exceptions.ConnectionError: 72 estr = "connreset" 73 retries += 1 74 except requests.exceptions.ReadTimeout: 75 continue 76 except requests.exceptions.ChunkedEncodingError: 77 continue 78 except lxml.etree.XMLSyntaxError: 79 estr = "xml error" 80 retries += 1 81 except requests.exceptions.TooManyRedirects: 82 estr = "redirects" 83 retries += 1 84 except OSError as err: 85 estr = err.strerror 86 retries += 1 87 88 if rcode == 404: 89 estr = "404" 90 retries += 1 91 92 if curfeed == None: 93 continue 94 95 # retry handling 96 if estr != None: 97 if retries > 2: 98 sys.stderr.write("pause %s %s\n" % \ 99 (estr, feeduri)) 100 db.pause(feeduri) 101 db.setretry(feeduri, retries) 102 continue 103 elif retries > 0: 104 db.setretry(feeduri, 0) 105 106 try: 107 clen = len(curfeed["articles"]) 108 except AttributeError: 109 continue 110 111 if clen == 0: 112 # This is no target anymore. Thanks NATO for your 113 # crappy RSS feed! 114 #print("0 articles -> pause %s" % (feeduri)) 115 #db.pause(feeduri) 116 continue 117 118 db.mergefeed(feeduri, curfeed) 119 ufeed = db.unreadarticles(feeduri) 120 if len(ufeed["articles"]) > 0 and onlychanges != True: 121 print("cur %d unread %d" % (clen, \ 122 len(ufeed["articles"]))) 123 debug(ufeed) 124 if dryrun == False: 125 try: 126 sendfeed(db, ufeed) 127 db.setreadarticles(feeduri, ufeed) 128 except smtplib.SMTPDataError: 129 return 130 except smtplib.SMTPSenderRefused: 131 return 132 133 class ExceptionHook: 134 instance = None 135 136 def __call__(self, *args, **kwargs): 137 if self.instance is None: 138 from IPython.core import ultratb 139 self.instance = ultratb.FormattedTB(mode='Verbose', 140 color_scheme='Linux', call_pdb=1) 141 return self.instance(*args, **kwargs) 142 143 def usage(app): 144 app = os.path.basename(app) 145 sys.stderr.write("usage: %s [-dhs] cmd\n" % (app)) 146 sys.exit(1) 147 148 def main(args): 149 global dodebug 150 retval = 0 151 152 try: 153 opts, largs = getopt.getopt(args[1:], "hds") 154 except getopt.GetoptError as err: 155 print(str(err)) 156 usage(args[0]) 157 158 silent = False 159 for o, a in opts: 160 if o == "-h": 161 usage(args[0]) 162 elif o == "-d": 163 dodebug = True 164 elif o == "-s": 165 silent = True 166 else: 167 usage(args[0]) 168 169 if len(largs) < 1: 170 usage(args[0]) 171 172 if dodebug == True: 173 sys.excepthook = ExceptionHook() 174 175 if largs[0] == "testfeed": 176 if len(largs) < 2: 177 print("usage: %s testfeed URI\n" % \ 178 (os.path.basename(args[0]))) 179 return 1 180 181 fe = feed.fetch(largs[1]) 182 pprint.pprint(fe) 183 return 0 184 185 db = feeddb.feeddb() 186 187 if largs[0] == "run": 188 if len(largs) > 1: 189 run(db, largs[1], onlychanges=silent) 190 else: 191 run(db, onlychanges=silent) 192 193 elif largs[0] == "dryrun": 194 if len(largs) > 1: 195 run(db, largs[1], dryrun=True, onlychanges=silent) 196 else: 197 run(db, dryrun=True, onlychanges=silent) 198 199 elif largs[0] == "cfg": 200 if len(largs) < 2: 201 for k in db.cfg: 202 print("%s = '%s'" % (k, db.cfg[k])) 203 elif len(args) < 3: 204 if largs[1] in db.cfg: 205 print("%s = '%s'" % (largs[1], \ 206 db.cfg[largs[1]])) 207 else: 208 retval = 1 209 else: 210 db.cfg[largs[1]] = largs[2] 211 print("%s = '%s'" % (largs[1], db.cfg[largs[1]])) 212 db.default = False 213 214 elif largs[0] == "cfgdel": 215 if len(largs) < 2: 216 usage(args[0]) 217 if largs[1] in db.cfg: 218 del db.cfg[largs[1]] 219 220 elif largs[0] == "add": 221 if len(largs) < 2: 222 usage(args[0]) 223 db.addfeed(largs[1]) 224 225 elif largs[0] == "list": 226 print("\n".join(db.listfeeds())) 227 228 elif largs[0] == "listuuids": 229 if len(largs) < 2: 230 usage(args[0]) 231 rfeed = db.readfeed(largs[1]) 232 for art in rfeed["articles"]: 233 print("%s: %s: %s" % (art["uuid"], art["link"],\ 234 art["title"])) 235 236 elif largs[0] == "unread": 237 if len(largs) < 3: 238 usage(args[0]) 239 db.setarticleunread(largs[1], largs[2]) 240 241 elif largs[0] == "resend": 242 if len(largs) < 2: 243 usage(args[0]) 244 ufeed = db.unreadarticles(largs[1]) 245 sendfeed(db, ufeed) 246 db.setreadarticles(largs[1], ufeed) 247 248 elif largs[0] == "del": 249 if len(largs) < 2: 250 usage(args[0]) 251 if db.delfeed(largs[1]) == True: 252 print("'%s' has been deleted." % (largs[1])) 253 254 elif largs[0] == "reset": 255 if len(largs) < 2: 256 usage(args[0]) 257 db.resetarticles(largs[1]) 258 259 elif largs[0] == "retry": 260 if len(largs) < 3: 261 usage(args[0]) 262 db.setretry(largs[1], int(largs[2])) 263 264 elif largs[0] == "pause": 265 if len(largs) < 2: 266 usage(args[0]) 267 db.pause(largs[1]) 268 269 elif largs[0] == "unpause": 270 if len(largs) < 2: 271 usage(args[0]) 272 db.unpause(largs[1]) 273 274 elif largs[0] == "opmlexport": 275 if len(largs) > 1: 276 filen = open(largs[1], "w") 277 else: 278 filen = sys.stdout 279 filen.write(opml.write(db.listfeeds())) 280 281 elif largs[0] == "opmlimport": 282 if len(largs) > 1: 283 filen = open(largs[1], "r") 284 else: 285 filen = sys.stdin 286 feedlist = db.listfeeds() 287 nfeedlist = opml.read(filen.read().encode("utf-8")) 288 for f in nfeedlist: 289 if not f in feedlist: 290 print("import feed: %s" % (f)) 291 db.addfeed(f) 292 293 del db 294 return retval 295 296 if __name__ == "__main__": 297 sys.exit(main(sys.argv)) 298