-
Notifications
You must be signed in to change notification settings - Fork 25
/
whatsnew-q.py
executable file
·109 lines (92 loc) · 3.45 KB
/
whatsnew-q.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
#!/usr/bin/env python3
# Copyright © 2012-13 Qtrac Ltd. All rights reserved.
# This program or module is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version. It is provided for
# educational purposes and is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
import argparse
import multiprocessing
import os
import queue
import tempfile
import threading
import webbrowser
import Feed
import Qtrac
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
jobs = queue.Queue()
results = queue.Queue()
create_threads(limit, jobs, results, concurrency)
todo = add_jobs(filename, jobs)
process(todo, jobs, results, concurrency)
def handle_commandline():
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--limit", type=int, default=0,
help="the maximum items per feed [default: unlimited]")
parser.add_argument("-c", "--concurrency", type=int,
default=multiprocessing.cpu_count() * 4,
help="specify the concurrency (for debugging and "
"timing) [default: %(default)d]")
args = parser.parse_args()
return args.limit, args.concurrency
def create_threads(limit, jobs, results, concurrency):
for _ in range(concurrency):
thread = threading.Thread(target=worker, args=(limit, jobs,
results))
thread.daemon = True
thread.start()
def worker(limit, jobs, results):
while True:
try:
feed = jobs.get()
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
elif result is not None:
Qtrac.report("read {}".format(result[0][4:-6]))
results.put(result)
finally:
jobs.task_done()
def add_jobs(filename, jobs):
for todo, feed in enumerate(Feed.iter(filename), start=1):
jobs.put(feed)
return todo
def process(todo, jobs, results, concurrency):
canceled = False
try:
jobs.join() # Wait for all the work to be done
except KeyboardInterrupt: # May not work on Windows
Qtrac.report("canceling...")
canceled = True
if canceled:
done = results.qsize()
else:
done, filename = output(results)
Qtrac.report("read {}/{} feeds using {} threads{}".format(done, todo,
concurrency, " [canceled]" if canceled else ""))
print()
if not canceled:
webbrowser.open(filename)
def output(results):
done = 0
filename = os.path.join(tempfile.gettempdir(), "whatsnew.html")
with open(filename, "wt", encoding="utf-8") as file:
file.write("<!doctype html>\n")
file.write("<html><head><title>What's New</title></head>\n")
file.write("<body><h1>What's New</h1>\n")
while not results.empty(): # Safe because all jobs have finished
result = results.get_nowait()
done += 1
for item in result:
file.write(item)
file.write("</body></html>\n")
return done, filename
if __name__ == "__main__":
main()