file_getter.py

import sys
import requests
import threading
import Queue


class Worker(threading.Thread):

	def __init__(self,queue, target, suffix = ""):
		threading.Thread.__init__(self)
		self.queue = queue
		self.target = target
		self.suffix = suffix
		
	#Process data
	def run(self):
		while True:
			piece = self.queue.get()
			words = piece.split("\n")
			for word in words:
				url = self.target + word + self.suffix
				#print "[*] Trying: " + url
				try:
					r = requests.head(url)
					if r.status_code == 200:
						print "[+] 200 - " + url
				except:
					print "[*] Request Error: " + url
			self.queue.task_done()

# http://stackoverflow.com/questions/519633/lazy-method-for-reading-big-file-in-python
def read_in_chunks(file_object, chunk_size=1024):
    """Lazy function (generator) to read a file piece by piece.
    Default chunk size: 1k."""
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data

def create_circular_seq(elements):
    i = 0
    while True:
        yield elements[i]
        i = (i + 1)%len(elements)
	
def main():
	
	if len(sys.argv) < 4:
		print "Usage:%s http://<target-host>/ <dictionary-file> <threads> <optional: suffix>" % sys.argv[0]
		exit(1)
	
	target = sys.argv[1]  # target-host
	f = open(sys.argv[2]) # dictionary to use
	threads = int(sys.argv[3]) # number of threads
	
	if len(sys.argv) == 5:
		suffix = sys.argv[4]
	else:
		suffix = ""
		
	#Initialize queues
	queues = []	
	for i in range(0,threads):
		q = Queue.Queue(maxsize = 1) # one job at a time
		queues.append(q)
	
	queue_list = create_circular_seq(queues)
	
	for q in queues:
		worker = Worker(q, target,suffix)
		worker.setDaemon(True)
		worker.start()
	
	for piece in read_in_chunks(f):
		nq = queue_list.next() # next queue
		nq.put(piece)
		
	for q in queues:
		q.join()

if __name__ == "__main__":	
    main()