scanWeb.py

# -*- coding: utf-8 -*-
import requests
import re
import threading
import queue
import time
import sys
from config import headers, cookie

urls = []

timeout = 3# 超时时间
dic="database\\dir.txt"
i=0
sum=0

class Boom(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
    
    def run(self):
        global urls, headers, timeout
        while True:
            if self.queue.empty():
                break
            try:
                i+=1
                print('percent: {:.2%}\r'.format(i/sum),end='',flush=True)
                url = self.queue.get_nowait()
                res = requests.get(url, headers = headers, timeout = timeout, cookies = cookie)
                if res.status_code == 200:
                    print(url)
                    urls.append(url)
            except KeyboardInterrupt:
                break
            except:
                pass

# pylint:disable=unused-variable

def main(url, threadsCount):
    threads_count = threadsCount
    global sum
    global host
    host = url

    with open(dic, 'r') as f:
        q = queue.Queue()
        for eachline in f:
            q.put(url + eachline[:-1])
            sum+=1

        threads = []
        for i in range(threads_count):
            threads.append(Boom(q))

        for t in threads:
            t.start()

        for t in threads:
            t.join()
    return urls
    
if __name__ == "__main__":
    if len(sys.argv) == 2:
        main(sys.argv[1], 10)
        #print(urls)
        exit(0)
    else:
        exit(-1)