python多线程下载豆瓣相册

发布时间:2020-07-14编辑:脚本学堂
有关python多线程的例子,python多线程下载豆瓣相册,用到了python threading线程模块。

文件download.py,代码:    
 

复制代码 代码示例:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2014-12-22 14:46:40
# @Author  : kuas (hukuas@gmail.com)
# @Version : $Id$
 
import _thread
from http import cookiejar
import os
import random
import re
import threading
import time
import urllib.request
 
userAgents = [{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0'},
    {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1) Gecko/20090624 Firefox/3.5"},
    {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER"},
    {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)"},
    {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0) Gecko/20121026 Firefox/16.0"},
    {"User-Agent":"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre"},
    {"User-Agent":"Mozilla/5.0 (X11; Ubuntu; linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0"},
    {"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15"},
    {"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11"},
    {"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133"},
    {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)"},
    {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"},
    {"User-Agent":"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"}, 
    {"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"},
    {"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"},
    {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101"},
    {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"}]
 
##只需要设置豆瓣相册地址ActivityURL即可
ActivityURL = "http://www.douban.com/online/11748222/album/131483485"
SAVE_DIR = "D:downPic" ##保存的文件目录
downPicCount = 16 ##下载图片线程数
clawThreadCount = 1 ##扒取图片链接线程数
 
##以下参数不用设置
pics = []
urls = []
openers = []
exitFlag = 0
picLock = threading.Lock()
urlLock = threading.Lock()
pageNum = 0
ActDir = "tmp"
PageSize = 18 #每页照片数量
MaxPageNum = 10  ##总共几页
 
def getRandomHeaders():
    headers = []
    headers.append(("User-Agent", random.choice(userAgents)["User-Agent"]))
    headers.append(("Accept-Language", "zh-cn,zh;q=0.8;"))
    headers.append(("Cache-Control", "max-age=0"))
    headers.append(("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"))
    return headers
 
def initOpeners(openerCount):
    for i in range(0,openerCount):
        cj = cookiejar.CookieJar()
        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
        opener.addheaders = getRandomHeaders()
        urllib.request.install_opener(opener)
        openers.append(opener)
        
def getRandomOpener():
    return random.choice(openers)
 
def downLoadImage(url, savePath):
    testCount = 0
    while testCount < 3:
        try:
            f = getRandomOpener().open(url)
            file = open(savePath, 'wb')
            file.write(f.read())
            file.close()
            f.close()
            break
        except Exception as e:
            print("DownLoad image %s Error:%s"%(url,str(e)))
            testCount += 1
    
    
def getHtml(url):
    testCount = 0
    html = ""
    while testCount < 3:
        try:
            f = getRandomOpener().open(url)
            html = f.read().decode('utf-8')
            f.close()
            break
        except Exception as e:
            print("getHtml %s Error:%s"%(url,str(e)))
            testCount += 1
    return html
 
class ClawThread(threading.Thread):
    def __init__(self, name):
        threading.Thread.__init__(self)
        self.name = name
    def run(self):
        global urls,pageNum
        while exitFlag == 0:
            time.sleep(0.1 * random.randint(0, 10))
            urlLock.acquire()
            pUrl = ""
            if len(urls) > 0:
                pUrl = urls[0]
                urls.remove(pUrl)
            else:
                if pageNum == MaxPageNum:
                    break
                endAddPage = pageNum + 5;
                endAddPage = min(endAddPage,MaxPageNum)
                while pageNum < endAddPage:
                    urls.append(ActivityURL % (pageNum*PageSize))
                    pageNum += 1
            urlLock.release()
            if pUrl != "":
                html = getHtml(pUrl)
                getPicURL(html)
 
class DownPicThread(threading.Thread):
    def __init__(self, name):
        threading.Thread.__init__(self)
        self.name = name
    def run(self):
        global pics
        while exitFlag == 0:
            time.sleep(0.01 * random.randint(0, 10))
            picLock.acquire()
            picUrl = ""
            if len(pics) > 0:
                picUrl = pics[0]
                pics.remove(picUrl)
            picLock.release()
            if picUrl != "":
                fileName = picUrl[picUrl.rindex('/')+1:]
                filePath = SAVE_DIR + ActDir + fileName
                if not os.path.exists(filePath):
                    downLoadImage(picUrl, filePath)
 
def getPicURL(html):
    reg = r"http://imgd.douban.com/view/photo/thumb/public/pd+.jpg"
    picURLs = re.findall(reg, html)
    picLock.acquire()
    for picurl in picURLs:
        pics.append(picurl.replace("thumb", "photo"))
    picLock.release()
 
def initData():
    global ActivityURL,ActDir,SAVE_DIR,PageSize,MaxPageNum
    if not os.path.exists(SAVE_DIR):
        os.mkdir(SAVE_DIR)
 
    regNum = "/d+"
    html = getHtml(ActivityURL)
    regStartNum = "?start=d+"
    startStrs = re.findall(regStartNum,html)
    maxStartInt = 0
    for startStr in startStrs:
        maxStartInt = max(int(startStr[7:]),maxStartInt)
    nums = re.findall(regNum,ActivityURL)
    
    if len(nums) == 2:#线上活动相册
        ActDir = nums[1][1:]+"/"
        PageSize = 90
        ActivityURL = ActivityURL +"?start=%d&sortby=popularity"
    elif len(nums) == 1:#个人相册
        ActDir = nums[0][1:]+"/"
        PageSize = 18
        ActivityURL = ActivityURL +"?start=%d"
    MaxPageNum = maxStartInt/PageSize + 1
    print("总页数:%d"%(MaxPageNum))
    print("PageSize:%d"%(PageSize))
    
if __name__ == '__main__':
    
    initOpeners(10)
    initData()
    
    threads = []
    
    if not os.path.exists(SAVE_DIR + ActDir):
        os.mkdir(SAVE_DIR + ActDir)
     
    for i in range(0,clawThreadCount):
        thread = ClawThread("%d"%(i))
        thread.start()
        threads.append(thread)   
 
    for i in range(0,downPicCount):
        thread = DownPicThread("%d"%(i))
        thread.start()
        threads.append(thread)
        
    while True:
        time.sleep(1)
        print("Downing:%d  ----------- Finished:%d"%(len(pics),len(os.listdir(SAVE_DIR + ActDir))))
        if len(urls) == 0 and len(pics) == 0 and pageNum == MaxPageNum:
            exitFlag = 1
            break
    print("Have DownLoaded %d files!"%(len(os.listdir(SAVE_DIR + ActDir))))
    print("Waiting for all thread exit...")
    for thread in threads:
        thread.join()