import requestsfrom lxml import etreeimport reimport osfrom multiprocessing.dummy import Pool as ThreadPool def gethtml(url,encode): #获取网页源码 r = requests.get(url) r.encoding = encode return r.text def filterFName(FName): #文件名过滤特殊字符 rstr = r"[\/\\\:\*\?\"\<\>\|]" new_name = re.sub(rstr, "_", FName) return new_name def mkdir(path): #创建文件夹 path = path.strip() isExists = os.path.exists(path) if not isExists: os.makedirs(path) def geturl(url): #获取网站各个分类链接、分类名称 html = gethtml(url,'utf-8') ehtml = etree.HTML(html) nurl = ehtml.xpath('//*[@id="chenxing_menu"]/li/a/@href') ntitle = ehtml.xpath('//*[@id="chenxing_menu"]/li/a/text()') urldata=[] for i in range(1,len(nurl)-1): urldata.append(nurl[i] '|' ntitle[i]) return urldata def downPic(url,savepath): #下载图片 picname=url.split('/')[-1] print('download: ' url) picdata = requests.get(url) with open(savepath picname, 'wb') as file: file.write(picdata.content) def getdata(ulist): #根据网站分类链接、名称进一步解析 这段代码是下载图片主体代码 url = ulist.split('|')[0] til = ulist.split('|')[1] html = gethtml(url,'utf-8') ehtml = etree.HTML(html) count = ''.join(ehtml.xpath('//*[@class="more r"]/em/text()')) page = int(count) // 20 if int(count) % 20 > 0 : page =1 for i in range(1,page 1):#获取各分类下面所有图集链接 nurl = url 'page/' str(i) html = gethtml(nurl, 'utf-8') ehtml = etree.HTML(html) lvurl = ehtml.xpath('//*[@class="img"]/@href') for j in lvurl: #获取每个图集页数和链接 html = gethtml(j, 'utf-8') ehtml = etree.HTML(html) lvurl = ''.join(ehtml.xpath('//*[@id="imagecx"]/h1/span/text()')) # 获取图集页数 title = ''.join(ehtml.xpath('//*[@id="imagecx"]/h1/text()')) #获取图集名称 title = title.replace('()','') lvurl = lvurl.replace('1/','') path = savepath '\\' til '\\' filterFName(title) '\\' mkdir(path) for k in range(1,int(lvurl)): #获取每个图集页面的图片 zurl = j.replace('.html','_' str(k) '.html') html = gethtml(zurl,'urf-8') ehtml = etree.HTML(html) picurl = ehtml.xpath('//*[@class="image_cx_cont"]/img/@src') for l in picurl: downPic(l,path) #下载图片 if __name__ == '__main__': url = 'http://mzsock.com' savepath='C:\\mzsock' #保存位置 urllist=geturl(url) pool = ThreadPool(4) results = pool.map(getdata, urllist) pool.close() pool.join() print('任务全部完成~！')
评论