标签：back 截断 pen port 针对 content put width 如何
先看下我的爬取成果：
很多人学习python，不知道从何学起。
很多人学习python，掌握了基本语法过后，不知道在哪里寻找案例上手。
很多已经做案例的人，却不知道如何去学习更加高深的知识。
那么针对这三类人，我给大家提供一个好的学习平台，免费领取视频教程，电子书籍，以及课程的源代码！
QQ群：961562169
本视频的演示步骤：
这2个库的详细用法，请看我的其他视频课程
import requests
from bs4 import BeautifulSoup
import re
def download_all_htmls():
"""
下载所有列表页面的HTML，用于后续的分析
"""
htmls = []
for idx in range(200):
url = f"https://fabiaoqing.com/biaoqing/lists/page/{idx+1}.html"
print("craw html:", url)
r = requests.get(url)
if r.status_code != 200:
raise Exception("error")
htmls.append(r.text)
print("success")
return htmls
# 执行爬取
htmls = download_all_htmls()
craw html: https://fabiaoqing.com/biaoqing/lists/page/1.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/2.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/3.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/4.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/188.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/189.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/190.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/191.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/192.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/193.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/194.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/195.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/196.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/197.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/198.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/199.html
craw html: https://fabiaoqing.com/biaoqing/lists/page/200.html
success
htmls[0][:1000]
‘<html>\n\n<head>\n <meta charset="UTF-8">\n <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">\n <meta http-equiv="X-UA-Compatible" content="IE=edge">\n <meta http-equiv="content-type" content="text/html; charset=UTF-8">\n <title>热门表情_发表情，表情包大全fabiaoqing.com</title>\n <meta name="Keywords" content="热门表情,表情包,聊天表情,微信表情包,QQ表情包,发表情,表情包大全,表情包下载,表情下载,表情包大战,贴吧表情包,表情包集中营,斗图">\n <meta name="Description" content="全网热门表情。发表情，最大最全的表情包网站，分享最新最热的表情包、聊天表情、微信表情包、QQ表情包、金馆长表情包、蘑菇头表情包等各类表情。">\n <meta name="referrer" content="no-referrer" />\n <link rel="stylesheet" type="text/css" href="//lib.baomitu.com/semantic-ui/2.2.2/semantic.min.css" />\n <link rel="stylesheet" type="text/css" href="/Public/css/fbq.css?v=2018" />\n <script data-ad-client="ca-pub-5486123269162001" async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>\n <script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.‘
def parse_single_html(html):
"""
解析单个HTML，得到数据
@return list((img_title, img_url))
"""
soup = BeautifulSoup(html, ‘html.parser‘)
img_divs = soup.find_all("div", class_="tagbqppdiv")
datas = []
for img_div in img_divs:
img_node = img_div.find("img")
if not img_node: continue
datas.append((img_node["title"], img_node["data-original"]))
return datas
import pprint
pprint.pprint(parse_single_html(htmls[0])[:10])
[(‘阿弥陀佛，施主放下骂图，立地成佛！‘,
‘http://ww2.sinaimg.cn/bmiddle/9150e4e5gy1g6qlfb10avj20d70f7gmf.jpg‘),
(‘看见你就烦（草莓果酱ox白眼 GIF 动图表情包）‘,
‘http://wx1.sinaimg.cn/bmiddle/006APoFYly1g68tiftpbmg30bh0bh4o5.gif‘),
(‘我在哭‘, ‘http://wx3.sinaimg.cn/bmiddle/006qir4ogy1g54eoes2q2j309q09cdgh.jpg‘),
(‘我的人生只要这样躺着混日子就很幸福了‘,
‘http://ww4.sinaimg.cn/bmiddle/9150e4e5gy1g6qm7x6fiuj20mw0mmt9y.jpg‘),
(‘草莓果酱ox动图表情包‘,
‘http://wx1.sinaimg.cn/bmiddle/ceeb653ely1g64664qyc0g20bf0br4jn.gif‘),
(‘噗呲 放屁（沙雕羊驼动图表情包）‘,
‘http://wx1.sinaimg.cn/bmiddle/78b88159gy1g69cze2hkkg20bp0bpx0y.gif‘),
(‘来群里转转（熊猫头旋转 GIF 动图）‘,
‘http://wx1.sinaimg.cn/bmiddle/ceeb653ely1g68tzab8xng207608wwou.gif‘),
(‘我爱你（草莓果酱oxGIF 动图表情包）‘,
‘http://wx2.sinaimg.cn/bmiddle/006APoFYly1g68uwg8djlg30b60b6e57.gif‘),
(‘锁屏 带薪拉屎‘,
‘http://wx3.sinaimg.cn/bmiddle/ceeb653ely1g654hwdsjkg20dc0avgm4.gif‘),
(‘我要可爱死你（草莓果酱ox表情包）‘,
‘http://wx2.sinaimg.cn/bmiddle/bf976b12gy1g68hx2gtleg208c08bk8q.gif‘)]
# 执行所有的HTML页面的解析
all_imgs = []
for html in htmls:
all_imgs.extend(parse_single_html(html))
all_imgs[:10]
[(‘阿弥陀佛，施主放下骂图，立地成佛！‘,
‘http://ww2.sinaimg.cn/bmiddle/9150e4e5gy1g6qlfb10avj20d70f7gmf.jpg‘),
(‘看见你就烦（草莓果酱ox白眼 GIF 动图表情包）‘,
‘http://wx1.sinaimg.cn/bmiddle/006APoFYly1g68tiftpbmg30bh0bh4o5.gif‘),
(‘我在哭‘, ‘http://wx3.sinaimg.cn/bmiddle/006qir4ogy1g54eoes2q2j309q09cdgh.jpg‘),
(‘我的人生只要这样躺着混日子就很幸福了‘,
‘http://ww4.sinaimg.cn/bmiddle/9150e4e5gy1g6qm7x6fiuj20mw0mmt9y.jpg‘),
(‘草莓果酱ox动图表情包‘,
‘http://wx1.sinaimg.cn/bmiddle/ceeb653ely1g64664qyc0g20bf0br4jn.gif‘),
(‘噗呲 放屁（沙雕羊驼动图表情包）‘,
‘http://wx1.sinaimg.cn/bmiddle/78b88159gy1g69cze2hkkg20bp0bpx0y.gif‘),
(‘来群里转转（熊猫头旋转 GIF 动图）‘,
‘http://wx1.sinaimg.cn/bmiddle/ceeb653ely1g68tzab8xng207608wwou.gif‘),
(‘我爱你（草莓果酱oxGIF 动图表情包）‘,
‘http://wx2.sinaimg.cn/bmiddle/006APoFYly1g68uwg8djlg30b60b6e57.gif‘),
(‘锁屏 带薪拉屎‘,
‘http://wx3.sinaimg.cn/bmiddle/ceeb653ely1g654hwdsjkg20dc0avgm4.gif‘),
(‘我要可爱死你（草莓果酱ox表情包）‘,
‘http://wx2.sinaimg.cn/bmiddle/bf976b12gy1g68hx2gtleg208c08bk8q.gif‘)]
len(all_imgs)
8999
for idx, (title, img_url) in enumerate(all_imgs):
# 移除标点符号，只保留中文、大小写字母和阿拉伯数字
reg = "[^0-9A-Za-z\u4e00-\u9fa5]"
title = re.sub(reg, ‘‘, title)
# 发现了超长的图片标题，做截断
if len(title)>10: title = title[:10]
# 得到jpg还是gif后缀
post_fix = img_url[-3:]
filename = f"./output/{title}.{post_fix}"
print(idx, filename)
img_data = requests.get(img_url)
with open(filename,"wb")as f:
f.write(img_data.content)
print("success")
0 ./output/阿弥陀佛施主放下骂图.jpg
1 ./output/看见你就烦草莓果酱o.gif
2 ./output/我在哭.jpg
3 ./output/我的人生只要这样躺着.jpg
4 ./output/草莓果酱ox动图表情.gif
5 ./output/噗呲放屁沙雕羊驼动图.gif
6 ./output/来群里转转熊猫头旋转.gif
7 ./output/我爱你草莓果酱oxG.gif
8 ./output/锁屏带薪拉屎.gif
9 ./output/我要可爱死你草莓果酱.gif
10 ./output/我尼玛傻了都.jpg
11 ./output/你今天表现蛮好10分.gif
12 ./output/真烦人得找个理由做她.gif
13 ./output/哇哦草莓果酱ox表情.jpg
14 ./output/哥哥又说笑了乔碧萝表.gif
15 ./output/锁屏带薪拉屎.gif
16 ./output/我简直难上加难麻将表.jpg标签：back 截断 pen port 针对 content put width 如何
原文地址：https://www.cnblogs.com/41280a/p/13714877.html
