爬虫入门笔记

import re
import urllib.request
import urllib

from collections import deque

queue = deque()
visited = set()

url = ‘https://jecvay.com/‘

queue.append(url)
cnt = 0

while queue:
    url = queue.pop()
    visited.add(url)

    print(‘Count: ‘ + str(cnt) + ‘ visiting <--- ‘ + url)
    cnt += 1
    urlop = urllib.request.urlopen(url)
    if ‘html‘ not in urlop.getheader(‘Content-Type‘):
        continue

    try:
        data = urlop.read().decode(‘utf-8‘)
    except:
        continue

    linkre = re.compile(r‘href="(.+?)"‘)
    
    for x in linkre.findall(data):
        if ‘http‘ in x and x not in visited:
            queue.append(x)
            print(‘add---> ‘ + x)