爬虫实战 豆瓣top250
准备工具
- Visual Studio Code
- Microsoft Edge
- python
任务开始
网页爬虫就是对网页元素的，获取，解析，汇总。
这里我们所需要操作的就是网页的静态元素
请勿对网站频繁请求，增加目标站负载，礼貌爬虫哦~
代码
import csv import requests from bs4 import BeautifulSoup import pandas as pd import re # 设置请求头，避免被豆瓣屏蔽 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' } # 获取豆瓣电影Top250的前五页数据 base_url = "https://movie.douban.com/top250" movie_list = [] def get_page(url): response = requests.get(url, headers=headers) # 验证请求结果 if response.status_code != 200: print(f"请求失败，状态码: {response.status_code}") return f"请求失败，请勿频繁请求{response.status_code}" return response.text # 正则匹配导演的中文名和英文名 def extract_director_names(text): # 匹配中文名：寻找“导演:”后面的中文序列 chinese_name_pattern = r"导演:\s*([\u4e00-\u9fa5·]+)" chinese_match = re.search(chinese_name_pattern, text) chinese_name = chinese_match.group(1) if chinese_match else "未匹配到中文名" return chinese_name # 对电影链接请求 def get_movie_details(url): response = requests.get(url, headers=headers) soup = BeautifulSoup(response.text, 'html.parser') # 获取所有编剧 screenwriters = [] screenwriter_span = soup.find('span', class_='pl', string=re.compile('编剧')) if screenwriter_span: screenwriters_span = screenwriter_span.find_next_sibling('span', class_='attrs') if screenwriters_span: screenwriters = [a.text.strip() for a in screenwriters_span.find_all('a')] # 获取所有演员 actors = [] actor_span = soup.find('span', class_='pl', string=re.compile('主演')) if actor_span: actors_container = actor_span.find_next_sibling('span', class_='attrs') if actors_container: actors = [a.text.strip() for a in actors_container.find_all('a', rel='v:starring')] # 获取电影其他数据 # 类型 move_type_span = soup.find('span', class_='pl', string=re.compile('类型:')) if move_type_span: move_type = move_type_span.find_next_sibling('span',class_='v:genre').text.strip() # 制片 diqu_span = soup.find('span', class_='pl', string=re.compile('制片国家/地区:')) if diqu_span: diqu = diqu_span.find_next_sibling(text=True).strip() # 语言 language_span = soup.find('span', class_='pl', string=re.compile('语言:')) if language_span: language = language_span.find_next_sibling(text=True).strip() # 上映日期 release_date_span = soup.find('span', class_='pl', string=re.compile('上映日期:')) if release_date_span: release_date = release_date_span.find_next_sibling('span',property='v:initialReleaseDate').text.strip() # 片长 duration_span = soup.find('span', class_='pl', string=re.compile('片长:')) if duration_span: duration = duration_span.find_next_sibling('span', property='v:runtime').text.strip() # 别名 aka_span = soup.find('span', class_='pl', string=re.compile('又名:')) if aka_span: aka = aka_span.find_next_sibling(text=True).strip() # IMDb imdb_span = soup.find('span', class_='pl', string=re.compile('IMDb:')) if imdb_span: imdb = imdb_span.find_next_sibling(text=True).strip() # 剧情简介 summary = '' summary_span= soup.find('span', property='v:summary') if summary_span: summary = summary_span.text.strip() return { 'screenwriters': screenwriters, 'actors': actors, 'move_type': move_type, 'diqu': diqu, 'language': language, 'release_date': release_date, 'duration': duration, 'aka': aka, 'imdb': imdb, 'summary': summary } def parse_page(html): soup = BeautifulSoup(html, 'html.parser') movies = soup.find_all('div', class_='item') for movie in movies: try: # 安全地提取基本信息（添加空值检查） em_elem = movie.find('em') title_elem = movie.find('span', class_='title') link_elem = movie.find('a') rating_elem = movie.find('span', class_='rating_num') # 检查关键元素是否存在 if not all([em_elem, title_elem, link_elem, rating_elem]): print("跳过不完整的电影条目") continue em = em_elem.text.strip() title = title_elem.text.strip() link = link_elem.get('href', '').strip() rating = rating_elem.text.strip() # 安全提取导演和演员信息 bd_elem = movie.find('div', class_='bd') director_actors = '' if bd_elem: p_elem = bd_elem.find('p') if p_elem: director_actors = p_elem.text.strip() dacn = extract_director_names(director_actors) if director_actors else '' # 安全获取详细信息 data = get_movie_details(link) or {} # 确保data是字典 # 使用get方法安全访问字典，避免KeyError movie_info = { '排名': em, '名字': title, '别名': data.get('aka', ''), '链接': link, '评分': rating, '导演': dacn, '编剧': data.get('screenwriters', []), # 默认为空列表 '主演': data.get('actors', []), # 默认为空列表 '类型': data.get('move_type', ''), '制片国家/地区': data.get('diqu', ''), '语言': data.get('language', ''), '上映日期': data.get('release_date', ''), '片长': data.get('duration', ''), 'IMDb': data.get('imdb', ''), '剧情简介': data.get('summary', '') } # 确保movie_list已定义 if 'movie_list' not in globals(): global movie_list movie_list = [] movie_list.append(movie_info) except Exception as e: print(f"处理电影条目时出错: {e}") continue # 跳过当前条目，继续处理下一个 def main(): for start in range(0, 250, 25): url = f"{base_url}?start={start}" html = get_page(url) parse_page(html) # 输出结果 for movie in movie_list: print(movie) if __name__ == "__main__": main() # 保存数据到CSV文件 def save_to_csv(): if not movie_list: # 或者使用 len(movie_list) == 0 print("警告：movie_list为空，没有数据可保存。") return # 直接返回，不执行后续保存操作 keys = movie_list[0].keys() with open('douban_top250.csv', 'w', newline='', encoding='utf-8-sig') as output_file: dict_writer = csv.DictWriter(output_file, fieldnames=keys) dict_writer.writeheader() dict_writer.writerows(movie_list) save_to_csv()
