爬虫实战 豆瓣top250准备工具Visual Studio CodeMicrosoft Edgepython任务开始网页爬虫就是对网页元素的获取解析汇总。这里我们所需要操作的就是网页的静态元素请勿对网站频繁请求增加目标站负载礼貌爬虫哦~代码import csv import requests from bs4 import BeautifulSoup import pandas as pd import re # 设置请求头避免被豆瓣屏蔽 headers { User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 } # 获取豆瓣电影Top250的前五页数据 base_url https://movie.douban.com/top250 movie_list [] def get_page(url): response requests.get(url, headersheaders) # 验证请求结果 if response.status_code ! 200: print(f请求失败状态码: {response.status_code}) return f请求失败请勿频繁请求{response.status_code} return response.text # 正则匹配导演的中文名和英文名 def extract_director_names(text): # 匹配中文名寻找“导演:”后面的中文序列 chinese_name_pattern r导演:\s*([\u4e00-\u9fa5·]) chinese_match re.search(chinese_name_pattern, text) chinese_name chinese_match.group(1) if chinese_match else 未匹配到中文名 return chinese_name # 对电影链接请求 def get_movie_details(url): response requests.get(url, headersheaders) soup BeautifulSoup(response.text, html.parser) # 获取所有编剧 screenwriters [] screenwriter_span soup.find(span, class_pl, stringre.compile(编剧)) if screenwriter_span: screenwriters_span screenwriter_span.find_next_sibling(span, class_attrs) if screenwriters_span: screenwriters [a.text.strip() for a in screenwriters_span.find_all(a)] # 获取所有演员 actors [] actor_span soup.find(span, class_pl, stringre.compile(主演)) if actor_span: actors_container actor_span.find_next_sibling(span, class_attrs) if actors_container: actors [a.text.strip() for a in actors_container.find_all(a, relv:starring)] # 获取电影其他数据 # 类型 move_type_span soup.find(span, class_pl, stringre.compile(类型:)) if move_type_span: move_type move_type_span.find_next_sibling(span,class_v:genre).text.strip() # 制片 diqu_span soup.find(span, class_pl, stringre.compile(制片国家/地区:)) if diqu_span: diqu diqu_span.find_next_sibling(textTrue).strip() # 语言 language_span soup.find(span, class_pl, stringre.compile(语言:)) if language_span: language language_span.find_next_sibling(textTrue).strip() # 上映日期 release_date_span soup.find(span, class_pl, stringre.compile(上映日期:)) if release_date_span: release_date release_date_span.find_next_sibling(span,propertyv:initialReleaseDate).text.strip() # 片长 duration_span soup.find(span, class_pl, stringre.compile(片长:)) if duration_span: duration duration_span.find_next_sibling(span, propertyv:runtime).text.strip() # 别名 aka_span soup.find(span, class_pl, stringre.compile(又名:)) if aka_span: aka aka_span.find_next_sibling(textTrue).strip() # IMDb imdb_span soup.find(span, class_pl, stringre.compile(IMDb:)) if imdb_span: imdb imdb_span.find_next_sibling(textTrue).strip() # 剧情简介 summary summary_span soup.find(span, propertyv:summary) if summary_span: summary summary_span.text.strip() return { screenwriters: screenwriters, actors: actors, move_type: move_type, diqu: diqu, language: language, release_date: release_date, duration: duration, aka: aka, imdb: imdb, summary: summary } def parse_page(html): soup BeautifulSoup(html, html.parser) movies soup.find_all(div, class_item) for movie in movies: try: # 安全地提取基本信息添加空值检查 em_elem movie.find(em) title_elem movie.find(span, class_title) link_elem movie.find(a) rating_elem movie.find(span, class_rating_num) # 检查关键元素是否存在 if not all([em_elem, title_elem, link_elem, rating_elem]): print(跳过不完整的电影条目) continue em em_elem.text.strip() title title_elem.text.strip() link link_elem.get(href, ).strip() rating rating_elem.text.strip() # 安全提取导演和演员信息 bd_elem movie.find(div, class_bd) director_actors if bd_elem: p_elem bd_elem.find(p) if p_elem: director_actors p_elem.text.strip() dacn extract_director_names(director_actors) if director_actors else # 安全获取详细信息 data get_movie_details(link) or {} # 确保data是字典 # 使用get方法安全访问字典避免KeyError movie_info { 排名: em, 名字: title, 别名: data.get(aka, ), 链接: link, 评分: rating, 导演: dacn, 编剧: data.get(screenwriters, []), # 默认为空列表 主演: data.get(actors, []), # 默认为空列表 类型: data.get(move_type, ), 制片国家/地区: data.get(diqu, ), 语言: data.get(language, ), 上映日期: data.get(release_date, ), 片长: data.get(duration, ), IMDb: data.get(imdb, ), 剧情简介: data.get(summary, ) } # 确保movie_list已定义 if movie_list not in globals(): global movie_list movie_list [] movie_list.append(movie_info) except Exception as e: print(f处理电影条目时出错: {e}) continue # 跳过当前条目继续处理下一个 def main(): for start in range(0, 250, 25): url f{base_url}?start{start} html get_page(url) parse_page(html) # 输出结果 for movie in movie_list: print(movie) if __name__ __main__: main() # 保存数据到CSV文件 def save_to_csv(): if not movie_list: # 或者使用 len(movie_list) 0 print(警告movie_list为空没有数据可保存。) return # 直接返回不执行后续保存操作 keys movie_list[0].keys() with open(douban_top250.csv, w, newline, encodingutf-8-sig) as output_file: dict_writer csv.DictWriter(output_file, fieldnameskeys) dict_writer.writeheader() dict_writer.writerows(movie_list) save_to_csv()