102302134陈蔡裔数据采集第四次作业

第一题
核心代码和运行结果

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.options import Options
import time
import sqlite3# 配置浏览器
edge_options = Options()
edge_options.add_argument('--headless')
edge_options.add_argument('--disable-gpu')
driver = webdriver.Edge(options=edge_options)# 初始化数据库
def init_db():conn = sqlite3.connect('stock_market.db')conn.execute('''CREATE TABLE IF NOT EXISTS stocks (market_type TEXT,stock_code TEXT,stock_name TEXT,current_price REAL,change_rate REAL,change_amount REAL,volume INTEGER,turnover REAL,high_price REAL,low_price REAL,open_price REAL,prev_close REAL)''')conn.commit()conn.close()# 保存数据到数据库
def save_data(data):conn = sqlite3.connect('stock_market.db')conn.execute('''INSERT INTO stocks (market_type, stock_code, stock_name, current_price, change_rate, change_amount, volume, turnover, high_price, low_price, open_price, prev_close)VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', data)conn.commit()conn.close()# 显示数据库数据
def display_data():conn = sqlite3.connect('stock_market.db')cursor = conn.cursor()cursor.execute('SELECT * FROM stocks')rows = cursor.fetchall()print("\n股票市场数据:")print("市场\t代码\t名称\t现价\t涨跌幅%\t涨跌额\t成交量\t成交额\t最高\t最低\t开盘\t昨收")print("-" * 100)for row in rows:print(f"{row[0]}\t{row[1]}\t{row[2]}\t{row[3]}\t{row[4]}\t{row[5]}\t{row[6]}\t{row[7]}\t{row[8]}\t{row[9]}\t{row[10]}\t{row[11]}")conn.close()# 主爬虫函数
def crawl_stocks():markets = [('https://quote.eastmoney.com/center/gridlist.html#hs_a_board', '沪深A股'),('https://quote.eastmoney.com/center/gridlist.html#sh_a_board', '上证A股'), ('https://quote.eastmoney.com/center/gridlist.html#sz_a_board', '深证A股')]for url, market_name in markets:print(f"正在爬取{market_name}...")driver.get(url)time.sleep(4)# 获取股票行数据rows = driver.find_elements(By.CSS_SELECTOR, "table tbody tr")for row in rows[:25]:  # 每个市场取25条cells = row.find_elements(By.TAG_NAME, "td")if len(cells) > 10:stock_data = (market_name,cells[1].text,  # 股票代码cells[2].text,  # 股票名称cells[3].text,  # 最新价cells[4].text,  # 涨跌幅cells[5].text,  # 涨跌额cells[6].text,  # 成交量cells[7].text,  # 成交额cells[8].text,  # 最高cells[9].text,  # 最低cells[10].text, # 今开cells[11].text  # 昨收)save_data(stock_data)# 运行主程序
if __name__ == "__main__":init_db()crawl_stocks()driver.quit()display_data()print("\n数据爬取完成！")

作业心得
通过这次股票数据爬虫项目，我深刻体会到理论与实践的结合。在爬取东方财富网数据时，遇到了页面结构复杂、数据重复存储等问题，通过不断调试和优化代码，学会了如何更精准地定位元素和处理异常数据。特别是在处理数据库重复记录时，明白了数据清洗和去重的重要性。这次实践不仅提升了我的Python编程能力和Selenium自动化操作技巧，更培养了我解决实际问题的能力和耐心，认识到一个完整的数据采集系统需要考虑到数据质量、效率和稳定性等多个方面。
https://gitee.com/chen-caiyi041130/chen-caiyi/tree/master/作业4
第二题
核心代码和运行结果
模拟登录

# 模拟登录
#点击登录按钮
def click_login():login_xpaths = ["//button[contains(text(), '登录')]","//a[contains(text(), '登录')]","//*[contains(text(), '登录')]"]for xpath in login_xpaths:try:elements = driver.find_elements(By.XPATH, xpath)for element in elements:if element.is_displayed():element.click()print("点击登录按钮")return Trueexcept:continuereturn False
#在登录表中填写用户名和密码，处理iframe中的登录表单
def fill_login_form(username, password):iframes = driver.find_elements(By.TAG_NAME, "iframe")for iframe in iframes:try:driver.switch_to.frame(iframe)inputs = driver.find_elements(By.CSS_SELECTOR, "input")username_filled = Falsepassword_filled = Falsefor field in inputs:field_type = field.get_attribute('type') or ''placeholder = field.get_attribute('placeholder') or ''if not username_filled and field_type in ['text', 'tel'] and '手机' in placeholder:field.clear()field.send_keys(username)print("输入用户名")username_filled = Truecontinueif not password_filled and field_type == 'password' and '密码' in placeholder:field.clear()field.send_keys(password)print("输入密码")password_filled = Truecontinuedriver.switch_to.default_content()return username_filled and password_filledexcept:driver.switch_to.default_content()return False
#提交登录表单
def submit_login():iframes = driver.find_elements(By.TAG_NAME, "iframe")for iframe in iframes:try:driver.switch_to.frame(iframe)submit_selectors = ["button[type='submit']","input[type='submit']","button:contains('登')",".j-login-btn",".u-loginbtn"]for selector in submit_selectors:try:if "contains" in selector:for btn in driver.find_elements(By.XPATH, "//button[contains(text(), '登')]"):if btn.is_displayed():btn.click()print("提交登录")driver.switch_to.default_content()return Trueelse:submit_btn = driver.find_element(By.CSS_SELECTOR, selector)if submit_btn.is_displayed():submit_btn.click()print("提交登录")driver.switch_to.default_content()return Trueexcept:continuedriver.switch_to.default_content()except:driver.switch_to.default_content()return False
#检查登录是否成功
def check_login():time.sleep(3)user_selectors = [".user-info", ".user-name", "a[href*='user']"]for selector in user_selectors:try:element = driver.find_element(By.CSS_SELECTOR, selector)if element.is_displayed():print("登录成功")return Trueexcept:continuereturn False
#登录主函数
def login(username, password):print("开始登录...")driver.get("https://www.icourse163.org/")time.sleep(3)if not click_login():return Falsetime.sleep(3)if not fill_login_form(username, password):return Falseif not submit_login():return Falsereturn check_login()

爬取Ajax

#爬取Ajax
def get_course_info(url, course_id):driver.get(url)time.sleep(3)info = {"course_id": course_id, "course_name": f"课程{course_id}","university": "未知学校", "instructor": "未知教师","team_members": "未知教师", "student_count": "未知","schedule": "未知", "description": "暂无简介"}# 课程名称for selector in [".course-title", "h1", ".u-course-title"]:try:title = driver.find_element(By.CSS_SELECTOR, selector).text.strip()if title and len(title) > 2:info['course_name'] = titlebreakexcept:continue# 学校名称try:school = driver.find_element(By.XPATH, "//*[@id='j-teacher']/div/a/img").get_attribute('alt')if school:info['university'] = schoolexcept:for selector in [".u-course-org", ".school-name"]:try:school = driver.find_element(By.CSS_SELECTOR, selector).textif school:info['university'] = schoolbreakexcept:continue# 主讲教师teacher_selectors = [".f-fc3", ".teacher-name", ".instructor", "h3.f-fc3"]for selector in teacher_selectors:try:teacher_elems = driver.find_elements(By.CSS_SELECTOR, selector)for elem in teacher_elems:teacher = elem.text.strip()# 过滤掉"位授课老师"这样的文本，只保留真实姓名if (teacher and len(teacher) > 1 and '位授课老师' not in teacher and'课程团队' not in teacher andlen(teacher) < 10):  # 中文姓名通常2-4个字info['instructor'] = teacherbreakif info['instructor'] != "未知教师":breakexcept:continue# 团队成员team_members = []try:# 查找所有教师姓名元素name_elements = driver.find_elements(By.CSS_SELECTOR, ".f-fc3, .teacher-name, .t-name, .name")for elem in name_elements:name = elem.text.strip()if (name and len(name) > 1 and '位授课老师' not in name and'课程团队' not in name andlen(name) < 10 andname != info['instructor']):  # 排除主讲教师team_members.append(name)# 去重team_members = list(set(team_members))if team_members:info['team_members'] = ', '.join(team_members)else:info['team_members'] = info['instructor']  # 如果没有其他成员，使用主讲教师except Exception as e:print(f"抓取团队成员失败: {e}")info['team_members'] = info['instructor']# 参与人数try:count_text = driver.find_element(By.CSS_SELECTOR, "span.count").textmatch = re.search(r"\d+", count_text)if match:info['student_count'] = match.group()except:pass# 课程进度try:progress = driver.find_element(By.XPATH, "//*[@id='course-enroll-info']/div/div[1]/div[4]/span[@class='text']").textif progress:info['schedule'] = progressexcept:# 备用选择器for selector in [".course-schedule", ".progress", ".schedule"]:try:progress_elem = driver.find_element(By.CSS_SELECTOR, selector)if progress_elem.text:info['schedule'] = progress_elem.textbreakexcept:continue# 课程简介try:intro = driver.find_element(By.XPATH, "//*[@id='j-rectxt2']").textif intro:info['description'] = intro[:200] + "..." if len(intro) > 200 else introexcept:# 备用选择器for selector in [".course-description", ".intro", ".description", ".m-courseintro"]:try:intro_elem = driver.find_element(By.CSS_SELECTOR, selector)if intro_elem.text:intro_text = intro_elem.text.strip()info['description'] = intro_text[:200] + "..." if len(intro_text) > 200 else intro_textbreakexcept:continueprint(f"课程信息: {info['course_name']} | 学校: {info['university']} | 教师: {info['instructor']} | 参与人数: {info['student_count']}")print(f"团队成员: {info['team_members']}")print(f"课程进度: {info['schedule']}")print(f"课程简介: {info['description'][:50]}...")return info

数据库保存与查询

def save_course(conn, cursor, data):cursor.execute('''INSERT INTO course_info VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s)''', (data['course_id'], data['course_name'], data['university'],data['instructor'], data['team_members'], data['student_count'],data['schedule'], data['description']))conn.commit()print(f"已保存: {data['course_name']}")def display_data(cursor):cursor.execute("SELECT * FROM course_info")records = cursor.fetchall()print("\n" + "="*150)print("MySQL课程信息：")print("="*150)print(f"{'ID':<3} {'课程号':<8} {'课程名称':<25} {'学校':<15} {'主讲教师':<12} {'团队成员':<15} {'参与人数':<10} {'进度':<20} {'简介':<30}")print("-"*150)for record in records:# 截断简介长度避免显示过长description = record[8][:30] + "..." if record[8] and len(record[8]) > 30 else record[8]print(f"{record[0]:<3} {record[1]:<8} {record[2]:<25} {record[3]:<15} {record[4]:<12} {record[5]:<15} {record[6]:<10} {record[7]:<20} {description:<30}")print(f"\n总计: {len(records)} 门课程")

主函数

if __name__ == "__main__":print("中国大学MOOC课程爬虫")conn, cursor = setup_database()cursor.execute("DELETE FROM course_info")conn.commit()if login("18065097076", "ccy20041130"):print("登录成功，开始爬取课程...")else:print("登录失败，爬取公开课程...")courses = ["https://www.icourse163.org/course/ZJU-1472847200","https://www.icourse163.org/course/ZJU-1003377027", "https://www.icourse163.org/course/ZJU-1472836187","https://www.icourse163.org/course/DUT-1463110162","https://www.icourse163.org/course/BIT-1001870001",]print(f"爬取 {len(courses)} 个课程...")for i, url in enumerate(courses, 1):print(f"处理第 {i} 个课程...")data = get_course_info(url, f"C{i:03d}")save_course(conn, cursor, data)time.sleep(2)display_data(cursor)driver.quit()cursor.close()conn.close()print("执行完成！")

作业心得
在完成这个中国大学MOOC课程爬虫项目的过程中，我深刻体会到了网络爬虫开发的挑战与乐趣。最初遇到了诸多困难：登录按钮无法定位、iframe处理复杂、表单填写失败等问题频发。通过不断调试，我发现网站使用了动态iframe加载登录表单，需要先切换到正确的iframe才能操作输入框。在数据爬取阶段，教师信息经常被错误地识别为"1位授课老师"这样的描述文本，通过添加过滤条件和备用选择器才解决了这个问题。数据库操作也遇到了表结构不匹配的报错，通过重新设计表结构得以解决。这个项目让我学会了如何处理复杂的网页结构、优化选择器策略、完善错误处理机制，最终成功实现了稳定登录和完整数据爬取的功能。
https://gitee.com/chen-caiyi041130/chen-caiyi/tree/master/作业4
第三题
开通MapReduce服务

Python脚本生成测试数据

配置Kafka

安装Flume客户端


配置Flume采集数据


作业心得
实验将课堂上学习到的Hadoop、Hive、Spark、Flink等理论知识与云上的具体操作相结合，解决了“纸上谈兵”的问题，遇到了真实环境中才会出现的配置、网络、性能问题，极大地提升了解决问题的能力。