旅行-爬虫 数据分析源码.zip

旅游攻略选择

# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
import time
import random
from fake_useragent import UserAgent
#import re

#爬取每个网址的分页
fb = open（r‘url.txt‘‘w‘）
# --------------------所有城市------------------------- #
#url = ‘http://travel.qunar.com/travelbook/list.htm?page={}&order=hot_heat&avgPrice=1_2‘
#请求头，cookies在电脑网页中可以查到
#ua = UserAgent（）
#headers={‘user-agent‘:ua.random
#         ‘cookies‘:‘JSESSIONID=5E9DCED322523560401A95B8643B49DF; QN1=00002b80306c204d8c38c41b; QN300=s%3Dbaidu; QN99=2793; QN205=s%3Dbaidu; QN277=s%3Dbaidu; QunarGlobal=10.86.213.148_-3ad026b5_17074636b8f_-44df|1582508935699; QN601=64fd2a8e533e94d422ac3da458ee6e88; _i=RBTKSueZDCmVnmnwlQKbrHgrodMx; QN269=D32536A056A711EA8A2FFA163E642F8B; QN48=6619068f-3a3c-496c-9370-e033bd32cbcc; fid=ae39c42c-66b4-4e2d-880f-fb3f1bfe72d0; QN49=13072299; csrfToken=51sGhnGXCSQTDKWcdAWIeIrhZLG86cka; QN163=0; Hm_lvt_c56a2b5278263aa647778d304009eafc=1582513259158252993015825510991582588666; viewdist=298663-1; uld=1-300750-1-1582590496|1-300142-1-1582590426|1-298663-1-1582590281|1-300698-1-1582514815; _vi=6vK5Gry4UmXDT70IFohKyFF8R8Mu0SvtUfxawwaKYRTq9NKud1iKUt8qkTLGH74E80hXLLVOFPYqRGy52OuTFnhpWvBXWEbkOJaDGaX_5L6CnyiQPPOYb2lFVxrJXsVd-W4NGHRzYtRQ5cJmiAbasK8kbNgDDhkJVTC9YrY6Rfi2; viewbook=7562814|7470570|7575429|7470584|7473513; QN267=675454631c32674; Hm_lpvt_c56a2b5278263aa647778d304009eafc=1582591567; QN271=c8712b13-2065-4aa7-a70b-e6156f6fc216‘
#         ‘referer‘:‘http://travel.qunar.com/travelbook/list.htm?page=1&order=hot_heat&avgPrice=1‘}

# -------------------以西安为例-------------------------#
url = ‘http://travel.qunar.com/travelbook/list/22-xian-300100/hot_heat/{}.htm?avgPrice=1_2‘
#请求头，cookies在电脑网页中可以查到
ua = UserAgent（）
headers = {
        ‘user-agent‘:ua.random
        ‘cookies‘:‘JSESSIONID=5E9DCED322523560401A95B8643B49DF; QN1=00002b80306c204d8c38c41b; QN300=s%3Dbaidu; QN99=2793; QN205=s%3Dbaidu; QN277=s%3Dbaidu; QunarGlobal=10.86.213.148_-3ad026b5_17074636b8f_-44df|1582508935699; QN601=64fd2a8e533e94d422ac3da458ee6e88; _i=RBTKSueZDCmVnmnwlQKbrHgrodMx; QN269=D32536A056A711EA8A2FFA163E642F8B; QN48=6619068f-3a3c-496c-9370-e033bd32cbcc; fid=ae39c42c-66b4-4e2d-880f-fb3f1bfe72d0; QN49=13072299; csrfToken=51sGhnGXCSQTDKWcdAWIeIrhZLG86cka; QN163=0; Hm_lvt_c56a2b5278263aa647778d304009eafc=1582513259158252993015825510991582588666; viewdist=298663-1; uld=1-300750-1-1582590496|1-300142-1-1582590426|1-298663-1-1582590281|1-300698-1-1582514815; _vi=6vK5Gry4UmXDT70IFohKyFF8R8Mu0SvtUfxawwaKYRTq9NKud1iKUt8qkTLGH74E80hXLLVOFPYqRGy52OuTFnhpWvBXWEbkOJaDGaX_5L6CnyiQPPOYb2lFVxrJXsVd-W4NGHRzYtRQ5cJmiAbasK8kbNgDDhkJVTC9YrY6Rfi2; viewbook=7562814|7470570|7575429|7470584|7473513; QN267=675454631c32674; Hm_lpvt_c56a2b5278263aa647778d304009eafc=1582591567; QN271=c8712b13-2065-4aa7-a70b-e6156f6fc216‘
        ‘referer‘:‘http://travel.qunar.com/travelbook/list/22-xian-300100/hot_heat/1.htm?avgPrice=1_2‘
        }

# -------------------网址爬取-------------------------

 属性            大小     日期    时间   名称
----------- ---------  ---------- -----  ----
     文件      244291  2020-03-25 20:56  The_Travel.csv
     文件       80730  2020-04-03 16:30  Travel.csv
     文件      123147  2020-03-25 20:56  Travel数据分析可视化.ipynb
     文件       16396  2020-04-03 09:23  url - Copy.txt
     文件        9567  2020-04-03 15:12  url.txt
     文件        4188  2020-04-03 15:06  爬虫上.py
     文件        5014  2020-04-03 15:59  爬虫下.py
     文件         170  2020-03-25 20:56  简介说明.txt