爬取瓜子二手车.py

# !/usr/bin/python
# ctrl+alt+L自动加空格格式化
# --king--

import requests
from lxml import html
import time

# 获取详情页面的函数
headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 （Windows NT 10.0; Win64; x64） AppleWebKit/537.36 （KHTML like Gecko） Chrome/81.0.4044.138 Safari/537.36‘
    ‘Cookie‘: ‘track_id=78612551481065472; uuid=078b88fa-951c-43c4-b101-6aa943f02548; user_city_id=103; ganji_uuid=7491503126491343275062; lg=1; cityDomain=zz; antipas=I93s6b4649487681198n608P1u; clueSourceCode=%2A%2300; sessionid=e88583b7-7f25-44bf-a7c7-325da0041169; cainfo=%7B%22ca_a%22%3A%22-%22%2C%22ca_b%22%3A%22-%22%2C%22ca_s%22%3A%22pz_baidu%22%2C%22ca_n%22%3A%22pcbiaoti%22%2C%22ca_medium%22%3A%22-%22%2C%22ca_term%22%3A%22-%22%2C%22ca_content%22%3A%22%22%2C%22ca_campaign%22%3A%22%22%2C%22ca_kw%22%3A%22-%22%2C%22ca_i%22%3A%22-%22%2C%22scode%22%3A%22-%22%2C%22keyword%22%3A%22-%22%2C%22ca_keywordid%22%3A%22-%22%2C%22ca_transid%22%3A%22%22%2C%22platform%22%3A%221%22%2C%22version%22%3A1%2C%22track_id%22%3A%2278612551481065472%22%2C%22display_finance_flag%22%3A%22-%22%2C%22client_ab%22%3A%22-%22%2C%22guid%22%3A%22078b88fa-951c-43c4-b101-6aa943f02548%22%2C%22ca_city%22%3A%22zz%22%2C%22sessionid%22%3A%22e88583b7-7f25-44bf-a7c7-325da0041169%22%7D; preTime=%7B%22last%22%3A1589953591%2C%22this%22%3A1589897893%2C%22pre%22%3A1589897893%7D‘}


def get_detail_urls（url）:
    resp = requests.get（url headers=headers）
    text = resp.content.decode（‘utf-8 ‘）
    # 由于etree要通过html引出所以变量名尽量不要使用html否则会报错
    htmls = html.etree.HTML（text）
    # 输出是一个l.etree._Element‘>一个class类

    # 确认上级ul标签的carlist clearfix js-top属性是否唯一
    # 确认后直接抓取这个ul标签下所有内容
    # 利用[0]切片从列表里取出第一个l.etree._Element‘>一个class类

    ul = htmls.xpath（‘//ul[@class=“carlist clearfix js-top“]‘）[0]
    lis = ul.xpath（‘./li‘）
    # 输出是由多个组成的列表所以命名为li+s=lis表示多个li
    # [...]
    # 说明里边包含的是多个Element元素
    # 用for循环取出里边所有的元素
    # 建立空列表把完整的detail_url添加进去
    detail_urls = []
    for li in lis:
        detail_url = li.xpath（“./a/@href“）
        detail_url = ‘https://www.guazi.com‘ + detail_url[0]
        # href[0]代表把当前列表里的数据以字符串的形式切片出来
        # 由于href是从li中取的