爱卡_xcar_汽车详解抓取

摘要：爬虫练手，主要运用，由于要对内部进行分析，所以就直接用了正则匹配，平时也可以用用，应该更加方便思路读取首页，就是如为了全部抓取，我们这里都是结尾找到页面里面的标签里面的，这个这个地址就是页面自动播放的下个页面同时读出即图片地址，然后保存我这

爬虫练手，主要运用requests，由于要对script内部进行分析，所以就直接用了 re 正则匹配，平时也可以用用Beautifulsoup， 应该更加方便

  
读取首页，就是如 http://newcar.xcar.com.cn/2674/2015/detail/1.htm；为了全部抓取，我们这里都是 1.htm 结尾

  
找到页面里面的 script 标签里面的 " var nextUrl " ，这个这个地址就是页面自动播放的下个页面;同时读出 img src 即 图片地址，然后保存（我这里是把 说明 等内容全部放进 目标文件的名称中了，其实大家只要图片的话完全不需要）

  
递归抓取 全部页面

这里另外做的一点实际 弄了个xcar_lst 记录所有页面、图片等信息，只是留作记录，暂时没用

python
# coding:utf-8



__author__ = "BONFY CHEN"



import requests

import re



PROXIES = None



HEADERS = {

    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"

    , "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

    , "Accept-Encoding": "gzip,deflate,sdch"

    , "Accept-Language": "zh-CN,zh;q=0.8"

}



BASE_FOLDER = "D:/xxx_folder/"





class xcarDown(object):



    _base_folder = None

    _proxies = None

    _headers = None

    _website = "http://newcar.xcar.com.cn"

    _xcar_lst = []



    def set_base_folder(self, base_folder):

        self._base_folder = base_folder



    def set_headers(self, headers):

        self._headers = headers



    def set_proxies(self, proxies):

        self._proxies = proxies



    def __init__(self, base_folder=BASE_FOLDER, proxies=PROXIES, headers=HEADERS):

        self.set_base_folder(base_folder)

        self.set_headers(headers)

        self.set_proxies(proxies)



    def download_image_from_url(self, url, name=None):



        """

        download_image_from_url

        :param url: the resource image url

        :param name: he destination file name

        :return:

        """



        local_filename = name + "_" + url.split("/")[-1]

        r = requests.get(url, proxies=self._proxies, headers=self._headers, stream=True)

        with open(self._base_folder + local_filename, "wb") as f:

            for chunk in r.iter_content(chunk_size=1024):

                if chunk:

                    f.write(chunk)

                    f.flush()

            f.close()

        return local_filename





    def download_xcar(self, url):



        """



        :param url: the source url in xcar.com.cn

                    http://newcar.xcar.com.cn/2674/2015/detail/1.htm

        :return:

        """



        r = requests.get(url, proxies=self._proxies, headers=self._headers)

        # print r.encoding

        r.encoding = "gbk"



        m1 = re.search(r"var nextUrl = "(?P.*.htm)"", r.text)

        next_url = m1.groupdict()["n_url"] if m1 else None



        m2 = re.search(r"
(?P", r.text)

        model = m5.groupdict()["model"] if m5 else ""



        if pic_url:

            try:

                self.download_image_from_url(pic_url, name="_".join([model, title, cont]))

                print "download complete: pic from {} ".format(pic_url)

            except IOError:

                print "file name IOERROR"

                self.download_image_from_url(pic_url, name=model)

                print "download complete: pic from {} ".format(pic_url)

            except Exception as e:

                print e



        dct = dict(pic_url=pic_url, next_url=next_url, title=title, cont=cont, model=model)

        self._xcar_lst.append(dct)

       if next_url[-4:] == ".htm":

            self.download_xcar(self._website + next_url)





if __name__ == "__main__":

    print("Welcome to the Pic Download for xcar.com")

    print("Downloaded files in the folder: " + BASE_FOLDER )

    print("---------------------------------------")



    id_modell = int(input("Please enter the modell id(eg.2674): "))

    year = int(input("Please enter the year (eg.2015): "))



    url = "http://newcar.xcar.com.cn/{}/{}/detail/1.htm".format(id_modell, year)

    xcar = xcarDown()

    xcar.download_xcar(url)