Just for fun——写个爬虫抓取whois信息

摘要：代码需要的字段模仿获取西部数码信息域名代理模拟执行代码解析出错添加代理解析出错查询西部数码失败请求西部数码失败生成失败提取西部数码数据使用结果另外这个域名是我的，有意出售。

爬取的网站是西部数码，该网站在https://www.west.cn/web/whois...可以查询whois信息，通过chrome调试知道，数据是从接口：https://www.west.cn/web/whois...中获取的

分析该请求发现，qtoken2016这个key是最为重要（反爬虫的标记，我之前写的时候，还是qtoken），这个token的生成不在本页，而在https://www.west.cn/main/whoi...这一页点击查看whois的时候，发送请求：https://www.west.cn/services/...来生成新的token，得到的结果是混淆的js代码：

var l=[119,98,115,33,117,116,101,112,98,62,92,50,50,54,45,50,49,50,45,50,50,52,45,50,50,49,45,50,50,55,45,50,51,51,45,50,49,58,45,50,50,54,45,50,50,52,45,50,49,55,45,50,50,54,45,50,49,50,45,50,50,51,45,50,50,54,45,50,50,52,45,50,50,51,45,50,51,51,45,50,50,51,45,50,50,58,45,50,49,55,45,50,50,55,45,50,49,50,94,60,119,98,115,33,101,99,105,107,114,62,92,57,45,53,45,50,56,45,50,49,45,50,50,45,50,57,45,58,45,50,51,45,51,49,45,50,54,45,49,45,50,52,45,55,45,54,45,50,55,45,51,50,45,52,45,50,58,45,50,53,45,50,45,56,45,51,94,60,119,98,115,33,99,62,35,35,60,103,112,115,33,41,100,62,49,60,100,61,101,99,105,107,114,47,109,102,111,104,117,105,60,100,44,44,42,124,99,44,62,84,117,115,106,111,104,47,103,115,112,110,68,105,98,115,68,112,101,102,41,117,116,101,112,98,92,101,99,105,107,114,92,100,94,94,42,126,60,37,47,100,112,112,108,106,102,41,40,114,117,112,108,102,111,51,49,50,55,40,45,99,45,124,113,98,117,105,59,40,48,40,126,42,60];eval(function(p,a,c,k,e,d){e=function(c){return(c35?String.fromCharCode(c+29):c.toString(36))};if(!"".replace(/^/,String)){while(c--)d[e(c)]=k[c]||e(c);k=[function(e){return d[e]}];e=function(){return"w+"};c=1;};while(c--)if(k[c])p=p.replace(new RegExp(""+e(c)+"","g"),k[c]);return p;}("6 3="";7(2=0;2<4.5;2++){3+=8.a(4[2]-1)};9(3)",11,11,"||i|t|l|length|var|for|String|eval|fromCharCode".split("|"),0,{}))

对于这一点，用js的eval函数模拟一下就可以解决。

/**
 * Created by salamander on 2016/11/8.
 */
let request = require("request");
let Q = require("q");
let datetime = require("locutus/php/datetime");

let getTokenUrl = "http://www.west.cn/main/whois.asp?act=gettok&_=";
let whoisUrl = "http://www.west.cn/web/whois/whoisinfo?domain=";
// 需要的字段
let needFields = ["domain", "registrar", "country", "mail", "whoisinfo", "add_time", "registrant_name", "expire_date"];

// 模仿jquery
let jQuery, $;
$ = jQuery = {
    token: ""
};

jQuery.cookie =  function(name, value, options) {
    this.token = value;
};

/**
 * 获取西部数码whois信息
 * @param domain 域名
 * @param proxy 代理
 */
function getWestWhois(domain, proxy) {
    let defer = Q.defer();
    let firstOptions = {
        url: getTokenUrl + (new Date()).getTime()
    };
    if(proxy) {
        firstOptions.proxy = "http://" + proxy.trim();
    }
    request(firstOptions, function (error, response, body) {
        if(!error && response.statusCode === 200) {
            // 模拟执行js代码
            try {
                eval(body);
            } catch (err) {
                defer.reject("解析json出错：" + err);
                return;
            }
            if($.token) {
                let options = {
                    url: whoisUrl +  domain + "&server=&refresh=1",
                    headers: {
                        "Cookie": "qtoken=" + $.token,
                        "X-Requested-With": "XMLHttpRequest"
                    }
                };
                // 添加代理
                if(proxy) {
                    options.proxy = "http://" + proxy.trim();
                }
                request(options, function (error, response, body) {
                    if(!error && response.statusCode === 200) {
                        let data = null;
                        try {
                            data = JSON.parse(body);
                        } catch (err) {
                            defer.reject("解析json出错：" + err);
                            return;
                        }
                        if(data["code"] === 200) {
                            defer.resolve(extractWestData(domain, data));
                        } else {
                            defer.reject("查询西部数码whois失败")
                        }
                    } else {
                        defer.reject("请求西部数码whois失败");
                    }
                })
            } else {
                defer.reject("生成token失败");
                return defer.promise;
            }
        } else {
            defer.reject(error);
        }
    });
    return defer.promise;
}

/**
 * 提取西部数码数据
 * @param domain
 * @param data
 */
function extractWestData(domain, data) {
    let country = solveCountry(domain, data["body"]);
    return {
        domain: domain,
        mail: data["dom_em"],
        errcode: 0,
        country: country,
        registrant_name: data["dom_org"],
        registrar: data["registrer"],
        expire: data["expdate"],
        whoisinfo: JSON.stringify({
            domain: domain,
            mail: data["dom_em"],
            errcode: 0,
            country: country,
            registrant_name: data["dom_org"],
            registrar: data["registrer"],
            expire: data["expdate"]
        })
    };

    function solveCountry(domain, html) {
        if(domain && domain.substr(-1, 3) === ".cn") {
            return  "CN";
        }
        let result = html.match(/Registrant Country: (S+?)
/);
        if(result) {
            return result[1].trim();
        }
        return "";
    }
}


module.exports.getWestWhois = getWestWhois;

let westWhois = require("./west_whois.js");

westWhois.getWestWhois("51nazi.com").then((info) => {
    console.log(info);
});

结果：

51nazi.com这个域名是我的，有意出售。