java版网络爬虫webmagic爬虫，爬去国家统计局区县划分信息

java版网络爬虫webmagic爬虫，爬去国家统计局区县划分信息

由于公司业务需求需要一份最新的城市区域划分信息，所以就考虑到国家统计局去获取一份最新区域信息。经过调研选用了java版本爬虫框架webmagic来处理这个需求。下面介绍下爬虫逻辑以及代码

web magic官方地址：

http://webmagic.io/

maven依赖

<!--爬虫-->
<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-core</artifactId>
    <version>0.7.3</version>
    <exclusions>
        <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
    </exclusions>
</dependency>

<dependency>
    <groupId>us.codecraft</groupId>
    <artifactId>webmagic-extension</artifactId>
    <version>0.7.3</version>
</dependency>

保存结果的实体 StaticDto.java

/**
 * Created by xiehui1956(@)gmail.com on 17-12-25.
 */
public class StaticDto {

    /**
     * 省份编码
     */
    private String provinceCode;

    /**
     * 省份名字
     */
    private String provinceName;

    /**
     * 城市编码
     */
    private String cityCode;

    /**
     * 城市名称
     */
    private String cityName;

    /**
     * 区县编码
     */
    private String countyCode;

    /**
     * 区县名称
     */
    private String countyName;

    /**
     * 乡镇编码
     */
    private String townCode;

    /**
     * 乡镇名称
     */
    private String townName;

    /**
     * 村编码
     */
    private String villageCode;

    /**
     * 村名称
     */
    private String villageName;

    /**
     * 城市分类代码
     */
    private String typeCode;

    public String getProvinceCode() {
        return provinceCode;
    }

    public void setProvinceCode(String provinceCode) {
        this.provinceCode = provinceCode;
    }

    public String getProvinceName() {
        return provinceName;
    }

    public void setProvinceName(String provinceName) {
        this.provinceName = provinceName;
    }

    public String getCityCode() {
        return cityCode;
    }

    public void setCityCode(String cityCode) {
        this.cityCode = cityCode;
    }

    public String getCityName() {
        return cityName;
    }

    public void setCityName(String cityName) {
        this.cityName = cityName;
    }

    public String getCountyCode() {
        return countyCode;
    }

    public void setCountyCode(String countyCode) {
        this.countyCode = countyCode;
    }

    public String getCountyName() {
        return countyName;
    }

    public void setCountyName(String countyName) {
        this.countyName = countyName;
    }

    public String getTownCode() {
        return townCode;
    }

    public void setTownCode(String townCode) {
        this.townCode = townCode;
    }

    public String getTownName() {
        return townName;
    }

    public void setTownName(String townName) {
        this.townName = townName;
    }

    public String getVillageCode() {
        return villageCode;
    }

    public void setVillageCode(String villageCode) {
        this.villageCode = villageCode;
    }

    public String getVillageName() {
        return villageName;
    }

    public void setVillageName(String villageName) {
        this.villageName = villageName;
    }

    public String getTypeCode() {
        return typeCode;
    }

    public void setTypeCode(String typeCode) {
        this.typeCode = typeCode;
    }

    @Override
    public String toString() {
        return "StaticDto{" +
                "provinceCode='" + provinceCode + '\'' +
                ", provinceName='" + provinceName + '\'' +
                ", cityCode='" + cityCode + '\'' +
                ", cityName='" + cityName + '\'' +
                ", countyCode='" + countyCode + '\'' +
                ", countyName='" + countyName + '\'' +
                ", townCode='" + townCode + '\'' +
                ", townName='" + townName + '\'' +
                ", villageCode='" + villageCode + '\'' +
                ", villageName='" + villageName + '\'' +
                ", typeCode='" + typeCode + '\'' +
                '}';
    }
}

爬虫核心逻辑代码 ChinaStatisticProcessor.java

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 国家统计局
 * Created by xiehui1956(@)gmail.com on 2017/12/24.
 */
public class ChinaStatisticProcessor implements PageProcessor {

    private static final String BASE_LINK = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/";

    private Site site = Site
            .me()
            .setDomain("www.stats.gov.cn")
            .setSleepTime(3000)
            .setCharset("gb2312")
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    public void process(Page page) {
        // 省份
        if (page.getUrl().regex("http://www\\.stats\\.gov\\.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/index\\.html").match()) {
            provincetableProcess(page);

            // 区(直辖市)
        } else if (page.getUrl().regex("http://www\\.stats\\.gov\\.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/\\d{2}\\.html").match()) {
            citytableProcess(page);

            //城市-区/县城
        } else if (page.getUrl().regex("http://www\\.stats\\.gov\\.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/\\d{2}/\\d{4}\\.html").match()) {
            countytableProcess(page);

            //乡镇
        } else if (page.getUrl().regex("http://www\\.stats\\.gov\\.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/\\d{2}/\\d{2}/\\d{6}\\.html").match()) {
            towntableProcess(page);

            //村
        } else if (page.getUrl().regex("http://www\\.stats\\.gov\\.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/\\d{2}/\\d{2}/\\d{2}/\\d{9}\\.html").match()) {
            villagetableProcess(page);

        }
    }


    //村
    private void villagetableProcess(Page page) {
        List<String> villagetables = page.getHtml().xpath("//tr[@class='villagetr']").all();
        //<tr class="villagetr"> <td>370102001002</td> <td>111</td> <td>解放桥社区居委会</td></tr>
        Pattern pattern = Pattern.compile("<tr class=\"villagetr\"> <td>(\\d+)</td> <td>(\\d+)</td> <td>(.+)</td></tr>");
        for (String villagetable : villagetables) {
            Matcher matcher = pattern.matcher(villagetable.replaceAll("\n", ""));
            while (matcher.find()) {
                String villagetableCode = matcher.group(1),
                        typeCode = matcher.group(2),
                        villagetableName = matcher.group(3);
                Request villagetableRequest = page.getRequest();
                page.putField("provinceCode", villagetableRequest.getExtra("provincetableCode"));
                page.putField("provinceName", villagetableRequest.getExtra("provincetableName"));
                page.putField("cityCode", villagetableRequest.getExtra("cityCode"));
                page.putField("cityName", villagetableRequest.getExtra("cityName"));
                page.putField("countyCode", villagetableRequest.getExtra("countyCode"));
                page.putField("countyName", villagetableRequest.getExtra("countyName"));
                page.putField("townCode", villagetableRequest.getExtra("towntrCode"));
                page.putField("townName", villagetableRequest.getExtra("towntrName"));
                page.putField("villageCode", villagetableCode);
                page.putField("villageName", villagetableName);
                page.putField("typeCode", typeCode);
                StaticDto staticDto = new StaticDto();
                staticDto.setProvinceCode(String.valueOf(villagetableRequest.getExtra("provincetableCode")));
                staticDto.setProvinceName(String.valueOf(villagetableRequest.getExtra("provincetableName")));
                staticDto.setCityCode(String.valueOf(villagetableRequest.getExtra("cityCode")));
                staticDto.setCityName(String.valueOf(villagetableRequest.getExtra("cityName")));
                staticDto.setCountyCode(String.valueOf(villagetableRequest.getExtra("countyCode")));
                staticDto.setCountyName(String.valueOf(villagetableRequest.getExtra("countyName")));
                staticDto.setTownCode(String.valueOf(villagetableRequest.getExtra("towntrCode")));
                staticDto.setTownName(String.valueOf(villagetableRequest.getExtra("towntrName")));
                staticDto.setVillageCode(String.valueOf(villagetableCode));
                staticDto.setVillageName(String.valueOf(villagetableName));
                staticDto.setTypeCode(String.valueOf(typeCode));
                System.out.println(staticDto.toString());
            }
        }
        return;
    }

    //乡镇
    private void towntableProcess(Page page) {
        List<String> towntrs = page.getHtml().xpath("//tr[@class='towntr']").all();
        //<tr class="citytr"><td><a href="32/3201.html">320100000000</a></td><td><a href="32/3201.html">南京市</a></td></tr>
        //<tr class="towntr"> <td><a href="02/220802001.html">220802001000</a></td> <td><a href="02/220802001.html">海明街道办事处</a></td></tr>
        Pattern pattern = Pattern.compile("<tr class=\"towntr\"> <td><a href=\"(\\d{2}/\\d{9}\\.html)\">(\\d+)</a></td> <td><a href=\"(\\d{2}/\\d{9}\\.html)\">(.+)</a></td></tr>");
        for (String towntr : towntrs) {
            Matcher matcher = pattern.matcher(towntr.replaceAll("\n", ""));
            while (matcher.find()) {
                String towntrCode = matcher.group(2),
                        towntrName = matcher.group(4);
                Request countryRequest = page.getRequest();
                String link = countryRequest.getExtra("provincetableBaseUrl") + "/" + countryRequest.getExtra("countryBaseCode") + matcher.group(1);
                Request request =
                        new Request(link)
                                .setPriority(3)
                                .putExtra("provincetableBaseUrl", countryRequest.getExtra("provincetableBaseUrl"))
                                .putExtra("provincetableCode", countryRequest.getExtra("provincetableCode"))
                                .putExtra("provincetableName", countryRequest.getExtra("provincetableName"))
                                .putExtra("cityCode", countryRequest.getExtra("cityCode"))
                                .putExtra("cityName", countryRequest.getExtra("cityName"))
                                .putExtra("countryBaseCode", countryRequest.getExtra("countryBaseCode"))
                                .putExtra("countryCode", countryRequest.getExtra("countryCode"))
                                .putExtra("countryName", countryRequest.getExtra("countryName"))
                                .putExtra("towntrCode", towntrCode)
                                .putExtra("towntrName", towntrName);
                page.addTargetRequest(request);
            }
        }
        return;
    }

    //区/县
    private void countytableProcess(Page page) {
        List<String> countrys = page.getHtml().xpath("//tr[@class='countytr']").all();
        //<tr class="countytr"> <td>150101000000</td> <td>市辖区</td></tr>
        //<tr class="countytr"> <td><a href="01/150102.html">150102000000</a></td> <td><a href="01/150102.html">新城区</a></td></tr>
        Pattern itemPattern = Pattern.compile("<tr class=\"countytr\"> <td><a href=\"(\\d{2}/\\d{6}\\.html)\">(\\d+)</a></td> <td><a href=\"(\\d{2}/\\d{6}\\.html)\">(.+)</a></td></tr>");
        for (String country : countrys) {
            Matcher matcher = itemPattern.matcher(country.replaceAll("\n", ""));
            //@TODO while
            while (matcher.find()) {
                String countryCode = matcher.group(2),
                        countryName = matcher.group(4);
                Request cityRequest = page.getRequest();
                String link = cityRequest.getExtra("provincetableBaseUrl") + "/" + matcher.group(1);
                Request request =
                        new Request(link)
                                .setPriority(2)
                                .putExtra("provincetableBaseUrl", cityRequest.getExtra("provincetableBaseUrl"))
                                .putExtra("provincetableCode", cityRequest.getExtra("provincetableCode"))
                                .putExtra("provincetableName", cityRequest.getExtra("provincetableName"))
                                .putExtra("cityCode", cityRequest.getExtra("cityCode"))
                                .putExtra("cityName", cityRequest.getExtra("cityName"))
                                .putExtra("countryBaseCode", matcher.group(1).substring(0, 3))
                                .putExtra("countryCode", countryCode)
                                .putExtra("countryName", countryName);
                page.addTargetRequest(request);
            }
        }
        return;
    }

    //市
    private void citytableProcess(Page page) {
        List<String> citys = page.getHtml().xpath("//tr[@class='citytr']").all();
        //<tr class="citytr"><td><a href="32/3201.html">320100000000</a></td><td><a href="32/3201.html">南京市</a></td></tr>
        Pattern pattern = Pattern.compile("<tr class=\"citytr\"> <td><a href=\"(\\d{2}/\\d{4}\\.html)\">(\\d+)</a></td> <td><a href=\"(\\d{2}/\\d{4}\\.html)\">(.+)</a></td></tr>");
        for (String city : citys) {
            Matcher matcher = pattern.matcher(city.replaceAll("\n", ""));
            //@TODO while
            while (matcher.find()) {
                String cityCode = matcher.group(2),
                        cityName = matcher.group(4);
                String link = BASE_LINK + matcher.group(1);
                Request provincetableRequest = page.getRequest();
                Request request =
                        new Request(link)
                                .setPriority(1)
                                .putExtra("provincetableBaseUrl", provincetableRequest.getExtra("provincetableBaseUrl"))
                                .putExtra("provincetableCode", provincetableRequest.getExtra("provincetableCode"))
                                .putExtra("provincetableName", provincetableRequest.getExtra("provincetableName"))
                                .putExtra("cityCode", cityCode)
                                .putExtra("cityName", cityName);
                page.addTargetRequest(request);
            }
        }
        return;
    }

    //省
    private void provincetableProcess(Page page) {
        List<String> provincetables = page.getHtml().xpath("//tr[@class='provincetr']//a").all();
        Pattern pattern = Pattern.compile("<a href=\"(\\d+)\\.html\">(.+)<br></a>");
        for (String provincetable : provincetables) {
            Matcher matcher = pattern.matcher(provincetable);
            //@TODO while
            while (matcher.find()) {
                String provincetableCode = matcher.group(1),
                        provincetableName = matcher.group(2);
                String link = BASE_LINK + matcher.group(1) + ".html";
                Request request =
                        new Request(link)
                                .setPriority(0)
                                .putExtra("provincetableBaseUrl", BASE_LINK + matcher.group(1))
                                .putExtra("provincetableCode", provincetableCode)
                                .putExtra("provincetableName", provincetableName);
                page.addTargetRequest(request);
            }
        }
        return;
    }


    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new ChinaStatisticProcessor())
                //从https://github.com/code4craft开始抓
                .addUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/index.html")
                //设置Pipeline，将结果以json方式保存到文件
//                .addPipeline(new JsonFilePipeline("/home/bls/work/myspace/workfile/statistic"))
                //开启5个线程同时执行
                .thread(30)
                //启动爬虫
                .run();
    }
}

核心爬虫逻辑说明

关于web magic的详细介绍请参考官方文档，官方文档描述的很清楚。这里我只对上面的逻辑做简单的描述介绍，看完官方文档在看这个逻辑会更好。如果有描述的不对的地方，请不吝赐教。

private Site site = Site
        .me()
        .setDomain("www.stats.gov.cn")
        .setSleepTime(3000)
        .setCharset("gb2312")
        .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

站点信息设置，包括设置请求域,间隔时间,编码方式,浏览器agent等信息

public void process(Page page)

process方法是爬虫爬去数据过程中系统自动循环调用的方法,这里使用了正则表达式针对路径做了业务区分。其他的几个方法是具体逻辑的处理，上面的注释很明显。

Spider.create(new ChinaStatisticProcessor())
        //从https://github.com/code4craft开始抓
        .addUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/index.html")
        //设置Pipeline，将结果以json方式保存到文件
        //.addPipeline(new JsonFilePipeline("/home/bls/work/myspace/workfile/statistic"))
        //开启5个线程同时执行
        .thread(30)
        //启动爬虫
        .run();

通过Spider创建执行入口,设置入口连接以及启动线程数。Pipeline这里可以设置输出形式，具体请参考官方文档。
以上，这里的代码留作备忘录。