某房产抢车位,java版网络爬虫,抓包

发表于

自动获取房产车位信息、抢车位

新房子入驻没多久又要买车位，上海的车位真是贵呀。而且车位数有限，想要喜欢的车位还要抢！车位都要抢！车位是通过微信小程序在线抢，提前一天给一次模拟的机会。看中了一个距离住的地方比较近的车位，有多个人加入了收藏。为了保险起见，程序员动手了。。。

为了减少不必要的麻烦，过程着重描述技术点，不加截图。最终如愿的抢中了喜欢的车位！

工具及软件环境

  1. iPhone8p
  2. MacBook Pro
  3. Charles4.2.7
  4. jdk1.8
  5. IDEA
  6. SpringBoot-下面附加maven信息
  7. Jsoup-下面附加maven信息

前期准备

  1. 手机、电脑，Charles抓包工具；
  2. Charles设置HTTPS协议数据包抓包配置，手机网络走电脑代理；
  3. 验证手机微信小程序数据包是否被Charles抓取成功，抓取成功；

分析抢车位页面逻辑

  1. 进入页面查看接口请求方式；
  2. 查看数据传输形式；
  3. 检查是否存在安全验证逻辑；
  4. 分析返回的数据，关联接口的请求参数获取位置<返回的数据可能有header中的cookie信息、页面中的csrf-token信息、还有其他的一些身份token>。这里建议把所有的内容都打印出来，查看是否存在有用的信息<关联业务的非公共报头信息>；

提取关键信息

  1. 关键的接口请求地址、接口请求形式、接口参数传递方式、请求头部信息；
  2. 接口请求参数的获取方式、标记id信息，分析完成后开始编码；
1
2
3
4
5
6
7
8
//全部待选信息
String all = "https://com/m/activity/CarFloorList/30897";
//我的关注
String myStart = "https://com/m/Activity/GetPartialMyFavoredHouse";
//具体车位
String detail = "https://com/m/activity/Details/";
//确认事件
String addAcivityTarget = "https://com/m1/ActivityTarget/AddPrice";

代码部分：

maven依赖

1
2
3
4
5
6
7
8
9
10
11
<parent>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-parent</artifactId>
    <version>1.5.9.RELEASE</version>
</parent>

<dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.12.1</version>
</dependency>

封装RestTemplate绕行SSL证书验证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
public RestTemplate restTemplate() {
    if (null != restTemplate) {
        return restTemplate;
    }

    //忽略证书
    SSLContext sslContext = null;
    try {
        sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
            @Override
            public boolean isTrusted(java.security.cert.X509Certificate[] x509Certificates, String s)
                    throws java.security.cert.CertificateException {
                return true;
            }
        }).build();
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();
    } catch (KeyManagementException e) {
        e.printStackTrace();
    } catch (KeyStoreException e) {
        e.printStackTrace();
    }

    SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
    CloseableHttpClient httpClient = HttpClients.custom()
            .setSSLSocketFactory(csf)
            .build();
    HttpComponentsClientHttpRequestFactory requestFactory =
            new HttpComponentsClientHttpRequestFactory();
    requestFactory.setHttpClient(httpClient);
    restTemplate = new RestTemplate(requestFactory);
    //处理中文乱码
    restTemplate.getMessageConverters().set(1, new StringHttpMessageConverter(StandardCharsets.UTF_8));
    return restTemplate;
}

确认选购

1
2
3
4
5
6
7
8
9
10
11
/**
 * 确认选购
 *
 * @param id
 * @TODO 部分请求参数需要替换
 */
public void activityTraget(String id, String price, String openType, String token) throws InterruptedException {
    String van = van(addAcivityTarget, HttpMethod.POST, getActivityTargetValueMap(id, price, openType, token));
    System.out.println("抢购结果 = " + van);
    Thread.sleep(10);
}

确认选房参数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * 确认选房参数
 *
 * @param id
 * @return
 */
public MultiValueMap<String, Object> getActivityTargetValueMap(String id, String price, String openType, String token) {
    MultiValueMap<String, Object> parameters = new LinkedMultiValueMap();
    parameters.add("id", id);
    parameters.add("site", "van");
    parameters.add("price", price);
    parameters.add("openType", openType);
    parameters.add("__RequestVerificationToken", token);
    return parameters;
}

房间详情获取确认token

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
/**
 * 房间详情获取确认token
 *
 * @param id
 * @return
 */
public String[] detail(long id) {
    String allHtml = van(detail + id, HttpMethod.GET, getHourseDetailValueMap(id));
    Document document = Jsoup.parse(allHtml);
    String token = document.select("input[name=__RequestVerificationToken]").first().attr("value");
    Elements elementsByTag = document.getElementsByTag("script");
    for (Element element : elementsByTag) {
        String[] data = element.data().toString().split("var");
        for (String datum : data) {
            if (datum.contains("=") && datum.contains("activityJson")) {
                String val = datum.split("=")[1].trim();
                Map parseObject = JSON.parseObject(val.substring(0, val.lastIndexOf(";")), Map.class);
                String rowId = parseObject.get("ID").toString();
                String CurrentPrice = parseObject.get("CurrentPrice").toString();
                String OpenType = parseObject.get("OpenType").toString();
                System.out.println(String.format("id = %s ; CurrentPrice = %s ; OpenType = %s", rowId, CurrentPrice, OpenType));
                return new String[]{rowId, CurrentPrice, OpenType, token};
            }
        }
    }
    return new String[]{};
}

待抢车位

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
/**
 * 所有待抢车位
 *
 * @return
 */
public Map<Long, Object> allList() {
    Map<Long, Object> idDes = new HashMap();
    Document document = Jsoup.parse(van(all, HttpMethod.GET, getHourseListValueMap()));
    Elements elementsByTag = document.getElementsByTag("a");
    for (Element element : elementsByTag) {
        String href = element.attr("href");
        String id;
        String desc;
        if (null == element.select("span").first()) {
            desc = element.select("a").first().text();
            id = "0";
        } else {
            id = href.substring(href.indexOf("Details") + 8, href.length());
            desc = element.select("span").first().text();
        }
        idDes.put(Long.parseLong(id), desc);
        System.out.println("id = " + id + " ; desc = " + desc);
    }
    return idDes;
}

构造请求参数

1
2
3
4
5
6
7
8
public MultiValueMap<String, Object> getHourseListValueMap() {
    MultiValueMap<String, Object> parameters = new LinkedMultiValueMap();
    parameters.add("id", "自己的id");
    parameters.add("categoryId", "自己的categoryId");
    parameters.add("unitId", "自己的id");
    parameters.add("isRandomMessageHideRandomInfo", "0");
    return parameters;
}

请求、解析封装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
public String van(String url, HttpMethod method, MultiValueMap<String, Object> parameters) {
    HttpHeaders headers = getVankHeaders();
    HttpEntity<MultiValueMap<String, Object>> httpEntity = new HttpEntity(parameters, headers);

    ResponseEntity<String> responseEntity = restTemplate().exchange(url, method, httpEntity, String.class);
    HttpHeaders entityHeaders = responseEntity.getHeaders();
    for (Map.Entry<String, List<String>> stringListEntry : entityHeaders.entrySet()) {
        StringBuilder content = new StringBuilder("head-key = " + stringListEntry.getKey());
        for (String value : stringListEntry.getValue()) {
            content.append(" ; value = " + value);
        }
        System.out.println(content.toString());
    }
    String postForObject = responseEntity.getBody();
    return postForObject;
}

构建请求报头

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
public HttpHeaders getVankHeaders() {
    HttpHeaders headers = new HttpHeaders();
    headers.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    headers.add("Accept-Encoding", "gzip, deflate, br");
    headers.add("Accept-Language", "zh-cn");
    headers.add("Content-Type", "application/x-www-form-urlencoded");

    headers.add("Cookie", "cookie信息");

    headers.add("Host", "域名主机");
    headers.add("Origin", "域名主机");
    headers.add("Pragma", "no-cache");
    headers.add("Referer", "域名主机");
    headers.add("Sec-Fetch-Mode", "navigate");
    headers.add("Sec-Fetch-Site", "same-origin");
    headers.add("Sec-Fetch-User", "?1");
    headers.add("Content-Length", "79");
    headers.add("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/7.0.8(0x17000820) NetType/WIFI Language/zh_CN miniProgram");
    headers.add("X-Requested-With", "XMLHttpRequest");
    return headers;
}

以上仅做技术记录，如有疑问或不妥之处请联系我删除<18501633176>