跳至主要內容

抖音罗盘-合作达人视频数据接口逆向分析

dragons962024年10月30日大约 3 分钟PythonJavaScript爬虫逆向

特此声明：所有爬虫文章均用于记录学习与交流, 请勿用于其非法用途或侵害他人利益， 若非法使用作者概不负责。

接口地址：

https://compass.jinritemai.com/compass_api/shop/video/detail_list/download

请求头参数：

{
    "cookie": "${cookie}", // 登录后的cookie
    "accept": "application/json, text/plain, */*",
    "accept-encoding": "gzip, deflate, br, zstd",
    "accept-language": "zh-CN,zh;q=0.9",
    "priority": "u=1, i",
    "referer": "https://compass.jinritemai.com/shop/short-video-analysis/list",
    "sec-ch-ua": "\"Chromium\";v=\"130\", \"Not=A?Brand\";v=\"8\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\"",
    "sec-fetch-dest": "empty",
    "sec-fetch-mode": "cors",
    "sec-fetch-site": "same-origin",
    "user-agent": "${ua}" // ua
}

接口参数：

{
    "date_type": "20",
    "begin_date": "${start_timestamp}", // 开始日期, 毫秒0点时间戳, 例如2024-10-29则填 1730131200
    "end_date": "${end_timestamp}", // 结束日期, 毫秒0点时间戳， 例如2024-10-29则填 1730131200
    "tab_type": "20",
    "is_activity": "false",
    "activity_id": "",
    "sort_field": "pay_amt",
    "is_asc": "false",
    "author_id": "",
    "account_type": "3",
    "selected_type": "1",
    "query": "",
    "index_selected": "watch_cnt,finish_watch_rate,pay_amt,pay_cnt,live_watch_cnt",
    "index_pinned": "",
    "video_type": "0",
    "_lid": "${lid}", 
    "verifyFp": "${s_v_web_id}", // s_v_web_id
    "fp": "${s_v_web_id}", // s_v_web_id
    "msToken": "${ms_token}", // common接口获取 https://mssdk.bytedance.com/web/common
    "a_bogus": "{a_bogus}" // 加密参数a_bogus
}

分析

  1. 经过接口分析发现需要自己计算的参数仅_lid, msToken, a_bogus参数
  2. lid 通过debug发现是一个随机数，生成逻辑如下:
function newLid() {
    return "".concat(String(Date.now()).slice(5)).concat(String(Math.random()).slice(2, 6))
}
  1. msToken在https://mssdk.bytedance.com/web/common接口获取这里暂不处理, https://mssdk.bytedance.com/web/common 接口获取到msToken后会放入localStorage中的xmst参数, 直接从中取即可 (todo: 接口实现)
  2. 经过debug可以发现a_bogus在bdms.js中, 且跟踪调用栈发现最后调用此处内容获取到a_bogus参数 image.png
  3. 所以我们只需在该处将u暴露出去即可, 代码修改如下: image.png
  4. 补充nodejs缺失的环境，代码如下:
const jsdom = require("jsdom"); // 需要npm install jsdom
const {JSDOM} = jsdom;
const dom = new JSDOM(`<!DOCTYPE html><p>Hello world</p>`, {
    userAgent: "${ua}" // 此处填写ua值
});

window = global;
document = dom.window.document
window.requestAnimationFrame = function (res) {
    console.log("window.requestAnimationFrame ->", res)
}
window._sdkGlueVersionMap = {
    "sdkGlueVersion": "1.0.0.49", // glue sdk 版本
    "bdmsVersion": "1.0.1.1", // bdms 版本
    "captchaVersion": "4.0.2" // captcha 版本
}
window.onwheelx = {
    "_Ax": "0X21" // 固定值
}
window.innerWidth = 1707 
window.innerHeight = 791
window.outerWidth = 1707
window.outerHeight = 912
window.screenX = 0
window.screenY = 0
window.pageYOffset = 0
window.fetch = function (res) {
    console.log("window.fetch ->", res)
}

screen = {
    availWidth: 1707,
    availHeight: 912,
    width: 1707,
    height: 960,
    colorDepth: 24,
    pixelDepth: 24,
}
XMLHttpRequest = dom.window.XMLHttpRequest;
span = []
document = {
    createElement: function (res) {
        console.log("document.createElement ->", res)
        return []
    },
    documentElement: function (res) {
        console.log("document.documentElement ->", res)
    },
    createEvent: function (res) {
        console.log("document.createEvent ->", res)
    },
    body: []
}

setTimeout = function () {
}
navigator = {
    userAgent: "${ua}", // ua
    vendorSubs: {
        "ink": ${ink} // 生成的ink值, todo: 具体生成规则待探索, 可通过登录后浏览器driver.execute_script("return navigator.vendorSubs.ink") 获取
    },
    platform: 'Win32' // 系统
}
document.cookie = "${cookie}" // cookie值
  1. 编写生成a_bogus的代码, 代码如下:
function newABogus(arguments) {
    var r = window._U._v;
    return window._U._u(r[0], arguments, r[1], r[2], null)
}
  1. 通过debug可知arguments参数结构如下:
[
    0,
    1,
    8, "date_type=20&begin_date=1730131200&end_date=1730131200&tab_type=20&is_activity=false&activity_id=&sort_field=pay_amt&is_asc=false&author_id=&account_type=3&selected_type=1&query=&index_selected=watch_cnt%2Cfinish_watch_rate%2Cpay_amt%2Cpay_cnt%2Clive_watch_cnt&index_pinned=&video_type=0&_lid=602961301030&verifyFp=verify_m2vbpa6z_aVU3rcbx_AOLw_40QC_93bv_RLnPkzg4DX1u&fp=verify_m2vbpa6z_aVU3rcbx_AOLw_40QC_93bv_RLnPkzg4DX1u&msToken=rKcWIrjX9vKHtf-Y5pozZHUnedpfJKVGGskfAzoCv-7LkFUQerKVG_IqVYXiQBNxFeEKjQi50hzd0tbd-Gt8KRkjOYJZSbRaEwannDx9GinB78IDNuUI9HKwy9Ph0s85", // 此处为除了a_bogus以外的参数, 值做了urlencode编码
    "",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36" // 此处为ua
]
  1. 到此生成a_bogus逻辑已分析完成

原创声明

平台文章均为原创文章，未经许可，禁止转载。

如需转载，请联系作者获取授权，并注明来源及原文链接。