http://192.168.31.82:8050/render.html?url=http://www.baidu.com

http://192.168.31.82:8050/render.html?url=https://www.endata.com.cn/BoxOffice/BO/Year/index.html&wait=5

import requests
"""
# splash提供的api接口
渲染html的接口
http://192.168.31.184:8050/render.html?url=你的url&wait=等待时间&time_out=超时时间

截图的接口
http://192.168.31.184:8050/render.png  参数和render.html基本一致, 可选width, height

加载过程接口
http://192.168.31.184:8050/render.har  参数和render.html基本一致

json接口
http://192.168.31.184:8050/render.json  参数和render.html基本一致

执行lua脚本的接口
http://192.168.31.184:8050/execute?lua_source=你要执行的lua脚本
"""

# 最简单的调用splash的render.html
url = "http://192.168.31.184:8050/render.html?url=https://www.baidu.com&wait=5"
resp = requests.get(url)
print(resp.text)

# 执行lua脚本
lua = """
function main(splash, args)
    assert(splash:go(args.url))
    assert(splash:wait(0.5))
    -- 加载一段js, 后面作为lua函数进行调用. 
    -- 在这个脚本中, 主要返回了"加载更多"按钮的状态
    get_display_style = splash:jsfunc([[
      function(){
        return document.getElementsByClassName('load_more_btn')[0].style.display;
      }
    ]])
    -- lua中的循环语句. 和python的while功能一样. 
    while (true)
    do  -- 语法规定. 相当于开始
        -- 直接运行js代码, 滚动到'加载更多'按钮
        splash:runjs("document.getElementsByClassName('load_more_btn')[0].scrollIntoView(true)")
        -- 等待
        splash:wait(1)
        -- 找到该按钮. 点击它
        splash:select(".load_more_btn").click()
        -- 调用上方预制的js脚本, 获取'正在加载按钮'的状态
        display_style = get_display_style()
        -- 如果不显示了. 也就结束了
        if(display_style== 'none')
        then
            break  -- 同python中的break. 打断循环
        end
    end
    assert(splash:wait(2)) -- 不在乎多等2秒
    return {
        html = splash:html(),    -- 拿到页面源代码
        cookies = splash:get_cookies()  -- 拿到cookies
    }
end
"""


# 准备能够执行lua脚本的url  -> splash服务地址
url = "http://192.168.31.82:8050/execute"

from lxml import etree

# 远程访问splash, 执行脚本
resp = requests.get(url, params={
    "url":"https://news.163.com",
    "lua_source": lua
})

result = resp.json()

# 提取结果
tree = etree.HTML(result.get('html'))
# print(resp.text)
divs = tree.xpath("//ul[@class='newsdata_list fixed_bar_padding noloading']/li[1]/div[2]/div")

for div in divs:
    a = div.xpath("./div/div/h3/a")
    if not a:
        continue
    a = a[0]
    href = a.xpath('./@href')
    title = a.xpath('./text()')

    print(title, href)

print(result.get("cookies"))

pip install scrapy_splash

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# scrapy_splash
# 渲染服务的url, 这里换成你自己的
SPLASH_URL = 'http://192.168.31.82:8050'
# 下载器中间件, 这个必须要配置
DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}

# 这个可由可无
# SPIDER_MIDDLEWARES = {
#     'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
# }
# 去重过滤器, 这个必须要配置
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
# 使用Splash的Http缓存, 这个必须要配置
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

import scrapy
from scrapy_splash.request import SplashRequest


# splash的lua脚本
lua_source = """
function main(splash, args)  -- 主函数
    assert(splash:go("https://news.163.com/"))  -- 访问url
    assert(splash:wait(2))  -- 等待
    -- 预存一个js函数
    get_btn_display = splash:jsfunc([[
        function(){
            return document.getElementsByClassName('load_more_btn')[0].style.display;
        }
    ]])
    -- lua的while循环
    while(true)
    do
        -- 直接执行一个js脚本
        -- 向下拉动滚动条. 
        splash:runjs("document.getElementsByClassName('load_more_btn')[0].scrollIntoView(true)")
        assert(splash:wait(1))
        -- 选择 "加载更多"
        btn = splash:select(".load_more_btn")
        -- 点它
        btn:click()
        -- 判断是否可见 调用上方预制的js函数
        ss = get_btn_display()
        -- 如果是none. 就没有数据了(网易自己设计的)
        if (ss == 'none')
        then
            break
        end
    end
    return {
        html = splash:html(),
        cookies = splash:get_cookies()
    }
    end
"""


class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    allowed_domains = ['163.com']
    start_urls = ['http://news.163.com/']

    def start_requests(self):
        # 发送splash请求
        yield SplashRequest(
            url=self.start_urls[0],
            callback=self.parse,
            endpoint="execute",
            args={"lua_source": lua_source, }
        )

    def parse(self, resp, **kwargs):
        divs = resp.xpath("//ul[@class='newsdata_list fixed_bar_padding noloading']/li[1]/div[2]/div")
        for div in divs:
            a = div.xpath("./div/div/h3/a")
            if not a:
                continue

            href = a.xpath('./@href').extract_first()
            title = a.xpath('./text()').extract_first()
            print(href)
            # 可以采用正常的抓取方案
            yield scrapy.Request(
                url=href,
                callback=self.details
            )

            break

    def details(self, resp):
        with open("data.txt", mode='w') as f:
            f.write("____".join(resp.xpath("//div[@class='post_body']//p/text()").extract()))