网页抓取 API
同步抓取
scrape 端点运行浏览器会话、执行 JavaScript,并在单次同步请求中返回渲染后的 HTML。
POST
/
v1
/
scrape
同步抓取
curl --request POST \
--url https://scrape.cleariflow.com/v1/scrape \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"api_key": "<string>",
"session_id": "<string>",
"fingerprint": "<string>",
"render": {},
"render.wait_until": "<string>",
"render.timeout_ms": 123,
"render.post_load_wait_ms": 123,
"render.ignore_https_errors": true,
"resources": {},
"resources.block": [
{}
],
"actions": [
{}
],
"actions[].type": "<string>",
"actions[].selector": "<string>",
"actions[].text": "<string>",
"actions[].to": "<string>",
"actions[].wait_ms": 123,
"actions[].timeout_ms": 123,
"cookies": [
{}
]
}
'import requests
url = "https://scrape.cleariflow.com/v1/scrape"
payload = {
"url": "<string>",
"api_key": "<string>",
"session_id": "<string>",
"fingerprint": "<string>",
"render": {},
"render.wait_until": "<string>",
"render.timeout_ms": 123,
"render.post_load_wait_ms": 123,
"render.ignore_https_errors": True,
"resources": {},
"resources.block": [{}],
"actions": [{}],
"actions[].type": "<string>",
"actions[].selector": "<string>",
"actions[].text": "<string>",
"actions[].to": "<string>",
"actions[].wait_ms": 123,
"actions[].timeout_ms": 123,
"cookies": [{}]
}
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
api_key: '<string>',
session_id: '<string>',
fingerprint: '<string>',
render: {},
'render.wait_until': '<string>',
'render.timeout_ms': 123,
'render.post_load_wait_ms': 123,
'render.ignore_https_errors': true,
resources: {},
'resources.block': [{}],
actions: [{}],
'actions[].type': '<string>',
'actions[].selector': '<string>',
'actions[].text': '<string>',
'actions[].to': '<string>',
'actions[].wait_ms': 123,
'actions[].timeout_ms': 123,
cookies: [{}]
})
};
fetch('https://scrape.cleariflow.com/v1/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://scrape.cleariflow.com/v1/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'api_key' => '<string>',
'session_id' => '<string>',
'fingerprint' => '<string>',
'render' => [
],
'render.wait_until' => '<string>',
'render.timeout_ms' => 123,
'render.post_load_wait_ms' => 123,
'render.ignore_https_errors' => true,
'resources' => [
],
'resources.block' => [
[
]
],
'actions' => [
[
]
],
'actions[].type' => '<string>',
'actions[].selector' => '<string>',
'actions[].text' => '<string>',
'actions[].to' => '<string>',
'actions[].wait_ms' => 123,
'actions[].timeout_ms' => 123,
'cookies' => [
[
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://scrape.cleariflow.com/v1/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://scrape.cleariflow.com/v1/scrape")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://scrape.cleariflow.com/v1/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}"
response = http.request(request)
puts response.read_body{
"ok": true,
"html": "<!DOCTYPE html><html>...</html>",
"meta": {
"elapsed_ms": 4521
}
}
入门
基础 URL
https://scrape.cleariflow.com/v1/scrape
请求示例
curl -X POST 'https://scrape.cleariflow.com/v1/scrape' \
-H 'Content-Type: application/json' \
-d '{
"api_key": "YOUR_UNIQUE_API_KEY",
"url": "https://example.com",
"render": {
"wait_until": "networkidle",
"timeout_ms": 60000
}
}'
{
"ok": true,
"html": "<!DOCTYPE html><html>...</html>",
"meta": {
"elapsed_ms": 4521
}
}
渲染选项
render 对象控制浏览器在捕获 HTML 前如何加载页面。所有字段均为可选,省略时使用服务器默认值。
"render": {
"wait_until": "networkidle",
"timeout_ms": 60000,
"post_load_wait_ms": 2000,
"ignore_https_errors": false
}
| 字段 | 类型 | 默认值 | 说明 |
|---|---|---|---|
wait_until | String | domcontentloaded | 何时视为导航完成。需要更快结果用 domcontentloaded;初始 HTML 后仍通过 XHR/fetch 加载数据的页面用 networkidle。 |
timeout_ms | Integer | 60000 | 等待页面加载的最长时间(毫秒),超时则请求失败。 |
post_load_wait_ms | Integer | 0 | wait_until 之后、捕获 HTML 前的额外延迟(毫秒)。适用于动画、懒加载组件或 networkidle 后仍在客户端渲染的内容。 |
ignore_https_errors | Boolean | false | 为 true 时忽略目标页面的 TLS 证书错误。 |
curl -X POST 'https://scrape.cleariflow.com/v1/scrape' \
-H 'Content-Type: application/json' \
-d '{
"api_key": "YOUR_UNIQUE_API_KEY",
"url": "https://quotes.toscrape.com/js/",
"render": {
"wait_until": "networkidle",
"timeout_ms": 60000,
"post_load_wait_ms": 1500,
"ignore_https_errors": false
}
}'
资源选项
resources 对象控制浏览器在抓取过程中加载哪些资源类型。当您只需要 HTML 文本和结构时,阻止重型资源可加快请求速度。
"resources": {
"block": ["images", "fonts", "media"]
}
| 值 | 阻止 |
|---|---|
images | 图片(<img>、CSS 背景、作为图片加载的 SVG 图标) |
fonts | Web 字体 |
media | 视频和音频流 |
resources 时,除非部署配置了服务器端默认值,否则不会阻止任何资源类型。
示例 — 跳过图片和字体以加快抓取:
curl -X POST 'https://scrape.cleariflow.com/v1/scrape' \
-H 'Content-Type: application/json' \
-d '{
"api_key": "YOUR_UNIQUE_API_KEY",
"url": "https://quotes.toscrape.com/",
"resources": {
"block": ["images", "fonts"]
}
}'
请求参数
String
必填
要抓取的目标 URL。必须是公开的 HTTP 或 HTTPS URL。对 localhost 和私有 IP 的请求会被 SSRF 防护阻止。
String
必填
您的唯一 API 密钥。
String
可选会话 ID,用于在多次抓取请求间复用浏览器状态(Cookie、本地存储)。
String
浏览器指纹预设。支持值:
desktop_en_us、desktop_ru_ru、mobile_en_us。Object
浏览器会话的渲染选项。
String
判定导航完成的时机。值:
domcontentloaded、networkidle。默认:domcontentloaded。Integer
等待页面加载的最长时间(毫秒)。默认:60000。
Integer
页面加载事件后、捕获内容前的额外等待时间(毫秒)。
Boolean
为
true 时忽略目标页面的 TLS 证书错误。Object
资源加载控制。
Array
要阻止的资源类型。支持值:
images、fonts、media。Array
捕获内容前执行的浏览器操作有序列表。每个操作是包含
type 字段的对象。String
必填
操作类型。支持值:
wait、wait_for、click、type、scroll。String
用于
wait_for、click 或 type 操作的 CSS 选择器。String
type 操作要输入的文本。String
scroll 操作的滚动目标(如 bottom)。Integer
wait 操作的等待时长(毫秒)。Integer
wait_for 操作的超时时间(毫秒)。Array
导航前注入的 Cookie。每个 Cookie 对象需要
name 和 value;可选字段:domain、path。响应参数
API 响应以通用轻量的 JSON 格式 返回。Boolean
抓取是否成功完成。
String
渲染后的页面 HTML。
Object
抓取元数据,包括最终 URL、HTTP 状态码和处理时长。
Object
当
ok 为 false 时的错误详情。⌘I
同步抓取
curl --request POST \
--url https://scrape.cleariflow.com/v1/scrape \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"api_key": "<string>",
"session_id": "<string>",
"fingerprint": "<string>",
"render": {},
"render.wait_until": "<string>",
"render.timeout_ms": 123,
"render.post_load_wait_ms": 123,
"render.ignore_https_errors": true,
"resources": {},
"resources.block": [
{}
],
"actions": [
{}
],
"actions[].type": "<string>",
"actions[].selector": "<string>",
"actions[].text": "<string>",
"actions[].to": "<string>",
"actions[].wait_ms": 123,
"actions[].timeout_ms": 123,
"cookies": [
{}
]
}
'import requests
url = "https://scrape.cleariflow.com/v1/scrape"
payload = {
"url": "<string>",
"api_key": "<string>",
"session_id": "<string>",
"fingerprint": "<string>",
"render": {},
"render.wait_until": "<string>",
"render.timeout_ms": 123,
"render.post_load_wait_ms": 123,
"render.ignore_https_errors": True,
"resources": {},
"resources.block": [{}],
"actions": [{}],
"actions[].type": "<string>",
"actions[].selector": "<string>",
"actions[].text": "<string>",
"actions[].to": "<string>",
"actions[].wait_ms": 123,
"actions[].timeout_ms": 123,
"cookies": [{}]
}
headers = {"Content-Type": "application/json"}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
api_key: '<string>',
session_id: '<string>',
fingerprint: '<string>',
render: {},
'render.wait_until': '<string>',
'render.timeout_ms': 123,
'render.post_load_wait_ms': 123,
'render.ignore_https_errors': true,
resources: {},
'resources.block': [{}],
actions: [{}],
'actions[].type': '<string>',
'actions[].selector': '<string>',
'actions[].text': '<string>',
'actions[].to': '<string>',
'actions[].wait_ms': 123,
'actions[].timeout_ms': 123,
cookies: [{}]
})
};
fetch('https://scrape.cleariflow.com/v1/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://scrape.cleariflow.com/v1/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'api_key' => '<string>',
'session_id' => '<string>',
'fingerprint' => '<string>',
'render' => [
],
'render.wait_until' => '<string>',
'render.timeout_ms' => 123,
'render.post_load_wait_ms' => 123,
'render.ignore_https_errors' => true,
'resources' => [
],
'resources.block' => [
[
]
],
'actions' => [
[
]
],
'actions[].type' => '<string>',
'actions[].selector' => '<string>',
'actions[].text' => '<string>',
'actions[].to' => '<string>',
'actions[].wait_ms' => 123,
'actions[].timeout_ms' => 123,
'cookies' => [
[
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://scrape.cleariflow.com/v1/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://scrape.cleariflow.com/v1/scrape")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://scrape.cleariflow.com/v1/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"api_key\": \"<string>\",\n \"session_id\": \"<string>\",\n \"fingerprint\": \"<string>\",\n \"render\": {},\n \"render.wait_until\": \"<string>\",\n \"render.timeout_ms\": 123,\n \"render.post_load_wait_ms\": 123,\n \"render.ignore_https_errors\": true,\n \"resources\": {},\n \"resources.block\": [\n {}\n ],\n \"actions\": [\n {}\n ],\n \"actions[].type\": \"<string>\",\n \"actions[].selector\": \"<string>\",\n \"actions[].text\": \"<string>\",\n \"actions[].to\": \"<string>\",\n \"actions[].wait_ms\": 123,\n \"actions[].timeout_ms\": 123,\n \"cookies\": [\n {}\n ]\n}"
response = http.request(request)
puts response.read_body{
"ok": true,
"html": "<!DOCTYPE html><html>...</html>",
"meta": {
"elapsed_ms": 4521
}
}