Async jobs

POST

jobs

Async jobs

curl --request POST \
  --url https://scrape.cleariflow.com/v1/jobs \
  --header 'Content-Type: application/json' \
  --data '
{
  "request": {},
  "priority": 123
}
'

import requests

url = "https://scrape.cleariflow.com/v1/jobs"

payload = {
    "request": {},
    "priority": 123
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({request: {}, priority: 123})
};

fetch('https://scrape.cleariflow.com/v1/jobs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrape.cleariflow.com/v1/jobs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'request' => [
        
    ],
    'priority' => 123
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://scrape.cleariflow.com/v1/jobs"

	payload := strings.NewReader("{\n  \"request\": {},\n  \"priority\": 123\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://scrape.cleariflow.com/v1/jobs")
  .header("Content-Type", "application/json")
  .body("{\n  \"request\": {},\n  \"priority\": 123\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://scrape.cleariflow.com/v1/jobs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"request\": {},\n  \"priority\": 123\n}"

response = http.request(request)
puts response.read_body

{
  "job_id": "550e8400-e29b-41d4-a716-446655440000"
}

Getting started

Base URL

https://scrape.cleariflow.com/v1/jobs

Create a job

Enqueue an async scrape job with the same ScrapeRequest payload used by the synchronous endpoint.

Example request

curl -X POST 'https://scrape.cleariflow.com/v1/jobs' \
  -H 'Content-Type: application/json' \
  -d '{
    "request": {
      "api_key": "YOUR_UNIQUE_API_KEY",
      "url": "https://example.com"
    },
    "priority": 10
  }'

This successful request returns a job identifier:

{
  "job_id": "550e8400-e29b-41d4-a716-446655440000"
}

Create job parameters

Object

required

A ScrapeRequest object with the same fields as the synchronous scrape endpoint (url, render, actions, cookies, etc.).

Integer

Job priority. Higher values are processed first. Defaults to 0.

Get job status

Poll a job by ID to check its status and retrieve results when complete.

Base URL

https://scrape.cleariflow.com/v1/jobs/{job_id}

Example request

curl 'https://scrape.cleariflow.com/v1/jobs/550e8400-e29b-41d4-a716-446655440000'

While the job is running:

{
  "job_id": "550e8400-e29b-41d4-a716-446655440000",
  "status": "running"
}

When the job completes successfully:

{
  "job_id": "550e8400-e29b-41d4-a716-446655440000",
  "status": "done",
  "result": {
    "ok": true,
    "html": "<!DOCTYPE html><html>...</html>",
    "meta": {
      "elapsed_ms": 8123
    }
  }
}

Job status values

Status	Description
`queued`	Job is waiting in the queue.
`running`	Browser session is active.
`done`	Scrape completed; `result` contains the output.
`failed`	Scrape failed; `error` contains a description.

Response parameters

String

Unique identifier for the async job.

String

Current job status: queued, running, done, or failed.

Object

The scrape result object (same shape as the synchronous endpoint response). Present when status is done.

String

Error message. Present when status is failed.

Object

Additional metadata about the job execution.

Async jobs

curl --request POST \
  --url https://scrape.cleariflow.com/v1/jobs \
  --header 'Content-Type: application/json' \
  --data '
{
  "request": {},
  "priority": 123
}
'

import requests

url = "https://scrape.cleariflow.com/v1/jobs"

payload = {
    "request": {},
    "priority": 123
}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({request: {}, priority: 123})
};

fetch('https://scrape.cleariflow.com/v1/jobs', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrape.cleariflow.com/v1/jobs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'request' => [
        
    ],
    'priority' => 123
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://scrape.cleariflow.com/v1/jobs"

	payload := strings.NewReader("{\n  \"request\": {},\n  \"priority\": 123\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://scrape.cleariflow.com/v1/jobs")
  .header("Content-Type", "application/json")
  .body("{\n  \"request\": {},\n  \"priority\": 123\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://scrape.cleariflow.com/v1/jobs")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Content-Type"] = 'application/json'
request.body = "{\n  \"request\": {},\n  \"priority\": 123\n}"

response = http.request(request)
puts response.read_body

{
  "job_id": "550e8400-e29b-41d4-a716-446655440000"
}

Verify

Lookup

Create

Getting started

Base URL

Create a job

Example request

Create job parameters

Get job status

Base URL

Example request

Job status values

Response parameters

​Getting started

​Base URL

​Create a job

​Example request

​Create job parameters

​Get job status

​Base URL

​Example request

​Job status values

​Response parameters

Getting started

Base URL

Create a job

Example request

Create job parameters

Get job status

Base URL

Example request

Job status values

Response parameters