curl --request POST \
--url https://api.kadoa.com/v4/crawl/ \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"startUrls": [
"<string>"
],
"pathsFilterIn": [
"<string>"
],
"pathsFilterOut": [
"<string>"
],
"proxyType": "<string>",
"proxyCountry": "<string>",
"timeout": 2,
"maxDepth": 2,
"maxPages": 2,
"maxMatches": 2,
"concurrency": 2,
"strictDomain": true,
"loadImages": true,
"safeMode": false,
"callbackUrl": "<string>",
"processDuringCrawl": true,
"crawlMethod": {
"mode": "auto"
},
"matchThreshold": 0.5,
"blueprint": [
{
"name": "<string>",
"description": "<string>",
"selector": "<string>",
"type": "<string>"
}
],
"extractionOptions": {
"extractions": {},
"schema": "<unknown>",
"entity": "<unknown>",
"mainContextSelector": "<string>",
"xhrExtractorConfigs": [
null
]
},
"navigationOptions": {
"browserActions": [
{}
],
"preBrowserActions": [
{}
],
"scrollHtml": true,
"scrollHtmlTimeout": 1,
"visualHtml": true,
"navigationStrategy": "<string>",
"navigationStrategies": [
"<string>"
],
"limit": 1,
"disableNavigation": true,
"ignoreIframes": true,
"navigationExploration": {},
"loadHtmlOnly": true,
"acceptCookies": true,
"cachedCookieAccept": true
},
"artifactOptions": {
"screenshot": true,
"screenshotFull": true,
"screenshotCache": true,
"screenshotPublic": true,
"screenshotLink": "<string>",
"archivePdf": true
},
"rawMode": true,
"outputOptions": {
"includeHtml": true,
"includeMarkdown": true,
"includeScreenshots": true,
"includeJson": true
},
"jobId": "<string>",
"workflowId": "<string>",
"dataKey": "<string>",
"billingSource": "<string>"
}
'import requests
url = "https://api.kadoa.com/v4/crawl/"
payload = {
"url": "<string>",
"startUrls": ["<string>"],
"pathsFilterIn": ["<string>"],
"pathsFilterOut": ["<string>"],
"proxyType": "<string>",
"proxyCountry": "<string>",
"timeout": 2,
"maxDepth": 2,
"maxPages": 2,
"maxMatches": 2,
"concurrency": 2,
"strictDomain": True,
"loadImages": True,
"safeMode": False,
"callbackUrl": "<string>",
"processDuringCrawl": True,
"crawlMethod": { "mode": "auto" },
"matchThreshold": 0.5,
"blueprint": [
{
"name": "<string>",
"description": "<string>",
"selector": "<string>",
"type": "<string>"
}
],
"extractionOptions": {
"extractions": {},
"schema": "<unknown>",
"entity": "<unknown>",
"mainContextSelector": "<string>",
"xhrExtractorConfigs": [None]
},
"navigationOptions": {
"browserActions": [{}],
"preBrowserActions": [{}],
"scrollHtml": True,
"scrollHtmlTimeout": 1,
"visualHtml": True,
"navigationStrategy": "<string>",
"navigationStrategies": ["<string>"],
"limit": 1,
"disableNavigation": True,
"ignoreIframes": True,
"navigationExploration": {},
"loadHtmlOnly": True,
"acceptCookies": True,
"cachedCookieAccept": True
},
"artifactOptions": {
"screenshot": True,
"screenshotFull": True,
"screenshotCache": True,
"screenshotPublic": True,
"screenshotLink": "<string>",
"archivePdf": True
},
"rawMode": True,
"outputOptions": {
"includeHtml": True,
"includeMarkdown": True,
"includeScreenshots": True,
"includeJson": True
},
"jobId": "<string>",
"workflowId": "<string>",
"dataKey": "<string>",
"billingSource": "<string>"
}
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
startUrls: ['<string>'],
pathsFilterIn: ['<string>'],
pathsFilterOut: ['<string>'],
proxyType: '<string>',
proxyCountry: '<string>',
timeout: 2,
maxDepth: 2,
maxPages: 2,
maxMatches: 2,
concurrency: 2,
strictDomain: true,
loadImages: true,
safeMode: false,
callbackUrl: '<string>',
processDuringCrawl: true,
crawlMethod: {mode: 'auto'},
matchThreshold: 0.5,
blueprint: [
{
name: '<string>',
description: '<string>',
selector: '<string>',
type: '<string>'
}
],
extractionOptions: {
extractions: {},
schema: '<unknown>',
entity: '<unknown>',
mainContextSelector: '<string>',
xhrExtractorConfigs: [null]
},
navigationOptions: {
browserActions: [{}],
preBrowserActions: [{}],
scrollHtml: true,
scrollHtmlTimeout: 1,
visualHtml: true,
navigationStrategy: '<string>',
navigationStrategies: ['<string>'],
limit: 1,
disableNavigation: true,
ignoreIframes: true,
navigationExploration: {},
loadHtmlOnly: true,
acceptCookies: true,
cachedCookieAccept: true
},
artifactOptions: {
screenshot: true,
screenshotFull: true,
screenshotCache: true,
screenshotPublic: true,
screenshotLink: '<string>',
archivePdf: true
},
rawMode: true,
outputOptions: {
includeHtml: true,
includeMarkdown: true,
includeScreenshots: true,
includeJson: true
},
jobId: '<string>',
workflowId: '<string>',
dataKey: '<string>',
billingSource: '<string>'
})
};
fetch('https://api.kadoa.com/v4/crawl/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.kadoa.com/v4/crawl/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'startUrls' => [
'<string>'
],
'pathsFilterIn' => [
'<string>'
],
'pathsFilterOut' => [
'<string>'
],
'proxyType' => '<string>',
'proxyCountry' => '<string>',
'timeout' => 2,
'maxDepth' => 2,
'maxPages' => 2,
'maxMatches' => 2,
'concurrency' => 2,
'strictDomain' => true,
'loadImages' => true,
'safeMode' => false,
'callbackUrl' => '<string>',
'processDuringCrawl' => true,
'crawlMethod' => [
'mode' => 'auto'
],
'matchThreshold' => 0.5,
'blueprint' => [
[
'name' => '<string>',
'description' => '<string>',
'selector' => '<string>',
'type' => '<string>'
]
],
'extractionOptions' => [
'extractions' => [
],
'schema' => '<unknown>',
'entity' => '<unknown>',
'mainContextSelector' => '<string>',
'xhrExtractorConfigs' => [
null
]
],
'navigationOptions' => [
'browserActions' => [
[
]
],
'preBrowserActions' => [
[
]
],
'scrollHtml' => true,
'scrollHtmlTimeout' => 1,
'visualHtml' => true,
'navigationStrategy' => '<string>',
'navigationStrategies' => [
'<string>'
],
'limit' => 1,
'disableNavigation' => true,
'ignoreIframes' => true,
'navigationExploration' => [
],
'loadHtmlOnly' => true,
'acceptCookies' => true,
'cachedCookieAccept' => true
],
'artifactOptions' => [
'screenshot' => true,
'screenshotFull' => true,
'screenshotCache' => true,
'screenshotPublic' => true,
'screenshotLink' => '<string>',
'archivePdf' => true
],
'rawMode' => true,
'outputOptions' => [
'includeHtml' => true,
'includeMarkdown' => true,
'includeScreenshots' => true,
'includeJson' => true
],
'jobId' => '<string>',
'workflowId' => '<string>',
'dataKey' => '<string>',
'billingSource' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.kadoa.com/v4/crawl/"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.kadoa.com/v4/crawl/")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.kadoa.com/v4/crawl/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"message": "<string>",
"sessionId": "<string>",
"error": "<string>",
"configId": "<string>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}Start crawl
Create a crawling configuration and start a session in one operation (equivalent to v4/crawl)
curl --request POST \
--url https://api.kadoa.com/v4/crawl/ \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"startUrls": [
"<string>"
],
"pathsFilterIn": [
"<string>"
],
"pathsFilterOut": [
"<string>"
],
"proxyType": "<string>",
"proxyCountry": "<string>",
"timeout": 2,
"maxDepth": 2,
"maxPages": 2,
"maxMatches": 2,
"concurrency": 2,
"strictDomain": true,
"loadImages": true,
"safeMode": false,
"callbackUrl": "<string>",
"processDuringCrawl": true,
"crawlMethod": {
"mode": "auto"
},
"matchThreshold": 0.5,
"blueprint": [
{
"name": "<string>",
"description": "<string>",
"selector": "<string>",
"type": "<string>"
}
],
"extractionOptions": {
"extractions": {},
"schema": "<unknown>",
"entity": "<unknown>",
"mainContextSelector": "<string>",
"xhrExtractorConfigs": [
null
]
},
"navigationOptions": {
"browserActions": [
{}
],
"preBrowserActions": [
{}
],
"scrollHtml": true,
"scrollHtmlTimeout": 1,
"visualHtml": true,
"navigationStrategy": "<string>",
"navigationStrategies": [
"<string>"
],
"limit": 1,
"disableNavigation": true,
"ignoreIframes": true,
"navigationExploration": {},
"loadHtmlOnly": true,
"acceptCookies": true,
"cachedCookieAccept": true
},
"artifactOptions": {
"screenshot": true,
"screenshotFull": true,
"screenshotCache": true,
"screenshotPublic": true,
"screenshotLink": "<string>",
"archivePdf": true
},
"rawMode": true,
"outputOptions": {
"includeHtml": true,
"includeMarkdown": true,
"includeScreenshots": true,
"includeJson": true
},
"jobId": "<string>",
"workflowId": "<string>",
"dataKey": "<string>",
"billingSource": "<string>"
}
'import requests
url = "https://api.kadoa.com/v4/crawl/"
payload = {
"url": "<string>",
"startUrls": ["<string>"],
"pathsFilterIn": ["<string>"],
"pathsFilterOut": ["<string>"],
"proxyType": "<string>",
"proxyCountry": "<string>",
"timeout": 2,
"maxDepth": 2,
"maxPages": 2,
"maxMatches": 2,
"concurrency": 2,
"strictDomain": True,
"loadImages": True,
"safeMode": False,
"callbackUrl": "<string>",
"processDuringCrawl": True,
"crawlMethod": { "mode": "auto" },
"matchThreshold": 0.5,
"blueprint": [
{
"name": "<string>",
"description": "<string>",
"selector": "<string>",
"type": "<string>"
}
],
"extractionOptions": {
"extractions": {},
"schema": "<unknown>",
"entity": "<unknown>",
"mainContextSelector": "<string>",
"xhrExtractorConfigs": [None]
},
"navigationOptions": {
"browserActions": [{}],
"preBrowserActions": [{}],
"scrollHtml": True,
"scrollHtmlTimeout": 1,
"visualHtml": True,
"navigationStrategy": "<string>",
"navigationStrategies": ["<string>"],
"limit": 1,
"disableNavigation": True,
"ignoreIframes": True,
"navigationExploration": {},
"loadHtmlOnly": True,
"acceptCookies": True,
"cachedCookieAccept": True
},
"artifactOptions": {
"screenshot": True,
"screenshotFull": True,
"screenshotCache": True,
"screenshotPublic": True,
"screenshotLink": "<string>",
"archivePdf": True
},
"rawMode": True,
"outputOptions": {
"includeHtml": True,
"includeMarkdown": True,
"includeScreenshots": True,
"includeJson": True
},
"jobId": "<string>",
"workflowId": "<string>",
"dataKey": "<string>",
"billingSource": "<string>"
}
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
startUrls: ['<string>'],
pathsFilterIn: ['<string>'],
pathsFilterOut: ['<string>'],
proxyType: '<string>',
proxyCountry: '<string>',
timeout: 2,
maxDepth: 2,
maxPages: 2,
maxMatches: 2,
concurrency: 2,
strictDomain: true,
loadImages: true,
safeMode: false,
callbackUrl: '<string>',
processDuringCrawl: true,
crawlMethod: {mode: 'auto'},
matchThreshold: 0.5,
blueprint: [
{
name: '<string>',
description: '<string>',
selector: '<string>',
type: '<string>'
}
],
extractionOptions: {
extractions: {},
schema: '<unknown>',
entity: '<unknown>',
mainContextSelector: '<string>',
xhrExtractorConfigs: [null]
},
navigationOptions: {
browserActions: [{}],
preBrowserActions: [{}],
scrollHtml: true,
scrollHtmlTimeout: 1,
visualHtml: true,
navigationStrategy: '<string>',
navigationStrategies: ['<string>'],
limit: 1,
disableNavigation: true,
ignoreIframes: true,
navigationExploration: {},
loadHtmlOnly: true,
acceptCookies: true,
cachedCookieAccept: true
},
artifactOptions: {
screenshot: true,
screenshotFull: true,
screenshotCache: true,
screenshotPublic: true,
screenshotLink: '<string>',
archivePdf: true
},
rawMode: true,
outputOptions: {
includeHtml: true,
includeMarkdown: true,
includeScreenshots: true,
includeJson: true
},
jobId: '<string>',
workflowId: '<string>',
dataKey: '<string>',
billingSource: '<string>'
})
};
fetch('https://api.kadoa.com/v4/crawl/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.kadoa.com/v4/crawl/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'startUrls' => [
'<string>'
],
'pathsFilterIn' => [
'<string>'
],
'pathsFilterOut' => [
'<string>'
],
'proxyType' => '<string>',
'proxyCountry' => '<string>',
'timeout' => 2,
'maxDepth' => 2,
'maxPages' => 2,
'maxMatches' => 2,
'concurrency' => 2,
'strictDomain' => true,
'loadImages' => true,
'safeMode' => false,
'callbackUrl' => '<string>',
'processDuringCrawl' => true,
'crawlMethod' => [
'mode' => 'auto'
],
'matchThreshold' => 0.5,
'blueprint' => [
[
'name' => '<string>',
'description' => '<string>',
'selector' => '<string>',
'type' => '<string>'
]
],
'extractionOptions' => [
'extractions' => [
],
'schema' => '<unknown>',
'entity' => '<unknown>',
'mainContextSelector' => '<string>',
'xhrExtractorConfigs' => [
null
]
],
'navigationOptions' => [
'browserActions' => [
[
]
],
'preBrowserActions' => [
[
]
],
'scrollHtml' => true,
'scrollHtmlTimeout' => 1,
'visualHtml' => true,
'navigationStrategy' => '<string>',
'navigationStrategies' => [
'<string>'
],
'limit' => 1,
'disableNavigation' => true,
'ignoreIframes' => true,
'navigationExploration' => [
],
'loadHtmlOnly' => true,
'acceptCookies' => true,
'cachedCookieAccept' => true
],
'artifactOptions' => [
'screenshot' => true,
'screenshotFull' => true,
'screenshotCache' => true,
'screenshotPublic' => true,
'screenshotLink' => '<string>',
'archivePdf' => true
],
'rawMode' => true,
'outputOptions' => [
'includeHtml' => true,
'includeMarkdown' => true,
'includeScreenshots' => true,
'includeJson' => true
],
'jobId' => '<string>',
'workflowId' => '<string>',
'dataKey' => '<string>',
'billingSource' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.kadoa.com/v4/crawl/"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.kadoa.com/v4/crawl/")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.kadoa.com/v4/crawl/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"startUrls\": [\n \"<string>\"\n ],\n \"pathsFilterIn\": [\n \"<string>\"\n ],\n \"pathsFilterOut\": [\n \"<string>\"\n ],\n \"proxyType\": \"<string>\",\n \"proxyCountry\": \"<string>\",\n \"timeout\": 2,\n \"maxDepth\": 2,\n \"maxPages\": 2,\n \"maxMatches\": 2,\n \"concurrency\": 2,\n \"strictDomain\": true,\n \"loadImages\": true,\n \"safeMode\": false,\n \"callbackUrl\": \"<string>\",\n \"processDuringCrawl\": true,\n \"crawlMethod\": {\n \"mode\": \"auto\"\n },\n \"matchThreshold\": 0.5,\n \"blueprint\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"selector\": \"<string>\",\n \"type\": \"<string>\"\n }\n ],\n \"extractionOptions\": {\n \"extractions\": {},\n \"schema\": \"<unknown>\",\n \"entity\": \"<unknown>\",\n \"mainContextSelector\": \"<string>\",\n \"xhrExtractorConfigs\": [\n null\n ]\n },\n \"navigationOptions\": {\n \"browserActions\": [\n {}\n ],\n \"preBrowserActions\": [\n {}\n ],\n \"scrollHtml\": true,\n \"scrollHtmlTimeout\": 1,\n \"visualHtml\": true,\n \"navigationStrategy\": \"<string>\",\n \"navigationStrategies\": [\n \"<string>\"\n ],\n \"limit\": 1,\n \"disableNavigation\": true,\n \"ignoreIframes\": true,\n \"navigationExploration\": {},\n \"loadHtmlOnly\": true,\n \"acceptCookies\": true,\n \"cachedCookieAccept\": true\n },\n \"artifactOptions\": {\n \"screenshot\": true,\n \"screenshotFull\": true,\n \"screenshotCache\": true,\n \"screenshotPublic\": true,\n \"screenshotLink\": \"<string>\",\n \"archivePdf\": true\n },\n \"rawMode\": true,\n \"outputOptions\": {\n \"includeHtml\": true,\n \"includeMarkdown\": true,\n \"includeScreenshots\": true,\n \"includeJson\": true\n },\n \"jobId\": \"<string>\",\n \"workflowId\": \"<string>\",\n \"dataKey\": \"<string>\",\n \"billingSource\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"message": "<string>",
"sessionId": "<string>",
"error": "<string>",
"configId": "<string>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}{
"error": true,
"message": "<string>",
"details": "<unknown>"
}Authorizations
API key for authentication
Body
Body
Schema for starting a crawling session with support for both single URL and multiple URLs
Single URL to start crawling (for backward compatibility)
List of URLs for crawling
1Regex patterns to include specific full URLs. Accepts array or JSON-stringified array of regex source strings.
1Regex patterns to exclude specific full URLs. Accepts array or JSON-stringified array of regex source strings.
1Type of proxy to use
Country for proxy selection
Timeout in milliseconds
x >= 1Maximum crawling depth
x >= 1Maximum number of pages to crawl
x >= 1Maximum number of matched pages to crawl before stopping
x >= 1Number of concurrent crawlers
x >= 1Whether to stay within the same domain
Whether to load images during crawling
Enable safe mode for crawling
Webhook URL for completion notifications
Whether to run preprocessing and extraction during the crawl phase
Crawl method configuration
Show child attributes
Show child attributes
Match threshold override for blueprint filtering
0 <= x <= 1Blueprint fields applied during crawling
Show child attributes
Show child attributes
Extraction-related options derived from legacy launch summary
Show child attributes
Show child attributes
Navigation-related options derived from legacy launch summary
Show child attributes
Show child attributes
Artifact capture options derived from legacy launch summary
Show child attributes
Show child attributes
Whether this is a raw data mode crawl
Output options for raw mode
Show child attributes
Show child attributes
Internal: Job ID for workflow tracking
Internal: Owning workflow (scraperId) for BrightData cost attribution
Internal: Data key for Parquet storage path
Internal: Billing source identifier