Firecrawl API

GET Check Crawl Job Status

{{baseUrl}}/crawl/:crawl_id

QUERY PARAMS

crawl_id

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "GET");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/crawl/:crawl_id");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/get "{{baseUrl}}/crawl/:crawl_id")

require "http/client"

url = "{{baseUrl}}/crawl/:crawl_id"

response = HTTP::Client.get url
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Get,
    RequestUri = new Uri("{{baseUrl}}/crawl/:crawl_id"),
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/crawl/:crawl_id");
var request = new RestRequest("", Method.Get);
var response = client.Execute(request);

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/crawl/:crawl_id"

	req, _ := http.NewRequest("GET", url, nil)

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

GET /baseUrl/crawl/:crawl_id HTTP/1.1
Host: example.com

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("GET", "{{baseUrl}}/crawl/:crawl_id")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/crawl/:crawl_id"))
    .method("GET", HttpRequest.BodyPublishers.noBody())
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("{{baseUrl}}/crawl/:crawl_id")
  .get()
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.get("{{baseUrl}}/crawl/:crawl_id")
  .asString();

const data = null;

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('GET', '{{baseUrl}}/crawl/:crawl_id');

xhr.send(data);

import axios from 'axios';

const options = {method: 'GET', url: '{{baseUrl}}/crawl/:crawl_id'};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/crawl/:crawl_id';
const options = {method: 'GET'};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/crawl/:crawl_id',
  method: 'GET',
  headers: {}
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val request = Request.Builder()
  .url("{{baseUrl}}/crawl/:crawl_id")
  .get()
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'GET',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/crawl/:crawl_id',
  headers: {}
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.end();

const request = require('request');

const options = {method: 'GET', url: '{{baseUrl}}/crawl/:crawl_id'};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('GET', '{{baseUrl}}/crawl/:crawl_id');

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {method: 'GET', url: '{{baseUrl}}/crawl/:crawl_id'};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/crawl/:crawl_id';
const options = {method: 'GET'};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/crawl/:crawl_id"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"GET"];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/crawl/:crawl_id" in

Client.call `GET uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/crawl/:crawl_id",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('GET', '{{baseUrl}}/crawl/:crawl_id');

echo $response->getBody();

setUrl('{{baseUrl}}/crawl/:crawl_id');
$request->setMethod(HTTP_METH_GET);

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

setRequestUrl('{{baseUrl}}/crawl/:crawl_id');
$request->setRequestMethod('GET');
$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$response = Invoke-WebRequest -Uri '{{baseUrl}}/crawl/:crawl_id' -Method GET

$response = Invoke-RestMethod -Uri '{{baseUrl}}/crawl/:crawl_id' -Method GET

import http.client

conn = http.client.HTTPSConnection("example.com")

conn.request("GET", "/baseUrl/crawl/:crawl_id")

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/crawl/:crawl_id"

response = requests.get(url)

print(response.json())

library(httr)

url <- "{{baseUrl}}/crawl/:crawl_id"

response <- VERB("GET", url, content_type("application/octet-stream"))

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/crawl/:crawl_id")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
)

response = conn.get('/baseUrl/crawl/:crawl_id') do |req|
end

puts response.status
puts response.body

use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/crawl/:crawl_id";

    let client = reqwest::Client::new();
    let response = client.get(url)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request GET \
  --url {{baseUrl}}/crawl/:crawl_id

http GET {{baseUrl}}/crawl/:crawl_id

wget --quiet \
  --method GET \
  --output-document \
  - {{baseUrl}}/crawl/:crawl_id

import Foundation

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/crawl/:crawl_id")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "GET"

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()

POST Crawl URL with WebSocket Monitoring

{{baseUrl}}/crawl/websocket

BODY json

{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/crawl/websocket");

struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);

curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/post "{{baseUrl}}/crawl/websocket" {:content-type :json
                                                            :form-params {:url ""
                                                                          :excludePaths []
                                                                          :limit 0
                                                                          :scrapeOptions {:formats []
                                                                                          :extract {:schema ""
                                                                                                    :systemPrompt ""
                                                                                                    :prompt ""}}}})

require "http/client"

url = "{{baseUrl}}/crawl/websocket"
headers = HTTP::Headers{
  "content-type" => "application/json"
}
reqBody = "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = HTTP::Client.post url, headers: headers, body: reqBody
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Post,
    RequestUri = new Uri("{{baseUrl}}/crawl/websocket"),
    Content = new StringContent("{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
    {
        Headers =
        {
            ContentType = new MediaTypeHeaderValue("application/json")
        }
    }
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/crawl/websocket");
var request = new RestRequest("", Method.Post);
request.AddHeader("content-type", "application/json");
request.AddParameter("application/json", "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}", ParameterType.RequestBody);
var response = client.Execute(request);

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/crawl/websocket"

	payload := strings.NewReader("{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

POST /baseUrl/crawl/websocket HTTP/1.1
Content-Type: application/json
Host: example.com
Content-Length: 184

{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("POST", "{{baseUrl}}/crawl/websocket")
  .setHeader("content-type", "application/json")
  .setBody("{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/crawl/websocket"))
    .header("content-type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"))
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");
Request request = new Request.Builder()
  .url("{{baseUrl}}/crawl/websocket")
  .post(body)
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.post("{{baseUrl}}/crawl/websocket")
  .header("content-type", "application/json")
  .body("{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .asString();

const data = JSON.stringify({
  url: '',
  excludePaths: [],
  limit: 0,
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('POST', '{{baseUrl}}/crawl/websocket');
xhr.setRequestHeader('content-type', 'application/json');

xhr.send(data);

import axios from 'axios';

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/websocket',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    excludePaths: [],
    limit: 0,
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/crawl/websocket';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","excludePaths":[],"limit":0,"scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/crawl/websocket',
  method: 'POST',
  headers: {
    'content-type': 'application/json'
  },
  processData: false,
  data: '{\n  "url": "",\n  "excludePaths": [],\n  "limit": 0,\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}'
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val mediaType = MediaType.parse("application/json")
val body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
val request = Request.Builder()
  .url("{{baseUrl}}/crawl/websocket")
  .post(body)
  .addHeader("content-type", "application/json")
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'POST',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/crawl/websocket',
  headers: {
    'content-type': 'application/json'
  }
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.write(JSON.stringify({
  url: '',
  excludePaths: [],
  limit: 0,
  scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
}));
req.end();

const request = require('request');

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/websocket',
  headers: {'content-type': 'application/json'},
  body: {
    url: '',
    excludePaths: [],
    limit: 0,
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  },
  json: true
};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('POST', '{{baseUrl}}/crawl/websocket');

req.headers({
  'content-type': 'application/json'
});

req.type('json');
req.send({
  url: '',
  excludePaths: [],
  limit: 0,
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/websocket',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    excludePaths: [],
    limit: 0,
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/crawl/websocket';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","excludePaths":[],"limit":0,"scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSDictionary *headers = @{ @"content-type": @"application/json" };
NSDictionary *parameters = @{ @"url": @"",
                              @"excludePaths": @[  ],
                              @"limit": @0,
                              @"scrapeOptions": @{ @"formats": @[  ], @"extract": @{ @"schema": @"", @"systemPrompt": @"", @"prompt": @"" } } };

NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/crawl/websocket"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/crawl/websocket" in
let headers = Header.add (Header.init ()) "content-type" "application/json" in
let body = Cohttp_lwt_body.of_string "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}" in

Client.call ~headers ~body `POST uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/crawl/websocket",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '',
    'excludePaths' => [
        
    ],
    'limit' => 0,
    'scrapeOptions' => [
        'formats' => [
                
        ],
        'extract' => [
                'schema' => '',
                'systemPrompt' => '',
                'prompt' => ''
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "content-type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('POST', '{{baseUrl}}/crawl/websocket', [
  'body' => '{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}',
  'headers' => [
    'content-type' => 'application/json',
  ],
]);

echo $response->getBody();

setUrl('{{baseUrl}}/crawl/websocket');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$request->setContentType('application/json');
$request->setBody(json_encode([
  'url' => '',
  'excludePaths' => [
    
  ],
  'limit' => 0,
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

append(json_encode([
  'url' => '',
  'excludePaths' => [
    
  ],
  'limit' => 0,
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));
$request->setRequestUrl('{{baseUrl}}/crawl/websocket');
$request->setRequestMethod('POST');
$request->setBody($body);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-WebRequest -Uri '{{baseUrl}}/crawl/websocket' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-RestMethod -Uri '{{baseUrl}}/crawl/websocket' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

import http.client

conn = http.client.HTTPSConnection("example.com")

payload = "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

headers = { 'content-type': "application/json" }

conn.request("POST", "/baseUrl/crawl/websocket", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/crawl/websocket"

payload = {
    "url": "",
    "excludePaths": [],
    "limit": 0,
    "scrapeOptions": {
        "formats": [],
        "extract": {
            "schema": "",
            "systemPrompt": "",
            "prompt": ""
        }
    }
}
headers = {"content-type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

library(httr)

url <- "{{baseUrl}}/crawl/websocket"

payload <- "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

encode <- "json"

response <- VERB("POST", url, body = payload, content_type("application/json"), encode = encode)

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/crawl/websocket")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["content-type"] = 'application/json'
request.body = "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
  headers: {'Content-Type' => 'application/json'}
)

response = conn.post('/baseUrl/crawl/websocket') do |req|
  req.body = "{\n  \"url\": \"\",\n  \"excludePaths\": [],\n  \"limit\": 0,\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"
end

puts response.status
puts response.body

use serde_json::json;
use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/crawl/websocket";

    let payload = json!({
        "url": "",
        "excludePaths": (),
        "limit": 0,
        "scrapeOptions": json!({
            "formats": (),
            "extract": json!({
                "schema": "",
                "systemPrompt": "",
                "prompt": ""
            })
        })
    });

    let mut headers = reqwest::header::HeaderMap::new();
    headers.insert("content-type", "application/json".parse().unwrap());

    let client = reqwest::Client::new();
    let response = client.post(url)
        .headers(headers)
        .json(&payload)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request POST \
  --url {{baseUrl}}/crawl/websocket \
  --header 'content-type: application/json' \
  --data '{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

echo '{
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}' |  \
  http POST {{baseUrl}}/crawl/websocket \
  content-type:application/json

wget --quiet \
  --method POST \
  --header 'content-type: application/json' \
  --body-data '{\n  "url": "",\n  "excludePaths": [],\n  "limit": 0,\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}' \
  --output-document \
  - {{baseUrl}}/crawl/websocket

import Foundation

let headers = ["content-type": "application/json"]
let parameters = [
  "url": "",
  "excludePaths": [],
  "limit": 0,
  "scrapeOptions": [
    "formats": [],
    "extract": [
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    ]
  ]
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/crawl/websocket")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()

POST Map a website and get URLs

{{baseUrl}}/map

BODY json

{
  "url": "",
  "search": ""
}

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/map");

struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);

curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "{\n  \"url\": \"\",\n  \"search\": \"\"\n}");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/post "{{baseUrl}}/map" {:content-type :json
                                                :form-params {:url ""
                                                              :search ""}})

require "http/client"

url = "{{baseUrl}}/map"
headers = HTTP::Headers{
  "content-type" => "application/json"
}
reqBody = "{\n  \"url\": \"\",\n  \"search\": \"\"\n}"

response = HTTP::Client.post url, headers: headers, body: reqBody
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Post,
    RequestUri = new Uri("{{baseUrl}}/map"),
    Content = new StringContent("{\n  \"url\": \"\",\n  \"search\": \"\"\n}")
    {
        Headers =
        {
            ContentType = new MediaTypeHeaderValue("application/json")
        }
    }
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/map");
var request = new RestRequest("", Method.Post);
request.AddHeader("content-type", "application/json");
request.AddParameter("application/json", "{\n  \"url\": \"\",\n  \"search\": \"\"\n}", ParameterType.RequestBody);
var response = client.Execute(request);

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/map"

	payload := strings.NewReader("{\n  \"url\": \"\",\n  \"search\": \"\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

POST /baseUrl/map HTTP/1.1
Content-Type: application/json
Host: example.com
Content-Length: 31

{
  "url": "",
  "search": ""
}

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("POST", "{{baseUrl}}/map")
  .setHeader("content-type", "application/json")
  .setBody("{\n  \"url\": \"\",\n  \"search\": \"\"\n}")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/map"))
    .header("content-type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"url\": \"\",\n  \"search\": \"\"\n}"))
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"search\": \"\"\n}");
Request request = new Request.Builder()
  .url("{{baseUrl}}/map")
  .post(body)
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.post("{{baseUrl}}/map")
  .header("content-type", "application/json")
  .body("{\n  \"url\": \"\",\n  \"search\": \"\"\n}")
  .asString();

const data = JSON.stringify({
  url: '',
  search: ''
});

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('POST', '{{baseUrl}}/map');
xhr.setRequestHeader('content-type', 'application/json');

xhr.send(data);

import axios from 'axios';

const options = {
  method: 'POST',
  url: '{{baseUrl}}/map',
  headers: {'content-type': 'application/json'},
  data: {url: '', search: ''}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/map';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","search":""}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/map',
  method: 'POST',
  headers: {
    'content-type': 'application/json'
  },
  processData: false,
  data: '{\n  "url": "",\n  "search": ""\n}'
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val mediaType = MediaType.parse("application/json")
val body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"search\": \"\"\n}")
val request = Request.Builder()
  .url("{{baseUrl}}/map")
  .post(body)
  .addHeader("content-type", "application/json")
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'POST',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/map',
  headers: {
    'content-type': 'application/json'
  }
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.write(JSON.stringify({url: '', search: ''}));
req.end();

const request = require('request');

const options = {
  method: 'POST',
  url: '{{baseUrl}}/map',
  headers: {'content-type': 'application/json'},
  body: {url: '', search: ''},
  json: true
};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('POST', '{{baseUrl}}/map');

req.headers({
  'content-type': 'application/json'
});

req.type('json');
req.send({
  url: '',
  search: ''
});

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: '{{baseUrl}}/map',
  headers: {'content-type': 'application/json'},
  data: {url: '', search: ''}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/map';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","search":""}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSDictionary *headers = @{ @"content-type": @"application/json" };
NSDictionary *parameters = @{ @"url": @"",
                              @"search": @"" };

NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/map"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/map" in
let headers = Header.add (Header.init ()) "content-type" "application/json" in
let body = Cohttp_lwt_body.of_string "{\n  \"url\": \"\",\n  \"search\": \"\"\n}" in

Client.call ~headers ~body `POST uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/map",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '',
    'search' => ''
  ]),
  CURLOPT_HTTPHEADER => [
    "content-type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('POST', '{{baseUrl}}/map', [
  'body' => '{
  "url": "",
  "search": ""
}',
  'headers' => [
    'content-type' => 'application/json',
  ],
]);

echo $response->getBody();

setUrl('{{baseUrl}}/map');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$request->setContentType('application/json');
$request->setBody(json_encode([
  'url' => '',
  'search' => ''
]));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

append(json_encode([
  'url' => '',
  'search' => ''
]));
$request->setRequestUrl('{{baseUrl}}/map');
$request->setRequestMethod('POST');
$request->setBody($body);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-WebRequest -Uri '{{baseUrl}}/map' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "search": ""
}'

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-RestMethod -Uri '{{baseUrl}}/map' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "search": ""
}'

import http.client

conn = http.client.HTTPSConnection("example.com")

payload = "{\n  \"url\": \"\",\n  \"search\": \"\"\n}"

headers = { 'content-type': "application/json" }

conn.request("POST", "/baseUrl/map", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/map"

payload = {
    "url": "",
    "search": ""
}
headers = {"content-type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

library(httr)

url <- "{{baseUrl}}/map"

payload <- "{\n  \"url\": \"\",\n  \"search\": \"\"\n}"

encode <- "json"

response <- VERB("POST", url, body = payload, content_type("application/json"), encode = encode)

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/map")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["content-type"] = 'application/json'
request.body = "{\n  \"url\": \"\",\n  \"search\": \"\"\n}"

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
  headers: {'Content-Type' => 'application/json'}
)

response = conn.post('/baseUrl/map') do |req|
  req.body = "{\n  \"url\": \"\",\n  \"search\": \"\"\n}"
end

puts response.status
puts response.body

use serde_json::json;
use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/map";

    let payload = json!({
        "url": "",
        "search": ""
    });

    let mut headers = reqwest::header::HeaderMap::new();
    headers.insert("content-type", "application/json".parse().unwrap());

    let client = reqwest::Client::new();
    let response = client.post(url)
        .headers(headers)
        .json(&payload)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request POST \
  --url {{baseUrl}}/map \
  --header 'content-type: application/json' \
  --data '{
  "url": "",
  "search": ""
}'

echo '{
  "url": "",
  "search": ""
}' |  \
  http POST {{baseUrl}}/map \
  content-type:application/json

wget --quiet \
  --method POST \
  --header 'content-type: application/json' \
  --body-data '{\n  "url": "",\n  "search": ""\n}' \
  --output-document \
  - {{baseUrl}}/map

import Foundation

let headers = ["content-type": "application/json"]
let parameters = [
  "url": "",
  "search": ""
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/map")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()

RESPONSE HEADERS

Content-Type

application/json

RESPONSE BODY json

{
  "status": "success",
  "links": [
    "https://firecrawl.dev",
    "https://www.firecrawl.dev/pricing",
    "https://www.firecrawl.dev/blog"
  ]
}

POST Scrape a URL and get its content.

{{baseUrl}}/scrape

BODY json

{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/scrape");

struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);

curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/post "{{baseUrl}}/scrape" {:content-type :json
                                                   :form-params {:url ""
                                                                 :formats []
                                                                 :extract {:schema ""
                                                                           :systemPrompt ""
                                                                           :prompt ""}
                                                                 :actions [{:type ""
                                                                            :selector ""
                                                                            :milliseconds 0
                                                                            :text ""
                                                                            :key ""}]}})

require "http/client"

url = "{{baseUrl}}/scrape"
headers = HTTP::Headers{
  "content-type" => "application/json"
}
reqBody = "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"

response = HTTP::Client.post url, headers: headers, body: reqBody
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Post,
    RequestUri = new Uri("{{baseUrl}}/scrape"),
    Content = new StringContent("{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}")
    {
        Headers =
        {
            ContentType = new MediaTypeHeaderValue("application/json")
        }
    }
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/scrape");
var request = new RestRequest("", Method.Post);
request.AddHeader("content-type", "application/json");
request.AddParameter("application/json", "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}", ParameterType.RequestBody);
var response = client.Execute(request);

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/scrape"

	payload := strings.NewReader("{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

POST /baseUrl/scrape HTTP/1.1
Content-Type: application/json
Host: example.com
Content-Length: 242

{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("POST", "{{baseUrl}}/scrape")
  .setHeader("content-type", "application/json")
  .setBody("{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/scrape"))
    .header("content-type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"))
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}");
Request request = new Request.Builder()
  .url("{{baseUrl}}/scrape")
  .post(body)
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.post("{{baseUrl}}/scrape")
  .header("content-type", "application/json")
  .body("{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}")
  .asString();

const data = JSON.stringify({
  url: '',
  formats: [],
  extract: {
    schema: '',
    systemPrompt: '',
    prompt: ''
  },
  actions: [
    {
      type: '',
      selector: '',
      milliseconds: 0,
      text: '',
      key: ''
    }
  ]
});

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('POST', '{{baseUrl}}/scrape');
xhr.setRequestHeader('content-type', 'application/json');

xhr.send(data);

import axios from 'axios';

const options = {
  method: 'POST',
  url: '{{baseUrl}}/scrape',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    formats: [],
    extract: {schema: '', systemPrompt: '', prompt: ''},
    actions: [{type: '', selector: '', milliseconds: 0, text: '', key: ''}]
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/scrape';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""},"actions":[{"type":"","selector":"","milliseconds":0,"text":"","key":""}]}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/scrape',
  method: 'POST',
  headers: {
    'content-type': 'application/json'
  },
  processData: false,
  data: '{\n  "url": "",\n  "formats": [],\n  "extract": {\n    "schema": "",\n    "systemPrompt": "",\n    "prompt": ""\n  },\n  "actions": [\n    {\n      "type": "",\n      "selector": "",\n      "milliseconds": 0,\n      "text": "",\n      "key": ""\n    }\n  ]\n}'
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val mediaType = MediaType.parse("application/json")
val body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}")
val request = Request.Builder()
  .url("{{baseUrl}}/scrape")
  .post(body)
  .addHeader("content-type", "application/json")
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'POST',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/scrape',
  headers: {
    'content-type': 'application/json'
  }
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.write(JSON.stringify({
  url: '',
  formats: [],
  extract: {schema: '', systemPrompt: '', prompt: ''},
  actions: [{type: '', selector: '', milliseconds: 0, text: '', key: ''}]
}));
req.end();

const request = require('request');

const options = {
  method: 'POST',
  url: '{{baseUrl}}/scrape',
  headers: {'content-type': 'application/json'},
  body: {
    url: '',
    formats: [],
    extract: {schema: '', systemPrompt: '', prompt: ''},
    actions: [{type: '', selector: '', milliseconds: 0, text: '', key: ''}]
  },
  json: true
};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('POST', '{{baseUrl}}/scrape');

req.headers({
  'content-type': 'application/json'
});

req.type('json');
req.send({
  url: '',
  formats: [],
  extract: {
    schema: '',
    systemPrompt: '',
    prompt: ''
  },
  actions: [
    {
      type: '',
      selector: '',
      milliseconds: 0,
      text: '',
      key: ''
    }
  ]
});

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: '{{baseUrl}}/scrape',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    formats: [],
    extract: {schema: '', systemPrompt: '', prompt: ''},
    actions: [{type: '', selector: '', milliseconds: 0, text: '', key: ''}]
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/scrape';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""},"actions":[{"type":"","selector":"","milliseconds":0,"text":"","key":""}]}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSDictionary *headers = @{ @"content-type": @"application/json" };
NSDictionary *parameters = @{ @"url": @"",
                              @"formats": @[  ],
                              @"extract": @{ @"schema": @"", @"systemPrompt": @"", @"prompt": @"" },
                              @"actions": @[ @{ @"type": @"", @"selector": @"", @"milliseconds": @0, @"text": @"", @"key": @"" } ] };

NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/scrape"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/scrape" in
let headers = Header.add (Header.init ()) "content-type" "application/json" in
let body = Cohttp_lwt_body.of_string "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}" in

Client.call ~headers ~body `POST uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '',
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ],
    'actions' => [
        [
                'type' => '',
                'selector' => '',
                'milliseconds' => 0,
                'text' => '',
                'key' => ''
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "content-type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('POST', '{{baseUrl}}/scrape', [
  'body' => '{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}',
  'headers' => [
    'content-type' => 'application/json',
  ],
]);

echo $response->getBody();

setUrl('{{baseUrl}}/scrape');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$request->setContentType('application/json');
$request->setBody(json_encode([
  'url' => '',
  'formats' => [
    
  ],
  'extract' => [
    'schema' => '',
    'systemPrompt' => '',
    'prompt' => ''
  ],
  'actions' => [
    [
        'type' => '',
        'selector' => '',
        'milliseconds' => 0,
        'text' => '',
        'key' => ''
    ]
  ]
]));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

append(json_encode([
  'url' => '',
  'formats' => [
    
  ],
  'extract' => [
    'schema' => '',
    'systemPrompt' => '',
    'prompt' => ''
  ],
  'actions' => [
    [
        'type' => '',
        'selector' => '',
        'milliseconds' => 0,
        'text' => '',
        'key' => ''
    ]
  ]
]));
$request->setRequestUrl('{{baseUrl}}/scrape');
$request->setRequestMethod('POST');
$request->setBody($body);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-WebRequest -Uri '{{baseUrl}}/scrape' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}'

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-RestMethod -Uri '{{baseUrl}}/scrape' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}'

import http.client

conn = http.client.HTTPSConnection("example.com")

payload = "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"

headers = { 'content-type': "application/json" }

conn.request("POST", "/baseUrl/scrape", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/scrape"

payload = {
    "url": "",
    "formats": [],
    "extract": {
        "schema": "",
        "systemPrompt": "",
        "prompt": ""
    },
    "actions": [
        {
            "type": "",
            "selector": "",
            "milliseconds": 0,
            "text": "",
            "key": ""
        }
    ]
}
headers = {"content-type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

library(httr)

url <- "{{baseUrl}}/scrape"

payload <- "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"

encode <- "json"

response <- VERB("POST", url, body = payload, content_type("application/json"), encode = encode)

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["content-type"] = 'application/json'
request.body = "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
  headers: {'Content-Type' => 'application/json'}
)

response = conn.post('/baseUrl/scrape') do |req|
  req.body = "{\n  \"url\": \"\",\n  \"formats\": [],\n  \"extract\": {\n    \"schema\": \"\",\n    \"systemPrompt\": \"\",\n    \"prompt\": \"\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"\",\n      \"selector\": \"\",\n      \"milliseconds\": 0,\n      \"text\": \"\",\n      \"key\": \"\"\n    }\n  ]\n}"
end

puts response.status
puts response.body

use serde_json::json;
use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/scrape";

    let payload = json!({
        "url": "",
        "formats": (),
        "extract": json!({
            "schema": "",
            "systemPrompt": "",
            "prompt": ""
        }),
        "actions": (
            json!({
                "type": "",
                "selector": "",
                "milliseconds": 0,
                "text": "",
                "key": ""
            })
        )
    });

    let mut headers = reqwest::header::HeaderMap::new();
    headers.insert("content-type", "application/json".parse().unwrap());

    let client = reqwest::Client::new();
    let response = client.post(url)
        .headers(headers)
        .json(&payload)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request POST \
  --url {{baseUrl}}/scrape \
  --header 'content-type: application/json' \
  --data '{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}'

echo '{
  "url": "",
  "formats": [],
  "extract": {
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  },
  "actions": [
    {
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    }
  ]
}' |  \
  http POST {{baseUrl}}/scrape \
  content-type:application/json

wget --quiet \
  --method POST \
  --header 'content-type: application/json' \
  --body-data '{\n  "url": "",\n  "formats": [],\n  "extract": {\n    "schema": "",\n    "systemPrompt": "",\n    "prompt": ""\n  },\n  "actions": [\n    {\n      "type": "",\n      "selector": "",\n      "milliseconds": 0,\n      "text": "",\n      "key": ""\n    }\n  ]\n}' \
  --output-document \
  - {{baseUrl}}/scrape

import Foundation

let headers = ["content-type": "application/json"]
let parameters = [
  "url": "",
  "formats": [],
  "extract": [
    "schema": "",
    "systemPrompt": "",
    "prompt": ""
  ],
  "actions": [
    [
      "type": "",
      "selector": "",
      "milliseconds": 0,
      "text": "",
      "key": ""
    ]
  ]
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/scrape")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()

POST Submit a crawl job with a webhook

{{baseUrl}}/crawl/webhook

BODY json

{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/crawl/webhook");

struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);

curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/post "{{baseUrl}}/crawl/webhook" {:content-type :json
                                                          :form-params {:url ""
                                                                        :limit 0
                                                                        :webhook ""
                                                                        :excludePaths []
                                                                        :scrapeOptions {:formats []
                                                                                        :extract {:schema ""
                                                                                                  :systemPrompt ""
                                                                                                  :prompt ""}}}})

require "http/client"

url = "{{baseUrl}}/crawl/webhook"
headers = HTTP::Headers{
  "content-type" => "application/json"
}
reqBody = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = HTTP::Client.post url, headers: headers, body: reqBody
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Post,
    RequestUri = new Uri("{{baseUrl}}/crawl/webhook"),
    Content = new StringContent("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
    {
        Headers =
        {
            ContentType = new MediaTypeHeaderValue("application/json")
        }
    }
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/crawl/webhook");
var request = new RestRequest("", Method.Post);
request.AddHeader("content-type", "application/json");
request.AddParameter("application/json", "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}", ParameterType.RequestBody);
var response = client.Execute(request);

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/crawl/webhook"

	payload := strings.NewReader("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

POST /baseUrl/crawl/webhook HTTP/1.1
Content-Type: application/json
Host: example.com
Content-Length: 201

{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("POST", "{{baseUrl}}/crawl/webhook")
  .setHeader("content-type", "application/json")
  .setBody("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/crawl/webhook"))
    .header("content-type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"))
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");
Request request = new Request.Builder()
  .url("{{baseUrl}}/crawl/webhook")
  .post(body)
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.post("{{baseUrl}}/crawl/webhook")
  .header("content-type", "application/json")
  .body("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .asString();

const data = JSON.stringify({
  url: '',
  limit: 0,
  webhook: '',
  excludePaths: [],
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('POST', '{{baseUrl}}/crawl/webhook');
xhr.setRequestHeader('content-type', 'application/json');

xhr.send(data);

import axios from 'axios';

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/webhook',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    limit: 0,
    webhook: '',
    excludePaths: [],
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/crawl/webhook';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","limit":0,"webhook":"","excludePaths":[],"scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/crawl/webhook',
  method: 'POST',
  headers: {
    'content-type': 'application/json'
  },
  processData: false,
  data: '{\n  "url": "",\n  "limit": 0,\n  "webhook": "",\n  "excludePaths": [],\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}'
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val mediaType = MediaType.parse("application/json")
val body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
val request = Request.Builder()
  .url("{{baseUrl}}/crawl/webhook")
  .post(body)
  .addHeader("content-type", "application/json")
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'POST',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/crawl/webhook',
  headers: {
    'content-type': 'application/json'
  }
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.write(JSON.stringify({
  url: '',
  limit: 0,
  webhook: '',
  excludePaths: [],
  scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
}));
req.end();

const request = require('request');

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/webhook',
  headers: {'content-type': 'application/json'},
  body: {
    url: '',
    limit: 0,
    webhook: '',
    excludePaths: [],
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  },
  json: true
};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('POST', '{{baseUrl}}/crawl/webhook');

req.headers({
  'content-type': 'application/json'
});

req.type('json');
req.send({
  url: '',
  limit: 0,
  webhook: '',
  excludePaths: [],
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl/webhook',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    limit: 0,
    webhook: '',
    excludePaths: [],
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/crawl/webhook';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","limit":0,"webhook":"","excludePaths":[],"scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSDictionary *headers = @{ @"content-type": @"application/json" };
NSDictionary *parameters = @{ @"url": @"",
                              @"limit": @0,
                              @"webhook": @"",
                              @"excludePaths": @[  ],
                              @"scrapeOptions": @{ @"formats": @[  ], @"extract": @{ @"schema": @"", @"systemPrompt": @"", @"prompt": @"" } } };

NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/crawl/webhook"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/crawl/webhook" in
let headers = Header.add (Header.init ()) "content-type" "application/json" in
let body = Cohttp_lwt_body.of_string "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}" in

Client.call ~headers ~body `POST uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/crawl/webhook",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '',
    'limit' => 0,
    'webhook' => '',
    'excludePaths' => [
        
    ],
    'scrapeOptions' => [
        'formats' => [
                
        ],
        'extract' => [
                'schema' => '',
                'systemPrompt' => '',
                'prompt' => ''
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "content-type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('POST', '{{baseUrl}}/crawl/webhook', [
  'body' => '{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}',
  'headers' => [
    'content-type' => 'application/json',
  ],
]);

echo $response->getBody();

setUrl('{{baseUrl}}/crawl/webhook');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$request->setContentType('application/json');
$request->setBody(json_encode([
  'url' => '',
  'limit' => 0,
  'webhook' => '',
  'excludePaths' => [
    
  ],
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

append(json_encode([
  'url' => '',
  'limit' => 0,
  'webhook' => '',
  'excludePaths' => [
    
  ],
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));
$request->setRequestUrl('{{baseUrl}}/crawl/webhook');
$request->setRequestMethod('POST');
$request->setBody($body);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-WebRequest -Uri '{{baseUrl}}/crawl/webhook' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-RestMethod -Uri '{{baseUrl}}/crawl/webhook' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

import http.client

conn = http.client.HTTPSConnection("example.com")

payload = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

headers = { 'content-type': "application/json" }

conn.request("POST", "/baseUrl/crawl/webhook", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/crawl/webhook"

payload = {
    "url": "",
    "limit": 0,
    "webhook": "",
    "excludePaths": [],
    "scrapeOptions": {
        "formats": [],
        "extract": {
            "schema": "",
            "systemPrompt": "",
            "prompt": ""
        }
    }
}
headers = {"content-type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

library(httr)

url <- "{{baseUrl}}/crawl/webhook"

payload <- "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

encode <- "json"

response <- VERB("POST", url, body = payload, content_type("application/json"), encode = encode)

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/crawl/webhook")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["content-type"] = 'application/json'
request.body = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
  headers: {'Content-Type' => 'application/json'}
)

response = conn.post('/baseUrl/crawl/webhook') do |req|
  req.body = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"webhook\": \"\",\n  \"excludePaths\": [],\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"
end

puts response.status
puts response.body

use serde_json::json;
use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/crawl/webhook";

    let payload = json!({
        "url": "",
        "limit": 0,
        "webhook": "",
        "excludePaths": (),
        "scrapeOptions": json!({
            "formats": (),
            "extract": json!({
                "schema": "",
                "systemPrompt": "",
                "prompt": ""
            })
        })
    });

    let mut headers = reqwest::header::HeaderMap::new();
    headers.insert("content-type", "application/json".parse().unwrap());

    let client = reqwest::Client::new();
    let response = client.post(url)
        .headers(headers)
        .json(&payload)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request POST \
  --url {{baseUrl}}/crawl/webhook \
  --header 'content-type: application/json' \
  --data '{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

echo '{
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}' |  \
  http POST {{baseUrl}}/crawl/webhook \
  content-type:application/json

wget --quiet \
  --method POST \
  --header 'content-type: application/json' \
  --body-data '{\n  "url": "",\n  "limit": 0,\n  "webhook": "",\n  "excludePaths": [],\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}' \
  --output-document \
  - {{baseUrl}}/crawl/webhook

import Foundation

let headers = ["content-type": "application/json"]
let parameters = [
  "url": "",
  "limit": 0,
  "webhook": "",
  "excludePaths": [],
  "scrapeOptions": [
    "formats": [],
    "extract": [
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    ]
  ]
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/crawl/webhook")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()

POST Submit a crawl job

{{baseUrl}}/crawl

BODY json

{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

Examples

REQUEST

CURL *hnd = curl_easy_init();

curl_easy_setopt(hnd, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(hnd, CURLOPT_URL, "{{baseUrl}}/crawl");

struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "content-type: application/json");
curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, headers);

curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");

CURLcode ret = curl_easy_perform(hnd);

(require '[clj-http.client :as client])

(client/post "{{baseUrl}}/crawl" {:content-type :json
                                                  :form-params {:url ""
                                                                :limit 0
                                                                :excludePaths []
                                                                :allowBackwardLinks false
                                                                :webhook ""
                                                                :scrapeOptions {:formats []
                                                                                :extract {:schema ""
                                                                                          :systemPrompt ""
                                                                                          :prompt ""}}}})

require "http/client"

url = "{{baseUrl}}/crawl"
headers = HTTP::Headers{
  "content-type" => "application/json"
}
reqBody = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = HTTP::Client.post url, headers: headers, body: reqBody
puts response.body

using System.Net.Http.Headers;
var client = new HttpClient();
var request = new HttpRequestMessage
{
    Method = HttpMethod.Post,
    RequestUri = new Uri("{{baseUrl}}/crawl"),
    Content = new StringContent("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
    {
        Headers =
        {
            ContentType = new MediaTypeHeaderValue("application/json")
        }
    }
};
using (var response = await client.SendAsync(request))
{
    response.EnsureSuccessStatusCode();
    var body = await response.Content.ReadAsStringAsync();
    Console.WriteLine(body);
}

var client = new RestClient("{{baseUrl}}/crawl");
var request = new RestRequest("", Method.Post);
request.AddHeader("content-type", "application/json");
request.AddParameter("application/json", "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}", ParameterType.RequestBody);
var response = client.Execute(request);

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "{{baseUrl}}/crawl"

	payload := strings.NewReader("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

POST /baseUrl/crawl HTTP/1.1
Content-Type: application/json
Host: example.com
Content-Length: 232

{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}

AsyncHttpClient client = new DefaultAsyncHttpClient();
client.prepare("POST", "{{baseUrl}}/crawl")
  .setHeader("content-type", "application/json")
  .setBody("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .execute()
  .toCompletableFuture()
  .thenAccept(System.out::println)
  .join();

client.close();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("{{baseUrl}}/crawl"))
    .header("content-type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"))
    .build();
HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
System.out.println(response.body());

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}");
Request request = new Request.Builder()
  .url("{{baseUrl}}/crawl")
  .post(body)
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

HttpResponse response = Unirest.post("{{baseUrl}}/crawl")
  .header("content-type", "application/json")
  .body("{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
  .asString();

const data = JSON.stringify({
  url: '',
  limit: 0,
  excludePaths: [],
  allowBackwardLinks: false,
  webhook: '',
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

const xhr = new XMLHttpRequest();
xhr.withCredentials = true;

xhr.addEventListener('readystatechange', function () {
  if (this.readyState === this.DONE) {
    console.log(this.responseText);
  }
});

xhr.open('POST', '{{baseUrl}}/crawl');
xhr.setRequestHeader('content-type', 'application/json');

xhr.send(data);

import axios from 'axios';

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    limit: 0,
    excludePaths: [],
    allowBackwardLinks: false,
    webhook: '',
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const url = '{{baseUrl}}/crawl';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","limit":0,"excludePaths":[],"allowBackwardLinks":false,"webhook":"","scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

const settings = {
  async: true,
  crossDomain: true,
  url: '{{baseUrl}}/crawl',
  method: 'POST',
  headers: {
    'content-type': 'application/json'
  },
  processData: false,
  data: '{\n  "url": "",\n  "limit": 0,\n  "excludePaths": [],\n  "allowBackwardLinks": false,\n  "webhook": "",\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}'
};

$.ajax(settings).done(function (response) {
  console.log(response);
});

val client = OkHttpClient()

val mediaType = MediaType.parse("application/json")
val body = RequestBody.create(mediaType, "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}")
val request = Request.Builder()
  .url("{{baseUrl}}/crawl")
  .post(body)
  .addHeader("content-type", "application/json")
  .build()

val response = client.newCall(request).execute()

const http = require('https');

const options = {
  method: 'POST',
  hostname: 'example.com',
  port: null,
  path: '/baseUrl/crawl',
  headers: {
    'content-type': 'application/json'
  }
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on('data', function (chunk) {
    chunks.push(chunk);
  });

  res.on('end', function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.write(JSON.stringify({
  url: '',
  limit: 0,
  excludePaths: [],
  allowBackwardLinks: false,
  webhook: '',
  scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
}));
req.end();

const request = require('request');

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl',
  headers: {'content-type': 'application/json'},
  body: {
    url: '',
    limit: 0,
    excludePaths: [],
    allowBackwardLinks: false,
    webhook: '',
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  },
  json: true
};

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

const unirest = require('unirest');

const req = unirest('POST', '{{baseUrl}}/crawl');

req.headers({
  'content-type': 'application/json'
});

req.type('json');
req.send({
  url: '',
  limit: 0,
  excludePaths: [],
  allowBackwardLinks: false,
  webhook: '',
  scrapeOptions: {
    formats: [],
    extract: {
      schema: '',
      systemPrompt: '',
      prompt: ''
    }
  }
});

req.end(function (res) {
  if (res.error) throw new Error(res.error);

  console.log(res.body);
});

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: '{{baseUrl}}/crawl',
  headers: {'content-type': 'application/json'},
  data: {
    url: '',
    limit: 0,
    excludePaths: [],
    allowBackwardLinks: false,
    webhook: '',
    scrapeOptions: {formats: [], extract: {schema: '', systemPrompt: '', prompt: ''}}
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

const fetch = require('node-fetch');

const url = '{{baseUrl}}/crawl';
const options = {
  method: 'POST',
  headers: {'content-type': 'application/json'},
  body: '{"url":"","limit":0,"excludePaths":[],"allowBackwardLinks":false,"webhook":"","scrapeOptions":{"formats":[],"extract":{"schema":"","systemPrompt":"","prompt":""}}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

#import 

NSDictionary *headers = @{ @"content-type": @"application/json" };
NSDictionary *parameters = @{ @"url": @"",
                              @"limit": @0,
                              @"excludePaths": @[  ],
                              @"allowBackwardLinks": @NO,
                              @"webhook": @"",
                              @"scrapeOptions": @{ @"formats": @[  ], @"extract": @{ @"schema": @"", @"systemPrompt": @"", @"prompt": @"" } } };

NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];

NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"{{baseUrl}}/crawl"]
                                                       cachePolicy:NSURLRequestUseProtocolCachePolicy
                                                   timeoutInterval:10.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];

NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
                                            completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
                                                if (error) {
                                                    NSLog(@"%@", error);
                                                } else {
                                                    NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
                                                    NSLog(@"%@", httpResponse);
                                                }
                                            }];
[dataTask resume];

open Cohttp_lwt_unix
open Cohttp
open Lwt

let uri = Uri.of_string "{{baseUrl}}/crawl" in
let headers = Header.add (Header.init ()) "content-type" "application/json" in
let body = Cohttp_lwt_body.of_string "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}" in

Client.call ~headers ~body `POST uri
>>= fun (res, body_stream) ->
  (* Do stuff with the result *)

 "{{baseUrl}}/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '',
    'limit' => 0,
    'excludePaths' => [
        
    ],
    'allowBackwardLinks' => null,
    'webhook' => '',
    'scrapeOptions' => [
        'formats' => [
                
        ],
        'extract' => [
                'schema' => '',
                'systemPrompt' => '',
                'prompt' => ''
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "content-type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

request('POST', '{{baseUrl}}/crawl', [
  'body' => '{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}',
  'headers' => [
    'content-type' => 'application/json',
  ],
]);

echo $response->getBody();

setUrl('{{baseUrl}}/crawl');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$request->setContentType('application/json');
$request->setBody(json_encode([
  'url' => '',
  'limit' => 0,
  'excludePaths' => [
    
  ],
  'allowBackwardLinks' => null,
  'webhook' => '',
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

append(json_encode([
  'url' => '',
  'limit' => 0,
  'excludePaths' => [
    
  ],
  'allowBackwardLinks' => null,
  'webhook' => '',
  'scrapeOptions' => [
    'formats' => [
        
    ],
    'extract' => [
        'schema' => '',
        'systemPrompt' => '',
        'prompt' => ''
    ]
  ]
]));
$request->setRequestUrl('{{baseUrl}}/crawl');
$request->setRequestMethod('POST');
$request->setBody($body);

$request->setHeaders([
  'content-type' => 'application/json'
]);

$client->enqueue($request)->send();
$response = $client->getResponse();

echo $response->getBody();

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-WebRequest -Uri '{{baseUrl}}/crawl' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

$headers=@{}
$headers.Add("content-type", "application/json")
$response = Invoke-RestMethod -Uri '{{baseUrl}}/crawl' -Method POST -Headers $headers -ContentType 'application/json' -Body '{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

import http.client

conn = http.client.HTTPSConnection("example.com")

payload = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

headers = { 'content-type': "application/json" }

conn.request("POST", "/baseUrl/crawl", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

import requests

url = "{{baseUrl}}/crawl"

payload = {
    "url": "",
    "limit": 0,
    "excludePaths": [],
    "allowBackwardLinks": False,
    "webhook": "",
    "scrapeOptions": {
        "formats": [],
        "extract": {
            "schema": "",
            "systemPrompt": "",
            "prompt": ""
        }
    }
}
headers = {"content-type": "application/json"}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

library(httr)

url <- "{{baseUrl}}/crawl"

payload <- "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

encode <- "json"

response <- VERB("POST", url, body = payload, content_type("application/json"), encode = encode)

content(response, "text")

require 'uri'
require 'net/http'

url = URI("{{baseUrl}}/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["content-type"] = 'application/json'
request.body = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"

response = http.request(request)
puts response.read_body

require 'faraday'

conn = Faraday.new(
  url: 'https://example.com',
  headers: {'Content-Type' => 'application/json'}
)

response = conn.post('/baseUrl/crawl') do |req|
  req.body = "{\n  \"url\": \"\",\n  \"limit\": 0,\n  \"excludePaths\": [],\n  \"allowBackwardLinks\": false,\n  \"webhook\": \"\",\n  \"scrapeOptions\": {\n    \"formats\": [],\n    \"extract\": {\n      \"schema\": \"\",\n      \"systemPrompt\": \"\",\n      \"prompt\": \"\"\n    }\n  }\n}"
end

puts response.status
puts response.body

use serde_json::json;
use reqwest;

#[tokio::main]
pub async fn main() {
    let url = "{{baseUrl}}/crawl";

    let payload = json!({
        "url": "",
        "limit": 0,
        "excludePaths": (),
        "allowBackwardLinks": false,
        "webhook": "",
        "scrapeOptions": json!({
            "formats": (),
            "extract": json!({
                "schema": "",
                "systemPrompt": "",
                "prompt": ""
            })
        })
    });

    let mut headers = reqwest::header::HeaderMap::new();
    headers.insert("content-type", "application/json".parse().unwrap());

    let client = reqwest::Client::new();
    let response = client.post(url)
        .headers(headers)
        .json(&payload)
        .send()
        .await;

    let results = response.unwrap()
        .json::()
        .await
        .unwrap();

    dbg!(results);
}

curl --request POST \
  --url {{baseUrl}}/crawl \
  --header 'content-type: application/json' \
  --data '{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}'

echo '{
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": {
    "formats": [],
    "extract": {
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    }
  }
}' |  \
  http POST {{baseUrl}}/crawl \
  content-type:application/json

wget --quiet \
  --method POST \
  --header 'content-type: application/json' \
  --body-data '{\n  "url": "",\n  "limit": 0,\n  "excludePaths": [],\n  "allowBackwardLinks": false,\n  "webhook": "",\n  "scrapeOptions": {\n    "formats": [],\n    "extract": {\n      "schema": "",\n      "systemPrompt": "",\n      "prompt": ""\n    }\n  }\n}' \
  --output-document \
  - {{baseUrl}}/crawl

import Foundation

let headers = ["content-type": "application/json"]
let parameters = [
  "url": "",
  "limit": 0,
  "excludePaths": [],
  "allowBackwardLinks": false,
  "webhook": "",
  "scrapeOptions": [
    "formats": [],
    "extract": [
      "schema": "",
      "systemPrompt": "",
      "prompt": ""
    ]
  ]
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "{{baseUrl}}/crawl")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()