Sitemap Scraper API

# 1. Create the task (returns 202 Accepted and the task "@id")
curl --request POST 'https://api.datablist.com/v1/sources/tasks' \
  --header 'X-API-Key: YOUR_PERSONAL_API_KEY' \
  --header 'Content-Type: application/json' \
  --data '{
    "sourceId": "sitemap-source",
    "uniqueId": "sitemap-source-run-1",
    "settings": {
      "sitemapUrl": "https://northpeak-coffee.de/sitemap.xml",
      "filterRegex": "/blog/"
    }
  }'

# 2. Poll every few seconds until status is completed, failed or stopped
curl 'https://api.datablist.com/v1/sources/tasks/TASK_ID' \
  --header 'X-API-Key: YOUR_PERSONAL_API_KEY'

# 3. Read the results, then follow nextCursor until it is null
curl 'https://api.datablist.com/v1/sources/tasks/TASK_ID/results?limit=50' \
  --header 'X-API-Key: YOUR_PERSONAL_API_KEY'

import time
import requests

BASE = "https://api.datablist.com/v1/sources"
HEADERS = {"X-API-Key": "YOUR_PERSONAL_API_KEY"}

# 1. Create the task
response = requests.post(f"{BASE}/tasks", headers=HEADERS, json={
  "sourceId": "sitemap-source",
  "uniqueId": "sitemap-source-run-1",
  "settings": {
    "sitemapUrl": "https://northpeak-coffee.de/sitemap.xml",
    "filterRegex": "/blog/"
  }
})
response.raise_for_status()
task_id = response.json()["@id"]

# 2. Poll until the task finishes
while True:
    task = requests.get(f"{BASE}/tasks/{task_id}", headers=HEADERS).json()
    if task["status"] in ("completed", "failed", "stopped"):
        break
    time.sleep(5)

# 3. Read every result page
params = {"limit": 50}
while True:
    page = requests.get(f"{BASE}/tasks/{task_id}/results", headers=HEADERS, params=params).json()
    for result in page["results"]:
        print(result)
    if not page["nextCursor"]:
        break
    params["cursor"] = page["nextCursor"]

// Node.js 18+ or any runtime with fetch. Keep your key on the server.
const BASE = "https://api.datablist.com/v1/sources";
const HEADERS = { "X-API-Key": "YOUR_PERSONAL_API_KEY" };
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// 1. Create the task
const created = await fetch(`${BASE}/tasks`, {
  method: "POST",
  headers: { ...HEADERS, "Content-Type": "application/json" },
  body: JSON.stringify({
    "sourceId": "sitemap-source",
    "uniqueId": "sitemap-source-run-1",
    "settings": {
      "sitemapUrl": "https://northpeak-coffee.de/sitemap.xml",
      "filterRegex": "/blog/"
    }
  }),
});
if (!created.ok) {
  throw new Error(`Create failed: ${created.status}`);
}
const taskId = (await created.json())["@id"];

// 2. Poll until the task finishes
let task;
do {
  await sleep(5000);
  task = await (await fetch(`${BASE}/tasks/${taskId}`, { headers: HEADERS })).json();
} while (!["completed", "failed", "stopped"].includes(task.status));

// 3. Read every result page
let cursor = null;
do {
  const url = new URL(`${BASE}/tasks/${taskId}/results`);
  url.searchParams.set("limit", "50");
  if (cursor) {
    url.searchParams.set("cursor", cursor);
  }
  const page = await (await fetch(url, { headers: HEADERS })).json();
  for (const result of page.results) {
    console.log(result);
  }
  cursor = page.nextCursor;
} while (cursor);