Crawling

import { KadoaClient } from '@kadoa/node-sdk';

const client = new KadoaClient({ apiKey: 'YOUR_API_KEY' });

const result = await client.crawler.session.start({
  url: "https://demo.vercel.store/",
  maxDepth: 10,
  maxPages: 50,
});

console.log(result.sessionId);

const result = await client.crawler.session.start({
  startUrls: [
    "https://demo.vercel.store/",
    "https://demo.vercel.store/collections",
    "https://demo.vercel.store/about",
  ],
  maxDepth: 10,
  maxPages: 50,
});

const status = await client.crawler.session.getSessionStatus(sessionId);

console.log(status.payload.crawledPages);
console.log(status.payload.finished);

const pages = await client.crawler.session.getPages(sessionId, {
  currentPage: 1,
  pageSize: 100,
});

for (const page of pages.payload) {
  console.log(page.id, page.url, page.status);
}

// Get as markdown
const markdown = await client.crawler.session.getPage(sessionId, pageId, {
  format: "markdown",
});

console.log(markdown.payload);

// Get as HTML
const html = await client.crawler.session.getPage(sessionId, pageId, {
  format: "html",
});