scrape-config.json

{
  "index_name": "kagi-docs",
  "start_urls": [
    "https://help.kagi.com/kagi",
    "https://help.kagi.com/orion"
  ],
  "rateLimit": 8,
  "maxDepth": 10,
  "selectors": {
    "lvl0": {
      "selector": "",
      "defaultValue": "Documentation"
    },
    "lvl1": "header h1, article h1, main h1, h1, head > title",
    "lvl2": "article h2, main h2, h2",
    "lvl3": "article h3, main h3, h3",
    "lvl4": "article h4, main h4, h4",
    "lvl5": "article h5, main h5, h5",
    "content": "article p, article li, main p, main li, p, li",
    "lang": {
      "selector": "/html/@lang",
      "type": "xpath",
      "global": true
    }
  },
  "custom_settings": {
      "attributesForFaceting": ["type", "lang"],
      "attributesToRetrieve": [
        "hierarchy",
        "content",
        "anchor",
        "url",
        "url_without_anchor",
        "type"
      ],
      "attributesToHighlight": ["hierarchy", "content"],
      "attributesToSnippet": ["content:10"],
      "camelCaseAttributes": ["hierarchy", "content"],
      "searchableAttributes": [
        "unordered(hierarchy.lvl0)",
        "unordered(hierarchy.lvl1)",
        "unordered(hierarchy.lvl2)",
        "unordered(hierarchy.lvl3)",
        "unordered(hierarchy.lvl4)",
        "unordered(hierarchy.lvl5)",
        "unordered(hierarchy.lvl6)",
        "content"
      ],
      "distinct": true,
      "attributeForDistinct": "url",
      "customRanking": [
        "desc(weight.pageRank)",
        "desc(weight.level)",
        "asc(weight.position)"
      ],
      "ranking": [
        "words",
        "filters",
        "typo",
        "attribute",
        "proximity",
        "exact",
        "custom"
      ],
      "highlightPreTag": "<span class='algolia-docsearch-suggestion--highlight'>",
      "highlightPostTag": "</span>",
      "minWordSizefor1Typo": 3,
      "minWordSizefor2Typos": 7,
      "allowTyposOnNumericTokens": false,
      "minProximity": 1,
      "ignorePlurals": true,
      "advancedSyntax": true,
      "attributeCriteriaComputedByMinProximity": true,
      "removeWordsIfNoResults": "allOptional"
  },
  "js_render": false
}