Page Not Found
We could not find what you were looking for 😢
We could not find what you were looking for 😢
playwright.gotoExtended",id:"playwrightgotoextended",level:2}],u={toc:c},g="wrapper";function m(e){var t=e.components,r=(0,a.Z)(e,o);return(0,i.kt)(g,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("a",{name:"playwright"}),(0,i.kt)("p",null,"A namespace that contains various utilities for ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/microsoft/playwright",target:"_blank",rel:"noopener"},"Playwright")," - the headless Chrome Node API."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Example usage:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-javascript"},"const Apify = require('apify');\nconst { playwright } = Apify.utils;\n\n// Navigate to https://www.example.com in Playwright with a POST request\nconst browser = await Apify.launchPlaywright();\nconst page = await browser.newPage();\nawait playwright.gotoExtended(page, {\n url: 'https://example.com,\n method: 'POST',\n});\n")),(0,i.kt)("hr",null),(0,i.kt)("a",{name:"gotoextended"}),(0,i.kt)("h2",{id:"playwrightgotoextended"},(0,i.kt)("inlineCode",{parentName:"h2"},"playwright.gotoExtended")),(0,i.kt)("p",null,"Extended version of Playwright's ",(0,i.kt)("inlineCode",{parentName:"p"},"page.goto()")," allowing to perform requests with HTTP method other than GET, with custom headers and POST payload.\nURL, method, headers and payload are taken from request parameter that must be an instance of Apify.Request class."),(0,i.kt)("p",null,(0,i.kt)("em",{parentName:"p"},"NOTE:")," In recent versions of Playwright using requests other than GET, overriding headers and adding payloads disables browser cache which degrades\nperformance."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"page")),": ",(0,i.kt)("inlineCode",{parentName:"li"},"Page")," - Puppeteer ",(0,i.kt)("a",{parentName:"li",href:"https://playwright.dev/docs/api/class-page",target:"_blank",rel:"noopener"},(0,i.kt)("inlineCode",{parentName:"a"},"Page"))," object."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"request")),": ",(0,i.kt)("a",{parentName:"li",href:"../api/request",target:null,rel:null},(0,i.kt)("inlineCode",{parentName:"a"},"Request"))),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"[gotoOptions]")),": ",(0,i.kt)("a",{parentName:"li",href:"../typedefs/direct-navigation-options",target:null,rel:null},(0,i.kt)("inlineCode",{parentName:"a"},"DirectNavigationOptions"))," - Custom options for ",(0,i.kt)("inlineCode",{parentName:"li"},"page.goto()"),".")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Returns"),":"),(0,i.kt)("p",null,(0,i.kt)("inlineCode",{parentName:"p"},"Promise<(Response|null)>")),(0,i.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>d,kt:()=>m});var n=r(67294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;tplaywright.gotoExtended",id:"playwrightgotoextended",level:2}],u={toc:c},g="wrapper";function m(e){var t=e.components,r=(0,a.Z)(e,o);return(0,i.kt)(g,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("a",{name:"playwright"}),(0,i.kt)("p",null,"A namespace that contains various utilities for ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/microsoft/playwright",target:"_blank",rel:"noopener"},"Playwright")," - the headless Chrome Node API."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Example usage:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-javascript"},"const Apify = require('apify');\nconst { playwright } = Apify.utils;\n\n// Navigate to https://www.example.com in Playwright with a POST request\nconst browser = await Apify.launchPlaywright();\nconst page = await browser.newPage();\nawait playwright.gotoExtended(page, {\n url: 'https://example.com,\n method: 'POST',\n});\n")),(0,i.kt)("hr",null),(0,i.kt)("a",{name:"gotoextended"}),(0,i.kt)("h2",{id:"playwrightgotoextended"},(0,i.kt)("inlineCode",{parentName:"h2"},"playwright.gotoExtended")),(0,i.kt)("p",null,"Extended version of Playwright's ",(0,i.kt)("inlineCode",{parentName:"p"},"page.goto()")," allowing to perform requests with HTTP method other than GET, with custom headers and POST payload.\nURL, method, headers and payload are taken from request parameter that must be an instance of Apify.Request class."),(0,i.kt)("p",null,(0,i.kt)("em",{parentName:"p"},"NOTE:")," In recent versions of Playwright using requests other than GET, overriding headers and adding payloads disables browser cache which degrades\nperformance."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"page")),": ",(0,i.kt)("inlineCode",{parentName:"li"},"Page")," - Puppeteer ",(0,i.kt)("a",{parentName:"li",href:"https://playwright.dev/docs/api/class-page",target:"_blank",rel:"noopener"},(0,i.kt)("inlineCode",{parentName:"a"},"Page"))," object."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"request")),": ",(0,i.kt)("a",{parentName:"li",href:"../api/request",target:null,rel:null},(0,i.kt)("inlineCode",{parentName:"a"},"Request"))),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},(0,i.kt)("inlineCode",{parentName:"strong"},"[gotoOptions]")),": ",(0,i.kt)("a",{parentName:"li",href:"../typedefs/direct-navigation-options",target:null,rel:null},(0,i.kt)("inlineCode",{parentName:"a"},"DirectNavigationOptions"))," - Custom options for ",(0,i.kt)("inlineCode",{parentName:"li"},"page.goto()"),".")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Returns"),":"),(0,i.kt)("p",null,(0,i.kt)("inlineCode",{parentName:"p"},"Promise<(Response|null)>")),(0,i.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>d,kt:()=>m});var n=r(67294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;teventTypes",id:"eventtypes",level:3},{value:"requestUrl",id:"requesturl",level:3},{value:"idempotencyKey",id:"idempotencykey",level:3},{value:"payloadTemplate",id:"payloadtemplate",level:3}],u={toc:s},y="wrapper";function m(e){var t=e.components,r=(0,o.Z)(e,p);return(0,a.kt)(y,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("a",{name:"adhocwebhook"}),(0,a.kt)("h2",{id:"properties"},"Properties"),(0,a.kt)("h3",{id:"eventtypes"},(0,a.kt)("inlineCode",{parentName:"h3"},"eventTypes")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("a",{parentName:"p",href:"../typedefs/event-types",target:null,rel:null},(0,a.kt)("inlineCode",{parentName:"a"},"EventTypes"))),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"requesturl"},(0,a.kt)("inlineCode",{parentName:"h3"},"requestUrl")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"idempotencykey"},(0,a.kt)("inlineCode",{parentName:"h3"},"idempotencyKey")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"payloadtemplate"},(0,a.kt)("inlineCode",{parentName:"h3"},"payloadTemplate")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>c,kt:()=>m});var n=r(67294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function p(e){for(var t=1;teventTypes",id:"eventtypes",level:3},{value:"requestUrl",id:"requesturl",level:3},{value:"idempotencyKey",id:"idempotencykey",level:3},{value:"payloadTemplate",id:"payloadtemplate",level:3}],u={toc:s},y="wrapper";function m(e){var t=e.components,r=(0,o.Z)(e,p);return(0,a.kt)(y,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("a",{name:"adhocwebhook"}),(0,a.kt)("h2",{id:"properties"},"Properties"),(0,a.kt)("h3",{id:"eventtypes"},(0,a.kt)("inlineCode",{parentName:"h3"},"eventTypes")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("a",{parentName:"p",href:"../typedefs/event-types",target:null,rel:null},(0,a.kt)("inlineCode",{parentName:"a"},"EventTypes"))),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"requesturl"},(0,a.kt)("inlineCode",{parentName:"h3"},"requestUrl")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"idempotencykey"},(0,a.kt)("inlineCode",{parentName:"h3"},"idempotencyKey")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null),(0,a.kt)("h3",{id:"payloadtemplate"},(0,a.kt)("inlineCode",{parentName:"h3"},"payloadTemplate")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"string")),(0,a.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>c,kt:()=>m});var n=r(67294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function p(e){for(var t=1;tlaunchOptions",id:"launchoptions",level:3},{value:"proxyUrl",id:"proxyurl",level:3},{value:"userAgent",id:"useragent",level:3},{value:"useChrome",id:"usechrome",level:3},{value:"launcher",id:"launcher",level:3},{value:"useIncognitoPages",id:"useincognitopages",level:3},{value:"stealth",id:"stealth",level:3},{value:"stealthOptions",id:"stealthoptions",level:3}],d={toc:c},h="wrapper";function m(e){var t=e.components,n=(0,o.Z)(e,p);return(0,a.kt)(h,(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("a",{name:"puppeteerlaunchcontext"}),(0,a.kt)("h1",{id:"puppeteerlaunchcontext"},(0,a.kt)("del",{parentName:"h1"},(0,a.kt)("inlineCode",{parentName:"del"},"PuppeteerLaunchContext"))),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},(0,a.kt)("em",{parentName:"strong"},"Deprecated"))),(0,a.kt)("p",null,"Apify extends the launch options of Puppeteer. You can use any of the Puppeteer compatible\n",(0,a.kt)("a",{parentName:"p",href:"https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions",target:"_blank",rel:"noopener"},(0,a.kt)("inlineCode",{parentName:"a"},"LaunchOptions"))," options by providing the ",(0,a.kt)("inlineCode",{parentName:"p"},"launchOptions")," property."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Example:")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-js"},"// launch a headless Chrome (not Chromium)\nconst launchContext = {\n // Apify helpers\n useChrome: true,\n proxyUrl: 'http://user:password@some.proxy.com'\n // Native Puppeteer options\n launchOptions: {\n headless: true,\n args: ['--some-flag'],\n }\n}\n")),(0,a.kt)("h2",{id:"properties"},"Properties"),(0,a.kt)("h3",{id:"launchoptions"},(0,a.kt)("inlineCode",{parentName:"h3"},"launchOptions")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Type"),": ",(0,a.kt)("inlineCode",{parentName:"p"},"RecordlaunchOptions",id:"launchoptions",level:3},{value:"proxyUrl",id:"proxyurl",level:3},{value:"userAgent",id:"useragent",level:3},{value:"useChrome",id:"usechrome",level:3},{value:"launcher",id:"launcher",level:3},{value:"useIncognitoPages",id:"useincognitopages",level:3},{value:"stealth",id:"stealth",level:3},{value:"stealthOptions",id:"stealthoptions",level:3}],d={toc:c},h="wrapper";function m(e){var t=e.components,n=(0,a.Z)(e,p);return(0,o.kt)(h,(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("a",{name:"puppeteerlaunchcontext"}),(0,o.kt)("h1",{id:"puppeteerlaunchcontext"},(0,o.kt)("del",{parentName:"h1"},(0,o.kt)("inlineCode",{parentName:"del"},"PuppeteerLaunchContext"))),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},(0,o.kt)("em",{parentName:"strong"},"Deprecated"))),(0,o.kt)("p",null,"Apify extends the launch options of Puppeteer. You can use any of the Puppeteer compatible\n",(0,o.kt)("a",{parentName:"p",href:"https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions",target:"_blank",rel:"noopener"},(0,o.kt)("inlineCode",{parentName:"a"},"LaunchOptions"))," options by providing the ",(0,o.kt)("inlineCode",{parentName:"p"},"launchOptions")," property."),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Example:")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-js"},"// launch a headless Chrome (not Chromium)\nconst launchContext = {\n // Apify helpers\n useChrome: true,\n proxyUrl: 'http://user:password@some.proxy.com'\n // Native Puppeteer options\n launchOptions: {\n headless: true,\n args: ['--some-flag'],\n }\n}\n")),(0,o.kt)("h2",{id:"properties"},"Properties"),(0,o.kt)("h3",{id:"launchoptions"},(0,o.kt)("inlineCode",{parentName:"h3"},"launchOptions")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"Recordsocial.LINKEDIN_REGEX",id:"sociallinkedin_regex",level:2},{value:"social.LINKEDIN_REGEX_GLOBAL",id:"sociallinkedin_regex_global",level:2},{value:"social.INSTAGRAM_REGEX",id:"socialinstagram_regex",level:2},{value:"social.INSTAGRAM_REGEX_GLOBAL",id:"socialinstagram_regex_global",level:2},{value:"social.TWITTER_REGEX",id:"socialtwitter_regex",level:2},{value:"social.TWITTER_REGEX_GLOBAL",id:"socialtwitter_regex_global",level:2},{value:"social.FACEBOOK_REGEX",id:"socialfacebook_regex",level:2},{value:"social.FACEBOOK_REGEX_GLOBAL",id:"socialfacebook_regex_global",level:2},{value:"social.YOUTUBE_REGEX",id:"socialyoutube_regex",level:2},{value:"social.YOUTUBE_REGEX_GLOBAL",id:"socialyoutube_regex_global",level:2},{value:"social.EMAIL_REGEX",id:"socialemail_regex",level:2},{value:"social.EMAIL_REGEX_GLOBAL",id:"socialemail_regex_global",level:2},{value:"social.emailsFromText(text)",id:"socialemailsfromtexttext",level:2},{value:"social.emailsFromUrls(urls)",id:"socialemailsfromurlsurls",level:2},{value:"social.phonesFromText(text)",id:"socialphonesfromtexttext",level:2},{value:"social.phonesFromUrls(urls)",id:"socialphonesfromurlsurls",level:2},{value:"social.parseHandlesFromHtml(html, [data])",id:"socialparsehandlesfromhtmlhtml-data",level:2}],u={toc:m},d="wrapper";function h(e){var t=e.components,a=(0,l.Z)(e,r);return(0,o.kt)(d,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("a",{name:"social"}),(0,o.kt)("p",null,"A namespace that contains various utilities to help you extract social handles from text, URLs and and HTML documents."),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Example usage:")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-javascript"},"const Apify = require('apify');\n\nconst emails = Apify.utils.social.emailsFromText('alice@example.com bob@example.com');\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"linkedin_regex"}),(0,o.kt)("h2",{id:"sociallinkedin_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.LINKEDIN_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single LinkedIn profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/alan-turing\nen.linkedin.com/in/alan-turing\nlinkedin.com/in/alan-turing\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds/latest-activity\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"linkedin_regex_global"}),(0,o.kt)("h2",{id:"sociallinkedin_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.LINKEDIN_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple LinkedIn profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/alan-turing\nen.linkedin.com/in/alan-turing\nlinkedin.com/in/alan-turing\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds/latest-activity\n")),(0,o.kt)("p",null,"the expression extracts just the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.LINKEDIN_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} LinkedIn profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"instagram_regex"}),(0,o.kt)("h2",{id:"socialinstagram_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.INSTAGRAM_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Instagram profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/old_prague\nwww.instagram.com/old_prague/\ninstagr.am/old_prague\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano/followers\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"instagram_regex_global"}),(0,o.kt)("h2",{id:"socialinstagram_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.INSTAGRAM_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Instagram profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/old_prague\nwww.instagram.com/old_prague/\ninstagr.am/old_prague\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano/followers\n")),(0,o.kt)("p",null,"the expression extracts just the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.INSTAGRAM_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Instagram profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"twitter_regex"}),(0,o.kt)("h2",{id:"socialtwitter_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.TWITTER_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Twitter profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/apify\ntwitter.com/apify\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump/following\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"twitter_regex_global"}),(0,o.kt)("h2",{id:"socialtwitter_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.TWITTER_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Twitter profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/apify\ntwitter.com/apify\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump/following\n")),(0,o.kt)("p",null,"the expression extracts only the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.TWITTER_REGEX_STRING);\nif (matches) console.log(`${matches.length} Twitter profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"facebook_regex"}),(0,o.kt)("h2",{id:"socialfacebook_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.FACEBOOK_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Facebook profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\nfacebook.com/apifytech\nfb.com/apifytech\nhttps://www.facebook.com/profile.php?id=123456789\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech/photos\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"facebook_regex_global"}),(0,o.kt)("h2",{id:"socialfacebook_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.FACEBOOK_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Facebook profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\nfacebook.com/apifytech\nfb.com/apifytech\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech/photos\n")),(0,o.kt)("p",null,"the expression extracts only the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.FACEBOOK_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Facebook profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"youtube_regex"}),(0,o.kt)("h2",{id:"socialyoutube_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.YOUTUBE_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Youtube video URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.youtube.com/watch?v=kM7YfhfkiEE\nhttps://youtu.be/kM7YfhfkiEE\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"youtube_regex_global"}),(0,o.kt)("h2",{id:"socialyoutube_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.YOUTUBE_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Youtube video URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.youtube.com/watch?v=kM7YfhfkiEE\nhttps://youtu.be/kM7YfhfkiEE\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Youtube videos found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"email_regex"}),(0,o.kt)("h2",{id:"socialemail_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.EMAIL_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single email address. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i"),"."),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"email_regex_global"}),(0,o.kt)("h2",{id:"socialemail_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.EMAIL_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple email addresses in a text. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig"),"."),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"emailsfromtext"}),(0,o.kt)("h2",{id:"socialemailsfromtexttext"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.emailsFromText(text)")),(0,o.kt)("p",null,"The function extracts email addresses from a plain text. Note that the function preserves the order of emails and keep duplicates."),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},(0,o.kt)("inlineCode",{parentName:"strong"},"text")),": ",(0,o.kt)("inlineCode",{parentName:"li"},"string")," - Text to search in.")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Returns"),":"),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"Arraysocial.LINKEDIN_REGEX",id:"sociallinkedin_regex",level:2},{value:"social.LINKEDIN_REGEX_GLOBAL",id:"sociallinkedin_regex_global",level:2},{value:"social.INSTAGRAM_REGEX",id:"socialinstagram_regex",level:2},{value:"social.INSTAGRAM_REGEX_GLOBAL",id:"socialinstagram_regex_global",level:2},{value:"social.TWITTER_REGEX",id:"socialtwitter_regex",level:2},{value:"social.TWITTER_REGEX_GLOBAL",id:"socialtwitter_regex_global",level:2},{value:"social.FACEBOOK_REGEX",id:"socialfacebook_regex",level:2},{value:"social.FACEBOOK_REGEX_GLOBAL",id:"socialfacebook_regex_global",level:2},{value:"social.YOUTUBE_REGEX",id:"socialyoutube_regex",level:2},{value:"social.YOUTUBE_REGEX_GLOBAL",id:"socialyoutube_regex_global",level:2},{value:"social.EMAIL_REGEX",id:"socialemail_regex",level:2},{value:"social.EMAIL_REGEX_GLOBAL",id:"socialemail_regex_global",level:2},{value:"social.emailsFromText(text)",id:"socialemailsfromtexttext",level:2},{value:"social.emailsFromUrls(urls)",id:"socialemailsfromurlsurls",level:2},{value:"social.phonesFromText(text)",id:"socialphonesfromtexttext",level:2},{value:"social.phonesFromUrls(urls)",id:"socialphonesfromurlsurls",level:2},{value:"social.parseHandlesFromHtml(html, [data])",id:"socialparsehandlesfromhtmlhtml-data",level:2}],u={toc:m},d="wrapper";function h(e){var t=e.components,a=(0,l.Z)(e,r);return(0,o.kt)(d,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("a",{name:"social"}),(0,o.kt)("p",null,"A namespace that contains various utilities to help you extract social handles from text, URLs and and HTML documents."),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Example usage:")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-javascript"},"const Apify = require('apify');\n\nconst emails = Apify.utils.social.emailsFromText('alice@example.com bob@example.com');\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"linkedin_regex"}),(0,o.kt)("h2",{id:"sociallinkedin_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.LINKEDIN_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single LinkedIn profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/alan-turing\nen.linkedin.com/in/alan-turing\nlinkedin.com/in/alan-turing\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds/latest-activity\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"linkedin_regex_global"}),(0,o.kt)("h2",{id:"sociallinkedin_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.LINKEDIN_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple LinkedIn profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/alan-turing\nen.linkedin.com/in/alan-turing\nlinkedin.com/in/alan-turing\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds/latest-activity\n")),(0,o.kt)("p",null,"the expression extracts just the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.linkedin.com/in/linus-torvalds\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.LINKEDIN_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} LinkedIn profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"instagram_regex"}),(0,o.kt)("h2",{id:"socialinstagram_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.INSTAGRAM_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Instagram profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/old_prague\nwww.instagram.com/old_prague/\ninstagr.am/old_prague\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano/followers\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"instagram_regex_global"}),(0,o.kt)("h2",{id:"socialinstagram_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.INSTAGRAM_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Instagram profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/old_prague\nwww.instagram.com/old_prague/\ninstagr.am/old_prague\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano/followers\n")),(0,o.kt)("p",null,"the expression extracts just the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.instagram.com/cristiano\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.INSTAGRAM_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Instagram profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"twitter_regex"}),(0,o.kt)("h2",{id:"socialtwitter_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.TWITTER_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Twitter profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/apify\ntwitter.com/apify\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump/following\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"twitter_regex_global"}),(0,o.kt)("h2",{id:"socialtwitter_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.TWITTER_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Twitter profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/apify\ntwitter.com/apify\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump/following\n")),(0,o.kt)("p",null,"the expression extracts only the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.twitter.com/realdonaldtrump\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.TWITTER_REGEX_STRING);\nif (matches) console.log(`${matches.length} Twitter profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"facebook_regex"}),(0,o.kt)("h2",{id:"socialfacebook_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.FACEBOOK_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Facebook profile URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\nfacebook.com/apifytech\nfb.com/apifytech\nhttps://www.facebook.com/profile.php?id=123456789\n")),(0,o.kt)("p",null,"The regular expression does NOT match URLs with additional subdirectories or query parameters, such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech/photos\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"facebook_regex_global"}),(0,o.kt)("h2",{id:"socialfacebook_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.FACEBOOK_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Facebook profile URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\nfacebook.com/apifytech\nfb.com/apifytech\n")),(0,o.kt)("p",null,"If the profile URL contains subdirectories or query parameters, the regular expression extracts just the base part of the profile URL. For example,\nfrom text such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech/photos\n")),(0,o.kt)("p",null,"the expression extracts only the following base URL:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.facebook.com/apifytech\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.FACEBOOK_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Facebook profiles found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"youtube_regex"}),(0,o.kt)("h2",{id:"socialyoutube_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.YOUTUBE_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single Youtube video URL. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.youtube.com/watch?v=kM7YfhfkiEE\nhttps://youtu.be/kM7YfhfkiEE\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"if (Apify.utils.social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {\n console.log('Match!');\n}\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"youtube_regex_global"}),(0,o.kt)("h2",{id:"socialyoutube_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.YOUTUBE_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple Youtube video URLs in a text or HTML. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig")," and matches URLs such as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"https://www.youtube.com/watch?v=kM7YfhfkiEE\nhttps://youtu.be/kM7YfhfkiEE\n")),(0,o.kt)("p",null,"Example usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"const matches = text.match(Apify.utils.social.YOUTUBE_REGEX_GLOBAL);\nif (matches) console.log(`${matches.length} Youtube videos found!`);\n")),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"email_regex"}),(0,o.kt)("h2",{id:"socialemail_regex"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.EMAIL_REGEX")),(0,o.kt)("p",null,"Regular expression to exactly match a single email address. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/^...$/i"),"."),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"email_regex_global"}),(0,o.kt)("h2",{id:"socialemail_regex_global"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.EMAIL_REGEX_GLOBAL")),(0,o.kt)("p",null,"Regular expression to find multiple email addresses in a text. It has the following form: ",(0,o.kt)("inlineCode",{parentName:"p"},"/.../ig"),"."),(0,o.kt)("hr",null),(0,o.kt)("a",{name:"emailsfromtext"}),(0,o.kt)("h2",{id:"socialemailsfromtexttext"},(0,o.kt)("inlineCode",{parentName:"h2"},"social.emailsFromText(text)")),(0,o.kt)("p",null,"The function extracts email addresses from a plain text. Note that the function preserves the order of emails and keep duplicates."),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},(0,o.kt)("inlineCode",{parentName:"strong"},"text")),": ",(0,o.kt)("inlineCode",{parentName:"li"},"string")," - Text to search in.")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Returns"),":"),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"ArraywasAlreadyPresent",id:"wasalreadypresent",level:3},{value:"wasAlreadyHandled",id:"wasalreadyhandled",level:3},{value:"requestId",id:"requestid",level:3},{value:"request",id:"request",level:3}],c={toc:d},f="wrapper";function m(e){var t=e.components,r=(0,a.Z)(e,i);return(0,o.kt)(f,(0,n.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("a",{name:"queueoperationinfo"}),(0,o.kt)("p",null,"A helper class that is used to report results from various ",(0,o.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," functions as well as\n",(0,o.kt)("a",{parentName:"p",href:"../api/utils#enqueuelinks",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"utils.enqueueLinks()")),"."),(0,o.kt)("h2",{id:"properties"},"Properties"),(0,o.kt)("h3",{id:"wasalreadypresent"},(0,o.kt)("inlineCode",{parentName:"h3"},"wasAlreadyPresent")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"boolean")),(0,o.kt)("p",null,"Indicates if request was already present in the queue."),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"wasalreadyhandled"},(0,o.kt)("inlineCode",{parentName:"h3"},"wasAlreadyHandled")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"boolean")),(0,o.kt)("p",null,"Indicates if request was already marked as handled."),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"requestid"},(0,o.kt)("inlineCode",{parentName:"h3"},"requestId")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"string")),(0,o.kt)("p",null,"The ID of the added request"),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"request"},(0,o.kt)("inlineCode",{parentName:"h3"},"request")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"Request"))),(0,o.kt)("p",null,"The original ",(0,o.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"Request"))," object passed to the ",(0,o.kt)("inlineCode",{parentName:"p"},"RequestQueue")," function."),(0,o.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>m});var n=r(67294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;twasAlreadyPresent",id:"wasalreadypresent",level:3},{value:"wasAlreadyHandled",id:"wasalreadyhandled",level:3},{value:"requestId",id:"requestid",level:3},{value:"request",id:"request",level:3}],c={toc:d},f="wrapper";function m(e){var t=e.components,r=(0,a.Z)(e,i);return(0,o.kt)(f,(0,n.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("a",{name:"queueoperationinfo"}),(0,o.kt)("p",null,"A helper class that is used to report results from various ",(0,o.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," functions as well as\n",(0,o.kt)("a",{parentName:"p",href:"../api/utils#enqueuelinks",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"utils.enqueueLinks()")),"."),(0,o.kt)("h2",{id:"properties"},"Properties"),(0,o.kt)("h3",{id:"wasalreadypresent"},(0,o.kt)("inlineCode",{parentName:"h3"},"wasAlreadyPresent")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"boolean")),(0,o.kt)("p",null,"Indicates if request was already present in the queue."),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"wasalreadyhandled"},(0,o.kt)("inlineCode",{parentName:"h3"},"wasAlreadyHandled")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"boolean")),(0,o.kt)("p",null,"Indicates if request was already marked as handled."),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"requestid"},(0,o.kt)("inlineCode",{parentName:"h3"},"requestId")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("inlineCode",{parentName:"p"},"string")),(0,o.kt)("p",null,"The ID of the added request"),(0,o.kt)("hr",null),(0,o.kt)("h3",{id:"request"},(0,o.kt)("inlineCode",{parentName:"h3"},"request")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Type"),": ",(0,o.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"Request"))),(0,o.kt)("p",null,"The original ",(0,o.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,o.kt)("inlineCode",{parentName:"a"},"Request"))," object passed to the ",(0,o.kt)("inlineCode",{parentName:"p"},"RequestQueue")," function."),(0,o.kt)("hr",null))}m.isMDXComponent=!0},3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>m});var n=r(67294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;tstats",id:"stats",level:3},{value:"requestList",id:"requestlist",level:3},{value:"requestQueue",id:"requestqueue",level:3},{value:"sessionPool",id:"sessionpool",level:3},{value:"autoscaledPool",id:"autoscaledpool",level:3},{value:"new BasicCrawler(options)",id:"new-basiccrawleroptions",level:2},{value:"basicCrawler.optionsShape",id:"basiccrawleroptionsshape",level:2},{value:"basicCrawler.log",id:"basiccrawlerlog",level:2},{value:"basicCrawler.sessionPoolOptions",id:"basiccrawlersessionpooloptions",level:2},{value:"basicCrawler.run()",id:"basiccrawlerrun",level:2}],d={toc:c},m="wrapper";function k(e){var t=e.components,a=(0,n.Z)(e,i);return(0,l.kt)(m,(0,r.Z)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("a",{name:"basiccrawler"}),(0,l.kt)("p",null,"Provides a simple framework for parallel crawling of web pages. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of\nURLs enabling recursive crawling of websites."),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," is a low-level tool that requires the user to implement the page download and data extraction functionality themselves. If you want a\ncrawler that already facilitates this functionality, please consider using ",(0,l.kt)("a",{parentName:"p",href:"../api/cheerio-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"CheerioCrawler")),",\n",(0,l.kt)("a",{parentName:"p",href:"../api/puppeteer-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"PuppeteerCrawler"))," or ",(0,l.kt)("a",{parentName:"p",href:"../api/playwright-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"PlaywrightCrawler")),"."),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," invokes the user-provided ",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#handlerequestfunction",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.handleRequestFunction"))," for\neach ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," object, which represents a single URL to crawl. The ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," objects are fed from the\n",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," or the ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," instances provided by the\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestlist",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestList"))," or\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestqueue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestQueue"))," constructor options, respectively."),(0,l.kt)("p",null,"If both ",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestlist",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestList"))," and\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestqueue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestQueue"))," options are used, the instance first processes URLs from the\n",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," and automatically enqueues all of them to ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," before it starts their\nprocessing. This ensures that a single URL is not crawled multiple times."),(0,l.kt)("p",null,"The crawler finishes if there are no more ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," objects to crawl."),(0,l.kt)("p",null,"New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," class. All ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," configuration options can be passed to the\n",(0,l.kt)("inlineCode",{parentName:"p"},"autoscaledPoolOptions")," parameter of the ",(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," constructor. For user convenience, the ",(0,l.kt)("inlineCode",{parentName:"p"},"minConcurrency")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"maxConcurrency"),"\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," options are available directly in the ",(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," constructor."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Example usage:")),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-javascript"},"// Prepare a list of URLs to crawl\nconst requestList = new Apify.RequestList({\n sources: [{ url: 'http://www.example.com/page-1' }, { url: 'http://www.example.com/page-2' }],\n});\nawait requestList.initialize();\n\n// Crawl the URLs\nconst crawler = new Apify.BasicCrawler({\n requestList,\n handleRequestFunction: async ({ request }) => {\n // 'request' contains an instance of the Request class\n // Here we simply fetch the HTML of the page and store it to a dataset\n const { body } = await Apify.utils.requestAsBrowser(request);\n await Apify.pushData({\n url: request.url,\n html: body,\n });\n },\n});\n\nawait crawler.run();\n")),(0,l.kt)("h2",{id:"properties"},"Properties"),(0,l.kt)("h3",{id:"stats"},(0,l.kt)("inlineCode",{parentName:"h3"},"stats")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/statistics",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Statistics"))),(0,l.kt)("p",null,"Contains statistics about the current run."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"requestlist"},(0,l.kt)("inlineCode",{parentName:"h3"},"requestList")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"requestqueue"},(0,l.kt)("inlineCode",{parentName:"h3"},"requestQueue")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"sessionpool"},(0,l.kt)("inlineCode",{parentName:"h3"},"sessionPool")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/session-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"SessionPool"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/session-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"SessionPool"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/session",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Session")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"autoscaledpool"},(0,l.kt)("inlineCode",{parentName:"h3"},"autoscaledPool")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," class that manages the concurrency of the crawler. Note that this property is\nonly initialized after calling the ",(0,l.kt)("a",{parentName:"p",href:"../api/basic-crawler#run",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawler.run()"))," function. You can use it to change the concurrency settings on the\nfly, to pause the crawler by calling ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool#pause",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool.pause()"))," or to abort it by calling\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool#abort",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool.abort()")),"."),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"exports.basiccrawler"}),(0,l.kt)("h2",{id:"new-basiccrawleroptions"},(0,l.kt)("inlineCode",{parentName:"h2"},"new BasicCrawler(options)")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("strong",{parentName:"li"},(0,l.kt)("inlineCode",{parentName:"strong"},"options")),": ",(0,l.kt)("a",{parentName:"li",href:"../typedefs/basic-crawler-options",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions"))," - All ",(0,l.kt)("inlineCode",{parentName:"li"},"BasicCrawler")," parameters are passed via an options object.")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"optionsshape"}),(0,l.kt)("h2",{id:"basiccrawleroptionsshape"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.optionsShape")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Internal"),":"),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"log"}),(0,l.kt)("h2",{id:"basiccrawlerlog"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.log")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"sessionpooloptions"}),(0,l.kt)("h2",{id:"basiccrawlersessionpooloptions"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.sessionPoolOptions")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"run"}),(0,l.kt)("h2",{id:"basiccrawlerrun"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.run()")),(0,l.kt)("p",null,"Runs the crawler. Returns a promise that gets resolved once all the requests are processed."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Returns"),":"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"Promisestats",id:"stats",level:3},{value:"requestList",id:"requestlist",level:3},{value:"requestQueue",id:"requestqueue",level:3},{value:"sessionPool",id:"sessionpool",level:3},{value:"autoscaledPool",id:"autoscaledpool",level:3},{value:"new BasicCrawler(options)",id:"new-basiccrawleroptions",level:2},{value:"basicCrawler.optionsShape",id:"basiccrawleroptionsshape",level:2},{value:"basicCrawler.log",id:"basiccrawlerlog",level:2},{value:"basicCrawler.sessionPoolOptions",id:"basiccrawlersessionpooloptions",level:2},{value:"basicCrawler.run()",id:"basiccrawlerrun",level:2}],d={toc:c},m="wrapper";function k(e){var t=e.components,a=(0,n.Z)(e,i);return(0,l.kt)(m,(0,r.Z)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("a",{name:"basiccrawler"}),(0,l.kt)("p",null,"Provides a simple framework for parallel crawling of web pages. The URLs to crawl are fed either from a static list of URLs or from a dynamic queue of\nURLs enabling recursive crawling of websites."),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," is a low-level tool that requires the user to implement the page download and data extraction functionality themselves. If you want a\ncrawler that already facilitates this functionality, please consider using ",(0,l.kt)("a",{parentName:"p",href:"../api/cheerio-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"CheerioCrawler")),",\n",(0,l.kt)("a",{parentName:"p",href:"../api/puppeteer-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"PuppeteerCrawler"))," or ",(0,l.kt)("a",{parentName:"p",href:"../api/playwright-crawler",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"PlaywrightCrawler")),"."),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," invokes the user-provided ",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#handlerequestfunction",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.handleRequestFunction"))," for\neach ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," object, which represents a single URL to crawl. The ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," objects are fed from the\n",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," or the ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," instances provided by the\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestlist",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestList"))," or\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestqueue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestQueue"))," constructor options, respectively."),(0,l.kt)("p",null,"If both ",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestlist",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestList"))," and\n",(0,l.kt)("a",{parentName:"p",href:"../typedefs/basic-crawler-options#requestqueue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions.requestQueue"))," options are used, the instance first processes URLs from the\n",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," and automatically enqueues all of them to ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," before it starts their\nprocessing. This ensures that a single URL is not crawled multiple times."),(0,l.kt)("p",null,"The crawler finishes if there are no more ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request"))," objects to crawl."),(0,l.kt)("p",null,"New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," class. All ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," configuration options can be passed to the\n",(0,l.kt)("inlineCode",{parentName:"p"},"autoscaledPoolOptions")," parameter of the ",(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," constructor. For user convenience, the ",(0,l.kt)("inlineCode",{parentName:"p"},"minConcurrency")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"maxConcurrency"),"\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," options are available directly in the ",(0,l.kt)("inlineCode",{parentName:"p"},"BasicCrawler")," constructor."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Example usage:")),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-javascript"},"// Prepare a list of URLs to crawl\nconst requestList = new Apify.RequestList({\n sources: [{ url: 'http://www.example.com/page-1' }, { url: 'http://www.example.com/page-2' }],\n});\nawait requestList.initialize();\n\n// Crawl the URLs\nconst crawler = new Apify.BasicCrawler({\n requestList,\n handleRequestFunction: async ({ request }) => {\n // 'request' contains an instance of the Request class\n // Here we simply fetch the HTML of the page and store it to a dataset\n const { body } = await Apify.utils.requestAsBrowser(request);\n await Apify.pushData({\n url: request.url,\n html: body,\n });\n },\n});\n\nawait crawler.run();\n")),(0,l.kt)("h2",{id:"properties"},"Properties"),(0,l.kt)("h3",{id:"stats"},(0,l.kt)("inlineCode",{parentName:"h3"},"stats")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/statistics",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Statistics"))),(0,l.kt)("p",null,"Contains statistics about the current run."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"requestlist"},(0,l.kt)("inlineCode",{parentName:"h3"},"requestList")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/request-list",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestList"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"requestqueue"},(0,l.kt)("inlineCode",{parentName:"h3"},"requestQueue")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/request-queue",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"RequestQueue"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/request",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Request")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"sessionpool"},(0,l.kt)("inlineCode",{parentName:"h3"},"sessionPool")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/session-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"SessionPool"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/session-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"SessionPool"))," class that manages the crawler's ",(0,l.kt)("a",{parentName:"p",href:"../api/session",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"Session")),"s. Only available if\nused by the crawler."),(0,l.kt)("hr",null),(0,l.kt)("h3",{id:"autoscaledpool"},(0,l.kt)("inlineCode",{parentName:"h3"},"autoscaledPool")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Type"),": ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))),(0,l.kt)("p",null,"A reference to the underlying ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool"))," class that manages the concurrency of the crawler. Note that this property is\nonly initialized after calling the ",(0,l.kt)("a",{parentName:"p",href:"../api/basic-crawler#run",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawler.run()"))," function. You can use it to change the concurrency settings on the\nfly, to pause the crawler by calling ",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool#pause",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool.pause()"))," or to abort it by calling\n",(0,l.kt)("a",{parentName:"p",href:"../api/autoscaled-pool#abort",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"AutoscaledPool.abort()")),"."),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"exports.basiccrawler"}),(0,l.kt)("h2",{id:"new-basiccrawleroptions"},(0,l.kt)("inlineCode",{parentName:"h2"},"new BasicCrawler(options)")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("strong",{parentName:"li"},(0,l.kt)("inlineCode",{parentName:"strong"},"options")),": ",(0,l.kt)("a",{parentName:"li",href:"../typedefs/basic-crawler-options",target:null,rel:null},(0,l.kt)("inlineCode",{parentName:"a"},"BasicCrawlerOptions"))," - All ",(0,l.kt)("inlineCode",{parentName:"li"},"BasicCrawler")," parameters are passed via an options object.")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"optionsshape"}),(0,l.kt)("h2",{id:"basiccrawleroptionsshape"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.optionsShape")),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Internal"),":"),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"log"}),(0,l.kt)("h2",{id:"basiccrawlerlog"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.log")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"sessionpooloptions"}),(0,l.kt)("h2",{id:"basiccrawlersessionpooloptions"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.sessionPoolOptions")),(0,l.kt)("hr",null),(0,l.kt)("a",{name:"run"}),(0,l.kt)("h2",{id:"basiccrawlerrun"},(0,l.kt)("inlineCode",{parentName:"h2"},"basicCrawler.run()")),(0,l.kt)("p",null,"Runs the crawler. Returns a promise that gets resolved once all the requests are processed."),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Returns"),":"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"Promisedataset.pushData(data)",id:"datasetpushdatadata",level:2},{value:"dataset.getData([options])",id:"datasetgetdataoptions",level:2},{value:"dataset.getInfo()",id:"datasetgetinfo",level:2},{value:"dataset.forEach(iteratee, [options], [index])",id:"datasetforeachiteratee-options-index",level:2},{value:"dataset.map(iteratee, [options])",id:"datasetmapiteratee-options",level:2},{value:"dataset.reduce(iteratee, memo, [options])",id:"datasetreduceiteratee-memo-options",level:2},{value:"dataset.drop()",id:"datasetdrop",level:2}],u={toc:m},k="wrapper";function c(e){var t=e.components,a=(0,i.Z)(e,o);return(0,r.kt)(k,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("a",{name:"dataset"}),(0,r.kt)("p",null,"The ",(0,r.kt)("inlineCode",{parentName:"p"},"Dataset")," class represents a store for structured data where each object stored has the same attributes, such as online store products or real\nestate offers. You can imagine it as a table, where each object is a row and its attributes are columns. Dataset is an append-only storage - you can\nonly add new records to it but you cannot modify or remove existing records. Typically it is used to store crawling results."),(0,r.kt)("p",null,"Do not instantiate this class directly, use the ",(0,r.kt)("a",{parentName:"p",href:"../api/apify#opendataset",target:null,rel:null},(0,r.kt)("inlineCode",{parentName:"a"},"Apify.openDataset()"))," function instead."),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"Dataset")," stores its data either on local disk or in the Apify cloud, depending on whether the ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_LOCAL_STORAGE_DIR")," or ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_TOKEN")," environment\nvariables are set."),(0,r.kt)("p",null,"If the ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_LOCAL_STORAGE_DIR")," environment variable is set, the data is stored in the local directory in the following files:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},"{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\n")),(0,r.kt)("p",null,"Note that ",(0,r.kt)("inlineCode",{parentName:"p"},"{DATASET_ID}")," is the name or ID of the dataset. The default dataset has ID: ",(0,r.kt)("inlineCode",{parentName:"p"},"default"),", unless you override it by setting the\n",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_DEFAULT_DATASET_ID")," environment variable. Each dataset item is stored as a separate JSON file, where ",(0,r.kt)("inlineCode",{parentName:"p"},"{INDEX}")," is a zero-based index of the\nitem in the dataset."),(0,r.kt)("p",null,"If the ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_TOKEN")," environment variable is set but ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_LOCAL_STORAGE_DIR")," not, the data is stored in the\n",(0,r.kt)("a",{parentName:"p",href:"https://docs.apify.com/storage/dataset",target:"_blank",rel:"noopener"},"Apify Dataset")," cloud storage. Note that you can force usage of the cloud storage also by passing the\n",(0,r.kt)("inlineCode",{parentName:"p"},"forceCloud")," option to ",(0,r.kt)("a",{parentName:"p",href:"../api/apify#opendataset",target:null,rel:null},(0,r.kt)("inlineCode",{parentName:"a"},"Apify.openDataset()"))," function, even if the ",(0,r.kt)("inlineCode",{parentName:"p"},"APIFY_LOCAL_STORAGE_DIR")," variable is set."),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Example usage:")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-javascript"},"// Write a single row to the default dataset\nawait Apify.pushData({ col1: 123, col2: 'val2' });\n\n// Open a named dataset\nconst dataset = await Apify.openDataset('some-name');\n\n// Write a single row\nawait dataset.pushData({ foo: 'bar' });\n\n// Write multiple rows\nawait dataset.pushData([{ foo: 'bar2', col2: 'val2' }, { col3: 123 }]);\n")),(0,r.kt)("hr",null),(0,r.kt)("a",{name:"pushdata"}),(0,r.kt)("h2",{id:"datasetpushdatadata"},(0,r.kt)("inlineCode",{parentName:"h2"},"dataset.pushData(data)")),(0,r.kt)("p",null,"Stores an object or an array of objects to the dataset. The function returns a promise that resolves when the operation finishes. It has no result,\nbut throws on invalid args or other errors."),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"IMPORTANT"),": Make sure to use the ",(0,r.kt)("inlineCode",{parentName:"p"},"await")," keyword when calling ",(0,r.kt)("inlineCode",{parentName:"p"},"pushData()"),", otherwise the actor process might finish before the data is stored!"),(0,r.kt)("p",null,"The size of the data is limited by the receiving API and therefore ",(0,r.kt)("inlineCode",{parentName:"p"},"pushData()")," will only allow objects whose JSON representation is smaller than 9MB.\nWhen an array is passed, none of the included objects may be larger than 9MB, but the array itself may be of any size."),(0,r.kt)("p",null,"The function internally chunks the array into separate items and pushes them sequentially. The chunking process is stable (keeps order of data), but\nit does not provide a transaction safety mechanism. Therefore, in the event of an uploading error (after several automatic retries), the function's\nPromise will reject and the dataset will be left in a state where some of the items have already been saved to the dataset while other items from the\nsource array were not. To overcome this limitation, the developer may, for example, read the last item saved in the dataset and re-attempt the save of\nthe data from this item onwards to prevent duplicates."),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Parameters"),":"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"strong"},"data")),": ",(0,r.kt)("inlineCode",{parentName:"li"},"object")," | ",(0,r.kt)("inlineCode",{parentName:"li"},"Array