diff --git a/src/providers/WebScraper/single_url.ts b/src/providers/WebScraper/single_url.ts index 76cfd58..cfba88f 100644 --- a/src/providers/WebScraper/single_url.ts +++ b/src/providers/WebScraper/single_url.ts @@ -35,6 +35,8 @@ export async function scrapSingleUrl(urlToScrap: string, toMarkdown: boolean = t try { let content = await scrapWithScrapingBee(urlToScrap); + + if (!content) { const res = await attemptScrapWithRequests(urlToScrap); @@ -72,8 +74,14 @@ export async function scrapSingleUrl(urlToScrap: string, toMarkdown: boolean = t } }); + if (formattedText.length < 1) { + formattedText = markdownContent; + } + const text = sanitizeText(formattedText.trim()); + + if (metadata) { // console.log(markdownContent) // console.log("here", toMarkdown)