Switch to app router edge runtime

apps/web/app/api/edge/metatags/route-wip.ts

@@ -0,0 +1,48 @@
+import { ratelimit } from "@/lib/upstash";
+import { LOCALHOST_IP, isValidUrl } from "@dub/utils";
+import { ipAddress } from "@vercel/edge";
+import { getToken } from "next-auth/jwt";
+import { NextFetchEvent, NextRequest } from "next/server";
+import { getMetaTags } from "./utils";
+
+export const runtime = "edge";
+
+// TODO: waitUntil() is not supported in App Router yet: https://vercel.com/docs/functions/edge-functions/edge-functions-api#waituntil
+export async function GET(req: NextRequest, ev: NextFetchEvent) {
+  const url = req.nextUrl.searchParams.get("url");
+  if (!url || !isValidUrl(url)) {
+    return new Response("Invalid URL", { status: 400 });
+  }
+
+  // Rate limit if user is not logged in
+  const session = await getToken({
+    req,
+    secret: process.env.NEXTAUTH_SECRET,
+  });
+  if (!session?.email) {
+    const ip = ipAddress(req) || LOCALHOST_IP;
+    const { success } = await ratelimit().limit(ip);
+    if (!success) {
+      return new Response("Don't DDoS me pls 🥺", { status: 429 });
+    }
+  }
+
+  const metatags = await getMetaTags(url, ev);
+  return new Response(JSON.stringify(metatags), {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+      "Access-Control-Allow-Origin": "*",
+    },
+  });
+}
+
+export function OPTIONS() {
+  return new Response(null, {
+    status: 204,
+    headers: {
+      "Access-Control-Allow-Origin": "*",
+      "Access-Control-Allow-Methods": "GET, OPTIONS",
+    },
+  });
+}

apps/web/app/api/edge/metatags/utils.ts

@@ -0,0 +1,111 @@
+import { parse } from "node-html-parser";
+import { recordMetatags } from "@/lib/upstash";
+import { isValidUrl } from "@dub/utils";
+import { NextFetchEvent } from "next/server";
+
+export const getHtml = async (url: string) => {
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 5000); // timeout if it takes longer than 5 seconds
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        "User-Agent": "dub-bot/1.0",
+      },
+    });
+    clearTimeout(timeoutId);
+    return await response.text();
+  } catch (error) {
+    if (error.name === "AbortError") {
+      // Handle fetch request abort (e.g., due to timeout)
+      console.error("Fetch request aborted due to timeout.");
+    } else {
+      // Handle other fetch errors
+      console.error("Fetch request failed:", error);
+    }
+    return null;
+  }
+};
+
+export const getHeadChildNodes = (html) => {
+  const ast = parse(html); // parse the html into AST format with node-html-parser
+  const metaTags = ast.querySelectorAll("meta").map(({ attributes }) => {
+    const property = attributes.property || attributes.name || attributes.href;
+    return {
+      property,
+      content: attributes.content,
+    };
+  });
+  const title = ast.querySelector("title")?.innerText;
+  const linkTags = ast.querySelectorAll("link").map(({ attributes }) => {
+    const { rel, href } = attributes;
+    return {
+      rel,
+      href,
+    };
+  });
+
+  return { metaTags, title, linkTags };
+};
+
+export const getRelativeUrl = (url: string, imageUrl: string) => {
+  if (!imageUrl) {
+    return null;
+  }
+  if (isValidUrl(imageUrl)) {
+    return imageUrl;
+  }
+  const { protocol, host } = new URL(url);
+  const baseURL = `${protocol}//${host}`;
+  return new URL(imageUrl, baseURL).toString();
+};
+
+export const getMetaTags = async (url: string, ev?: NextFetchEvent) => {
+  const html = await getHtml(url);
+  if (!html) {
+    return {
+      title: url,
+      description: "No description",
+      image: null,
+    };
+  }
+  const { metaTags, title: titleTag, linkTags } = getHeadChildNodes(html);
+
+  let object = {};
+
+  for (let k in metaTags) {
+    let { property, content } = metaTags[k];
+
+    property && (object[property] = content);
+  }
+
+  for (let m in linkTags) {
+    let { rel, href } = linkTags[m];
+
+    rel && (object[rel] = href);
+  }
+
+  const title = object["og:title"] || object["twitter:title"] || titleTag;
+
+  const description =
+    object["description"] ||
+    object["og:description"] ||
+    object["twitter:description"];
+
+  const image =
+    object["og:image"] ||
+    object["twitter:image"] ||
+    object["image_src"] ||
+    object["icon"] ||
+    object["shortcut icon"];
+
+  ev?.waitUntil(
+    recordMetatags(url, title && description && image ? false : true),
+  );
+
+  return {
+    title: title || url,
+    description: description || "No description",
+    image: getRelativeUrl(url, image),
+  };
+};

apps/web/app/api/edge/stats/[endpoint]/route.ts

@@ -7,8 +7,7 @@ import { LOCALHOST_IP } from "@dub/utils";
 import { ipAddress } from "@vercel/edge";
 import { NextResponse, type NextRequest } from "next/server";
 
-// TODO: switch to 'edge' after https://github.com/vercel/next.js/issues/48295 is fixed
-// export const runtime = "edge";
+export const runtime = "edge";
 
 export const GET = async (
   req: NextRequest,

apps/web/app/api/route.ts

@@ -1,8 +1,7 @@
 import { NextResponse } from "next/server";
 import { OpenAPIV3 } from "openapi-types";
 
-// TODO: switch to 'edge' after https://github.com/vercel/next.js/issues/48295 is fixed
-// export const runtime = "edge";
+export const runtime = "edge";
 
 export function GET(): NextResponse<OpenAPIV3.Document> {
   return NextResponse.json({

apps/web/app/rewrite/[url]/page.tsx

@@ -1,9 +1,9 @@
-import { getMetaTags } from "@/pages/api/edge/metatags";
 import {
   GOOGLE_FAVICON_URL,
   constructMetadata,
   getApexDomain,
 } from "@dub/utils";
+import { getMetaTags } from "app/api/edge/metatags/utils";
 
 export const runtime = "edge";
 

apps/web/lib/tinybird.ts

@@ -75,7 +75,6 @@ export async function recordClick({
     ).then((res) => res.json()),
     // increment the click count for the link if key is specified (not root click)
     // also increment the usage count for the project, and then we have a cron that will reset it at the start of new billing cycle
-    // TODO: might wanna include root clicks in the usage count as well?
     ...(conn
       ? [
           key

apps/web/lib/upstash.ts

@@ -1,4 +1,4 @@
-import { nanoid } from "@dub/utils";
+import { getDomainWithoutWWW, nanoid } from "@dub/utils";
 import { Ratelimit } from "@upstash/ratelimit";
 import { Redis } from "@upstash/redis";
 
@@ -8,6 +8,17 @@ export const redis = new Redis({
   token: process.env.UPSTASH_REDIS_REST_TOKEN || "",
 });
 
+export const ratelimitRedis = new Redis({
+  url:
+    process.env.RATELIMIT_UPSTASH_REDIS_REST_URL ||
+    process.env.UPSTASH_REDIS_REST_URL ||
+    "",
+  token:
+    process.env.RATELIMIT_UPSTASH_REDIS_REST_TOKEN ||
+    process.env.UPSTASH_REDIS_REST_TOKEN ||
+    "",
+});
+
 // Create a new ratelimiter, that allows 10 requests per 10 seconds by default
 export const ratelimit = (
   requests: number = 10,
@@ -19,16 +30,7 @@ export const ratelimit = (
     | `${number} d` = "10 s",
 ) => {
   return new Ratelimit({
-    redis: new Redis({
-      url:
-        process.env.RATELIMIT_UPSTASH_REDIS_REST_URL ||
-        process.env.UPSTASH_REDIS_REST_URL ||
-        "",
-      token:
-        process.env.RATELIMIT_UPSTASH_REDIS_REST_TOKEN ||
-        process.env.UPSTASH_REDIS_REST_TOKEN ||
-        "",
-    }),
+    redis: ratelimitRedis,
     limiter: Ratelimit.slidingWindow(requests, seconds),
     analytics: true,
     prefix: "dub",
@@ -67,9 +69,12 @@ export async function recordMetatags(url: string, error: boolean) {
   if (url === "https://github.com/steven-tey/dub") {
     // don't log metatag generation for default URL
     return null;
-  } else {
-    return await redis.lpush(error ? "metatags-errors" : "metatags", {
-      url,
-    });
   }
+
+  if (error) {
+    return await ratelimitRedis.zincrby("metatags-error-zset", 1, url);
+  }
+
+  const domain = getDomainWithoutWWW(url);
+  return await ratelimitRedis.zincrby("metatags-zset", 1, domain);
 }

apps/web/package.json

@@ -52,7 +52,7 @@
     "js-cookie": "^3.0.5",
     "lucide-react": "^0.269.0",
     "nanoid": "^5.0.1",
-    "next": "14.0.3-canary.8",
+    "next": "14.0.4-canary.39",
     "next-auth": "^4.24.4",
     "node-html-parser": "^6.1.4",
     "nodemailer": "^6.9.3",

apps/web/pages/api/edge/metatags.ts

@@ -1,9 +1,9 @@
-import { ratelimit, recordMetatags } from "@/lib/upstash";
+import { ratelimit } from "@/lib/upstash";
 import { LOCALHOST_IP, isValidUrl } from "@dub/utils";
 import { ipAddress } from "@vercel/edge";
+import { getMetaTags } from "app/api/edge/metatags/utils";
 import { getToken } from "next-auth/jwt";
 import { NextFetchEvent, NextRequest } from "next/server";
-import { parse } from "node-html-parser";
 
 export const config = {
   runtime: "edge",
@@ -49,110 +49,3 @@ export default async function handler(req: NextRequest, ev: NextFetchEvent) {
     return new Response(`Method ${req.method} Not Allowed`, { status: 405 });
   }
 }
-
-const getHtml = async (url: string) => {
-  try {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 5000); // timeout if it takes longer than 5 seconds
-    const response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        "User-Agent": "dub-bot/1.0",
-      },
-    });
-    clearTimeout(timeoutId);
-    return await response.text();
-  } catch (error) {
-    if (error.name === "AbortError") {
-      // Handle fetch request abort (e.g., due to timeout)
-      console.error("Fetch request aborted due to timeout.");
-    } else {
-      // Handle other fetch errors
-      console.error("Fetch request failed:", error);
-    }
-    return null;
-  }
-};
-
-const getHeadChildNodes = (html) => {
-  const ast = parse(html); // parse the html into AST format with node-html-parser
-  const metaTags = ast.querySelectorAll("meta").map(({ attributes }) => {
-    const property = attributes.property || attributes.name || attributes.href;
-    return {
-      property,
-      content: attributes.content,
-    };
-  });
-  const title = ast.querySelector("title")?.innerText;
-  const linkTags = ast.querySelectorAll("link").map(({ attributes }) => {
-    const { rel, href } = attributes;
-    return {
-      rel,
-      href,
-    };
-  });
-
-  return { metaTags, title, linkTags };
-};
-
-const getRelativeUrl = (url: string, imageUrl: string) => {
-  if (!imageUrl) {
-    return null;
-  }
-  if (isValidUrl(imageUrl)) {
-    return imageUrl;
-  }
-  const { protocol, host } = new URL(url);
-  const baseURL = `${protocol}//${host}`;
-  return new URL(imageUrl, baseURL).toString();
-};
-
-export const getMetaTags = async (url: string, ev?: NextFetchEvent) => {
-  const html = await getHtml(url);
-  if (!html) {
-    return {
-      title: url,
-      description: "No description",
-      image: null,
-    };
-  }
-  const { metaTags, title: titleTag, linkTags } = getHeadChildNodes(html);
-
-  let object = {};
-
-  for (let k in metaTags) {
-    let { property, content } = metaTags[k];
-
-    property && (object[property] = content);
-  }
-
-  for (let m in linkTags) {
-    let { rel, href } = linkTags[m];
-
-    rel && (object[rel] = href);
-  }
-
-  const title = object["og:title"] || object["twitter:title"] || titleTag;
-
-  const description =
-    object["description"] ||
-    object["og:description"] ||
-    object["twitter:description"];
-
-  const image =
-    object["og:image"] ||
-    object["twitter:image"] ||
-    object["image_src"] ||
-    object["icon"] ||
-    object["shortcut icon"];
-
-  ev?.waitUntil(
-    recordMetatags(url, title && description && image ? false : true),
-  );
-
-  return {
-    title: title || url,
-    description: description || "No description",
-    image: getRelativeUrl(url, image),
-  };
-};