[add] Crawler API based on idea2app/Web-file-cache (#27)

Signed-off-by: TechQuery <shiy2008@gmail.com>

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: CI & CD
 on:
   push:
     branches:
-      - main
+      - '*'
 jobs:
   Build-and-Deploy:
     env:
@@ -15,32 +15,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: pnpm/action-setup@v4
-        with:
-          version: 9
-
-      - uses: actions/setup-node@v4
-        if: ${{ !env.VERCEL_TOKEN || !env.VERCEL_ORG_ID || !env.VERCEL_PROJECT_ID }}
-        with:
-          node-version: 20
-          cache: pnpm
-
-      - name: Install Dependencies
-        if: ${{ !env.VERCEL_TOKEN || !env.VERCEL_ORG_ID || !env.VERCEL_PROJECT_ID }}
-        run: pnpm i --frozen-lockfile
-
-      - name: Export Static pages
-        if: ${{ !env.VERCEL_TOKEN || !env.VERCEL_ORG_ID || !env.VERCEL_PROJECT_ID }}
-        run: pnpm export
-
-      - name: Deploy to GitHub pages
-        uses: peaceiris/actions-gh-pages@v4
-        if: ${{ !env.VERCEL_TOKEN || !env.VERCEL_ORG_ID || !env.VERCEL_PROJECT_ID }}
-        with:
-          publish_dir: ./out
-          personal_token: ${{ secrets.GITHUB_TOKEN }}
-          force_orphan: true
-
       - name: Deploy to Vercel
         uses: amondnet/vercel-action@v25
         if: ${{ env.VERCEL_TOKEN && env.VERCEL_ORG_ID && env.VERCEL_PROJECT_ID }}
@@ -50,4 +24,4 @@ jobs:
           vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
           vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
           working-directory: ./
-          vercel-args: --prod
+          vercel-args: ${{ github.ref == 'refs/heads/main' && ' --prod' || '' }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20-slim AS base
+FROM node:22-slim AS base
 RUN apt-get update && \
     apt-get install ca-certificates curl libjemalloc-dev -y --no-install-recommends  && \
     rm -rf /var/lib/apt/lists/*

components/Git/ArticleEditor.tsx

@@ -12,12 +12,11 @@ import { blobOf, formatDate, uniqueID } from 'web-utility';
 import YAML from 'yaml';
 
 import { GitRepositoryModel, userStore } from '../../models/Repository';
-import { i18n } from '../../models/Translation';
+import { t } from '../../models/Translation';
 import { ListField } from '../Form/JSONEditor';
 import { PathSelect } from './PathSelect';
 import { RepositorySelect } from './RepositorySelect';
 
-const { t } = i18n;
 const HTMLEditor = dynamic(() => import('../Form/HTMLEditor'), { ssr: false });
 
 export const fileType = {

components/License/helper.ts

@@ -1,8 +1,6 @@
 import { FeatureAttitude, InfectionRange } from 'license-filter';
 
-import { i18n } from '../../models/Translation';
-
-const { t } = i18n;
+import { t } from '../../models/Translation';
 
 type OptionValue = Record<string, { value: number; text: string }[]>;
 

components/data.ts

@@ -1,6 +1,4 @@
-import { i18n } from '../models/Translation';
-
-const { t } = i18n;
+import { t } from '../models/Translation';
 
 export interface Link {
   title: string;

eslint.config.mjs

@@ -17,7 +17,6 @@ export default tsEslint.config(
   {
     plugins: {
       '@typescript-eslint': tsEslint.plugin,
-      // @ts-expect-error https://github.com/jsx-eslint/eslint-plugin-react/issues/3699
       react: fixupPluginRules(reactPlugin),
       'simple-import-sort': simpleImportSortPlugin,
     },

models/Translation.ts

@@ -8,6 +8,8 @@ export const i18n = new TranslationModel({
   'en-US': () => import('../translation/en-US'),
 });
 
+export const { t } = i18n;
+
 export const LanguageName: Record<(typeof i18n)['currentLanguage'], string> = {
   'zh-CN': '简体中文',
   'zh-TW': '繁體中文',

next-env.d.ts

@@ -2,4 +2,4 @@
 /// <reference types="next/image-types/global" />
 
 // NOTE: This file should not be edited
-// see https://nextjs.org/docs/pages/building-your-application/configuring/typescript for more information.
+// see https://nextjs.org/docs/pages/api-reference/config/typescript for more information.

package.json

@@ -4,39 +4,39 @@
   "description": "React project scaffold based on TypeScript, Next.js, Bootstrap & Workbox.",
   "private": true,
   "engines": {
-    "node": ">=20"
+    "node": ">=22"
   },
   "dependencies": {
     "@mdx-js/loader": "^3.1.0",
     "@mdx-js/react": "^3.1.0",
-    "@next/mdx": "^15.0.3",
-    "@sentry/nextjs": "^8.41.0",
+    "@next/mdx": "^15.1.6",
+    "@sentry/nextjs": "^8.53.0",
     "@swc/helpers": "^0.5.15",
     "classnames": "^2.5.1",
     "copy-webpack-plugin": "^12.0.2",
-    "core-js": "^3.39.0",
-    "file-type": "^19.6.0",
+    "core-js": "^3.40.0",
+    "file-type": "^20.0.1",
     "idea-react": "^2.0.0-rc.8",
-    "koajax": "^3.1.0",
-    "less": "^4.2.1",
+    "koajax": "^3.1.1",
+    "less": "^4.2.2",
     "less-loader": "^12.2.0",
     "license-filter": "^0.2.4",
     "lodash": "^4.17.21",
     "markdown-ime": "^1.0.3",
-    "marked": "^15.0.3",
-    "mobx": "^6.13.5",
+    "marked": "^15.0.6",
+    "mobx": "^6.13.6",
     "mobx-github": "^0.3.5",
     "mobx-i18n": "^0.6.0",
-    "mobx-react": "^9.1.1",
-    "mobx-restful": "^2.0.0",
+    "mobx-react": "^9.2.0",
+    "mobx-restful": "^2.1.0",
     "mobx-restful-table": "^2.0.1",
-    "next": "^15.0.3",
+    "next": "^15.1.6",
     "next-pwa": "~5.6.0",
     "next-ssr-middleware": "^0.8.9",
     "next-with-less": "^3.0.1",
-    "primereact": "^10.8.5",
+    "primereact": "^10.9.2",
     "react": "^18.3.1",
-    "react-bootstrap": "^2.10.6",
+    "react-bootstrap": "^2.10.9",
     "react-bootstrap-editor": "^2.0.4",
     "react-dom": "^18.3.1",
     "remark-frontmatter": "^5.0.0",
@@ -46,36 +46,36 @@
     "turndown": "^7.2.0",
     "turndown-plugin-gfm": "^1.0.2",
     "web-utility": "^4.4.2",
-    "webpack": "^5.96.1",
-    "yaml": "^2.6.1"
+    "webpack": "^5.97.1",
+    "yaml": "^2.7.0"
   },
   "devDependencies": {
     "@babel/plugin-proposal-decorators": "^7.25.9",
-    "@babel/plugin-transform-typescript": "^7.25.9",
-    "@babel/preset-react": "^7.25.9",
-    "@eslint/compat": "^1.2.3",
+    "@babel/plugin-transform-typescript": "^7.26.7",
+    "@babel/preset-react": "^7.26.3",
+    "@eslint/compat": "^1.2.6",
     "@eslint/eslintrc": "^3.2.0",
     "@softonus/prettier-plugin-duplicate-remover": "^1.1.2",
     "@types/eslint-config-prettier": "^6.11.3",
     "@types/eslint__eslintrc": "^2.1.2",
-    "@types/lodash": "^4.17.13",
+    "@types/lodash": "^4.17.15",
     "@types/next-pwa": "^5.6.9",
-    "@types/node": "^20.17.9",
-    "@types/react": "^18.3.12",
+    "@types/node": "^22.13.0",
+    "@types/react": "^18.3.18",
     "@types/turndown": "^5.0.5",
-    "eslint": "^9.16.0",
-    "eslint-config-next": "^15.0.3",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-react": "^7.37.2",
+    "eslint": "^9.19.0",
+    "eslint-config-next": "^15.1.6",
+    "eslint-config-prettier": "^10.0.1",
+    "eslint-plugin-react": "^7.37.4",
     "eslint-plugin-simple-import-sort": "^12.1.1",
     "get-git-folder": "^0.1.2",
-    "globals": "^15.13.0",
+    "globals": "^15.14.0",
     "husky": "^9.1.7",
-    "lint-staged": "^15.2.10",
-    "prettier": "^3.4.1",
+    "lint-staged": "^15.4.3",
+    "prettier": "^3.4.2",
     "prettier-plugin-css-order": "^2.1.2",
-    "typescript": "~5.7.2",
-    "typescript-eslint": "^8.16.0"
+    "typescript": "~5.7.3",
+    "typescript-eslint": "^8.22.0"
   },
   "resolutions": {
     "@types/node": "$@types/node",
@@ -91,17 +91,15 @@
     ]
   },
   "lint-staged": {
-    "*.{html,md,less,json,yml,js,mjs,ts,tsx}": "prettier --write",
-    "*.{js,mjs,ts,tsx}": "eslint --fix"
+    "*.{html,md,less,json,yml,js,mjs,ts,tsx}": "prettier --write"
   },
   "scripts": {
     "prepare": "husky",
     "install": "get-git-folder https://github.com/kaiyuanshe/service-configuration main OSS-toolbox  ||  true",
     "dev": "next dev",
     "build": "next build",
-    "export": "next build  &&  next export",
     "start": "next start",
-    "lint": "next lint",
+    "lint": "next lint --fix  &&  git add .",
     "test": "lint-staged  &&  npm run lint  &&  tsc --noEmit",
     "pack-image": "docker build -t kaiyuanshe/oss-toolbox:latest .",
     "container": "docker rm -f oss-toolbox  &&  docker run --name oss-toolbox -p 3000:3000 -d kaiyuanshe/oss-toolbox:latest"

pages/_app.tsx

@@ -11,14 +11,12 @@ import { Col, Container, Image, Row } from 'react-bootstrap';
 import { MainNavigator } from '../components/MainNavigator';
 import { MDXLayout } from '../components/MDXLayout';
 import { isServer } from '../models/Base';
-import { i18n } from '../models/Translation';
+import { t } from '../models/Translation';
 
 configure({ enforceActions: 'never' });
 
 enableStaticRendering(isServer());
 
-const { t } = i18n;
-
 const AppShell: FC<AppProps> = observer(({ Component, pageProps, router }) => (
   <>
     <Head>

pages/api/crawler/task.ts

@@ -0,0 +1,32 @@
+import { githubClient } from 'mobx-github';
+
+import { safeAPI } from '../core';
+
+export interface CrawlerTask {
+  URI: string;
+  title?: string;
+}
+
+const CRAWLER_TOKEN = process.env.CRAWLER_TOKEN;
+
+export default safeAPI(async ({ method, headers, body }, response) => {
+  if (
+    !CRAWLER_TOKEN ||
+    CRAWLER_TOKEN !== headers.authorization?.split(/\s+/)[1]
+  )
+    return void response.status(401).end();
+
+  if (method !== 'POST') return void response.status(405).end();
+
+  const { URI, title = URI } = body as CrawlerTask;
+
+  const { status, body: data } = await githubClient.post(
+    'repos/kaiyuanshe/OWS-cache/issues',
+    {
+      title,
+      body: `### URL\n\n${URI}`,
+      labels: ['crawler'],
+    },
+  );
+  response.status(status).send(data);
+});

pages/index.tsx

@@ -9,9 +9,7 @@ import { Col, Container, Row } from 'react-bootstrap';
 import { GitCard } from '../components/Git/Card';
 import { PageHead } from '../components/PageHead';
 import { repositoryStore } from '../models/Repository';
-import { i18n } from '../models/Translation';
-
-const { t } = i18n;
+import { i18n, t } from '../models/Translation';
 
 export const getServerSideProps = compose(
   cache(),

pages/issue.tsx

@@ -9,7 +9,7 @@ import { Breadcrumb, Container, Row } from 'react-bootstrap';
 import { IssueModule } from '../components/Git/Issue/IssueModule';
 import { PageHead } from '../components/PageHead';
 import { repositoryStore } from '../models/Repository';
-import { i18n } from '../models/Translation';
+import { i18n, t } from '../models/Translation';
 
 export const getServerSideProps = compose(
   cache(),
@@ -23,8 +23,6 @@ export const getServerSideProps = compose(
   },
 );
 
-const { t } = i18n;
-
 const IssuesPage: FC<{ list: GitRepository[] }> = observer(({ list }) => (
   <Container className="py-5">
     <PageHead title="GitHub issues" />

pages/license-filter.tsx

@@ -16,15 +16,13 @@ import {
 
 import { licenseTips, optionValue } from '../components/License/helper';
 import { PageHead } from '../components/PageHead';
-import { i18n } from '../models/Translation';
+import { t } from '../models/Translation';
 
 interface List {
   license: License;
   score: number;
 }
 
-const { t } = i18n;
-
 const choiceSteps = [
   'popularity',
   'reuseCondition',

pages/polyfill.tsx

@@ -17,13 +17,11 @@ import {
 import { PageHead } from '../components/PageHead';
 import { PolyfillHost } from '../models/Base';
 import polyfillStore from '../models/Polyfill';
-import { i18n } from '../models/Translation';
+import { i18n, t } from '../models/Translation';
 import { UserAgent } from './api/polyfill';
 
 export const getServerSideProps = compose(translator(i18n));
 
-const { t } = i18n;
-
 @observer
 export default class PolyfillPage extends Component {
   @computed