diff --git a/package.json b/package.json index 0b50f67..6d58099 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "moodle-typed-ws": "^0.4.0", "object-to-formdata": "^4.5.1", "orval": "^6.31.0", + "p-limit": "^6.1.0", "react": "^18.2.0", "react-dom": "^18.2.0", "tailwind-merge": "^2.3.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c29ecdf..a3ea27f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: orval: specifier: ^6.31.0 version: 6.31.0(openapi-types@12.1.3)(typescript@5.4.5) + p-limit: + specifier: ^6.1.0 + version: 6.1.0 react: specifier: ^18.2.0 version: 18.3.1 @@ -38,7 +41,7 @@ importers: devDependencies: '@antfu/eslint-config': specifier: ^2.21.1 - version: 2.21.1(@eslint-react/eslint-plugin@1.5.15)(@unocss/eslint-plugin@0.60.4)(@vue/compiler-sfc@3.4.29)(eslint-plugin-react-hooks@4.6.2)(eslint-plugin-react-refresh@0.4.7)(eslint@8.57.0)(typescript@5.4.5) + version: 2.21.1(@eslint-react/eslint-plugin@1.5.15(eslint@8.57.0)(typescript@5.4.5))(@unocss/eslint-plugin@0.60.4(eslint@8.57.0)(typescript@5.4.5))(@vue/compiler-sfc@3.4.29)(eslint-plugin-react-hooks@4.6.2(eslint@8.57.0))(eslint-plugin-react-refresh@0.4.7(eslint@8.57.0))(eslint@8.57.0)(typescript@5.4.5) '@crxjs/vite-plugin': specifier: ^2.0.0-beta.19 version: 2.0.0-beta.23 @@ -65,7 +68,7 @@ importers: version: 0.61.0 '@vitejs/plugin-react': specifier: ^4.1.0 - version: 4.3.1(vite@4.5.3) + version: 4.3.1(vite@4.5.3(@types/node@20.14.2)) bumpp: specifier: ^9.4.1 version: 9.4.1 @@ -101,7 +104,7 @@ importers: version: 5.4.5 unocss: specifier: ^0.61.0 - version: 0.61.0(postcss@8.4.38)(vite@4.5.3) + version: 0.61.0(postcss@8.4.38)(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2)) vite: specifier: ^4.4.11 version: 4.5.3(@types/node@20.14.2) @@ -3421,6 +3424,10 @@ packages: resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-limit@6.1.0: + resolution: {integrity: sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==} + engines: {node: '>=18'} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -4535,6 +4542,10 @@ packages: resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} engines: {node: '>=12.20'} + yocto-queue@1.1.1: + resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} + engines: {node: '>=12.20'} + snapshots: '@ampproject/remapping@2.3.0': @@ -4542,15 +4553,13 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 - '@antfu/eslint-config@2.21.1(@eslint-react/eslint-plugin@1.5.15)(@unocss/eslint-plugin@0.60.4)(@vue/compiler-sfc@3.4.29)(eslint-plugin-react-hooks@4.6.2)(eslint-plugin-react-refresh@0.4.7)(eslint@8.57.0)(typescript@5.4.5)': + '@antfu/eslint-config@2.21.1(@eslint-react/eslint-plugin@1.5.15(eslint@8.57.0)(typescript@5.4.5))(@unocss/eslint-plugin@0.60.4(eslint@8.57.0)(typescript@5.4.5))(@vue/compiler-sfc@3.4.29)(eslint-plugin-react-hooks@4.6.2(eslint@8.57.0))(eslint-plugin-react-refresh@0.4.7(eslint@8.57.0))(eslint@8.57.0)(typescript@5.4.5)': dependencies: '@antfu/install-pkg': 0.3.3 '@clack/prompts': 0.7.0 - '@eslint-react/eslint-plugin': 1.5.15(eslint@8.57.0)(typescript@5.4.5) '@stylistic/eslint-plugin': 2.2.0(eslint@8.57.0)(typescript@5.4.5) - '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0)(eslint@8.57.0)(typescript@5.4.5) + '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) '@typescript-eslint/parser': 7.13.0(eslint@8.57.0)(typescript@5.4.5) - '@unocss/eslint-plugin': 0.60.4(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-config-flat-gitignore: 0.1.5 eslint-flat-config-utils: 0.2.5 @@ -4564,14 +4573,12 @@ snapshots: eslint-plugin-markdown: 5.0.0(eslint@8.57.0) eslint-plugin-n: 17.9.0(eslint@8.57.0) eslint-plugin-no-only-tests: 3.1.0 - eslint-plugin-perfectionist: 2.11.0(eslint@8.57.0)(typescript@5.4.5)(vue-eslint-parser@9.4.3) - eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) - eslint-plugin-react-refresh: 0.4.7(eslint@8.57.0) + eslint-plugin-perfectionist: 2.11.0(eslint@8.57.0)(typescript@5.4.5)(vue-eslint-parser@9.4.3(eslint@8.57.0)) eslint-plugin-regexp: 2.6.0(eslint@8.57.0) eslint-plugin-toml: 0.11.0(eslint@8.57.0) eslint-plugin-unicorn: 53.0.0(eslint@8.57.0) - eslint-plugin-unused-imports: 3.2.0(@typescript-eslint/eslint-plugin@7.13.0)(eslint@8.57.0) - eslint-plugin-vitest: 0.5.4(@typescript-eslint/eslint-plugin@7.13.0)(eslint@8.57.0)(typescript@5.4.5) + eslint-plugin-unused-imports: 3.2.0(@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0) + eslint-plugin-vitest: 0.5.4(@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) eslint-plugin-vue: 9.26.0(eslint@8.57.0) eslint-plugin-yml: 1.14.0(eslint@8.57.0) eslint-processor-vue-blocks: 0.1.2(@vue/compiler-sfc@3.4.29)(eslint@8.57.0) @@ -4584,6 +4591,11 @@ snapshots: vue-eslint-parser: 9.4.3(eslint@8.57.0) yaml-eslint-parser: 1.2.3 yargs: 17.7.2 + optionalDependencies: + '@eslint-react/eslint-plugin': 1.5.15(eslint@8.57.0)(typescript@5.4.5) + '@unocss/eslint-plugin': 0.60.4(eslint@8.57.0)(typescript@5.4.5) + eslint-plugin-react-hooks: 4.6.2(eslint@8.57.0) + eslint-plugin-react-refresh: 0.4.7(eslint@8.57.0) transitivePeerDependencies: - '@vue/compiler-sfc' - supports-color @@ -5100,6 +5112,7 @@ snapshots: eslint-plugin-react-dom: 1.5.15(eslint@8.57.0)(typescript@5.4.5) eslint-plugin-react-hooks-extra: 1.5.15(eslint@8.57.0)(typescript@5.4.5) eslint-plugin-react-naming-convention: 1.5.15(eslint@8.57.0)(typescript@5.4.5) + optionalDependencies: typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -5395,11 +5408,13 @@ snapshots: estree-walker: 2.0.2 picomatch: 2.3.1 - '@rollup/pluginutils@5.1.0': + '@rollup/pluginutils@5.1.0(rollup@3.29.4)': dependencies: '@types/estree': 1.0.5 estree-walker: 2.0.2 picomatch: 2.3.1 + optionalDependencies: + rollup: 3.29.4 '@stoplight/better-ajv-errors@1.0.3(ajv@8.16.0)': dependencies: @@ -5703,7 +5718,7 @@ snapshots: '@types/expect': 1.20.4 '@types/node': 20.14.2 - '@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0)(eslint@8.57.0)(typescript@5.4.5)': + '@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)': dependencies: '@eslint-community/regexpp': 4.10.1 '@typescript-eslint/parser': 7.13.0(eslint@8.57.0)(typescript@5.4.5) @@ -5716,6 +5731,7 @@ snapshots: ignore: 5.3.1 natural-compare: 1.4.0 ts-api-utils: 1.3.0(typescript@5.4.5) + optionalDependencies: typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -5728,6 +5744,7 @@ snapshots: '@typescript-eslint/visitor-keys': 7.13.0 debug: 4.3.5 eslint: 8.57.0 + optionalDependencies: typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -5744,6 +5761,7 @@ snapshots: debug: 4.3.5 eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.4.5) + optionalDependencies: typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -5760,6 +5778,7 @@ snapshots: minimatch: 9.0.4 semver: 7.6.2 ts-api-utils: 1.3.0(typescript@5.4.5) + optionalDependencies: typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -5782,19 +5801,20 @@ snapshots: '@ungap/structured-clone@1.2.0': {} - '@unocss/astro@0.61.0(vite@4.5.3)': + '@unocss/astro@0.61.0(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2))': dependencies: '@unocss/core': 0.61.0 '@unocss/reset': 0.61.0 - '@unocss/vite': 0.61.0(vite@4.5.3) + '@unocss/vite': 0.61.0(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2)) + optionalDependencies: vite: 4.5.3(@types/node@20.14.2) transitivePeerDependencies: - rollup - '@unocss/cli@0.61.0': + '@unocss/cli@0.61.0(rollup@3.29.4)': dependencies: '@ampproject/remapping': 2.3.0 - '@rollup/pluginutils': 5.1.0 + '@rollup/pluginutils': 5.1.0(rollup@3.29.4) '@unocss/config': 0.61.0 '@unocss/core': 0.61.0 '@unocss/preset-uno': 0.61.0 @@ -5937,10 +5957,10 @@ snapshots: dependencies: '@unocss/core': 0.61.0 - '@unocss/vite@0.61.0(vite@4.5.3)': + '@unocss/vite@0.61.0(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2))': dependencies: '@ampproject/remapping': 2.3.0 - '@rollup/pluginutils': 5.1.0 + '@rollup/pluginutils': 5.1.0(rollup@3.29.4) '@unocss/config': 0.61.0 '@unocss/core': 0.61.0 '@unocss/inspector': 0.61.0 @@ -5953,7 +5973,7 @@ snapshots: transitivePeerDependencies: - rollup - '@vitejs/plugin-react@4.3.1(vite@4.5.3)': + '@vitejs/plugin-react@4.3.1(vite@4.5.3(@types/node@20.14.2))': dependencies: '@babel/core': 7.24.7 '@babel/plugin-transform-react-jsx-self': 7.24.7(@babel/core@7.24.7) @@ -6013,7 +6033,7 @@ snapshots: acorn@8.12.0: {} ajv-draft-04@1.0.0(ajv@8.16.0): - dependencies: + optionalDependencies: ajv: 8.16.0 ajv-errors@3.0.0(ajv@8.16.0): @@ -6021,7 +6041,7 @@ snapshots: ajv: 8.16.0 ajv-formats@2.1.1(ajv@8.16.0): - dependencies: + optionalDependencies: ajv: 8.16.0 ajv@6.12.6: @@ -6996,12 +7016,13 @@ snapshots: eslint-plugin-no-only-tests@3.1.0: {} - eslint-plugin-perfectionist@2.11.0(eslint@8.57.0)(typescript@5.4.5)(vue-eslint-parser@9.4.3): + eslint-plugin-perfectionist@2.11.0(eslint@8.57.0)(typescript@5.4.5)(vue-eslint-parser@9.4.3(eslint@8.57.0)): dependencies: '@typescript-eslint/utils': 7.13.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 minimatch: 9.0.4 natural-compare-lite: 1.4.0 + optionalDependencies: vue-eslint-parser: 9.4.3(eslint@8.57.0) transitivePeerDependencies: - supports-color @@ -7023,8 +7044,9 @@ snapshots: eslint: 8.57.0 string-ts: 2.1.1 ts-api-utils: 1.3.0(typescript@5.4.5) - typescript: 5.4.5 valibot: 0.31.0 + optionalDependencies: + typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -7042,8 +7064,9 @@ snapshots: '@typescript-eslint/utils': 7.13.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 string-ts: 2.1.1 - typescript: 5.4.5 valibot: 0.31.0 + optionalDependencies: + typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -7062,8 +7085,9 @@ snapshots: '@typescript-eslint/utils': 7.13.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 string-ts: 2.1.1 - typescript: 5.4.5 valibot: 0.31.0 + optionalDependencies: + typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -7085,8 +7109,9 @@ snapshots: '@typescript-eslint/utils': 7.13.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 string-ts: 2.1.1 - typescript: 5.4.5 valibot: 0.31.0 + optionalDependencies: + typescript: 5.4.5 transitivePeerDependencies: - supports-color @@ -7137,17 +7162,19 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-unused-imports@3.2.0(@typescript-eslint/eslint-plugin@7.13.0)(eslint@8.57.0): + eslint-plugin-unused-imports@3.2.0(@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0): dependencies: - '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0)(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-rule-composer: 0.3.0 + optionalDependencies: + '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) - eslint-plugin-vitest@0.5.4(@typescript-eslint/eslint-plugin@7.13.0)(eslint@8.57.0)(typescript@5.4.5): + eslint-plugin-vitest@0.5.4(@typescript-eslint/eslint-plugin@7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5): dependencies: - '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0)(eslint@8.57.0)(typescript@5.4.5) '@typescript-eslint/utils': 7.13.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 + optionalDependencies: + '@typescript-eslint/eslint-plugin': 7.13.0(@typescript-eslint/parser@7.13.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) transitivePeerDependencies: - supports-color - typescript @@ -7698,10 +7725,11 @@ snapshots: gulp-zip@6.0.0(gulp@4.0.2): dependencies: get-stream: 8.0.1 - gulp: 4.0.2 gulp-plugin-extras: 0.3.0 vinyl: 3.0.0 yazl: 2.5.1 + optionalDependencies: + gulp: 4.0.2 gulp@4.0.2: dependencies: @@ -8600,6 +8628,10 @@ snapshots: dependencies: yocto-queue: 1.0.0 + p-limit@6.1.0: + dependencies: + yocto-queue: 1.1.1 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -9406,7 +9438,7 @@ snapshots: ts-pattern@5.1.2: {} tsconfck@2.1.2(typescript@5.4.5): - dependencies: + optionalDependencies: typescript: 5.4.5 tslib@1.14.1: {} @@ -9519,10 +9551,10 @@ snapshots: universalify@2.0.1: {} - unocss@0.61.0(postcss@8.4.38)(vite@4.5.3): + unocss@0.61.0(postcss@8.4.38)(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2)): dependencies: - '@unocss/astro': 0.61.0(vite@4.5.3) - '@unocss/cli': 0.61.0 + '@unocss/astro': 0.61.0(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2)) + '@unocss/cli': 0.61.0(rollup@3.29.4) '@unocss/core': 0.61.0 '@unocss/extractor-arbitrary-variants': 0.61.0 '@unocss/postcss': 0.61.0(postcss@8.4.38) @@ -9540,7 +9572,8 @@ snapshots: '@unocss/transformer-compile-class': 0.61.0 '@unocss/transformer-directives': 0.61.0 '@unocss/transformer-variant-group': 0.61.0 - '@unocss/vite': 0.61.0(vite@4.5.3) + '@unocss/vite': 0.61.0(rollup@3.29.4)(vite@4.5.3(@types/node@20.14.2)) + optionalDependencies: vite: 4.5.3(@types/node@20.14.2) transitivePeerDependencies: - postcss @@ -9640,11 +9673,11 @@ snapshots: vite@4.5.3(@types/node@20.14.2): dependencies: - '@types/node': 20.14.2 esbuild: 0.18.20 postcss: 8.4.38 rollup: 3.29.4 optionalDependencies: + '@types/node': 20.14.2 fsevents: 2.3.3 vue-eslint-parser@9.4.3(eslint@8.57.0): @@ -9778,3 +9811,5 @@ snapshots: yocto-queue@0.1.0: {} yocto-queue@1.0.0: {} + + yocto-queue@1.1.1: {} diff --git a/src/features/search-sync/background.ts b/src/features/search-sync/background.ts index 0fbec3d..c1b0f5f 100644 --- a/src/features/search-sync/background.ts +++ b/src/features/search-sync/background.ts @@ -1,17 +1,29 @@ +import axios from 'axios' import type { MoodleClientFunctionTypes } from 'moodle-typed-ws' +import pLimit from 'p-limit' +import { sendMessage } from '@/shared/messages' import { getStored, setStored } from '@/shared/storage' import { downloadFileByUrl } from '@/shared/moodle-ws-api/download-file' -import type { InContentsInput, InContentsOutput } from '@/shared/innohassle-api/search' +import type { FlattenInContentsWithPresignedUrl, InContents } from '@/shared/innohassle-api/search' import { search } from '@/shared/innohassle-api/search' import { moodle } from '@/shared/moodle-ws-api' const DELAY = 24 * 60 * 60 * 1000 // 24 hours +const MAX_CONCURRENT_REQUESTS = 10 // Limit the number of concurrent requests const state = { // Lock to prevent multiple concurrent processes isSyncingCourses: false, + + // Limit the number of concurrent uploads + queue: pLimit(MAX_CONCURRENT_REQUESTS), + totalUploads: 0, + currentUploads: 0, } +type CourseContents = MoodleClientFunctionTypes.CoreCourseGetContentsWSResponse +type CourseModule = CourseContents[number]['modules'][number] + /** * Sync all courses with InNoHassle Search for indexing. */ @@ -36,11 +48,8 @@ export async function syncCourses() { console.log('Syncing all courses with InNoHassle Search') try { - // Retrieve courses list from Moodle - const { courses } = await moodle.core.course.getEnrolledCoursesByTimelineClassification({ classification: 'all' }) - - // Sync course data with InNoHassle Search - await Promise.all(courses.map(course => syncCourseData(course.id, course.fullname))) + // Sync all courses with InNoHassle Search + await syncAllCourses() // Save last sync time await setStored('syncCoursesLastUpdateMS', Date.now()) @@ -52,59 +61,43 @@ export async function syncCourses() { state.isSyncingCourses = false } -/** - * Send course info and files to InNoHassle Search for indexing. - */ -export async function syncCourseData(courseId: number, courseFullName: string) { - console.log(`Sending course info (courseId: ${courseId}, courseFullName: ${courseFullName})`) - try { - // Retrieve course structure from Moodle - const contents = await moodle.core.course.getContents({ courseid: courseId }) - - // Sync course structure with InNoHassle Search - await syncCourseInfo(courseId, courseFullName, contents) - - // Check whether the course files need to be uploaded - const needToUpload = await needToUploadContents(courseId, contents) - console.log(`Need to upload: ${needToUpload.length} modules`) - - // Upload course files to InNoHassle Search module by module - for (const module of needToUpload) { - if (module.contents.length === 0) { - continue - } - const section = contents.find(s => s.modules.find(m => m.id === module.module_id)) - const originalModule = section?.modules.find(m => m.id === module.module_id) - - // Collect file URLs for each content - const fileUrls: string[] = [] - for (const content of module.contents) { - const originalContent = originalModule?.contents?.find(c => c.filename === content.filename) - if (!originalContent || !originalContent.fileurl) { - break - } - fileUrls.push(originalContent.fileurl) - } - - // If all files are found, send them asynchronously - if (fileUrls.length === module.contents.length) { - sendFiles(module, fileUrls) - } - } - } - catch (e) { - console.log(`Error: Couldn't sync course data (${e})`) - } -} - -/** - * Sync course info (structure of modules) with InNoHassle Search. - */ -async function syncCourseInfo(courseId: number, courseFullName: string, contents: MoodleClientFunctionTypes.CoreCourseGetContentsWSResponse) { - return search.moodleCourseContent({ - course_id: courseId, - course_fullname: courseFullName, - sections: contents.map(s => ({ +async function syncAllCourses() { + state.queue.clearQueue() + state.totalUploads = 0 + state.currentUploads = 0 + + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + // Retrieve courses list from Moodle + const { courses } = await moodle.core.course.getEnrolledCoursesByTimelineClassification({ classification: 'all' }) + + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + // Retrieve course structure from Moodle + console.log('Fetching course contents from Moodle') + const contentsByCourseId: Record = {} + const modulesByCourseId: Record = {} + const limit = pLimit(MAX_CONCURRENT_REQUESTS) + await Promise.all(courses.map(async ({ id }) => limit(async () => { + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + const contents = await moodle.core.course.getContents({ courseid: id }) + contentsByCourseId[id] = contents + modulesByCourseId[id] = contents.flatMap(section => section.modules) + }))) + + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + // Sync course structure with InNoHassle Search + console.log('Syncing course info') + await search.moodleCourseContent(courses.map(course => ({ + course_id: course.id, + course_fullname: course.fullname, + sections: contentsByCourseId[course.id]?.map(s => ({ id: s.id, summary: s.summary, modules: s.modules.map(m => ({ @@ -118,24 +111,28 @@ async function syncCourseInfo(courseId: number, courseFullName: string, contents timemodified: c.timemodified, })), })), - })), - }) -} + })) ?? [], + }))) + + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + // Check whether the course files need to be uploaded + console.log('Checking if contents need to be uploaded') + const needToUpload = await search.moodleNeedToUploadContents(courses.flatMap((course) => { + const contents = contentsByCourseId[course.id] + const modules = modulesByCourseId[course.id] + if (!contents || !modules) { + return [] + } -/** - * Make request to InNoHassle Search to check if the contents need to be updated. - */ -export async function needToUploadContents(courseId: number, contents: MoodleClientFunctionTypes.CoreCourseGetContentsWSResponse) { - // Collect info for each module that has contents - const modules: InContentsInput[] = [] - for (const section of contents) { - for (const module of section.modules) { + return modules.reduce((acc, module) => { if (!module.contents || module.contents.length === 0) { - continue + return acc } - modules.push({ - course_id: courseId, + return [...acc, { + course_id: course.id, module_id: module.id, contents: module.contents.map(c => ({ type: c.type, @@ -143,24 +140,98 @@ export async function needToUploadContents(courseId: number, contents: MoodleCli timecreated: c.timecreated, timemodified: c.timemodified, })), - }) - } + }] + }, [] as InContents[]) + })) + console.log(`Need to upload: ${needToUpload.length} modules`) + + if (needToUpload.length === 0) { + // No files to upload + return } - // Make request to InNoHassle Search - return search.moodleNeedToUploadContents(modules) + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + // Find urls for each file + const toUpload = needToUpload.reduce((acc, module) => { + const originalModule = modulesByCourseId[module.course_id]?.find(m => m.id === module.module_id) + const originalContent = originalModule?.contents?.find(c => c.filename === module.content.filename) + if (!originalModule || !originalContent || !originalContent.fileurl) { + return acc + } + + return [...acc, { + module, + fileUrl: originalContent.fileurl, + }] + }, [] as { module: FlattenInContentsWithPresignedUrl, fileUrl: string }[]) + + // Upload course files to InNoHassle Search module by module + // Limit the number of concurrent uploads + console.log(`Uploading ${toUpload.length} files`) + state.totalUploads = toUpload.length + const promises = toUpload.map( + ({ module, fileUrl }) => state.queue(async () => { + if (!state.isSyncingCourses) + throw new Error('Syncing stopped') // Stop syncing if the user disabled this feature + + await sendFile(module, fileUrl) + }).then(() => { + state.currentUploads += 1 + console.log(`Uploaded files: ${state.currentUploads} / ${state.totalUploads}`) + }), + ) + await Promise.all(promises) + + if (!state.isSyncingCourses) + return // Stop syncing if the user disabled this feature + + console.log('Finished syncing all courses') } /** * Upload course module files to InNoHassle Search for indexing. */ -export async function sendFiles(module: InContentsOutput, fileUrls: string[]) { - console.log(`Downloading files for module ${module.module_id} (${fileUrls})`) - const blobs: Blob[] = await Promise.all(fileUrls.map(downloadFileByUrl)) - - console.log(`Uploading files for module ${module.module_id}`) - await search.moodleUploadContent({ - data: JSON.stringify(module), - files: blobs, +export async function sendFile(module: FlattenInContentsWithPresignedUrl, fileUrl: string) { + console.log(`Downloading file for course ${module.course_id} module ${module.module_id} (${fileUrl})`) + const blob = await downloadFileByUrl(fileUrl) + + console.log(`Uploading files for course ${module.course_id} module ${module.module_id}`) + const resp = await axios({ + url: module.presigned_url, + method: 'PUT', + data: blob, + }) + + if (resp.status !== 200) { + console.log(`Error: Couldn't upload file for course ${module.course_id} module ${module.module_id}`) + return + } + + console.log(`File uploaded for course ${module.course_id} module ${module.module_id}`) + await search.moodleContentUploaded({ + course_id: module.course_id, + module_id: module.module_id, + content: module.content, + }) +} + +/** + * Stop syncing courses. + */ +export function stopSync() { + state.queue.clearQueue() + state.isSyncingCourses = false +} + +/** + * Send syncing progress as message. + */ +export function sendSyncProgress() { + sendMessage('SYNCING_PROGRESS', { + isSyncing: state.isSyncingCourses, + current: state.currentUploads, + total: state.totalUploads, }) } diff --git a/src/shared/innohassle-api/search/__generated__.ts b/src/shared/innohassle-api/search/__generated__.ts index 8dc9ed9..40611e3 100644 --- a/src/shared/innohassle-api/search/__generated__.ts +++ b/src/shared/innohassle-api/search/__generated__.ts @@ -17,14 +17,23 @@ export interface MoodlePreviewMoodleParams { filename: string } +export type SearchAddUserFeedbackFeedback = typeof SearchAddUserFeedbackFeedback[keyof typeof SearchAddUserFeedbackFeedback] + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const SearchAddUserFeedbackFeedback = { + like: 'like', + dislike: 'dislike', +} as const + export interface SearchAddUserFeedbackParams { response_index: number - feedback: string + feedback: SearchAddUserFeedbackFeedback } export interface SearchSearchByQueryParams { query: string limit?: number + use_ai?: boolean } export type ValidationErrorLocItem = string | number @@ -58,98 +67,148 @@ export interface TelegramSource { type: TelegramSourceType } +export type SearchTaskStatus = typeof SearchTaskStatus[keyof typeof SearchTaskStatus] + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const SearchTaskStatus = { + pending: 'pending', + completed: 'completed', + failed: 'failed', +} as const + +export interface SearchTask { + query: string + status: SearchTaskStatus + task_id: string +} + +export type SearchResultStatus = typeof SearchResultStatus[keyof typeof SearchResultStatus] + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const SearchResultStatus = { + completed: 'completed', + failed: 'failed', +} as const + +export interface SearchResult { + result?: MoodleFileResult[] + status: SearchResultStatus + task_id: string +} + /** * Assigned search query index */ export type SearchResponsesSearchQueryId = string | null +export interface SearchResponses { + /** Responses to the search query. */ + responses: SearchResponse[] + /** Assigned search query index */ + search_query_id: SearchResponsesSearchQueryId + /** Text that was searched for. */ + searched_for: string +} + /** * Relevant source for the search. */ -export type SearchResponseSource = MoodleSource | TelegramSource +export type SearchResponseSource = MoodleFileSource | MoodleUrlSource | MoodleUnknownSource | TelegramSource /** - * Score of the search response. Optional. + * Score of the search response. Multiple scores if was an aggregation of multiple chunks. Optional. */ -export type SearchResponseScore = number | null +export type SearchResponseScore = number | number[] | null export interface SearchResponse { - /** Score of the search response. Optional. */ + /** Score of the search response. Multiple scores if was an aggregation of multiple chunks. Optional. */ score: SearchResponseScore /** Relevant source for the search. */ source: SearchResponseSource } -export interface SearchResponses { - /** Responses to the search query. */ - responses: SearchResponse[] - /** Assigned search query index */ - search_query_id: SearchResponsesSearchQueryId - /** Text that was searched for. */ - searched_for: string -} - export interface PdfLocation { /** Page index in the PDF file. Starts from 1. */ page_index: number } -export type MoodleSourceType = typeof MoodleSourceType[keyof typeof MoodleSourceType] +export type MoodleUrlSourceType = typeof MoodleUrlSourceType[keyof typeof MoodleUrlSourceType] // eslint-disable-next-line @typescript-eslint/no-redeclare -export const MoodleSourceType = { - moodle: 'moodle', +export const MoodleUrlSourceType = { + 'moodle-url': 'moodle-url', } as const -export type MoodleSourcePreviewLocation = PdfLocation | null +export interface MoodleUrlSource { + /** Breadcrumbs to the resource. */ + breadcrumbs: string[] + /** Display name of the resource. */ + display_name: string + /** Anchor URL to the resource on Moodle. */ + link: string + type: MoodleUrlSourceType + /** URL of the resource */ + url: string +} + +export type MoodleUnknownSourceType = typeof MoodleUnknownSourceType[keyof typeof MoodleUnknownSourceType] + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const MoodleUnknownSourceType = { + 'moodle-unknown': 'moodle-unknown', +} as const + +export interface MoodleUnknownSource { + /** Breadcrumbs to the resource. */ + breadcrumbs: string[] + /** Display name of the resource. */ + display_name: string + /** Anchor URL to the resource on Moodle. */ + link: string + type: MoodleUnknownSourceType +} + +export type MoodleFileSourceType = typeof MoodleFileSourceType[keyof typeof MoodleFileSourceType] + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const MoodleFileSourceType = { + 'moodle-file': 'moodle-file', +} as const + +/** + * URL to get the preview of the resource. + */ +export type MoodleFileSourceResourcePreviewUrl = string | null /** - * Filename of the resource. + * URL to download the resource. */ -export type MoodleSourceFilename = string | null +export type MoodleFileSourceResourceDownloadUrl = string | null -export interface MoodleSource { +export type MoodleFileSourcePreviewLocation = PdfLocation | null + +export interface MoodleFileSource { /** Breadcrumbs to the resource. */ breadcrumbs: string[] - /** Course ID in the Moodle system. */ - course_id: number - /** Course name in the Moodle system. */ - course_name: string /** Display name of the resource. */ display_name: string - /** Filename of the resource. */ - filename: MoodleSourceFilename /** Anchor URL to the resource on Moodle. */ link: string - /** Module ID in the Moodle system (resources). */ - module_id: number - /** Module name in the Moodle system. */ - module_name: string - preview_location: MoodleSourcePreviewLocation + preview_location: MoodleFileSourcePreviewLocation /** URL to download the resource. */ - resource_download_url: string + resource_download_url: MoodleFileSourceResourceDownloadUrl /** URL to get the preview of the resource. */ - resource_preview_url: string - /** Type of the resource. */ - resource_type: string - type: MoodleSourceType + resource_preview_url: MoodleFileSourceResourcePreviewUrl + type: MoodleFileSourceType } -export type MoodleEntrySectionSummary = string | null - -export type MoodleEntrySectionId = number | null +export type MoodleFileResultScore = number[] | number | null -export interface MoodleEntry { - contents: MoodleContentSchemaOutput[] - course_fullname: string +export interface MoodleFileResult { course_id: number - /** MongoDB document ObjectID */ - id: string + filename: string module_id: number - module_modname: string - module_name: string - section_id: MoodleEntrySectionId - section_summary: MoodleEntrySectionSummary + score?: MoodleFileResultScore } export interface MoodleCourse { @@ -171,6 +230,20 @@ export interface MoodleContentSchemaOutput { timecreated: MoodleContentSchemaOutputTimecreated timemodified: MoodleContentSchemaOutputTimemodified type: string + uploaded: boolean +} + +export interface MoodleEntry { + contents: MoodleContentSchemaOutput[] + course_fullname: string + course_id: number + /** MongoDB document ObjectID */ + id: string + module_id: number + module_modname: string + module_name: string + section_id: number + section_summary: string } export type MoodleContentSchemaInputTimemodified = number | null @@ -182,6 +255,20 @@ export interface MoodleContentSchemaInput { timecreated?: MoodleContentSchemaInputTimecreated timemodified?: MoodleContentSchemaInputTimemodified type: string + uploaded?: boolean +} + +export type MessageSchemaText = string | null + +export type MessageSchemaCaption = string | null + +export interface MessageSchema { + caption: MessageSchemaCaption + chat: Chat + date: string + id: number + sender_chat: Chat + text: MessageSchemaText } export interface InModule { @@ -215,14 +302,14 @@ export interface InCourses { courses: InCourse[] } -export interface InContentsOutput { - contents: MoodleContentSchemaOutput[] +export interface InContents { + contents: MoodleContentSchemaInput[] course_id: number module_id: number } -export interface InContentsInput { - contents: MoodleContentSchemaInput[] +export interface InContent { + content: MoodleContentSchemaInput course_id: number module_id: number } @@ -231,9 +318,37 @@ export interface HTTPValidationError { detail?: ValidationError[] } -export interface BodyMoodleUploadContent { - data: string - files: Blob[] +export interface FlattenInContentsWithPresignedUrl { + content: MoodleContentSchemaOutput + course_id: number + module_id: number + presigned_url: string +} + +export interface Detail { + detail: string +} + +export type DBMessageSchemaText = string | null + +export type DBMessageSchemaCaption = string | null + +export interface DBMessageSchema { + caption: DBMessageSchemaCaption + chat_id: number + chat_title: string + chat_username: string + date: string + link: string + message_id: number + text: DBMessageSchemaText +} + +export interface Chat { + id: number + title: string + type: string + username: string } type SecondParameter any> = Parameters[1] @@ -266,6 +381,20 @@ export function getInNoHassleSearch() { ) } + /** + * Determining whether to save the message or overwrite it + * @summary Save Or Update Message + */ + const telegramSaveOrUpdateMessage = ( + messageSchema: MessageSchema, + options?: SecondParameter, + ) => { + return searchQueryPromise( + { url: `/telegram/messages`, method: 'POST', headers: { 'Content-Type': 'application/json' }, data: messageSchema }, + options, + ) + } + /** * @summary Preview Moodle */ @@ -292,6 +421,19 @@ export function getInNoHassleSearch() { ) } + /** + * @summary Courses + */ + const moodleCourses = ( + + options?: SecondParameter) => { + return searchQueryPromise( + { url: `/moodle/courses`, method: 'GET', + }, + options, + ) + } + /** * @summary Batch Upsert Courses */ @@ -306,13 +448,13 @@ export function getInNoHassleSearch() { } /** - * @summary Courses + * @summary Courses Content */ - const moodleCourses = ( + const moodleCoursesContent = ( options?: SecondParameter) => { - return searchQueryPromise( - { url: `/moodle/courses`, method: 'GET', + return searchQueryPromise( + { url: `/moodle/courses-content`, method: 'GET', }, options, ) @@ -322,7 +464,7 @@ export function getInNoHassleSearch() { * @summary Course Content */ const moodleCourseContent = ( - inSections: InSections, + inSections: InSections[], options?: SecondParameter, ) => { return searchQueryPromise( @@ -332,57 +474,83 @@ export function getInNoHassleSearch() { } /** - * @summary Courses Content + * @summary Need To Upload Contents */ - const moodleCoursesContent = ( + const moodleNeedToUploadContents = ( + inContents: InContents[], + options?: SecondParameter, + ) => { + return searchQueryPromise( + { url: `/moodle/need-to-upload-contents`, method: 'POST', headers: { 'Content-Type': 'application/json' }, data: inContents }, + options, + ) + } + + /** + * @summary Content Uploaded + */ + const moodleContentUploaded = ( + inContent: InContent, + options?: SecondParameter, + ) => { + return searchQueryPromise( + { url: `/moodle/content-uploaded`, method: 'POST', headers: { 'Content-Type': 'application/json' }, data: inContent }, + options, + ) + } + + /** + * @summary Get Corpora + */ + const computeGetCorpora = ( options?: SecondParameter) => { - return searchQueryPromise( - { url: `/moodle/courses-content`, method: 'GET', + return searchQueryPromise( + { url: `/compute/corpora`, method: 'GET', }, options, ) } /** - * @summary Need To Upload Contents + * @summary Get Pending Search Queries */ - const moodleNeedToUploadContents = ( - inContentsInput: InContentsInput[], - options?: SecondParameter, - ) => { - return searchQueryPromise( - { url: `/moodle/need-to-upload-contents`, method: 'POST', headers: { 'Content-Type': 'application/json' }, data: inContentsInput }, + const computeGetPendingSearchQueries = ( + + options?: SecondParameter) => { + return searchQueryPromise( + { url: `/compute/pending-searchs`, method: 'GET', + }, options, ) } /** - * @summary Upload Content + * @summary Post Completed Search Queries */ - const moodleUploadContent = ( - bodyMoodleUploadContent: BodyMoodleUploadContent, + const computePostCompletedSearchQueries = ( + searchResult: SearchResult[], options?: SecondParameter, ) => { - const formData = new FormData() - bodyMoodleUploadContent.files.forEach(value => formData.append('files', value)) - formData.append('data', bodyMoodleUploadContent.data) - return searchQueryPromise( - { url: `/moodle/upload-contents`, method: 'POST', headers: { 'Content-Type': 'multipart/form-data' }, data: formData }, + { url: `/compute/completed-searchs`, method: 'POST', headers: { 'Content-Type': 'application/json' }, data: searchResult }, options, ) } - return { searchSearchByQuery, searchAddUserFeedback, moodlePreviewMoodle, moodleGetMoodleFiles, moodleBatchUpsertCourses, moodleCourses, moodleCourseContent, moodleCoursesContent, moodleNeedToUploadContents, moodleUploadContent } + return { searchSearchByQuery, searchAddUserFeedback, telegramSaveOrUpdateMessage, moodlePreviewMoodle, moodleGetMoodleFiles, moodleCourses, moodleBatchUpsertCourses, moodleCoursesContent, moodleCourseContent, moodleNeedToUploadContents, moodleContentUploaded, computeGetCorpora, computeGetPendingSearchQueries, computePostCompletedSearchQueries } } export type SearchSearchByQueryResult = NonNullable['searchSearchByQuery']>>> export type SearchAddUserFeedbackResult = NonNullable['searchAddUserFeedback']>>> +export type TelegramSaveOrUpdateMessageResult = NonNullable['telegramSaveOrUpdateMessage']>>> export type MoodlePreviewMoodleResult = NonNullable['moodlePreviewMoodle']>>> export type MoodleGetMoodleFilesResult = NonNullable['moodleGetMoodleFiles']>>> -export type MoodleBatchUpsertCoursesResult = NonNullable['moodleBatchUpsertCourses']>>> export type MoodleCoursesResult = NonNullable['moodleCourses']>>> -export type MoodleCourseContentResult = NonNullable['moodleCourseContent']>>> +export type MoodleBatchUpsertCoursesResult = NonNullable['moodleBatchUpsertCourses']>>> export type MoodleCoursesContentResult = NonNullable['moodleCoursesContent']>>> +export type MoodleCourseContentResult = NonNullable['moodleCourseContent']>>> export type MoodleNeedToUploadContentsResult = NonNullable['moodleNeedToUploadContents']>>> -export type MoodleUploadContentResult = NonNullable['moodleUploadContent']>>> +export type MoodleContentUploadedResult = NonNullable['moodleContentUploaded']>>> +export type ComputeGetCorporaResult = NonNullable['computeGetCorpora']>>> +export type ComputeGetPendingSearchQueriesResult = NonNullable['computeGetPendingSearchQueries']>>> +export type ComputePostCompletedSearchQueriesResult = NonNullable['computePostCompletedSearchQueries']>>> diff --git a/src/shared/messages/types.d.ts b/src/shared/messages/types.d.ts index 35fa696..a2694d6 100644 --- a/src/shared/messages/types.d.ts +++ b/src/shared/messages/types.d.ts @@ -6,4 +6,9 @@ export interface Messages { AUTOLOGIN_FAILED: void AUTOLOGIN_SUCCEEDED: void AUTOLOGIN_LAST_SUCCESS: number + + REQUEST_SYNC: void + STOP_SYNC: void + REQUEST_SYNC_PROGRESS: void + SYNCING_PROGRESS: { isSyncing: boolean, current: number, total: number } }