diff --git a/env.d/development/common.dist b/env.d/development/common.dist index 4b1389bf44..75e7460fa7 100644 --- a/env.d/development/common.dist +++ b/env.d/development/common.dist @@ -64,3 +64,4 @@ COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/ # Frontend FRONTEND_THEME=default +FRONTEND_URL=http://localhost:3000 diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 3230bd3ce7..ae02231881 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -5,7 +5,7 @@ import logging import uuid from collections import defaultdict -from urllib.parse import unquote, urlparse +from urllib.parse import unquote, urlencode, urlparse from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg @@ -18,6 +18,7 @@ from django.db.models.expressions import RawSQL from django.db.models.functions import Left, Length from django.http import Http404, StreamingHttpResponse +from django.shortcuts import redirect from django.utils.functional import cached_property from django.utils.text import capfirst, slugify from django.utils.translation import gettext_lazy as _ @@ -34,8 +35,17 @@ from core import authentication, choices, enums, models from core.services.ai_services import AIService from core.services.collaboration_services import CollaborationService +from core.services.converter_services import YdocConverter +from core.services.notion_import import ( + ImportedDocument, + build_notion_session, + fetch_all_pages, + import_page, + link_child_page_to_parent, +) from core.utils import extract_attachments, filter_descendants +from ..notion_schemas.notion_page import NotionPage from . import permissions, serializers, utils from .filters import DocumentFilter, ListDocumentFilter @@ -1817,3 +1827,175 @@ def _load_theme_customization(self): ) return theme_customization + + +@drf.decorators.api_view() +def notion_import_redirect(request): + query = urlencode( + { + "client_id": settings.NOTION_CLIENT_ID, + "response_type": "code", + "owner": "user", + "redirect_uri": settings.NOTION_REDIRECT_URI, + } + ) + return redirect("https://api.notion.com/v1/oauth/authorize?" + query) + + +@drf.decorators.api_view() +def notion_import_callback(request): + code = request.GET.get("code") + resp = requests.post( + "https://api.notion.com/v1/oauth/token", + auth=requests.auth.HTTPBasicAuth( + settings.NOTION_CLIENT_ID, settings.NOTION_CLIENT_SECRET + ), + headers={"Accept": "application/json"}, + data={ + "grant_type": "authorization_code", + "code": code, + "redirect_uri": settings.NOTION_REDIRECT_URI, + }, + ) + resp.raise_for_status() + data = resp.json() + request.session["notion_token"] = data["access_token"] + return redirect(f"{settings.FRONTEND_URL}/import-notion/") + + +def _import_notion_doc_content(imported_doc, obj, user): + for att in imported_doc.attachments: + extra_args = { + "Metadata": { + "owner": str(user.id), + "status": enums.DocumentAttachmentStatus.READY, # TODO + }, + } + file_id = uuid.uuid4() + key = f"{obj.key_base}/{enums.ATTACHMENTS_FOLDER:s}/{file_id!s}.raw" + with requests.get(att.file.file["url"], stream=True) as resp: + default_storage.connection.meta.client.upload_fileobj( + resp.raw, default_storage.bucket_name, key + ) + obj.attachments.append(key) + att.block["props"]["url"] = ( + f"{settings.MEDIA_BASE_URL}{settings.MEDIA_URL}{key}" + ) + + obj.content = YdocConverter().convert_blocks(imported_doc.blocks) + obj.save() + + +def _import_notion_child_page(imported_doc, parent_doc, user, imported_ids): + obj = parent_doc.add_child( + creator=user, + title=imported_doc.page.get_title() or "J'aime les carottes", + ) + + models.DocumentAccess.objects.create( + document=obj, + user=user, + role=models.RoleChoices.OWNER, + ) + + _import_notion_doc_content(imported_doc, obj, user) + + imported_ids.append(imported_doc.page.id) + + for child in imported_doc.children: + _import_notion_child_page(child, obj, user, imported_ids) + + +def _import_notion_root_page(imported_doc, user) -> list[str]: + obj = models.Document.add_root( + depth=1, + creator=user, + title=imported_doc.page.get_title() or "J'aime les courgettes", + link_reach=models.LinkReachChoices.RESTRICTED, + ) + + models.DocumentAccess.objects.create( + document=obj, + user=user, + role=models.RoleChoices.OWNER, + ) + + imported_ids = [imported_doc.page.id] + + _import_notion_doc_content(imported_doc, obj, user) + + for child in imported_doc.children: + _import_notion_child_page(child, obj, user, imported_ids) + + return imported_ids + + +def _generate_notion_progress( + all_pages: list[NotionPage], page_statuses: dict[str, str] +) -> str: + raw = json.dumps( + [ + { + "title": page.get_title(), + "status": page_statuses[page.id], + } + for page in all_pages + ] + ) + return f"data: {raw}\n\n" + + +def _notion_import_event_stream(request): + session = build_notion_session(request.session["notion_token"]) + all_pages = fetch_all_pages(session) + + page_statuses = {} + for page in all_pages: + page_statuses[page.id] = "pending" + + yield _generate_notion_progress(all_pages, page_statuses) + + docs_by_page_id: dict[str, ImportedDocument] = {} + child_page_blocs_ids_to_parent_page_ids: dict[str, str] = {} + + for page in all_pages: + docs_by_page_id[page.id] = import_page( + session, page, child_page_blocs_ids_to_parent_page_ids + ) + page_statuses[page.id] = "fetched" + yield _generate_notion_progress(all_pages, page_statuses) + + for page in all_pages: + link_child_page_to_parent( + page, docs_by_page_id, child_page_blocs_ids_to_parent_page_ids + ) + + root_docs = [doc for doc in docs_by_page_id.values() if doc.page.is_root()] + + for root_doc in root_docs: + imported_ids = _import_notion_root_page(root_doc, request.user) + for imported_id in imported_ids: + page_statuses[imported_id] = "imported" + + yield _generate_notion_progress(all_pages, page_statuses) + + +class IgnoreClientContentNegotiation(drf.negotiation.BaseContentNegotiation): + def select_parser(self, request, parsers): + return parsers[0] + + def select_renderer(self, request, renderers, format_suffix): + return (renderers[0], renderers[0].media_type) + + +class NotionImportRunView(drf.views.APIView): + content_negotiation_class = IgnoreClientContentNegotiation + + def get(self, request, format=None): + if "notion_token" not in request.session: + raise drf.exceptions.PermissionDenied() + + # return drf.response.Response({"sava": "oui et toi ?"}) + return StreamingHttpResponse( + _notion_import_event_stream(request), content_type="text/event-stream" + ) diff --git a/src/backend/core/notion_schemas/notion_block.py b/src/backend/core/notion_schemas/notion_block.py new file mode 100644 index 0000000000..74e7ea4896 --- /dev/null +++ b/src/backend/core/notion_schemas/notion_block.py @@ -0,0 +1,296 @@ +from datetime import datetime +from enum import StrEnum +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, Discriminator, Field, ValidationError, model_validator + +from .notion_color import NotionColor +from .notion_file import NotionFile +from .notion_rich_text import NotionRichText + +"""Usage: NotionBlock.model_validate(response.json())""" + + +class NotionBlock(BaseModel): + id: str + created_time: datetime + last_edited_time: datetime + archived: bool + specific: "NotionBlockSpecifics" + has_children: bool + children: list["NotionBlock"] = Field(init=False, default_factory=list) + # This is not part of the API response, but is used to store children blocks + + @model_validator(mode="before") + @classmethod + def move_type_inward_and_rename(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + if "type" not in data: + raise ValidationError("Type must be specified") + + data_type = data.pop("type") + data["specific"] = data.pop(data_type) + data["specific"]["block_type"] = data_type + + return data + + +class NotionBlockType(StrEnum): + """https://developers.notion.com/reference/block""" + + BOOKMARK = "bookmark" + BREADCRUMB = "breadcrumb" + BULLETED_LIST_ITEM = "bulleted_list_item" + CALLOUT = "callout" + CHILD_DATABASE = "child_database" + CHILD_PAGE = "child_page" + CODE = "code" + COLUMN = "column" + COLUMN_LIST = "column_list" + DIVIDER = "divider" + EMBED = "embed" + EQUATION = "equation" + FILE = "file" + HEADING_1 = "heading_1" + HEADING_2 = "heading_2" + HEADING_3 = "heading_3" + IMAGE = "image" + LINK_PREVIEW = "link_preview" + LINK_TO_PAGE = "link_to_page" + NUMBERED_LIST_ITEM = "numbered_list_item" + PARAGRAPH = "paragraph" + PDF = "pdf" + QUOTE = "quote" + SYNCED_BLOCK = "synced_block" + TABLE = "table" + TABLE_OF_CONTENTS = "table_of_contents" + TABLE_ROW = "table_row" + TEMPLATE = "template" + TO_DO = "to_do" + TOGGLE = "toggle" + UNSUPPORTED = "unsupported" + VIDEO = "video" + + +class NotionHeadingBase(BaseModel): + """https://developers.notion.com/reference/block#headings""" + + rich_text: list[NotionRichText] + color: NotionColor + is_toggleable: bool = False + + +class NotionHeading1(NotionHeadingBase): + block_type: Literal[NotionBlockType.HEADING_1] = NotionBlockType.HEADING_1 + + +class NotionHeading2(NotionHeadingBase): + block_type: Literal[NotionBlockType.HEADING_2] = NotionBlockType.HEADING_2 + + +class NotionHeading3(NotionHeadingBase): + block_type: Literal[NotionBlockType.HEADING_3] = NotionBlockType.HEADING_3 + + +class NotionParagraph(BaseModel): + """https://developers.notion.com/reference/block#paragraph""" + + block_type: Literal[NotionBlockType.PARAGRAPH] = NotionBlockType.PARAGRAPH + rich_text: list[NotionRichText] + color: NotionColor + children: list["NotionBlock"] = Field(default_factory=list) + + +class NotionBulletedListItem(BaseModel): + """https://developers.notion.com/reference/block#bulleted-list-item""" + + block_type: Literal[NotionBlockType.BULLETED_LIST_ITEM] = ( + NotionBlockType.BULLETED_LIST_ITEM + ) + rich_text: list[NotionRichText] + color: NotionColor + children: list["NotionBlock"] = Field(default_factory=list) + + +class NotionNumberedListItem(BaseModel): + """https://developers.notion.com/reference/block#numbered-list-item""" + + block_type: Literal[NotionBlockType.NUMBERED_LIST_ITEM] = ( + NotionBlockType.NUMBERED_LIST_ITEM + ) + rich_text: list[NotionRichText] + color: NotionColor + children: list["NotionBlock"] = Field(default_factory=list) + + +class NotionToDo(BaseModel): + """https://developers.notion.com/reference/block#to-do""" + + block_type: Literal[NotionBlockType.TO_DO] = NotionBlockType.TO_DO + rich_text: list[NotionRichText] + checked: bool + color: NotionColor + children: list["NotionBlock"] = Field(default_factory=list) + + +class NotionCode(BaseModel): + """https://developers.notion.com/reference/block#code""" + + block_type: Literal[NotionBlockType.CODE] = NotionBlockType.CODE + caption: list[NotionRichText] + rich_text: list[NotionRichText] + language: str # Actually an enum + + +class NotionCallout(BaseModel): + """https://developers.notion.com/reference/block#callout""" + + block_type: Literal[NotionBlockType.CALLOUT] = NotionBlockType.CALLOUT + rich_text: list[NotionRichText] + # icon: Any # could be an emoji or an image + color: NotionColor + + +class NotionDivider(BaseModel): + """https://developers.notion.com/reference/block#divider""" + + block_type: Literal[NotionBlockType.DIVIDER] = NotionBlockType.DIVIDER + + +class NotionEmbed(BaseModel): + """https://developers.notion.com/reference/block#embed""" + + block_type: Literal[NotionBlockType.EMBED] = NotionBlockType.EMBED + url: str + + +class NotionBlockFile(BaseModel): + # FIXME: this is actually another occurrence of type discriminating + """https://developers.notion.com/reference/block#file""" + + block_type: Literal[NotionBlockType.FILE] = NotionBlockType.FILE + # TODO: NotionFile + + +class NotionImage(BaseModel): + """https://developers.notion.com/reference/block#image""" + + block_type: Literal[NotionBlockType.IMAGE] = NotionBlockType.IMAGE + file: NotionFile + + @model_validator(mode="before") + @classmethod + def move_file_type_inward_and_rename(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + return {"block_type": "image", "file": data} + + +class NotionVideo(BaseModel): + """https://developers.notion.com/reference/block#video""" + + block_type: Literal[NotionBlockType.VIDEO] = NotionBlockType.VIDEO + # FIXME: this actually contains a file reference which will be defined for the above, but with the "video" attribute + + +class NotionLinkPreview(BaseModel): + """https://developers.notion.com/reference/block#link-preview""" + + block_type: Literal[NotionBlockType.LINK_PREVIEW] = NotionBlockType.LINK_PREVIEW + url: str + + +class NotionBookmark(BaseModel): + """https://developers.notion.com/reference/block#bookmark""" + + block_type: Literal[NotionBlockType.BOOKMARK] = NotionBlockType.BOOKMARK + url: str + caption: list[NotionRichText] = Field(default_factory=list) + + +class NotionTable(BaseModel): + """https://developers.notion.com/reference/block#table + + The children of this block are NotionTableRow blocks.""" + + block_type: Literal[NotionBlockType.TABLE] = NotionBlockType.TABLE + table_width: int + has_column_header: bool + has_row_header: bool + + +class NotionTableRow(BaseModel): + """https://developers.notion.com/reference/block#table-row""" + + block_type: Literal[NotionBlockType.TABLE_ROW] = NotionBlockType.TABLE_ROW + cells: list[list[NotionRichText]] # Each cell is a list of rich text objects + + +class NotionColumnList(BaseModel): + """https://developers.notion.com/reference/block#column-list-and-column""" + + block_type: Literal[NotionBlockType.COLUMN_LIST] = NotionBlockType.COLUMN_LIST + + +class NotionColumn(BaseModel): + """https://developers.notion.com/reference/block#column-list-and-column""" + + block_type: Literal[NotionBlockType.COLUMN] = NotionBlockType.COLUMN + + +class NotionChildPage(BaseModel): + """https://developers.notion.com/reference/block#child-page + + My guess is that the actual child page is a child of this block ? We don't have the id...""" + + block_type: Literal[NotionBlockType.CHILD_PAGE] = NotionBlockType.CHILD_PAGE + title: str + + +class NotionUnsupported(BaseModel): + block_type: str + raw: dict[str, Any] | None = None + + @model_validator(mode="before") + @classmethod + def put_all_in_raw(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + if "raw" not in data: + data["raw"] = data.copy() + + return data + + +NotionBlockSpecifics = Annotated[ + Annotated[ + NotionHeading1 + | NotionHeading2 + | NotionHeading3 + | NotionParagraph + | NotionNumberedListItem + | NotionBulletedListItem + | NotionToDo + | NotionCode + | NotionColumn + | NotionColumnList + | NotionDivider + | NotionEmbed + | NotionBlockFile + | NotionImage + | NotionVideo + | NotionLinkPreview + | NotionTable + | NotionTableRow + | NotionChildPage + | NotionCallout + | NotionLinkPreview + | NotionBookmark, + Discriminator(discriminator="block_type"), + ] + | NotionUnsupported, + Field(union_mode="left_to_right"), +] diff --git a/src/backend/core/notion_schemas/notion_color.py b/src/backend/core/notion_schemas/notion_color.py new file mode 100644 index 0000000000..881a6de941 --- /dev/null +++ b/src/backend/core/notion_schemas/notion_color.py @@ -0,0 +1,23 @@ +from enum import StrEnum + + +class NotionColor(StrEnum): + DEFAULT = "default" + BLUE = "blue" + BLUE_BACKGROUND = "blue_background" + BROWN = "brown" + BROWN_BACKGROUND = "brown_background" + GRAY = "gray" + GRAY_BACKGROUND = "gray_background" + GREEN = "green" + GREEN_BACKGROUND = "green_background" + ORANGE = "orange" + ORANGE_BACKGROUND = "orange_background" + YELLOW = "yellow" + YELLOW_BACKGROUND = "yellow_background" + PINK = "pink" + PINK_BACKGROUND = "pink_background" + PURPLE = "purple" + PURPLE_BACKGROUND = "purple_background" + RED = "red" + RED_BACKGROUND = "red_background" diff --git a/src/backend/core/notion_schemas/notion_file.py b/src/backend/core/notion_schemas/notion_file.py new file mode 100644 index 0000000000..7b1a11f397 --- /dev/null +++ b/src/backend/core/notion_schemas/notion_file.py @@ -0,0 +1,31 @@ +from enum import StrEnum +from typing import Annotated, Literal + +from pydantic import BaseModel, Discriminator + + +class NotionFileType(StrEnum): + HOSTED = "file" + UPLOAD = "file_upload" + EXTERNAL = "external" + + +class NotionFileHosted(BaseModel): + type: Literal[NotionFileType.HOSTED] = NotionFileType.HOSTED + file: dict # TODO + + +class NotionFileUpload(BaseModel): + type: Literal[NotionFileType.UPLOAD] = NotionFileType.UPLOAD + file_upload: dict # TODO + + +class NotionFileExternal(BaseModel): + type: Literal[NotionFileType.EXTERNAL] = NotionFileType.EXTERNAL + external: dict # TODO + + +NotionFile = Annotated[ + NotionFileHosted | NotionFileUpload | NotionFileExternal, + Discriminator(discriminator="type"), +] diff --git a/src/backend/core/notion_schemas/notion_page.py b/src/backend/core/notion_schemas/notion_page.py new file mode 100644 index 0000000000..4d98856c58 --- /dev/null +++ b/src/backend/core/notion_schemas/notion_page.py @@ -0,0 +1,61 @@ +from enum import StrEnum +from typing import Annotated, Literal + +from pydantic import BaseModel, Discriminator + + +class NotionParentType(StrEnum): + DATABASE = "database_id" + PAGE = "page_id" + WORKSPACE = "workspace" + BLOCK = "block_id" + + +class NotionParentDatabase(BaseModel): + type: Literal[NotionParentType.DATABASE] = NotionParentType.DATABASE + database_id: str + + +class NotionParentPage(BaseModel): + type: Literal[NotionParentType.PAGE] = NotionParentType.PAGE + page_id: str + + +class NotionParentWorkspace(BaseModel): + type: Literal[NotionParentType.WORKSPACE] = NotionParentType.WORKSPACE + + +class NotionParentBlock(BaseModel): + type: Literal[NotionParentType.BLOCK] = NotionParentType.BLOCK + block_id: str + + +NotionParent = Annotated[ + NotionParentDatabase | NotionParentPage | NotionParentWorkspace | NotionParentBlock, + Discriminator(discriminator="type"), +] + + +class NotionPage(BaseModel): + id: str + archived: bool + parent: NotionParent + + # created_time: datetime + # last_edited_time: datetime + # icon: NotionFile + # cover: NotionFile + + properties: dict # This is a very messy dict, with some RichText somewhere + + def get_title(self) -> str | None: + title_property: dict | None = self.properties.get("title") + if title_property is None: + return None + + # This could be parsed using NotionRichText + rich_text = title_property["title"][0] + return rich_text["plain_text"] + + def is_root(self): + return isinstance(self.parent, NotionParentWorkspace) diff --git a/src/backend/core/notion_schemas/notion_rich_text.py b/src/backend/core/notion_schemas/notion_rich_text.py new file mode 100644 index 0000000000..036a57d867 --- /dev/null +++ b/src/backend/core/notion_schemas/notion_rich_text.py @@ -0,0 +1,72 @@ +from enum import StrEnum +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, Discriminator, Field, ValidationError, model_validator + +from .notion_color import NotionColor + + +class NotionRichTextAnnotation(BaseModel): + """https://developers.notion.com/reference/rich-text#the-annotation-object""" + + bold: bool = False + italic: bool = False + strikethrough: bool = False + underline: bool = False + code: bool = False + color: NotionColor = NotionColor.DEFAULT + + +class NotionRichText(BaseModel): + """https://developers.notion.com/reference/rich-text, not a block""" + + annotations: NotionRichTextAnnotation + plain_text: str + href: str | None = None + specific: "NotionRichTextSpecifics" + + @model_validator(mode="before") + @classmethod + def move_type_inward_and_rename(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + if "type" not in data: + raise ValidationError("Type must be specified") + data_type = data.pop("type") + data["specific"] = data.pop(data_type) + data["specific"]["type"] = data_type + + return data + + +class NotionRichTextType(StrEnum): + TEXT = "text" + MENTION = "mention" + EQUATION = "equation" + + +class NotionLink(BaseModel): + url: str + + +class NotionRichTextText(BaseModel): + type: Literal[NotionRichTextType.TEXT] = NotionRichTextType.TEXT + content: str + link: NotionLink | None + + +class NotionRichTextMention(BaseModel): + type: Literal[NotionRichTextType.MENTION] = NotionRichTextType.MENTION + # Mention + + +class NotionRichTextEquation(BaseModel): + type: Literal[NotionRichTextType.EQUATION] = NotionRichTextType.EQUATION + expression: str # LaTeX expression + + +NotionRichTextSpecifics = Annotated[ + NotionRichTextText | NotionRichTextMention | NotionRichTextEquation, + Discriminator(discriminator="type"), +] diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index 5213bac86c..7fa603a118 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -76,3 +76,46 @@ def convert_markdown(self, text): ) from err return document_content + + def convert_blocks(self, blocks): + """Convert a list of blocks into our internal format using an external microservice.""" + + try: + response = requests.post( + f"{settings.Y_PROVIDER_API_BASE_URL}{settings.BLOCKS_CONVERSION_API_ENDPOINT}/", + json={ + "blocks": blocks, + }, + headers={ + "Authorization": self.auth_header, + "Content-Type": "application/json", + }, + timeout=settings.CONVERSION_API_TIMEOUT, + verify=settings.CONVERSION_API_SECURE, + ) + if not response.ok: + raise ValueError( + f"Conversion service returned an error: {response.status_code} - {response.text}" + ) + conversion_response = response.json() + + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to conversion service", + ) from err + + except ValueError as err: + raise InvalidResponseError( + "Could not parse conversion service response" + ) from err + + try: + document_content = conversion_response[ + settings.CONVERSION_API_CONTENT_FIELD + ] + except KeyError as err: + raise MissingContentError( + f"Response missing required field: {settings.CONVERSION_API_CONTENT_FIELD}" + ) from err + + return document_content diff --git a/src/backend/core/services/notion_import.py b/src/backend/core/services/notion_import.py new file mode 100644 index 0000000000..7af2c26f67 --- /dev/null +++ b/src/backend/core/services/notion_import.py @@ -0,0 +1,484 @@ +import json +import logging +from typing import Any + +from pydantic import BaseModel, Field, TypeAdapter +from requests import Session + +from ..notion_schemas.notion_block import ( + NotionBlock, + NotionBookmark, + NotionBulletedListItem, + NotionCallout, + NotionChildPage, + NotionCode, + NotionColumn, + NotionColumnList, + NotionDivider, + NotionHeading1, + NotionHeading2, + NotionHeading3, + NotionImage, + NotionNumberedListItem, + NotionParagraph, + NotionTable, + NotionTableRow, + NotionToDo, + NotionUnsupported, +) +from ..notion_schemas.notion_file import NotionFileExternal, NotionFileHosted +from ..notion_schemas.notion_page import ( + NotionPage, + NotionParentBlock, + NotionParentPage, + NotionParentWorkspace, +) +from ..notion_schemas.notion_rich_text import NotionRichText, NotionRichTextAnnotation + +logger = logging.getLogger(__name__) + + +def build_notion_session(token: str) -> Session: + session = Session() + session.headers = { + "Authorization": f"Bearer {token}", + "Notion-Version": "2022-06-28", + } + return session + + +def search_notion(session: Session, start_cursor: str) -> dict[str, Any]: + req_data = { + "filter": { + "value": "page", + "property": "object", + }, + } + if start_cursor: + req_data = { + "start_cursor": start_cursor, + "filter": { + "value": "page", + "property": "object", + }, + } + + response = session.post( + "https://api.notion.com/v1/search", + json=req_data, + ) + + if response.status_code != 200: + print(response.json()) + + response.raise_for_status() + return response.json() + + +def fetch_all_pages(session: Session) -> list[NotionPage]: + pages = [] + cursor = "" + has_more = True + + while has_more: + response = search_notion(session, start_cursor=cursor) + + for item in response["results"]: + if item["object"] != "page": + logger.warning(f"Skipping non-page object: {item['object']}") + continue + + pages.append(NotionPage.model_validate(item)) + + has_more = response.get("has_more", False) + cursor = response.get("next_cursor", "") + + return pages + + +def fetch_blocks(session: Session, block_id: str, start_cursor: str) -> dict[str, Any]: + response = session.get( + f"https://api.notion.com/v1/blocks/{block_id}/children", + params={ + "start_cursor": start_cursor if start_cursor else None, + }, + ) + + response.raise_for_status() + return response.json() + + +def fetch_block_children(session: Session, block_id: str) -> list[NotionBlock]: + blocks: list[NotionBlock] = [] + cursor = "" + has_more = True + + while has_more: + response = fetch_blocks(session, block_id, cursor) + + blocks.extend( + TypeAdapter(list[NotionBlock]).validate_python(response["results"]) + ) + + has_more = response.get("has_more", False) + cursor = response.get("next_cursor", "") + + for block in blocks: + if block.has_children: + block.children = fetch_block_children(session, block.id) + + return blocks + + +def convert_rich_texts(rich_texts: list[NotionRichText]) -> list[dict[str, Any]]: + content = [] + for rich_text in rich_texts: + if rich_text.href: + content.append( + { + "type": "link", + "content": [convert_rich_text(rich_text)], + "href": rich_text.href, # FIXME: if it was a notion link, we should convert it to a link to the document + } + ) + else: + content.append(convert_rich_text(rich_text)) + return content + + +def convert_rich_text(rich_text: NotionRichText) -> dict[str, Any]: + return { + "type": "text", + "text": rich_text.plain_text, + "styles": convert_annotations(rich_text.annotations), + } + + +class ImportedAttachment(BaseModel): + block: Any + file: NotionFileHosted + + +class ImportedChildPage(BaseModel): + child_page_block: NotionBlock + block_to_update: Any + + +def convert_image( + image: NotionImage, attachments: list[ImportedAttachment] +) -> list[dict[str, Any]]: + # TODO: NotionFileUpload + match image.file: + case NotionFileExternal(): + return [ + { + "type": "image", + "props": { + "url": image.file.external["url"], + }, + } + ] + case NotionFileHosted(): + block = { + "type": "image", + "props": { + "url": "about:blank", # populated later on + }, + } + attachments.append(ImportedAttachment(block=block, file=image.file)) + + return [block] + case _: + return [{"paragraph": {"content": "Unsupported image type"}}] + + +def convert_block( + block: NotionBlock, + attachments: list[ImportedAttachment], + child_page_blocks: list[ImportedChildPage], +) -> list[dict[str, Any]]: + match block.specific: + case NotionColumnList(): + columns_content = [] + for column in block.children: + columns_content.extend( + convert_block(column, attachments, child_page_blocks) + ) + return columns_content + case NotionColumn(): + return [ + convert_block(child_content, attachments, child_page_blocks)[0] + for child_content in block.children + ] + + case NotionParagraph(): + content = convert_rich_texts(block.specific.rich_text) + return [ + { + "type": "paragraph", + "content": content, + } + ] + case NotionImage(): + return convert_image(block.specific, attachments) + case NotionHeading1() | NotionHeading2() | NotionHeading3(): + return [ + { + "type": "heading", + "content": convert_rich_texts(block.specific.rich_text), + "props": { + "level": block.specific.block_type.value.split("_")[ + -1 + ], # e.g., "1", "2", or "3" + }, + } + ] + # case NotionDivider(): + # return [{"type": "divider"}] + case NotionCallout(): + return [ + { + "type": "quote", + "content": convert_rich_texts(block.specific.rich_text), + "props": { + "backgroundColor": "yellow", # TODO: use the callout color + }, + } + ] + case NotionTable(): + rows: list[NotionTableRow] = [child.specific for child in block.children] # type: ignore # I don't know how to assert properly + if len(rows) == 0: + return [ + { + "type": "paragraph", + "content": "Empty table ?!", + } + ] + + n_columns = len( + rows[0].cells + ) # I'll assume all rows have the same number of cells + if n_columns == 0: + return [{"type": "paragraph", "content": "Empty row ?!"}] + if not all(len(row.cells) == n_columns for row in rows): + return [ + { + "type": "paragraph", + "content": "Rows have different number of cells ?!", + } + ] + SEEMINGLY_DEFAULT_WIDTH = 128 + return [ + { + "type": "table", + "content": { + "type": "tableContent", + "columnWidths": [ + SEEMINGLY_DEFAULT_WIDTH for _ in range(n_columns) + ], + "headerRows": int(block.specific.has_column_header), + "headerColumns": int(block.specific.has_row_header), + "props": { + "textColor": "default", # TODO + }, + "rows": [ + { + "cells": [ + { + "type": "tableCell", + "content": convert_rich_texts(cell), + } + for cell in row.cells + ] + } + for row in rows + ], + }, + } + ] + case NotionBulletedListItem(): + return [ + { + "type": "bulletListItem", + "content": convert_rich_texts(block.specific.rich_text), + "children": convert_block_list( + block.children, + attachments, + child_page_blocks, + ), + } + ] + case NotionNumberedListItem(): + return [ + { + "type": "numberedListItem", + "content": convert_rich_texts(block.specific.rich_text), + "children": convert_block_list( + block.children, + attachments, + child_page_blocks, + ), + } + ] + case NotionToDo(): + return [ + { + "type": "checkListItem", + "content": convert_rich_texts(block.specific.rich_text), + "checked": block.specific.checked, + "children": convert_block_list( + block.children, + attachments, + child_page_blocks, + ), + } + ] + case NotionCode(): + return [ + { + "type": "codeBlock", + "content": "".join( + rich_text.plain_text for rich_text in block.specific.rich_text + ), + "props": {"language": block.specific.language}, + } + ] + case NotionBookmark(): + caption = convert_rich_texts(block.specific.caption) or block.specific.url + return [ + { + "type": "paragraph", + "content": [ + { + "type": "link", + "content": caption, + "href": block.specific.url, + }, + ], + } + ] + case NotionChildPage(): + # TODO: convert to a link + res = { + "type": "paragraph", + "content": [ + { + "type": "link", + "content": f"Child page: {block.specific.title}", + "href": "about:blank", # populated later on + }, + ], + } + child_page_blocks.append( + ImportedChildPage(child_page_block=block, block_to_update=res) + ) + return [res] + case NotionUnsupported(): + return [ + { + "type": "paragraph", + "content": f"This should be a {block.specific.block_type}, not yet supported in docs", + }, + # { + # "type": "quote", + # "content": json.dumps(block.specific.raw, indent=2), + # }, + ] + case _: + return [ + { + "type": "paragraph", + "content": f"This should be a {block.specific.block_type}, not yet handled by the importer", + } + ] + + +def convert_annotations(annotations: NotionRichTextAnnotation) -> dict[str, str]: + res = {} + if annotations.bold: + res["bold"] = "true" + if annotations.italic: + res["italic"] = "true" + if annotations.underline: + res["underline"] = "true" + if annotations.strikethrough: + res["strike"] = "true" + + if "_" in annotations.color: + res["backgroundColor"] = annotations.color.split("_")[0].lower() + else: + res["textColor"] = annotations.color.lower() + return res + + +def convert_block_list( + blocks: list[NotionBlock], + attachments: list[ImportedAttachment], + child_page_blocks: list[ImportedChildPage], +) -> list[dict[str, Any]]: + converted_blocks = [] + for block in blocks: + converted_blocks.extend(convert_block(block, attachments, child_page_blocks)) + return converted_blocks + + +class ImportedDocument(BaseModel): + page: NotionPage + blocks: list[dict[str, Any]] = Field(default_factory=list) + children: list["ImportedDocument"] = Field(default_factory=list) + attachments: list[ImportedAttachment] = Field(default_factory=list) + child_page_blocks: list[ImportedChildPage] = Field(default_factory=list) + + +def find_block_child_page(block_id: str, all_pages: list[NotionPage]): + for page in all_pages: + if ( + isinstance(page.parent, NotionParentBlock) + and page.parent.block_id == block_id + ): + return page + return None + + +def import_page( + session: Session, + page: NotionPage, + child_page_blocs_ids_to_parent_page_ids: dict[str, str], +) -> ImportedDocument: + blocks = fetch_block_children(session, page.id) + logger.info(f"Page {page.get_title()} (id {page.id})") + logger.info(blocks) + attachments: list[ImportedAttachment] = [] + + child_page_blocks: list[ImportedChildPage] = [] + + converted_blocks = convert_block_list(blocks, attachments, child_page_blocks) + + for child_page_block in child_page_blocks: + child_page_blocs_ids_to_parent_page_ids[ + child_page_block.child_page_block.id + ] = page.id + + return ImportedDocument( + page=page, + blocks=converted_blocks, + attachments=attachments, + child_page_blocks=child_page_blocks, + ) + + +def link_child_page_to_parent( + page: NotionPage, + docs_by_page_id: dict[str, ImportedDocument], + child_page_blocs_ids_to_parent_page_ids: dict[str, str], +): + if isinstance(page.parent, NotionParentPage): + docs_by_page_id[page.parent.page_id].children.append(docs_by_page_id[page.id]) + elif isinstance(page.parent, NotionParentBlock): + parent_page_id = child_page_blocs_ids_to_parent_page_ids.get(page.id) + if parent_page_id: + docs_by_page_id[parent_page_id].children.append(docs_by_page_id[page.id]) + else: + logger.warning( + f"Page {page.id} has a parent block, but no parent page found." + ) diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py index 0544189547..7c0f25943b 100644 --- a/src/backend/core/urls.py +++ b/src/backend/core/urls.py @@ -52,6 +52,11 @@ r"^templates/(?P[0-9a-z-]*)/", include(template_related_router.urls), ), + path("notion_import/", include([ + path("redirect", viewsets.notion_import_redirect), + path("callback", viewsets.notion_import_callback), + path("run", viewsets.NotionImportRunView.as_view()), + ])) ] ), ), diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 571d7052d8..2bb3b6f181 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -415,6 +415,9 @@ class Base(Configuration): ) # Frontend + FRONTEND_URL = values.Value( + None, environ_name="FRONTEND_URL", environ_prefix=None + ) FRONTEND_THEME = values.Value( None, environ_name="FRONTEND_THEME", environ_prefix=None ) @@ -628,6 +631,11 @@ class Base(Configuration): environ_name="CONVERSION_API_ENDPOINT", environ_prefix=None, ) + BLOCKS_CONVERSION_API_ENDPOINT = values.Value( + default="convert-blocks", + environ_name="BLOCKS_CONVERSION_API_ENDPOINT", + environ_prefix=None, + ) CONVERSION_API_CONTENT_FIELD = values.Value( default="content", environ_name="CONVERSION_API_CONTENT_FIELD", @@ -644,6 +652,22 @@ class Base(Configuration): environ_prefix=None, ) + NOTION_CLIENT_ID = values.Value( + default=None, + environ_name="NOTION_CLIENT_ID", + environ_prefix=None, + ) + NOTION_CLIENT_SECRET = values.Value( + default=None, + environ_name="NOTION_CLIENT_SECRET", + environ_prefix=None, + ) + NOTION_REDIRECT_URI = values.Value( + default=None, + environ_name="NOTION_REDIRECT_URI", + environ_prefix=None, + ) + # Logging # We want to make it easy to log to console but by default we log production # to Sentry and don't want to log to console. diff --git a/src/frontend/apps/impress/src/components/DropdownMenu.tsx b/src/frontend/apps/impress/src/components/DropdownMenu.tsx index 5513ccb78b..d2a7223162 100644 --- a/src/frontend/apps/impress/src/components/DropdownMenu.tsx +++ b/src/frontend/apps/impress/src/components/DropdownMenu.tsx @@ -13,6 +13,7 @@ export type DropdownMenuOption = { danger?: boolean; isSelected?: boolean; disabled?: boolean; + padding?: BoxProps['$padding']; show?: boolean; }; @@ -129,7 +130,9 @@ export const DropdownMenu = ({ $justify="space-between" $background={colorsTokens['greyscale-000']} $color={colorsTokens['primary-600']} - $padding={{ vertical: 'xs', horizontal: 'base' }} + $padding={ + option.padding ?? { vertical: 'xs', horizontal: 'base' } + } $width="100%" $gap={spacingsTokens['base']} $css={css` diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useImportNotion.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useImportNotion.tsx new file mode 100644 index 0000000000..e24479d0c5 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useImportNotion.tsx @@ -0,0 +1,74 @@ +import { useRouter } from 'next/navigation'; +import { useEffect, useState } from 'react'; + +import { baseApiUrl } from '@/api'; + +type ImportState = { + title: string; + status: 'pending' | 'fetched' | 'imported'; +}[]; + +const computeSuccessPercentage = (importState?: ImportState) => { + if (!importState) { + return 0; + } + if (!importState.length) { + return 100; + } + + let fetchedFiles = 0; + let importedFiles = 0; + + for (const file of importState) { + if (file.status === 'fetched') { + fetchedFiles += 1; + } else if (file.status === 'imported') { + fetchedFiles += 1; + importedFiles += 1; + } + } + + const filesNb = importState.length; + + return Math.round(((fetchedFiles + importedFiles) / (2 * filesNb)) * 100); +}; + +export function useImportNotion() { + const router = useRouter(); + + const [importState, setImportState] = useState(); + + useEffect(() => { + // send the request with an Event Source + const eventSource = new EventSource( + `${baseApiUrl('1.0')}notion_import/run`, + { + withCredentials: true, + }, + ); + + eventSource.onmessage = (event) => { + console.log('hello', event.data); + const files = JSON.parse(event.data as string) as ImportState; + + // si tous les fichiers sont chargés, rediriger vers la home page + if (files.some((file) => file.status === 'imported')) { + eventSource.close(); + router.push('/'); + } + + // mettre à jour le state d'import + setImportState(files); + }; + + return () => { + eventSource.close(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return { + importState, + percentageValue: computeSuccessPercentage(importState), + }; +} diff --git a/src/frontend/apps/impress/src/features/left-panel/components/LeftPanelHeader.tsx b/src/frontend/apps/impress/src/features/left-panel/components/LeftPanelHeader.tsx index 5733b0dff0..b13f690bcc 100644 --- a/src/frontend/apps/impress/src/features/left-panel/components/LeftPanelHeader.tsx +++ b/src/frontend/apps/impress/src/features/left-panel/components/LeftPanelHeader.tsx @@ -1,16 +1,16 @@ import { Button } from '@openfun/cunningham-react'; -import { useRouter } from 'next/router'; +import { t } from 'i18next'; +import { useRouter } from 'next/navigation'; import { PropsWithChildren, useCallback, useState } from 'react'; -import { Box, Icon, SeparatedSection } from '@/components'; -import { DocSearchModal, DocSearchTarget } from '@/docs/doc-search/'; +import { Box, DropdownMenu, Icon, SeparatedSection } from '@/components'; +import { useCreateDoc } from '@/docs/doc-management'; +import { DocSearchModal } from '@/docs/doc-search'; import { useAuth } from '@/features/auth'; import { useCmdK } from '@/hook/useCmdK'; import { useLeftPanelStore } from '../stores'; -import { LeftPanelHeaderButton } from './LeftPanelHeaderButton'; - export const LeftPanelHeader = ({ children }: PropsWithChildren) => { const router = useRouter(); const { authenticated } = useAuth(); @@ -35,11 +35,32 @@ export const LeftPanelHeader = ({ children }: PropsWithChildren) => { useCmdK(openSearchModal); const { togglePanel } = useLeftPanelStore(); + const { mutate: createDoc, isPending: isCreatingDoc } = useCreateDoc({ + onSuccess: (doc) => { + router.push(`/docs/${doc.id}`); + togglePanel(); + }, + }); + const goToHome = () => { - void router.push('/'); + router.push('/'); togglePanel(); }; + const createNewDoc = () => { + createDoc(); + }; + + const handleImportFilesystem = () => { + // TODO: Implement filesystem import + }; + + const handleImportNotion = () => { + const baseApiUrl = process.env.NEXT_PUBLIC_API_ORIGIN; + const notionAuthUrl = `${baseApiUrl}/api/v1.0/notion_import/redirect`; + window.location.href = notionAuthUrl; + }; + return ( <> @@ -71,8 +92,38 @@ export const LeftPanelHeader = ({ children }: PropsWithChildren) => { /> )} - - {authenticated && } + {authenticated && ( + + + + )} {children} diff --git a/src/frontend/apps/impress/src/features/service-worker/service-worker.ts b/src/frontend/apps/impress/src/features/service-worker/service-worker.ts index b4db83f6d2..42772df3db 100644 --- a/src/frontend/apps/impress/src/features/service-worker/service-worker.ts +++ b/src/frontend/apps/impress/src/features/service-worker/service-worker.ts @@ -112,6 +112,7 @@ const precacheResources = [ '/accessibility/', '/legal-notice/', '/personal-data-cookies/', + '/import-notion', FALLBACK.offline, FALLBACK.images, FALLBACK.docs, diff --git a/src/frontend/apps/impress/src/i18n/translations.json b/src/frontend/apps/impress/src/i18n/translations.json index fb906dea0d..9860fae69c 100644 --- a/src/frontend/apps/impress/src/i18n/translations.json +++ b/src/frontend/apps/impress/src/i18n/translations.json @@ -166,6 +166,7 @@ "No text selected": "Kein Text ausgewählt", "No versions": "Keine Versionen", "Nothing exceptional, no special privileges related to a .gouv.fr.": "Nichts Außergewöhnliches, keine besonderen Privilegien im Zusammenhang mit .gouv.fr.", + "Notion import in progress...": "Notion-Import in Arbeit...", "OK": "OK", "Offline ?!": "Offline?!", "Only invited people can access": "Nur eingeladene Personen haben Zugriff", @@ -182,6 +183,7 @@ "Pin document icon": "Pinne das Dokumentenlogo an", "Pinned documents": "Angepinnte Dokumente", "Please download it only if it comes from a trusted source.": "Bitte laden Sie es nur herunter, wenn es von einer vertrauenswürdigen Quelle stammt.", + "Please stay on this page and be patient": "Bitte bleiben Sie auf dieser Seite und haben Sie Geduld", "Private": "Privat", "Proconnect Login": "Proconnect-Anmeldung", "Public": "Öffentlich", @@ -399,6 +401,7 @@ "No text selected": "No hay texto seleccionado", "No versions": "No hay versiones", "Nothing exceptional, no special privileges related to a .gouv.fr.": "Nada excepcional, no hay privilegios especiales relacionados con un .gouv.fr.", + "Notion import in progress...": "Importación de Notion en curso...", "OK": "Ok", "Offline ?!": "¿¡Sin conexión!?", "Only invited people can access": "Solo las personas invitadas pueden acceder", @@ -415,6 +418,7 @@ "Pin document icon": "Icono para marcar el documento como favorito", "Pinned documents": "Documentos favoritos", "Please download it only if it comes from a trusted source.": "Por favor, descárguelo solo si viene de una fuente de confianza.", + "Please stay on this page and be patient": "Rimanete su questa pagina e siate pazienti", "Private": "Privado", "Proconnect Login": "Iniciar sesión ProConnect", "Public": "Público", @@ -624,6 +628,10 @@ "No text selected": "Aucun texte sélectionné", "No versions": "Aucune version", "Nothing exceptional, no special privileges related to a .gouv.fr.": "Rien d'exceptionnel, pas de privilèges spéciaux liés à un .gouv.fr.", + "Notion import in progress...": "Import Notion en cours...", + "Notion import fetched": "🔄 Page Notion récupérée", + "Notion import imported": "✅️ Importé", + "Notion import pending": "⏸️ En attente", "OK": "OK", "Offline ?!": "Hors-ligne ?!", "Only invited people can access": "Seules les personnes invitées peuvent accéder", @@ -640,6 +648,7 @@ "Pin document icon": "Icône épingler un document", "Pinned documents": "Documents épinglés", "Please download it only if it comes from a trusted source.": "Veuillez le télécharger uniquement s'il provient d'une source fiable.", + "Please stay on this page and be patient": "Merci de rester sur cette page et de patienter un peu", "Private": "Privé", "Proconnect Login": "Login Proconnect", "Public": "Public", @@ -828,6 +837,7 @@ "No text selected": "Non è stato selezionato nessun testo", "No versions": "Nessuna versione", "Nothing exceptional, no special privileges related to a .gouv.fr.": "Niente di eccezionale, nessun privilegio speciale legato a un .gouv.fr.", + "Notion import in progress...": "Importazione di nozioni in corso...", "OK": "OK", "Offline ?!": "Offline ?!", "Only invited people can access": "Solo le persone invitate possono accedere", @@ -844,6 +854,7 @@ "Pin document icon": "Icona \"fissa documento\"", "Pinned documents": "Documenti fissati", "Please download it only if it comes from a trusted source.": "Per favore scaricalo solo se proviene da una fonte attendibile", + "Please stay on this page and be patient": "Rimanete su questa pagina e siate pazienti", "Private": "Privato", "Public": "Pubblico", "Public document": "Documento pubblico", @@ -1033,6 +1044,7 @@ "No text selected": "Geen tekst geselecteerd", "No versions": "Geen versies", "Nothing exceptional, no special privileges related to a .gouv.fr.": "Niets uitzonderlijk, geen speciale privileges gerelateerd aan een .gouv.fr.", + "Notion import in progress...": "Notion import bezig...", "OK": "Ok", "Offline ?!": "Offline ?!", "Only invited people can access": "Alleen uitgenodigde gebruikers hebben toegang", @@ -1049,6 +1061,7 @@ "Pin document icon": "Document icoon vastzetten", "Pinned documents": "Vastgepinde documenten", "Please download it only if it comes from a trusted source.": "Alleen downloaden als het van een vertrouwde bron komt.", + "Please stay on this page and be patient": "Blijf op deze pagina en heb geduld", "Private": "Privé", "Proconnect Login": "Login", "Public": "Publiek", diff --git a/src/frontend/apps/impress/src/pages/import-notion/index.tsx b/src/frontend/apps/impress/src/pages/import-notion/index.tsx new file mode 100644 index 0000000000..1685f1a8cf --- /dev/null +++ b/src/frontend/apps/impress/src/pages/import-notion/index.tsx @@ -0,0 +1,47 @@ +import { Loader } from '@openfun/cunningham-react'; +import { ReactElement } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { Box, Text } from '@/components'; +import { useImportNotion } from '@/features/docs/doc-management/api/useImportNotion'; +import { MainLayout } from '@/layouts'; +import { NextPageWithLayout } from '@/types/next'; + +const Page: NextPageWithLayout = () => { + const { t } = useTranslation(); + + const { importState, percentageValue } = useImportNotion(); + + return ( + + + {t('Notion import in progress...')} + + + {t('Please stay on this page and be patient')} + + + + {percentageValue}% + + + {importState?.map((page) => ( + {`${page.title} - ${t(`Notion import ${page.status}`)}`} + ))} + + + ); +}; + +Page.getLayout = function getLayout(page: ReactElement) { + return {page}; +}; + +export default Page; diff --git a/src/frontend/servers/y-provider/src/handlers/convertBlocksHandler.ts b/src/frontend/servers/y-provider/src/handlers/convertBlocksHandler.ts new file mode 100644 index 0000000000..05665c3b60 --- /dev/null +++ b/src/frontend/servers/y-provider/src/handlers/convertBlocksHandler.ts @@ -0,0 +1,47 @@ +//import { PartialBlock } from '@blocknote/core'; +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import { Request, Response } from 'express'; +import * as Y from 'yjs'; + +import { logger, toBase64 } from '@/utils'; + +interface ConversionRequest { + blocks: any; // TODO: PartialBlock +} + +interface ConversionResponse { + content: string; +} + +interface ErrorResponse { + error: string; +} + +export const convertBlocksHandler = async ( + req: Request< + object, + ConversionResponse | ErrorResponse, + ConversionRequest, + object + >, + res: Response, +) => { + const blocks = req.body?.blocks; + if (!blocks) { + res.status(400).json({ error: 'Invalid request: missing content' }); + return; + } + + try { + const editor = ServerBlockNoteEditor.create(); + + // Create a Yjs Document from blocks, and encode it as a base64 string + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const content = toBase64(Y.encodeStateAsUpdate(yDocument)); + + res.status(200).json({ content }); + } catch (e) { + logger('conversion failed:', e); + res.status(500).json({ error: String(e) }); + } +}; diff --git a/src/frontend/servers/y-provider/src/handlers/index.ts b/src/frontend/servers/y-provider/src/handlers/index.ts index 75bd7f7bbf..167493a306 100644 --- a/src/frontend/servers/y-provider/src/handlers/index.ts +++ b/src/frontend/servers/y-provider/src/handlers/index.ts @@ -1,3 +1,4 @@ export * from './collaborationResetConnectionsHandler'; export * from './collaborationWSHandler'; export * from './convertMarkdownHandler'; +export * from './convertBlocksHandler'; diff --git a/src/frontend/servers/y-provider/src/routes.ts b/src/frontend/servers/y-provider/src/routes.ts index 98803b87f6..7b8d289bb8 100644 --- a/src/frontend/servers/y-provider/src/routes.ts +++ b/src/frontend/servers/y-provider/src/routes.ts @@ -2,4 +2,5 @@ export const routes = { COLLABORATION_WS: '/collaboration/ws/', COLLABORATION_RESET_CONNECTIONS: '/collaboration/api/reset-connections/', CONVERT_MARKDOWN: '/api/convert-markdown/', + CONVERT_BLOCKS: '/api/convert-blocks/', }; diff --git a/src/frontend/servers/y-provider/src/servers/appServer.ts b/src/frontend/servers/y-provider/src/servers/appServer.ts index 5c035db799..2f99db5b1e 100644 --- a/src/frontend/servers/y-provider/src/servers/appServer.ts +++ b/src/frontend/servers/y-provider/src/servers/appServer.ts @@ -9,6 +9,7 @@ import { collaborationResetConnectionsHandler, collaborationWSHandler, convertMarkdownHandler, + convertBlocksHandler, } from '../handlers'; import { corsMiddleware, httpSecurity, wsSecurity } from '../middlewares'; import { routes } from '../routes'; @@ -51,6 +52,8 @@ export const initServer = () => { */ app.post(routes.CONVERT_MARKDOWN, httpSecurity, convertMarkdownHandler); + app.post(routes.CONVERT_BLOCKS, httpSecurity, convertBlocksHandler); + Sentry.setupExpressErrorHandler(app); app.get('/ping', (req, res) => {