diff --git a/package.json b/package.json index 8ecf9a8..7555320 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "async": "^3.2.5", "axios": "^1.6.5", "cheerio": "^1.0.0-rc.12", + "confluence.js": "^1.7.2", "dotenv": "^16.4.1", "glob": "^10.3.10", "googleapis": "^131.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 552fa8b..b4b3ea2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ dependencies: cheerio: specifier: ^1.0.0-rc.12 version: 1.0.0-rc.12 + confluence.js: + specifier: ^1.7.2 + version: 1.7.2 dotenv: specifier: ^16.4.1 version: 16.4.1 @@ -2729,6 +2732,14 @@ packages: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} dev: false + /atlassian-jwt@2.0.3: + resolution: {integrity: sha512-G9oO3HHS1UKgsLRXj6nNKv2TY6g3PleBCdzHwbFeVKg+18GBFIMRz+ApxuOuWAgcL7RngNFF5rGNtw1Ss3hvTg==} + engines: {node: '>= 0.4.0'} + dependencies: + jsuri: 1.3.1 + lodash: 4.17.21 + dev: false + /axios-retry@3.9.1: resolution: {integrity: sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==} dependencies: @@ -3133,6 +3144,18 @@ packages: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} dev: true + /confluence.js@1.7.2: + resolution: {integrity: sha512-hCHC4tZNikLonSJdKjcc7CQa5XuWFVyzn5iCKYAtDs7lXWmQyzRoc7mTnKGgMDYNOOCuK+vVIwf9nEcecWXOtw==} + dependencies: + atlassian-jwt: 2.0.3 + axios: 1.6.5 + form-data: 4.0.0 + oauth: 0.10.0 + tslib: 2.6.2 + transitivePeerDependencies: + - debug + dev: false + /convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} dev: true @@ -4735,6 +4758,10 @@ packages: semver: 7.5.4 dev: false + /jsuri@1.3.1: + resolution: {integrity: sha512-LLdAeqOf88/X0hylAI7oSir6QUsz/8kOW0FcJzzu/SJRfORA/oPHycAOthkNp7eLPlTAbqVDFbqNRHkRVzEA3g==} + dev: false + /jwa@1.4.1: resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} dependencies: @@ -4860,6 +4887,10 @@ packages: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} dev: false + /lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /lru-cache@10.2.0: resolution: {integrity: sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==} engines: {node: 14 || >=16.14} @@ -5021,6 +5052,10 @@ packages: boolbase: 1.0.0 dev: false + /oauth@0.10.0: + resolution: {integrity: sha512-1orQ9MT1vHFGQxhuy7E/0gECD3fd2fCC+PIX+/jgmU/gI3EpRocXtmtvxCO5x3WZ443FLTLFWNDjl5MPJf9u+Q==} + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} diff --git a/src/__tests__/providers/Confluence/index.test.ts b/src/__tests__/providers/Confluence/index.test.ts new file mode 100644 index 0000000..6b16276 --- /dev/null +++ b/src/__tests__/providers/Confluence/index.test.ts @@ -0,0 +1,34 @@ +import { createDataConnector } from "../../../DataConnector"; +import dotenv from "dotenv"; +dotenv.config(); + +test( + "Confluence Provider Testing", + async () => { + const confluenceDataConnector = createDataConnector({ + provider: "confluence", + }); + + if (!process.env.NANGO_CONNECTION_ID_TEST) { + throw new Error( + "Please specify the NANGO_CONNECTION_ID_TEST environment variable." + ); + } + + await confluenceDataConnector.authorizeNango({ + nango_connection_id: process.env.NANGO_CONNECTION_ID_TEST, + }); + + const pages = await confluenceDataConnector.getDocuments(); + expect(pages.length).toBeGreaterThan(0); + pages.forEach((issue) => { + expect(issue.provider).toBe("confluence"); + expect(issue.type).toBe("page"); + expect(issue.content).not.toBe(null); + expect(issue.createdAt).not.toBe(undefined); + expect(issue.updatedAt).not.toBe(undefined); + expect(issue.metadata.sourceURL).not.toBe(null); + }); + }, + 10 * 1000 +); // 10 seconds diff --git a/src/providers/Confluence/index.ts b/src/providers/Confluence/index.ts index f254df2..d93af45 100644 --- a/src/providers/Confluence/index.ts +++ b/src/providers/Confluence/index.ts @@ -1,118 +1,151 @@ +import { Nango } from "@nangohq/node"; import { DataProvider } from "../DataProvider"; import { Document } from "../../entities/Document"; -import { Nango } from "@nangohq/node"; +import { NangoAuthorizationOptions } from "../GoogleDrive"; +import { ConfluenceClient, Config } from "confluence.js"; +import { Content } from "confluence.js/out/api/models"; +import axios from "axios"; -export interface ConfluenceInputOptions { - text: string; -} -export interface ConfluenceAuthorizeOptions { - access_token: string; - // client_id: string; - // client_secret: string; - // authorization_url: string; -} +export type ConfluenceInputOptions = object; -export interface NangoConfluenceAuthorizationOptions { - nango_connection_id: string; - nango_integration_id?: string; -} +export type ConfluenceAuthorizationOptions = { + /** + * Your Confluence host. Example: "https://your-domain.atlassian.net" + */ + host?: string; + + /** + * Your Confluence authentication method. [Read more here.](https://github.com/mrrefactoring/confluence.js/?tab=readme-ov-file#authentication) + */ + auth?: Config.Authentication; +}; -// create an interface thant join the two interfaces above export interface ConfluenceOptions extends ConfluenceInputOptions, - ConfluenceAuthorizeOptions, - NangoConfluenceAuthorizationOptions {} + ConfluenceAuthorizationOptions, + NangoAuthorizationOptions {} + +/** + * Retrieves all pages from Confluence. + */ +async function getAllPages( + confluence: ConfluenceClient, + start?: number +): Promise { + const content = await confluence.content.getContent({ + start, + expand: ["body.storage", "history", "history.lastUpdated", "ancestors"], + type: "page", + }); + if (content.size === content.limit) { + return (content.results ?? []).concat( + await getAllPages(confluence, content.start + content.size) + ); + } else { + return content.results ?? []; + } +} + +/** + * The Confluence Data Provider retrieves all pages from a Confluence workspace. + */ export class ConfluenceDataProvider implements DataProvider { - private client_id: string = ""; - private client_secret: string = ""; - // this is the guy we need to get from the user - private authorization_url: string = ""; - // - private scopes: string[] = [ - "read:confluence-space.summary", - "read:confluence-props", - "read:confluence-content.all", - "read:confluence-content.summary", - "read:confluence-content.permission", - "readonly:content.attachment:confluence", - "read:content:confluence", - "read:content-details:confluence", - "read:page:confluence", - "read:attachment:confluence", - "read:blogpost:confluence", - "read:custom-content:confluence", - "read:content.metadata:confluence", - ]; - private token_url: string = "https://auth.atlassian.com/oauth/token"; // ?? - private access_token: string = ""; - // Don't need this? - private redirect_uri: string = "https://X"; - - private using_nango: boolean = false; - private nango_integration_id: string = "confluence"; - private nango_connection_id: string = ""; - private nango: Nango; - - constructor() { - if (!process.env.NANGO_SECRET_KEY) { - throw new Error("Nango secret key is required"); + private confluence: ConfluenceClient = undefined; + + private cloudUrl: string = ""; + + /** + * Authorizes the Confluence Data Provider. + */ + async authorize(options: ConfluenceAuthorizationOptions): Promise { + if (options.host === undefined || options.host === null) { + throw new Error("options.host is required."); + } + + if (options.auth === undefined || options.auth === null) { + throw new Error("options.auth is required."); } - this.nango = new Nango({ secretKey: process.env.NANGO_SECRET_KEY }); - } - async authorize({ access_token }: { access_token: string }): Promise { - throw new Error("Method not implemented."); + this.confluence = new ConfluenceClient({ + host: options.host, + authentication: options.auth, + }); } - async authorizeNango( - authorizeOptions: NangoConfluenceAuthorizationOptions - ): Promise { - const connection = await this.nango.getConnection( - authorizeOptions.nango_integration_id || this.nango_integration_id, - authorizeOptions.nango_connection_id + /** + * Authorizes the Confluence Data Provider via Nango. + */ + async authorizeNango(options: NangoAuthorizationOptions): Promise { + if (!process.env.NANGO_SECRET_KEY) { + throw new Error( + "Nango secret key is required. Please specify it in the NANGO_SECRET_KEY environment variable." + ); + } + const nango = new Nango({ secretKey: process.env.NANGO_SECRET_KEY }); + + const connection = await nango.getConnection( + options.nango_integration_id ?? "confluence", + options.nango_connection_id ); - this.nango_connection_id = authorizeOptions.nango_connection_id; - this.access_token = connection.credentials.raw.access_token; - this.using_nango = true; + const access = await axios.get( + "https://api.atlassian.com/oauth/token/accessible-resources", + { + headers: { + Accept: "application/json", + Authorization: `Bearer ${connection.credentials.raw.access_token}`, + }, + } + ); - this.authorize({ access_token: this.access_token }); + const cloudId = access.data[0].id; + this.cloudUrl = access.data[0].url - return; + await this.authorize({ + host: `https://api.atlassian.com/ex/confluence/${cloudId}`, + auth: { + oauth2: { + accessToken: connection.credentials.raw.access_token, + }, + }, + }); } - async getDocuments(): Promise { - throw new Error("Method not implemented."); - - // if (this.using_nango) { - // return new Promise((resolve, reject) => { - // const intervalId = setInterval(async () => { - // const rcs = await this.nango.listRecords({ - // providerConfigKey: this.nango_integration_id, - // connectionId: this.nango_connection_id, - // model: "Document", - // }); - // const records = rcs.records as NangoDocument[]; - // if (records.length > 0) { - // clearInterval(intervalId); - // const documents = records.map((record) => { - // return record.transformToDocument("confluence"); - // }); - // resolve(documents); - // } - // }, 2000); - // }); - // } - // return Promise.resolve([]); - } + /** + * Retrieves all pages from the authorized Confluence workspace. + * The pages' content will be HTML. + */ + async getDocuments(): Promise { + if (this.confluence === undefined) { + throw Error( + "You must authorize the ConfluenceDataProvider before requesting documents." + ); + } - nangoPoolingDocs(): Promise { - // await for documents to be ready - return new Promise((resolve, reject) => []); - } + const pages = await getAllPages(this.confluence); - setOptions(): void { - throw new Error("Method not implemented."); + return await Promise.all( + pages.map(async (page) => { + const ancestor = (page.ancestors ?? [])[0]; + return { + provider: "confluence", + id: `${page.id}`, + content: `

${page.title}

\n${page.body.storage.value}`, + createdAt: new Date((page as any).history.createdDate), + updatedAt: new Date((page as any).history.lastUpdated.when), + metadata: { + sourceURL: this.cloudUrl + "/wiki" + page._links.webui, + ancestor: ancestor?.title, + }, + type: "page", + }; + }) + ); } + + /** + * Do not call. The Confluence Data Provider doesn't have any options. + */ + setOptions(_options: ConfluenceOptions): void {} } diff --git a/src/providers/providers.ts b/src/providers/providers.ts index ae29d1d..8a25f5a 100644 --- a/src/providers/providers.ts +++ b/src/providers/providers.ts @@ -1,8 +1,7 @@ import { - ConfluenceAuthorizeOptions, + ConfluenceAuthorizationOptions, ConfluenceDataProvider, ConfluenceInputOptions, - NangoConfluenceAuthorizationOptions, } from "./Confluence"; import { DataProvider } from "./DataProvider"; import { FileDataProvider, FileInputOptions } from "./File"; @@ -78,8 +77,8 @@ type ProviderConfig = { confluence: { DataProvider: ConfluenceDataProvider; Options: ConfluenceInputOptions; - AuthorizeOptions: ConfluenceAuthorizeOptions; - NangoAuthorizeOptions: NangoConfluenceAuthorizationOptions; + AuthorizeOptions: ConfluenceAuthorizationOptions; + NangoAuthorizeOptions: NangoAuthorizationOptions; }; github: { DataProvider: GitHubDataProvider;