feat(confluence): add Confluence Data Provider

package.json

@@ -63,6 +63,7 @@
         "async": "^3.2.5",
         "axios": "^1.6.5",
         "cheerio": "^1.0.0-rc.12",
+        "confluence.js": "^1.7.2",
         "dotenv": "^16.4.1",
         "glob": "^10.3.10",
         "googleapis": "^131.0.0",

src/__tests__/providers/Confluence/index.test.ts

@@ -0,0 +1,34 @@
+import { createDataConnector } from "../../../DataConnector";
+import dotenv from "dotenv";
+dotenv.config();
+
+test(
+  "Confluence Provider Testing",
+  async () => {
+    const confluenceDataConnector = createDataConnector({
+      provider: "confluence",
+    });
+
+    if (!process.env.NANGO_CONNECTION_ID_TEST) {
+      throw new Error(
+        "Please specify the NANGO_CONNECTION_ID_TEST environment variable."
+      );
+    }
+
+    await confluenceDataConnector.authorizeNango({
+      nango_connection_id: process.env.NANGO_CONNECTION_ID_TEST,
+    });
+
+    const pages = await confluenceDataConnector.getDocuments();
+    expect(pages.length).toBeGreaterThan(0);
+    pages.forEach((issue) => {
+      expect(issue.provider).toBe("confluence");
+      expect(issue.type).toBe("page");
+      expect(issue.content).not.toBe(null);
+      expect(issue.createdAt).not.toBe(undefined);
+      expect(issue.updatedAt).not.toBe(undefined);
+      expect(issue.metadata.sourceURL).not.toBe(null);
+    });
+  },
+  10 * 1000
+); // 10 seconds

src/providers/Confluence/index.ts

@@ -1,118 +1,151 @@
+import { Nango } from "@nangohq/node";
 import { DataProvider } from "../DataProvider";
 import { Document } from "../../entities/Document";
-import { Nango } from "@nangohq/node";
+import { NangoAuthorizationOptions } from "../GoogleDrive";
+import { ConfluenceClient, Config } from "confluence.js";
+import { Content } from "confluence.js/out/api/models";
+import axios from "axios";
 
-export interface ConfluenceInputOptions {
-  text: string;
-}
-export interface ConfluenceAuthorizeOptions {
-  access_token: string;
-  // client_id: string;
-  // client_secret: string;
-  // authorization_url: string;
-}
+export type ConfluenceInputOptions = object;
 
-export interface NangoConfluenceAuthorizationOptions {
-  nango_connection_id: string;
-  nango_integration_id?: string;
-}
+export type ConfluenceAuthorizationOptions = {
+  /**
+   * Your Confluence host. Example: "https://your-domain.atlassian.net"
+   */
+  host?: string;
+
+  /**
+   * Your Confluence authentication method. [Read more here.](https://github.com/mrrefactoring/confluence.js/?tab=readme-ov-file#authentication)
+   */
+  auth?: Config.Authentication;
+};
 
-// create an interface thant join the two interfaces above
 export interface ConfluenceOptions
   extends ConfluenceInputOptions,
-    ConfluenceAuthorizeOptions,
-    NangoConfluenceAuthorizationOptions {}
+    ConfluenceAuthorizationOptions,
+    NangoAuthorizationOptions {}
+
+/**
+ * Retrieves all pages from Confluence.
+ */
+async function getAllPages(
+  confluence: ConfluenceClient,
+  start?: number
+): Promise<Content[]> {
+  const content = await confluence.content.getContent({
+    start,
+    expand: ["body.storage", "history", "history.lastUpdated", "ancestors"],
+    type: "page",
+  });
 
+  if (content.size === content.limit) {
+    return (content.results ?? []).concat(
+      await getAllPages(confluence, content.start + content.size)
+    );
+  } else {
+    return content.results ?? [];
+  }
+}
+
+/**
+ * The Confluence Data Provider retrieves all pages from a Confluence workspace.
+ */
 export class ConfluenceDataProvider implements DataProvider<ConfluenceOptions> {
-  private client_id: string = "";
-  private client_secret: string = "";
-  // this is the guy we need to get from the user
-  private authorization_url: string = "";
-  //
-  private scopes: string[] = [
-    "read:confluence-space.summary",
-    "read:confluence-props",
-    "read:confluence-content.all",
-    "read:confluence-content.summary",
-    "read:confluence-content.permission",
-    "readonly:content.attachment:confluence",
-    "read:content:confluence",
-    "read:content-details:confluence",
-    "read:page:confluence",
-    "read:attachment:confluence",
-    "read:blogpost:confluence",
-    "read:custom-content:confluence",
-    "read:content.metadata:confluence",
-  ];
-  private token_url: string = "https://auth.atlassian.com/oauth/token"; // ??
-  private access_token: string = "";
-  // Don't need this?
-  private redirect_uri: string = "https://X";
-
-  private using_nango: boolean = false;
-  private nango_integration_id: string = "confluence";
-  private nango_connection_id: string = "";
-  private nango: Nango;
-
-  constructor() {
-    if (!process.env.NANGO_SECRET_KEY) {
-      throw new Error("Nango secret key is required");
+  private confluence: ConfluenceClient = undefined;
+
+  private cloudUrl: string = "";
+
+  /**
+   * Authorizes the Confluence Data Provider.
+   */
+  async authorize(options: ConfluenceAuthorizationOptions): Promise<void> {
+    if (options.host === undefined || options.host === null) {
+      throw new Error("options.host is required.");
+    }
+
+    if (options.auth === undefined || options.auth === null) {
+      throw new Error("options.auth is required.");
     }
-    this.nango = new Nango({ secretKey: process.env.NANGO_SECRET_KEY });
-  }
 
-  async authorize({ access_token }: { access_token: string }): Promise<void> {
-    throw new Error("Method not implemented.");
+    this.confluence = new ConfluenceClient({
+      host: options.host,
+      authentication: options.auth,
+    });
   }
 
-  async authorizeNango(
-    authorizeOptions: NangoConfluenceAuthorizationOptions
-  ): Promise<void> {
-    const connection = await this.nango.getConnection(
-      authorizeOptions.nango_integration_id || this.nango_integration_id,
-      authorizeOptions.nango_connection_id
+  /**
+   * Authorizes the Confluence Data Provider via Nango.
+   */
+  async authorizeNango(options: NangoAuthorizationOptions): Promise<void> {
+    if (!process.env.NANGO_SECRET_KEY) {
+      throw new Error(
+        "Nango secret key is required. Please specify it in the NANGO_SECRET_KEY environment variable."
+      );
+    }
+    const nango = new Nango({ secretKey: process.env.NANGO_SECRET_KEY });
+
+    const connection = await nango.getConnection(
+      options.nango_integration_id ?? "confluence",
+      options.nango_connection_id
     );
 
-    this.nango_connection_id = authorizeOptions.nango_connection_id;
-    this.access_token = connection.credentials.raw.access_token;
-    this.using_nango = true;
+    const access = await axios.get(
+      "https://api.atlassian.com/oauth/token/accessible-resources",
+      {
+        headers: {
+          Accept: "application/json",
+          Authorization: `Bearer ${connection.credentials.raw.access_token}`,
+        },
+      }
+    );
 
-    this.authorize({ access_token: this.access_token });
+    const cloudId = access.data[0].id;
+    this.cloudUrl = access.data[0].url
 
-    return;
+    await this.authorize({
+      host: `https://api.atlassian.com/ex/confluence/${cloudId}`,
+      auth: {
+        oauth2: {
+          accessToken: connection.credentials.raw.access_token,
+        },
+      },
+    });
   }
 
-  async getDocuments(): Promise<Document[] | []> {
-    throw new Error("Method not implemented.");
-
-    // if (this.using_nango) {
-    //   return new Promise<Document[] | []>((resolve, reject) => {
-    //     const intervalId = setInterval(async () => {
-    //       const rcs = await this.nango.listRecords({
-    //         providerConfigKey: this.nango_integration_id,
-    //         connectionId: this.nango_connection_id,
-    //         model: "Document",
-    //       });
-    //       const records = rcs.records as NangoDocument[];
-    //       if (records.length > 0) {
-    //         clearInterval(intervalId);
-    //         const documents = records.map((record) => {
-    //           return record.transformToDocument("confluence");
-    //         });
-    //         resolve(documents);
-    //       }
-    //     }, 2000);
-    //   });
-    // }
-    // return Promise.resolve([]);
-  }
+  /**
+   * Retrieves all pages from the authorized Confluence workspace.
+   * The pages' content will be HTML.
+   */
+  async getDocuments(): Promise<Document[]> {
+    if (this.confluence === undefined) {
+      throw Error(
+        "You must authorize the ConfluenceDataProvider before requesting documents."
+      );
+    }
 
-  nangoPoolingDocs(): Promise<Document[] | []> {
-    // await for documents to be ready
-    return new Promise((resolve, reject) => []);
-  }
+    const pages = await getAllPages(this.confluence);
 
-  setOptions(): void {
-    throw new Error("Method not implemented.");
+    return await Promise.all(
+      pages.map(async (page) => {
+        const ancestor = (page.ancestors ?? [])[0];
+        return {
+          provider: "confluence",
+          id: `${page.id}`,
+          content: `<h1>${page.title}</h1>\n${page.body.storage.value}`,
+          createdAt: new Date((page as any).history.createdDate),
+          updatedAt: new Date((page as any).history.lastUpdated.when),
+          metadata: {
+            sourceURL: this.cloudUrl + "/wiki" + page._links.webui,
+            ancestor: ancestor?.title,
+          },
+          type: "page",
+        };
+      })
+    );
   }
+
+  /**
+   * Do not call. The Confluence Data Provider doesn't have any options.
+   */
+  setOptions(_options: ConfluenceOptions): void {}
 }

src/providers/providers.ts

@@ -1,8 +1,7 @@
 import {
-  ConfluenceAuthorizeOptions,
+  ConfluenceAuthorizationOptions,
   ConfluenceDataProvider,
   ConfluenceInputOptions,
-  NangoConfluenceAuthorizationOptions,
 } from "./Confluence";
 import { DataProvider } from "./DataProvider";
 import { FileDataProvider, FileInputOptions } from "./File";
@@ -78,8 +77,8 @@ type ProviderConfig = {
   confluence: {
     DataProvider: ConfluenceDataProvider;
     Options: ConfluenceInputOptions;
-    AuthorizeOptions: ConfluenceAuthorizeOptions;
-    NangoAuthorizeOptions: NangoConfluenceAuthorizationOptions;
+    AuthorizeOptions: ConfluenceAuthorizationOptions;
+    NangoAuthorizeOptions: NangoAuthorizationOptions;
   };
   github: {
     DataProvider: GitHubDataProvider;