Merge pull request #35 from MerginMaps/driver-google-drive

Driver google drive

.github/workflows/tests_mergin_media_sync.yaml

@@ -8,19 +8,21 @@ on:
     - ".github/workflows/tests_mergin_media_sync.yaml"
 
 env:
-  TEST_MERGIN_URL: https://test.dev.merginmaps.com
+  TEST_MERGIN_URL: https://app.dev.merginmaps.com
   TEST_API_USERNAME: test_media_sync
   TEST_API_PASSWORD: ${{ secrets.TEST_API_PASSWORD }}
   TEST_API_WORKSPACE: test-media-sync
   TEST_MINIO_URL: 127.0.0.1:9000
   TEST_MINIO_ACCESS_KEY: minioaccesskey
   TEST_MINIO_SECRET_KEY: miniosecretkey
+  TEST_GOOGLE_DRIVE_FOLDER: SAVE_FOLDER
+  TEST_GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE: google-credentials.json
 
 jobs:
 
   Tests-for-Mergin-Maps-Media-Sync:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
 
@@ -29,6 +31,9 @@ jobs:
 
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Save Google Credentials File
+        run: echo "${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE }}" | base64 -d > $TEST_GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE
 
       - name: Install Python dependencies
         run: |

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-buster
+FROM python:3.11-slim-buster
 MAINTAINER Martin Dobias "martin.dobias@lutraconsulting.co.uk"
 
 # to fix issue with mod_spatialite.so

Pipfile

@@ -9,8 +9,9 @@ pytest-cov = "~=3.0"
 
 [packages]
 minio = "~=7.1"
-mergin-client = "==0.9.0"
+mergin-client = "==0.9.3"
 dynaconf = {extras = ["ini"],version = "~=3.1"}
+google-api-python-client = "==2.24"
 
 [requires]
 python_version = "3"

README.md

@@ -102,7 +102,7 @@ one with reference to local file and another for external URL where file can be
 
 Initialization:
 
-1. set up configuration in config.ini  (see config.ini.default for a sample)
+1. set up configuration in config.yaml  (see config.yaml.default for a sample)
 2. all settings can be overridden with env variables (see docker example above)
 3. run media-sync
 ```shell

config.py

@@ -9,6 +9,7 @@
 import pathlib
 
 from dynaconf import Dynaconf
+from drivers import DriverType
 
 config = Dynaconf(
     envvar_prefix=False,
@@ -28,7 +29,11 @@ def validate_config(config):
     ):
         raise ConfigError("Config error: Incorrect mergin settings")
 
-    if config.driver not in ["local", "minio"]:
+    if not (
+        config.driver == DriverType.LOCAL
+        or config.driver == DriverType.MINIO
+        or config.driver == DriverType.GOOGLE_DRIVE
+    ):
         raise ConfigError("Config error: Unsupported driver")
 
     if config.operation_mode not in ["move", "copy"]:
@@ -37,7 +42,7 @@ def validate_config(config):
     if config.driver == "local" and not config.local.dest:
         raise ConfigError("Config error: Incorrect Local driver settings")
 
-    if config.driver == "minio" and not (
+    if config.driver == DriverType.MINIO and not (
         config.minio.endpoint
         and config.minio.access_key
         and config.minio.secret_key
@@ -66,6 +71,13 @@ def validate_config(config):
         ):
             raise ConfigError("Config error: Incorrect media reference settings")
 
+    if config.driver == DriverType.GOOGLE_DRIVE and not (
+        hasattr(config.google_drive, "service_account_file")
+        and hasattr(config.google_drive, "folder")
+        and hasattr(config.google_drive, "share_with")
+    ):
+        raise ConfigError("Config error: Incorrect GoogleDrive driver settings")
+
 
 def update_config_path(
     path_param: str,

config.yaml.default

@@ -24,6 +24,11 @@ minio:
   region:
   bucket_subpath:
 
+google_drive:
+  service_account_file: 
+  folder:
+  share_with:
+
 references:
   - file: survey.gpkg
     table: notes

drivers.py

@@ -7,11 +7,36 @@
 """
 
 import os
+from pathlib import Path
 import shutil
+import typing
+import re
+import enum
+
 from minio import Minio
 from minio.error import S3Error
 from urllib.parse import urlparse, urlunparse
 
+from google.oauth2 import service_account
+from googleapiclient.discovery import build, Resource
+from googleapiclient.http import MediaFileUpload
+
+
+class DriverType(enum.Enum):
+    LOCAL = "local"
+    MINIO = "minio"
+    GOOGLE_DRIVE = "google_drive"
+
+    def __eq__(self, value):
+        if isinstance(value, str):
+            return self.value == value
+
+    def __str__(self):
+        return self.value
+
+    def __repr__(self):
+        return self.value
+
 
 class DriverError(Exception):
     pass
@@ -92,11 +117,172 @@ def upload_file(self, src, obj_path):
         return dest
 
 
+class GoogleDriveDriver(Driver):
+    """Driver to handle connection to Google Drive"""
+
+    def __init__(self, config):
+        super(GoogleDriveDriver, self).__init__(config)
+
+        try:
+            self._credentials = service_account.Credentials.from_service_account_file(
+                Path(config.google_drive.service_account_file),
+                scopes=["https://www.googleapis.com/auth/drive.file"],
+            )
+
+            self._service: Resource = build(
+                "drive", "v3", credentials=self._credentials
+            )
+
+            self._folder = config.google_drive.folder
+            self._folder_id = self._folder_exists(self._folder)
+
+            if not self._folder_id:
+                self._folder_id = self._create_folder(self._folder)
+
+            for email in self._get_share_with(config.google_drive):
+                if email:
+                    self._share_with(email)
+
+        except Exception as e:
+            raise DriverError("GoogleDrive driver init error: " + str(e))
+
+    def upload_file(self, src: str, obj_path: str) -> str:
+        try:
+            file_metadata = {
+                "name": obj_path,
+                "parents": [self._folder_id],
+            }
+            media = MediaFileUpload(src)
+
+            file = (
+                self._service.files()
+                .create(body=file_metadata, media_body=media, fields="id")
+                .execute()
+            )
+
+            file_id = file.get("id")
+
+        except Exception as e:
+            raise DriverError("GoogleDrive driver error: " + str(e))
+
+        return self._file_link(file_id)
+
+    def _folder_exists(self, folder_name: str) -> typing.Optional[str]:
+        """Check if a folder with the specified name exists. Return boolean and folder ID if exists."""
+
+        # Query to check if a folder with the specified name exists
+        try:
+            query = f"name = '{folder_name}' and mimeType = 'application/vnd.google-apps.folder'"
+            results = (
+                self._service.files().list(q=query, fields="files(id, name)").execute()
+            )
+            items = results.get("files", [])
+        except Exception as e:
+            raise DriverError("Google Drive folder exists error: " + str(e))
+
+        if len(items) > 1:
+            print(
+                f"Multiple folders with name '{folder_name}' found. Using the first one found."
+            )
+
+        if items:
+            return items[0]["id"]
+        else:
+            return None
+
+    def _create_folder(self, folder_name: str) -> str:
+        file_metadata = {
+            "name": folder_name,
+            "mimeType": "application/vnd.google-apps.folder",
+        }
+
+        try:
+            folder = (
+                self._service.files().create(body=file_metadata, fields="id").execute()
+            )
+            return folder.get("id")
+        except Exception as e:
+            raise DriverError("Google Drive create folder error: " + str(e))
+
+    def _file_link(self, file_id: str) -> str:
+        """Get a link to the file in Google Drive."""
+        try:
+            file = (
+                self._service.files()
+                .get(fileId=file_id, fields="webViewLink")
+                .execute()
+            )
+            return file.get("webViewLink")
+        except Exception as e:
+            raise DriverError("Google Drive file link error: " + str(e))
+
+    def _has_already_permission(self, email: str) -> bool:
+        """Check if email already has permission to the folder."""
+        try:
+            # List all permissions for the file
+            permissions = (
+                self._service.permissions()
+                .list(
+                    fileId=self._folder_id,
+                    fields="permissions(id, emailAddress, role, type)",
+                )
+                .execute()
+            )
+
+            return any(
+                permission.get("emailAddress", "").lower() == email.lower()
+                for permission in permissions.get("permissions", [])
+            )
+
+        except Exception as e:
+            raise DriverError("Google Drive has permission error: " + str(e))
+
+        return False
+
+    def _share_with(self, email: str) -> None:
+        """Share the folder with the specified email."""
+        if not self._has_already_permission(email):
+            try:
+                permission = {
+                    "type": "user",
+                    "role": "writer",
+                    "emailAddress": email,
+                }
+                self._service.permissions().create(
+                    fileId=self._folder_id, body=permission
+                ).execute()
+            except Exception as e:
+                raise DriverError("Google Drive sharing folder error: " + str(e))
+
+    def _get_share_with(self, config_google_drive) -> typing.List[str]:
+        email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
+
+        emails_to_share_with = []
+        if isinstance(config_google_drive.share_with, str):
+            if email_regex.match(config_google_drive.share_with):
+                emails_to_share_with.append(config_google_drive.share_with)
+        elif isinstance(config_google_drive.share_with, list):
+            for email in config_google_drive.share_with:
+                if email_regex.match(email):
+                    emails_to_share_with.append(email)
+        else:
+            raise DriverError(
+                "Google Drive sharing: Incorrect GoogleDrive shared_with settings"
+            )
+
+        if not emails_to_share_with:
+            print("Google Drive sharing: Not shared with any user")
+
+        return emails_to_share_with
+
+
 def create_driver(config):
     """Create driver object based on type defined in config"""
     driver = None
-    if config.driver == "local":
+    if config.driver == DriverType.LOCAL:
         driver = LocalDriver(config)
-    elif config.driver == "minio":
+    elif config.driver == DriverType.MINIO:
         driver = MinioDriver(config)
+    elif config.driver == DriverType.GOOGLE_DRIVE:
+        driver = GoogleDriveDriver(config)
     return driver

google-drive-setup.md

@@ -0,0 +1,47 @@
+# Google Drive Setup
+
+To set up Google Drive for use with the Google Drive API, follow these steps. This guide provides a step-by-step process for both personal and business accounts.
+
+1. **Create a Google Account**: Ensure you have a Google Account. This process works for both personal and business accounts.
+
+2. **Create a Project**:
+   - Visit [Google Cloud Console](https://console.cloud.google.com/).
+   - In the top left corner, click on `Select a project` (there might already be a project name selected if you have previously created a project) and then `New Project`.
+   - Assign a name to the project, select a location (which specifies where the project is created), and click `Create`.
+   - In the top left corner, you should see the name of your project, or click on `Select a project` and select the project you just created.
+
+3. **Enable Google Drive API**:
+   - In the left menu, click on `APIs & Services` and then `Dashboard`.
+   - Click on `Enable APIs and Services`.
+   - Search for `Google Drive API`, click on it, and then click `Enable`.
+
+4. **Create a Service Account**:
+   - On the left side of the screen, click on `Credentials`.
+   - Click on `Manage service accounts` and then `Create Service Account`.
+   - Specify the name, account ID, and description, and click `Done`.
+   - Click on the created credentials, select the `Keys` page, and then create a new key using `Add Key`.
+   - In the following dialog, select `JSON` and click `Create`. The key will be downloaded to your computer (store it safely, as it cannot be redownloaded). In case of a lost key, you can delete it and create a new one.
+
+The downloaded JSON file contains all the necessary information to authenticate with the Google Drive API. Provide this file to the media sync tool as the path to the file:
+
+```yaml
+google_drive:
+  service_account_file: path/to/your/service_account_file.json
+```
+
+Keep this file secret and do not share it with anyone.
+
+## Sharing the Folder with Data from Media Sync
+
+The data stored under the project's `Service Account` is counted towards the Google Drive storage quota of the user who created the project.
+
+To store data in Google Drive, use the following settings:
+
+```yaml
+google_drive:
+  service_account_file: path/to/your/service_account_file.json
+  folder: your_folder_name
+  share_with: [email1@example.com, email2@example.com]
+```
+
+This creates a `folder` in Google Drive under the `Service account`, accessible only by this specific user. To make it available to other users, use the `share_with` setting. The folder will be shared with all the email addresses specified in the list (the emails need to be Google Emails - business or free). Every user will have the same access rights as the user who created the folder and can create and delete files in the folder. For users with whom the folder is shared, it will be listed in their Google Drive under the `Shared with me` section.

test/conftest.py

@@ -16,6 +16,10 @@
 MINIO_URL = os.environ.get("TEST_MINIO_URL")
 MINIO_ACCESS_KEY = os.environ.get("TEST_MINIO_ACCESS_KEY")
 MINIO_SECRET_KEY = os.environ.get("TEST_MINIO_SECRET_KEY")
+GOOGLE_DRIVE_FOLDER = os.environ.get("TEST_GOOGLE_DRIVE_FOLDER")
+GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE = os.environ.get(
+    "TEST_GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE"
+)
 
 
 @pytest.fixture(scope="function")