From 2dd85f23693fe9c513ad14b8be0ecdf915ef4b03 Mon Sep 17 00:00:00 2001 From: ZuluPro Date: Thu, 28 Jul 2016 06:34:24 -0400 Subject: [PATCH 1/2] Removed storage and adapt code --- dbbackup/management/commands/dbbackup.py | 5 +- dbbackup/management/commands/dbrestore.py | 4 +- dbbackup/management/commands/listbackups.py | 4 +- dbbackup/management/commands/mediabackup.py | 5 +- dbbackup/settings.py | 23 -- dbbackup/{storage/base.py => storage.py} | 52 +++-- dbbackup/storage/__init__.py | 0 dbbackup/storage/builtin_django.py | 39 ---- dbbackup/storage/dropbox_storage.py | 196 ---------------- dbbackup/storage/filesystem_storage.py | 46 ---- dbbackup/storage/ftp_storage.py | 61 ----- dbbackup/storage/s3_storage.py | 27 --- dbbackup/storage/sftp_storage.py | 58 ----- dbbackup/tests/commands/test_dbbackup.py | 9 +- dbbackup/tests/commands/test_dbrestore.py | 9 +- dbbackup/tests/commands/test_listbackups.py | 17 +- dbbackup/tests/functional/test_commands.py | 5 - dbbackup/tests/settings.py | 2 +- .../test_base.py => test_storage.py} | 32 +-- dbbackup/tests/test_storages/__init__.py | 0 dbbackup/tests/test_storages/test_django.py | 40 ---- .../tests/test_storages/test_filesystem.py | 71 ------ dbbackup/tests/test_storages/test_s3.py | 62 ----- dbbackup/tests/utils.py | 47 ++-- docs/storage.rst | 212 ++++++++---------- functional.sh | 4 +- 26 files changed, 191 insertions(+), 839 deletions(-) rename dbbackup/{storage/base.py => storage.py} (83%) delete mode 100644 dbbackup/storage/__init__.py delete mode 100644 dbbackup/storage/builtin_django.py delete mode 100644 dbbackup/storage/dropbox_storage.py delete mode 100644 dbbackup/storage/filesystem_storage.py delete mode 100644 dbbackup/storage/ftp_storage.py delete mode 100644 dbbackup/storage/s3_storage.py delete mode 100644 dbbackup/storage/sftp_storage.py rename dbbackup/tests/{test_storages/test_base.py => test_storage.py} (82%) delete mode 100644 dbbackup/tests/test_storages/__init__.py delete mode 100644 dbbackup/tests/test_storages/test_django.py delete mode 100644 dbbackup/tests/test_storages/test_filesystem.py delete mode 100644 dbbackup/tests/test_storages/test_s3.py diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index 0a54c5ee..dc30b01e 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -10,7 +10,7 @@ from ._base import BaseDbBackupCommand from ...db.base import get_connector -from ...storage.base import BaseStorage, StorageError +from ...storage import get_storage, StorageError from ... import utils, settings as dbbackup_settings @@ -48,7 +48,7 @@ def handle(self, **options): self.encrypt = options.get('encrypt') self.filename = options.get('output_filename') self.path = options.get('output_path') - self.storage = BaseStorage.storage_factory() + self.storage = get_storage() database_keys = (self.database,) if self.database else dbbackup_settings.DATABASES for database_key in database_keys: self.connector = get_connector(database_key) @@ -78,6 +78,7 @@ def _save_new_backup(self, database): if not self.quiet: self.logger.info("Backup size: %s", utils.handle_size(outputfile)) # Store backup + outputfile.seek(0) if self.path is None: self.write_to_storage(outputfile, filename) else: diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 259ecadf..e2d97140 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -13,7 +13,7 @@ from ._base import BaseDbBackupCommand from ... import utils from ...db.base import get_connector -from ...storage.base import BaseStorage, StorageError +from ...storage import get_storage, StorageError class Command(BaseDbBackupCommand): @@ -47,7 +47,7 @@ def handle(self, *args, **options): self.passphrase = options.get('passphrase') self.interactive = options.get('interactive') self.database = self._get_database(options) - self.storage = BaseStorage.storage_factory() + self.storage = get_storage() self._restore_backup() except StorageError as err: raise CommandError(err) diff --git a/dbbackup/management/commands/listbackups.py b/dbbackup/management/commands/listbackups.py index bb1bb359..d0ab39bd 100644 --- a/dbbackup/management/commands/listbackups.py +++ b/dbbackup/management/commands/listbackups.py @@ -6,7 +6,7 @@ from optparse import make_option from ... import utils from ._base import BaseDbBackupCommand -from ...storage.base import BaseStorage, StorageError +from ...storage import get_storage ROW_TEMPLATE = '{name:40} {datetime:20}' FILTER_KEYS = ('encrypted', 'compressed', 'content_type', 'database') @@ -28,7 +28,7 @@ class Command(BaseDbBackupCommand): def handle(self, **options): self.quiet = options.get('quiet') - self.storage = BaseStorage.storage_factory() + self.storage = get_storage() files_attr = self.get_backup_attrs(options) if not self.quiet: title = ROW_TEMPLATE.format(name='Name', datetime='Datetime') diff --git a/dbbackup/management/commands/mediabackup.py b/dbbackup/management/commands/mediabackup.py index 1026e6ca..a828ef42 100644 --- a/dbbackup/management/commands/mediabackup.py +++ b/dbbackup/management/commands/mediabackup.py @@ -12,7 +12,8 @@ from ._base import BaseDbBackupCommand from ... import utils -from ...storage.base import BaseStorage, StorageError +from ...storage import get_storage, StorageError +from ... import settings class Command(BaseDbBackupCommand): @@ -43,7 +44,7 @@ def handle(self, *args, **options): self.path = options.get('output_path') try: self.media_storage = get_storage_class()() - self.storage = BaseStorage.storage_factory() + self.storage = get_storage() self.backup_mediafiles() if options.get('clean'): self._cleanup_old_backups() diff --git a/dbbackup/settings.py b/dbbackup/settings.py index 757ad1bc..5cddaebf 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -42,7 +42,6 @@ GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_RECIPIENT', None) STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'dbbackup.storage.filesystem_storage') -BUILTIN_STORAGE = getattr(settings, 'DBBACKUP_BUILTIN_STORAGE', None) STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {}) CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {}) @@ -63,25 +62,3 @@ if hasattr(settings, 'DBBACKUP_FAKE_HOST'): # pragma: no cover warnings.warn("DBBACKUP_FAKE_HOST is deprecated, use DBBACKUP_HOSTNAME", DeprecationWarning) HOSTNAME = settings.DBBACKUP_FAKE_HOST - -UNSED_AWS_SETTINGS = ('DIRECTORY',) -DEPRECATED_AWS_SETTINGS = ( - ('BUCKET', 'bucket_name'), - ('ACCESS_KEY', 'access_key'), - ('SECRET_KEY', 'secret_key'), - ('DOMAIN', 'host'), - ('IS_SECURE', 'use_ssl'), - ('SERVER_SIDE_ENCRYPTION', 'encryption'), -) -if hasattr(settings, 'DBBACKUP_S3_BUCKET'): - for old_suffix, new_key in DEPRECATED_AWS_SETTINGS: - old_key = 'DBBACKUP_S3_%s' % old_suffix - if hasattr(settings, old_key): - STORAGE_OPTIONS[new_key] = getattr(settings, old_key) - msg = "%s is deprecated, use DBBACKUP_STORAGE_OPTIONS['%s']" % (old_key, new_key) - warnings.warn(msg, DeprecationWarning) - for old_suffix in UNSED_AWS_SETTINGS: - if hasattr(settings, 'DBBACKUP_S3_%s' % old_suffix): - msg = "DBBACKUP_S3_%s is now useless" % old_suffix - warnings.warn(msg, DeprecationWarning) - del old_suffix, new_key diff --git a/dbbackup/storage/base.py b/dbbackup/storage.py similarity index 83% rename from dbbackup/storage/base.py rename to dbbackup/storage.py index 0f91b497..e91cb4b7 100644 --- a/dbbackup/storage/base.py +++ b/dbbackup/storage.py @@ -1,10 +1,10 @@ """ -Abstract Storage class. +Backup Storage class. """ import logging -from importlib import import_module from django.core.exceptions import ImproperlyConfigured -from .. import settings, utils +from django.core.files.storage import get_storage_class +from . import settings, utils def get_storage(path=None, options=None): @@ -23,13 +23,11 @@ def get_storage(path=None, options=None): :rtype: :class:`.Storage` """ path = path or settings.STORAGE - option = options or {} options = options or settings.STORAGE_OPTIONS if not path: raise ImproperlyConfigured('You must specify a storage class using ' 'DBBACKUP_STORAGE settings.') - storage_module = import_module(path) - return storage_module.Storage(**options) + return Storage(path, **options) class StorageError(Exception): @@ -40,38 +38,48 @@ class FileNotFound(StorageError): pass -class BaseStorage(object): +class Storage(object): """Abstract storage class.""" - @property def logger(self): if not hasattr(self, '_logger'): self._logger = logging.getLogger('dbbackup.storage') return self._logger - def __init__(self, server_name=None): - if not self.name: - raise Exception("Programming Error: storage.name not defined.") + def __init__(self, storage_path=None, **options): + """ + Initialize a Django Storage instance with given options. + + :param storage_path: Path to a Django Storage class with dot style + If ``None``, ``settings.DBBACKUP_BUILTIN_STORAGE`` + will be used. + :type storage_path: str + """ + self._storage_path = storage_path or settings.STORAGE + options = options.copy() + options.update(settings.STORAGE_OPTIONS) + options = dict([(key.lower(), value) for key, value in options.items()]) + self.storageCls = get_storage_class(self._storage_path) + self.storage = self.storageCls(**options) + self.name = self.storageCls.__name__ def __str__(self): return self.name - # TODO: Remove in favor of get_storage - @classmethod - def storage_factory(cls): - return get_storage() - - def backup_dir(self): - raise NotImplementedError("Programming Error: backup_dir() not defined.") - def delete_file(self, filepath): - raise NotImplementedError("Programming Error: delete_file() not defined.") + self.logger.debug('Deleting file %s', filepath) + self.storage.delete(name=filepath) + + def list_directory(self): + return self.storage.listdir('')[1] def write_file(self, filehandle, filename): - raise NotImplementedError("Programming Error: write_file() not defined.") + self.logger.debug('Writing file %s', filename) + self.storage.save(name=filename, content=filehandle) def read_file(self, filepath): - raise NotImplementedError("Programming Error: read_file() not defined.") + self.logger.debug('Reading file %s', filepath) + return self.storage.open(name=filepath, mode='rb') def list_backups(self, encrypted=None, compressed=None, content_type=None, database=None): diff --git a/dbbackup/storage/__init__.py b/dbbackup/storage/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dbbackup/storage/builtin_django.py b/dbbackup/storage/builtin_django.py deleted file mode 100644 index 7b95cb06..00000000 --- a/dbbackup/storage/builtin_django.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Wrapper around Django storage API. -""" -from django.core.files.storage import get_storage_class -from .base import BaseStorage -from .. import settings - - -class Storage(BaseStorage): - def __init__(self, storage_path=None, **options): - """ - Initialize a Django Storage instance with given options. - - :param storage_path: Path to a Django Storage class with dot style - If ``None``, ``settings.DBBACKUP_BUILTIN_STORAGE`` - will be used. - :type storage_path: str - """ - self._storage_path = storage_path or settings.BUILTIN_STORAGE - options = options.copy() - options.update(settings.STORAGE_OPTIONS) - self.storageCls = get_storage_class(self._storage_path) - self.storage = self.storageCls(**options) - self.name = self.storageCls.__name__ - - def delete_file(self, filepath): - self.logger.debug('Deleting file %s', filepath) - self.storage.delete(name=filepath) - - def list_directory(self): - return self.storage.listdir('')[1] - - def write_file(self, filehandle, filename): - self.logger.debug('Writing file %s', filename) - self.storage.save(name=filename, content=filehandle) - - def read_file(self, filepath): - self.logger.debug('Reading file %s', filepath) - return self.storage.open(name=filepath, mode='rb') diff --git a/dbbackup/storage/dropbox_storage.py b/dbbackup/storage/dropbox_storage.py deleted file mode 100644 index 9a5bd9e6..00000000 --- a/dbbackup/storage/dropbox_storage.py +++ /dev/null @@ -1,196 +0,0 @@ -""" -Dropbox API Storage object. -""" -from __future__ import (absolute_import, division, - print_function, unicode_literals) -import pickle -import os -import tempfile -import dropbox -from django.conf import settings -from shutil import copyfileobj -from .base import BaseStorage, StorageError -from .. import settings as dbbackup_settings - -MAX_SPOOLED_SIZE = 10 * 1024 * 1024 -FILE_SIZE_LIMIT = 10 * 1024 * 1024 * 1024 -CHUNK_SIZE = 10 * 1024 * 1024 -RETRY_COUNT = 2 - - -################################ -# Dropbox Storage Object -################################ - -class Storage(BaseStorage): - """ Dropbox API Storage. """ - name = 'Dropbox' - TOKENS_FILEPATH = getattr(settings, 'DBBACKUP_TOKENS_FILEPATH', None) - DROPBOX_DIRECTORY = getattr(settings, 'DBBACKUP_DROPBOX_DIRECTORY', '').strip('/') - DBBACKUP_DROPBOX_APP_KEY = getattr(settings, 'DBBACKUP_DROPBOX_APP_KEY', None) - DBBACKUP_DROPBOX_APP_SECRET = getattr(settings, 'DBBACKUP_DROPBOX_APP_SECRET', None) - DBBACKUP_DROPBOX_FILE_SIZE_LIMIT = getattr(settings, 'DBBACKUP_DROPBOX_FILE_SIZE_LIMIT', FILE_SIZE_LIMIT) - _access_token = None - - def __init__(self, server_name=None): - self._check_settings() - self.dropbox = self.get_dropbox_client() - super(Storage, self).__init__() - - def _check_settings(self): - """ Check we have all the required settings defined. """ - if not self.TOKENS_FILEPATH: - raise StorageError('Dropbox storage requires DBBACKUP_TOKENS_FILEPATH to be defined in settings.') - if not self.DBBACKUP_DROPBOX_APP_KEY: - raise StorageError('%s storage requires DBBACKUP_DROPBOX_APP_KEY to be defined in settings.' % self.name) - if not self.DBBACKUP_DROPBOX_APP_SECRET: - raise StorageError('%s storage requires DBBACKUP_DROPBOX_APP_SECRET to be specified.' % self.name) - - ################################### - # DBBackup Storage Attributes - ################################### - - @property - def backup_dir(self): - return self.DROPBOX_DIRECTORY - - def delete_file(self, filepath): - """ Delete the specified filepath. """ - files = self.list_directory(raw=True) - to_be_deleted = [x for x in files if os.path.splitext(x)[0] == filepath] - for name in to_be_deleted: - self.run_dropbox_action(self.dropbox.file_delete, name) - - def list_directory(self, raw=False): - """ List all stored backups for the specified. """ - metadata = self.run_dropbox_action(self.dropbox.metadata, self.DROPBOX_DIRECTORY) - filepaths = [x['path'] for x in metadata['contents'] if not x['is_dir']] - if not raw: - filepaths = [os.path.splitext(x)[0] for x in filepaths] - filepaths = list(set(filepaths)) - return sorted(filepaths) - - def get_numbered_path(self, path, number): - return "{0}.{1}".format(path, number) - - def write_file(self, file_object, filename): - """ Write the specified file. """ - - file_object.seek(0, os.SEEK_END) - file_size = file_object.tell() - file_object.seek(0) - - file_path = os.path.join(self.DROPBOX_DIRECTORY, filename) - file_number = 0 - - while file_object.tell() < file_size: - - upload_id = None - chunk = None - numbered_file_offset = 0 - - numbered_file_size = min(self.DBBACKUP_DROPBOX_FILE_SIZE_LIMIT, file_size - file_object.tell()) - - while numbered_file_offset < numbered_file_size: - chunk_size = min(CHUNK_SIZE, numbered_file_size - numbered_file_offset) - if chunk is None: - chunk = file_object.read(chunk_size) - - for try_number in range(RETRY_COUNT + 1): - try: - numbered_file_offset, upload_id = self.dropbox.upload_chunk(chunk, chunk_size, numbered_file_offset, upload_id) - chunk = None - except dropbox.rest.ErrorResponse: - print(' Chunk upload failed') - if try_number == RETRY_COUNT: - raise - else: - print(' Retry') - else: - break - - upload_progress = file_object.tell() / file_size * 100 - print(' Uploaded {:4.1f}%'.format(upload_progress)) - - numbered_file_path = self.get_numbered_path(file_path, file_number) - numbered_file_full_path = os.path.join(self.dropbox.session.root, numbered_file_path) - - print(' Commit to {}'.format(numbered_file_path)) - self.dropbox.commit_chunked_upload(numbered_file_full_path, upload_id) - - file_number += 1 - - def read_file(self, filepath): - """ Read the specified file and return it's handle. """ - total_files = 0 - filehandle = tempfile.SpooledTemporaryFile( - max_size=MAX_SPOOLED_SIZE, - dir=dbbackup_settings.TMP_DIR) - try: - while True: - response = self.run_dropbox_action(self.dropbox.get_file, - self.get_numbered_path(filepath, total_files), - ignore_404=(total_files > 0)) - if not response: - break - copyfileobj(response, filehandle) - total_files += 1 - except: - filehandle.close() - raise - return filehandle - - def run_dropbox_action(self, method, *args, **kwargs): - """ Check we have a valid 200 response from Dropbox. """ - ignore_404 = kwargs.pop("ignore_404", False) - try: - response = method(*args, **kwargs) - except dropbox.rest.ErrorResponse as e: - if ignore_404 and e.status == 404: - return None - errmsg = "ERROR %s" % (e,) - raise StorageError(errmsg) - return response - - ################################### - # Dropbox Client Methods - ################################### - - def get_dropbox_client(self): - """ Connect and return a Dropbox client object. """ - self.read_token_file() - flow = dropbox.client.DropboxOAuth2FlowNoRedirect(self.DBBACKUP_DROPBOX_APP_KEY, - self.DBBACKUP_DROPBOX_APP_SECRET) - access_token = self.get_access_token(flow) - client = dropbox.client.DropboxClient(access_token) - return client - - def get_access_token(self, flow): - """ Return Access Token. If not available, a new one will be created and saved. """ - if not self._access_token: - return self.create_access_token(flow) - return self._access_token - - def create_access_token(self, flow): - """ Create and save a new access token to self.TOKENFILEPATH. """ - authorize_url = flow.start() - print("1. Go to: %s" % authorize_url) - print("2. Click 'Allow' (you might have to log in first)") - print("3. Copy the authorization code.") - code = input("Enter the authorization code here: ").strip() - self._access_token, userid = flow.finish(code) - self.save_token_file() - return self._access_token - - def save_token_file(self): - """ Save the request and access tokens to disk. """ - tokendata = dict(access_token=self._access_token) - with open(self.TOKENS_FILEPATH, 'wb') as tokenhandle: - pickle.dump(tokendata, tokenhandle, -1) - - def read_token_file(self): - """ Reload the request and/or access tokens from disk. """ - if os.path.exists(self.TOKENS_FILEPATH): - with open(self.TOKENS_FILEPATH, 'rb') as tokenhandle: - tokendata = pickle.load(tokenhandle) - self._access_token = tokendata.get('access_token') diff --git a/dbbackup/storage/filesystem_storage.py b/dbbackup/storage/filesystem_storage.py deleted file mode 100644 index 2628ccdd..00000000 --- a/dbbackup/storage/filesystem_storage.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Filesystem Storage API. -""" -from __future__ import (absolute_import, division, - print_function, unicode_literals) -import warnings - -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured - -from .base import StorageError -from .builtin_django import Storage as DjangoStorage - -STORAGE_PATH = 'django.core.files.storage.FileSystemStorage' - - -class Storage(DjangoStorage): - """Filesystem API Storage.""" - name = 'Filesystem' - - def __init__(self, server_name=None, **options): - self._check_filesystem_errors(options) - super(Storage, self).__init__(storage_path=STORAGE_PATH, - **options) - - def _check_filesystem_errors(self, options): - """ Check we have all the required settings defined. """ - location = options.get('location') - if location is None: - raise StorageError('Filesystem storage requires ' - 'DBBACKUP_STORAGE_OPTIONS["location"] to be ' - 'defined in settings.') - if location == '': - raise StorageError('Filesystem storage requires ' - 'DBBACKUP_STORAGE_OPTIONS["location"] to be ' - 'defined with a non empty string.') - - if settings.MEDIA_ROOT and \ - options.get('location', '').startswith(settings.MEDIA_ROOT): - if not settings.DEBUG: - msg = "Backups can't be stored in MEDIA_ROOT if DEBUG is "\ - "False, Please use an another location for your storage." - raise ImproperlyConfigured(msg) - msg = "Backups are saved in MEDIA_ROOT, this is a critical issue "\ - "in production." - warnings.warn(msg) diff --git a/dbbackup/storage/ftp_storage.py b/dbbackup/storage/ftp_storage.py deleted file mode 100644 index 49dd43d7..00000000 --- a/dbbackup/storage/ftp_storage.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -FTP Storage object. -""" -from __future__ import (absolute_import, division, - print_function, unicode_literals) -import os -import tempfile -from ftplib import FTP - -from django.conf import settings - -from .. import settings as dbbackup_settings -from .base import BaseStorage, StorageError - - -class Storage(BaseStorage): - """FTP Storage.""" - name = 'FTP' - FTP_HOST = getattr(settings, 'DBBACKUP_FTP_HOST', None) - FTP_USER = getattr(settings, 'DBBACKUP_FTP_USER', None) - FTP_PASSWORD = getattr(settings, 'DBBACKUP_FTP_PASSWORD', None) - FTP_PATH = getattr(settings, 'DBBACKUP_FTP_PATH', ".") - FTP_PATH = '/%s/' % FTP_PATH.strip('/') - FTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_FTP_PASSIVE_MODE', False) - - def __init__(self, server_name=None): - self._check_settings() - self.ftp = FTP(self.FTP_HOST, self.FTP_USER, self.FTP_PASSWORD) - self.ftp.set_pasv(self.FTP_PASSIVE_MODE) - BaseStorage.__init__(self) - - def _check_settings(self): - """ Check we have all the required settings defined. """ - if not self.FTP_HOST: - raise StorageError('%s storage requires DBBACKUP_FTP_HOST to be defined in settings.' % self.name) - - @property - def backup_dir(self): - return self.FTP_PATH - - def delete_file(self, filepath): - """ Delete the specified filepath. """ - self.ftp.delete(filepath) - - def list_directory(self, raw=False): - """ List all stored backups for the specified. """ - return sorted(self.ftp.nlst(self.FTP_PATH)) - - def write_file(self, filehandle, filename): - """ Write the specified file. """ - filehandle.seek(0) - backuppath = os.path.join(self.FTP_PATH, filename) - self.ftp.storbinary('STOR ' + backuppath, filehandle) - - def read_file(self, filepath): - """ Read the specified file and return it's handle. """ - outputfile = tempfile.SpooledTemporaryFile( - max_size=dbbackup_settings.TMP_FILE_MAX_SIZE, - dir=dbbackup_settings.TMP_DIR) - self.ftp.retrbinary('RETR ' + filepath, outputfile.write) - return outputfile diff --git a/dbbackup/storage/s3_storage.py b/dbbackup/storage/s3_storage.py deleted file mode 100644 index 2ba43ec5..00000000 --- a/dbbackup/storage/s3_storage.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -S3 Storage object. -""" -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from .base import StorageError -from .builtin_django import Storage as DjangoStorage - -STORAGE_PATH = 'storages.backends.s3boto.S3BotoStorage' - - -class Storage(DjangoStorage): - """Filesystem API Storage.""" - def __init__(self, server_name=None, **options): - self.name = 'AmazonS3' - self._check_filesystem_errors(options) - super(Storage, self).__init__(storage_path=STORAGE_PATH, - bucket=options['bucket_name'], - **options) - - def _check_filesystem_errors(self, options): - """Check we have all the required settings defined.""" - required_args = ('bucket_name', 'access_key', 'secret_key') - err_msg = "%s storage requires settings.DBBACKUP_STORAGE_OPTIONS['%s'] to be define" - for arg in required_args: - if arg not in options: - raise StorageError(err_msg % (self.name, arg)) diff --git a/dbbackup/storage/sftp_storage.py b/dbbackup/storage/sftp_storage.py deleted file mode 100644 index f9a6042f..00000000 --- a/dbbackup/storage/sftp_storage.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -SFTP Storage object. -""" -import os -import tempfile -from django.conf import settings -from pysftp import Connection - -from .base import BaseStorage, StorageError -from .. import settings as dbbackup_settings - - -class Storage(BaseStorage): - """ SFTP Storage """ - name = 'SFTP' - SFTP_HOST = getattr(settings, 'DBBACKUP_SFTP_HOST', None) - SFTP_USER = getattr(settings, 'DBBACKUP_SFTP_USER', None) - SFTP_PASSWORD = getattr(settings, 'DBBACKUP_SFTP_PASSWORD', None) - SFTP_PATH = getattr(settings, 'DBBACKUP_SFTP_PATH', ".") - SFTP_PATH = '/%s/' % SFTP_PATH.strip('/') - SFTP_PASSIVE_MODE = getattr(settings, 'DBBACKUP_SFTP_PASSIVE_MODE', False) - - def __init__(self, server_name=None): - self._check_settings() - self.sftp = Connection(host=self.SFTP_HOST, username=self.SFTP_USER, - password=self.SFTP_PASSWORD) - - def _check_settings(self): - """Check we have all the required settings defined.""" - if not self.SFTP_HOST: - msg = '%s storage requires DBBACKUP_SFTP_HOST to be defined in settings.' % self.name - raise StorageError(msg) - - @property - def backup_dir(self): - return self.SFTP_PATH - - def delete_file(self, filepath): - """Delete the specified filepath.""" - self.sftp.remove(filepath) - - def list_directory(self, raw=False): - """List all stored backups for the specified.""" - return sorted(self.sftp.listdir(self.SFTP_PATH)) - - def write_file(self, filehandle, filename): - """Write the specified file.""" - filehandle.seek(0) - backuppath = os.path.join(self.SFTP_PATH, filename) - self.sftp.putfo(filehandle, backuppath) - - def read_file(self, filepath): - """Read the specified file and return it's handle.""" - outputfile = tempfile.SpooledTemporaryFile( - max_size=dbbackup_settings.TMP_FILE_MAX_SIZE, - dir=dbbackup_settings.TMP_DIR) - self.sftp.getfo(filepath, outputfile) - return outputfile diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index eb8f3458..19df725b 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -8,8 +8,9 @@ from dbbackup.management.commands.dbbackup import Command as DbbackupCommand from dbbackup.db.base import get_connector -from dbbackup.tests.utils import (FakeStorage, TEST_DATABASE, - add_public_gpg, clean_gpg_keys, DEV_NULL) +from dbbackup.storage import get_storage +from dbbackup.tests.utils import (TEST_DATABASE, add_public_gpg, clean_gpg_keys, + DEV_NULL) @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') @@ -21,7 +22,7 @@ def setUp(self): self.command.encrypt = False self.command.compress = False self.command.database = TEST_DATABASE['NAME'] - self.command.storage = FakeStorage() + self.command.storage = get_storage() self.command.connector = get_connector() self.command.stdout = DEV_NULL self.command.filename = None @@ -60,7 +61,7 @@ def setUp(self): self.command.servername = 'foo-server' self.command.encrypt = False self.command.compress = False - self.command.storage = FakeStorage() + self.command.storage = get_storage() self.command.stdout = DEV_NULL self.command.filename = None self.command.path = None diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index 923d660d..f45431bc 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -13,13 +13,12 @@ from dbbackup.db.base import get_connector from dbbackup.db.mongodb import MongoDumpConnector from dbbackup.management.commands.dbrestore import Command as DbrestoreCommand -from dbbackup.tests.utils import (FakeStorage, TEST_DATABASE, - add_private_gpg, DEV_NULL, +from dbbackup.storage import get_storage +from dbbackup.tests.utils import (TEST_DATABASE, add_private_gpg, DEV_NULL, clean_gpg_keys, HANDLED_FILES, TEST_MONGODB, TARED_FILE, get_dump, get_dump_name) -@patch('django.conf.settings.DATABASES', {'default': TEST_DATABASE}) @patch('dbbackup.management.commands._base.input', return_value='y') class DbrestoreCommandRestoreBackupTest(TestCase): def setUp(self): @@ -32,7 +31,7 @@ def setUp(self): self.command.database = TEST_DATABASE self.command.passphrase = None self.command.interactive = True - self.command.storage = FakeStorage() + self.command.storage = get_storage() self.command.connector = get_connector() HANDLED_FILES.clean() @@ -120,7 +119,7 @@ def setUp(self): self.command.database = TEST_MONGODB self.command.passphrase = None self.command.interactive = True - self.command.storage = FakeStorage() + self.command.storage = get_storage() self.command.connector = MongoDumpConnector() HANDLED_FILES.clean() add_private_gpg() diff --git a/dbbackup/tests/commands/test_listbackups.py b/dbbackup/tests/commands/test_listbackups.py index 5eb90e1e..58439564 100644 --- a/dbbackup/tests/commands/test_listbackups.py +++ b/dbbackup/tests/commands/test_listbackups.py @@ -3,13 +3,14 @@ from django.core.management import execute_from_command_line from django.utils.six import StringIO from dbbackup.management.commands.listbackups import Command as ListbackupsCommand -from dbbackup.tests.utils import HANDLED_FILES, FakeStorage, TEST_DATABASE +from dbbackup.storage import get_storage +from dbbackup.tests.utils import HANDLED_FILES class ListbackupsCommandTest(TestCase): def setUp(self): self.command = ListbackupsCommand() - self.command.storage = FakeStorage() + self.command.storage = get_storage() HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810.bak', '2015-02-07-042810.bak', @@ -22,19 +23,17 @@ def test_get_backup_attrs(self): self.assertEqual(len(HANDLED_FILES['written_files']), len(attrs)) -@patch('django.conf.settings.DATABASES', {'default': TEST_DATABASE}) -@patch('dbbackup.settings.STORAGE', 'dbbackup.tests.utils') class ListbackupsCommandArgComputingTest(TestCase): def setUp(self): HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810_foo.db', '2015-02-06-042810_foo.db.gz', '2015-02-06-042810_foo.db.gpg', '2015-02-06-042810_foo.db.gz.gpg', - '2015-02-06-042810_foo.media.tar', '2015-02-06-042810_foo.media.tar.gz', - '2015-02-06-042810_foo.media.tar.gpg', '2015-02-06-042810_foo.media.tar.gz.gpg', + '2015-02-06-042810_foo.tar', '2015-02-06-042810_foo.tar.gz', + '2015-02-06-042810_foo.tar.gpg', '2015-02-06-042810_foo.tar.gz.gpg', '2015-02-06-042810_bar.db', '2015-02-06-042810_bar.db.gz', '2015-02-06-042810_bar.db.gpg', '2015-02-06-042810_bar.db.gz.gpg', - '2015-02-06-042810_bar.media.tar', '2015-02-06-042810_bar.media.tar.gz', - '2015-02-06-042810_bar.media.tar.gpg', '2015-02-06-042810_bar.media.tar.tar.gz.gpg', + '2015-02-06-042810_bar.tar', '2015-02-06-042810_bar.tar.gz', + '2015-02-06-042810_bar.tar.gpg', '2015-02-06-042810_bar.tar.gz.gpg', ]] def test_list(self): @@ -92,4 +91,4 @@ def test_filter_media(self): stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertIn('.media', line) + self.assertIn('.tar', line) diff --git a/dbbackup/tests/functional/test_commands.py b/dbbackup/tests/functional/test_commands.py index 1ba21a95..8eb5d401 100644 --- a/dbbackup/tests/functional/test_commands.py +++ b/dbbackup/tests/functional/test_commands.py @@ -14,8 +14,6 @@ from dbbackup.tests.testapp import models -@patch('django.conf.settings.DATABASES', {'default': TEST_DATABASE}) -@patch('dbbackup.settings.STORAGE', 'dbbackup.tests.utils') class DbBackupCommandTest(TestCase): def setUp(self): HANDLED_FILES.clean() @@ -56,8 +54,6 @@ def test_compress_and_encrypt(self): self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) -@patch('django.conf.settings.DATABASES', {'default': TEST_DATABASE}) -@patch('dbbackup.settings.STORAGE', 'dbbackup.tests.utils') @patch('dbbackup.management.commands._base.input', return_value='y') class DbRestoreCommandTest(TestCase): def setUp(self): @@ -116,7 +112,6 @@ def test_available_but_not_compressed(self, *args): execute_from_command_line(['', 'dbrestore', '--uncompress']) -@patch('dbbackup.settings.STORAGE', 'dbbackup.tests.utils') class MediaBackupCommandTest(TestCase): def setUp(self): HANDLED_FILES.clean() diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 10071638..5d194add 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -35,7 +35,7 @@ DBBACKUP_GPG_RECIPIENT = "test@test" DBBACKUP_GPG_ALWAYS_TRUST = True, -DBBACKUP_STORAGE = os.environ.get('STORAGE', 'dbbackup.tests.utils') +DBBACKUP_STORAGE = os.environ.get('STORAGE', 'dbbackup.tests.utils.FakeStorage') DBBACKUP_STORAGE_OPTIONS = dict([keyvalue.split('=') for keyvalue in os.environ.get('STORAGE_OPTIONS', '').split(',') if keyvalue]) diff --git a/dbbackup/tests/test_storages/test_base.py b/dbbackup/tests/test_storage.py similarity index 82% rename from dbbackup/tests/test_storages/test_base.py rename to dbbackup/tests/test_storage.py index 45a60698..b45e20c6 100644 --- a/dbbackup/tests/test_storages/test_base.py +++ b/dbbackup/tests/test_storage.py @@ -1,10 +1,10 @@ from mock import patch from django.test import TestCase -from dbbackup.storage.base import get_storage, BaseStorage +from dbbackup.storage import get_storage, Storage from dbbackup.tests.utils import HANDLED_FILES, FakeStorage from dbbackup import utils -DEFAULT_STORAGE_PATH = 'dbbackup.storage.filesystem_storage' +DEFAULT_STORAGE_PATH = 'django.core.files.storage.FileSystemStorage' STORAGE_OPTIONS = {'location': '/tmp'} @@ -12,37 +12,37 @@ class Get_StorageTest(TestCase): @patch('dbbackup.settings.STORAGE', DEFAULT_STORAGE_PATH) @patch('dbbackup.settings.STORAGE_OPTIONS', STORAGE_OPTIONS) def test_func(self, *args): - storage = get_storage() - self.assertEqual(storage.__module__, DEFAULT_STORAGE_PATH) + self.assertIsInstance(get_storage(), Storage) def test_set_path(self): - storage = get_storage(path=FakeStorage.__module__) - self.assertIsInstance(storage, FakeStorage) + fake_storage_path = 'dbbackup.tests.utils.FakeStorage' + storage = get_storage(fake_storage_path) + self.assertIsInstance(storage.storage, FakeStorage) @patch('dbbackup.settings.STORAGE', DEFAULT_STORAGE_PATH) def test_set_options(self, *args): storage = get_storage(options=STORAGE_OPTIONS) - self.assertEqual(storage.__module__, DEFAULT_STORAGE_PATH) + self.assertEqual(storage.storage.__module__, 'django.core.files.storage') -class BaseStorageTest(TestCase): +class StorageTest(TestCase): def setUp(self): - self.storageCls = BaseStorage + self.storageCls = Storage self.storageCls.name = 'foo' - self.storage = BaseStorage() + self.storage = Storage() class StorageListBackupsTest(TestCase): def setUp(self): HANDLED_FILES.clean() - self.storage = FakeStorage() + self.storage = get_storage() HANDLED_FILES['written_files'] += [ (utils.filename_generate(ext, 'foo'), None) for ext in ('db', 'db.gz', 'db.gpg', 'db.gz.gpg') ] HANDLED_FILES['written_files'] += [ (utils.filename_generate(ext, 'foo', None, 'media'), None) for ext in - ('media.tar', 'media.tar.gz', 'media.tar.gpg', 'media.tar.gz.gpg') + ('tar', 'tar.gz', 'tar.gpg', 'tar.gz.gpg') ] HANDLED_FILES['written_files'] += [ ('file_without_date', None) @@ -82,12 +82,12 @@ def test_mediabackup(self): files = self.storage.list_backups(content_type='media') # self.assertEqual(8, len(files)) for file in files: - self.assertIn('.media', file) + self.assertIn('.tar', file) class StorageGetLatestTest(TestCase): def setUp(self): - self.storage = FakeStorage() + self.storage = get_storage() HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810.bak', '2015-02-07-042810.bak', @@ -104,7 +104,7 @@ def test_func(self): class StorageGetMostRecentTest(TestCase): def setUp(self): - self.storage = FakeStorage() + self.storage = get_storage() HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810.bak', '2015-02-07-042810.bak', @@ -121,7 +121,7 @@ def test_func(self): class StorageCleanOldBackupsTest(TestCase): def setUp(self): - self.storage = FakeStorage() + self.storage = get_storage() HANDLED_FILES.clean() HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810.bak', diff --git a/dbbackup/tests/test_storages/__init__.py b/dbbackup/tests/test_storages/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dbbackup/tests/test_storages/test_django.py b/dbbackup/tests/test_storages/test_django.py deleted file mode 100644 index 350da9cf..00000000 --- a/dbbackup/tests/test_storages/test_django.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import tempfile -import shutil -from io import BytesIO -from django.test import TestCase -from dbbackup.storage.builtin_django import Storage as DjangoStorage - - -class DjangoStorageTest(TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - self.storage = DjangoStorage(location=self.temp_dir) - - def tearDown(self): - shutil.rmtree(self.temp_dir) - - def test_delete_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - open(file_path, 'w').close() - self.storage.delete_file('foo') - self.assertFalse(os.path.exists(file_path)) - - def test_list_directory(self): - file_path = os.path.join(self.temp_dir, 'foo') - open(file_path, 'w').close() - files = self.storage.list_directory() - self.assertEqual(len(files), 1) - - def test_write_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - self.storage.write_file(BytesIO(b'bar'), 'foo') - self.assertTrue(os.path.exists(file_path)) - self.assertEqual(open(file_path).read(), 'bar') - - def test_read_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - with open(file_path, 'w') as fd: - fd.write('bar') - read_file = self.storage.read_file('foo') - self.assertEqual(read_file.read(), b'bar') diff --git a/dbbackup/tests/test_storages/test_filesystem.py b/dbbackup/tests/test_storages/test_filesystem.py deleted file mode 100644 index fbeffc91..00000000 --- a/dbbackup/tests/test_storages/test_filesystem.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import tempfile -import shutil -from io import BytesIO -from mock import patch -from django.test import TestCase -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured -from dbbackup.storage.filesystem_storage import Storage as FileSystemStorage - - -class FileSystemStorageTest(TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - self.storage = FileSystemStorage(location=self.temp_dir) - - def tearDown(self): - shutil.rmtree(self.temp_dir) - - def test_delete_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - open(file_path, 'w').close() - self.storage.delete_file('foo') - self.assertFalse(os.listdir(self.temp_dir)) - - def test_list_directory(self): - file_path1 = os.path.join(self.temp_dir, 'foo') - file_path2 = os.path.join(self.temp_dir, 'bar') - self.assertEqual(0, len(os.listdir(self.temp_dir))) - open(file_path1, 'w').close() - self.assertEqual(1, len(os.listdir(self.temp_dir))) - open(file_path2, 'w').close() - self.assertEqual(2, len(os.listdir(self.temp_dir))) - - def test_write_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - self.storage.write_file(BytesIO(b'bar'), 'foo') - self.assertTrue(os.path.exists(file_path)) - self.assertEqual(open(file_path).read(), 'bar') - - def test_read_file(self): - file_path = os.path.join(self.temp_dir, 'foo') - with open(file_path, 'w') as fd: - fd.write('bar') - read_file = self.storage.read_file('foo') - self.assertEqual(read_file.read(), b'bar') - - -class FileSystemStorageCheckTest(TestCase): - def test_fail_location_is_none(self): - with self.assertRaises(Exception): - self.storage = FileSystemStorage(location=None) - - def test_fail_location_is_empty_str(self): - with self.assertRaises(Exception): - self.storage = FileSystemStorage(location='') - - def test_fail_no_location(self): - with self.assertRaises(Exception): - self.storage = FileSystemStorage() - - def test_fail_backup_in_media_file(self): - with self.assertRaises(ImproperlyConfigured): - self.storage = FileSystemStorage(location=settings.MEDIA_ROOT) - - @patch('django.conf.settings.DEBUG', True) - def test_success_backup_in_media_file_debug(self): - self.storage = FileSystemStorage(location=settings.MEDIA_ROOT) - - def test_success(self): - self.storage = FileSystemStorage(location='foo') diff --git a/dbbackup/tests/test_storages/test_s3.py b/dbbackup/tests/test_storages/test_s3.py deleted file mode 100644 index 1cda0a84..00000000 --- a/dbbackup/tests/test_storages/test_s3.py +++ /dev/null @@ -1,62 +0,0 @@ -from io import BytesIO -from django.test import TestCase -import boto -try: - from moto import mock_s3 -except SyntaxError: - mock_s3 = None -from dbbackup.storage.s3_storage import Storage as S3Storage -from dbbackup.storage.base import StorageError -from dbbackup.tests.utils import skip_py3 - - -# Python 3.2 fix -if mock_s3 is None: - def mock_s3(obj): - return obj - - -@mock_s3 -@skip_py3 -class S3StorageTest(TestCase): - def setUp(self): - self.storage = S3Storage(bucket_name='foo_bucket', - access_key='foo_id', - secret_key='foo_secret') - # Create fixtures - self.conn = boto.connect_s3() - self.bucket = self.conn.create_bucket('foo_bucket') - key = boto.s3.key.Key(self.bucket) - key.key = 'foo_file' - key.set_contents_from_string('bar') - - def test_delete_file(self): - self.storage.delete_file('foo_file') - self.assertEqual(0, len(self.bucket.get_all_keys())) - - def test_list_directory(self): - files = self.storage.list_directory() - self.assertEqual(len(files), 1) - - def test_write_file(self): - self.storage.write_file(BytesIO(b'bar'), 'foo') - self.assertEqual(2, len(self.bucket.get_all_keys())) - key = self.bucket.get_key('foo') - self.assertEqual('bar', key.get_contents_as_string()) - - def test_read_file(self): - read_file = self.storage.read_file('foo_file') - self.assertEqual(read_file.read(), b'bar') - - def test_check(self): - with self.assertRaises(StorageError): - self.storage._check_filesystem_errors({ - 'bucket_name': '', 'access_key': ''}) - with self.assertRaises(StorageError): - self.storage._check_filesystem_errors({ - 'bucket_name': '', 'secret_key': ''}) - with self.assertRaises(StorageError): - self.storage._check_filesystem_errors({ - 'access_key': '', 'secret_key': ''}) - self.storage._check_filesystem_errors({ - 'bucket_name': '', 'access_key': '', 'secret_key': ''}) diff --git a/dbbackup/tests/utils.py b/dbbackup/tests/utils.py index cac72f3f..6339b407 100644 --- a/dbbackup/tests/utils.py +++ b/dbbackup/tests/utils.py @@ -1,9 +1,8 @@ import os import subprocess -import logging from django.conf import settings -from django.utils import six -from dbbackup.storage.base import BaseStorage +from django.utils import six, timezone +from django.core.files.storage import Storage from dbbackup.db.base import get_connector BASE_FILE = os.path.join(settings.BLOB_DIR, 'test.txt') @@ -37,29 +36,37 @@ def clean(self): HANDLED_FILES = handled_files() -class FakeStorage(BaseStorage): +class FakeStorage(Storage): name = 'FakeStorage' - logger = logging.getLogger('dbbackup.storage') - def __init__(self, *args, **kwargs): - super(FakeStorage, self).__init__(*args, **kwargs) - self.deleted_files = [] - self.written_files = [] + def exists(self, name): + return name in HANDLED_FILES['written_files'] - def delete_file(self, filepath): - self.logger.debug("Delete %s", filepath) - HANDLED_FILES['deleted_files'].append(filepath) - self.deleted_files.append(filepath) + def get_available_name(self, name, max_length=None): + return name[:max_length] - def list_directory(self, raw=False): - return [f[0] for f in HANDLED_FILES['written_files']] + def get_valid_name(self, name): + return name - def write_file(self, filehandle, filename): - self.logger.debug("Write %s", filename) - HANDLED_FILES['written_files'].append((filename, filehandle)) + def listdir(self, path): + return ([], [f[0] for f in HANDLED_FILES['written_files']]) - def read_file(self, filepath): - return [f[1] for f in HANDLED_FILES['written_files'] if f[0] == filepath][0] + def accessed_time(self, name): + return timezone.now() + created_time = modified_time = accessed_time + + def _open(self, name, mode='rb'): + file_ = [f[1] for f in HANDLED_FILES['written_files'] + if f[0] == name][0] + file_.seek(0) + return file_ + + def _save(self, name, content): + HANDLED_FILES['written_files'].append((name, content)) + return name + + def delete(self, name): + HANDLED_FILES['deleted_files'].append(name) Storage = FakeStorage diff --git a/docs/storage.rst b/docs/storage.rst index f62b1e44..9b160166 100644 --- a/docs/storage.rst +++ b/docs/storage.rst @@ -1,36 +1,31 @@ Storage ======= -django-dbbackup comes with a variety of remote storage options and it can deal -with Django Storage API for extend its possibilities. +One of the most helpful feature of django-dbbackup is the avaibility to store +and retrieve backups from a local or remote storage. This functionality is +mainly based on Django Storage API and extend its possibilities. -You can choose your storage backend by set ``settings.DBBACKUP_STORAGE``, -it must point to module containing the chosen Storage class. For example: -``dbbackup.storage.filesystem_storage`` for use file system storage. +You can choose your backup storage backend by set ``settings.DBBACKUP_STORAGE``, +it must be a full path of a storage class. For example: +``django.core.files.storage.FileSystemStorage`` for use file system storage. Below, we'll list some of the available solutions and their options. Storage's option are gathered in ``settings.DBBACKUP_STORAGE_OPTIONS`` which is a dictionary of keywords representing how to configure it. -.. note:: - - A lot of changes has been made for use Django Storage API as primary source of - backends and due to this task, some settings has been deprecated but always - functionnal until removing. Please take care of notes and warnings in this - documentation and at your project's launching. - .. warning:: Do not configure backup storage with the same configuration than your media files, you'll risk to share backups inside public directories. -Local disk ----------- +FileSystemStorage +----------------- +Django has a built-in filesystem storage helping to deal with local file. Dbbackup uses `built-in file system storage`_ to manage files on a local directory. -.. _`built-in file system storage`: https://docs.djangoproject.com/en/1.8/ref/files/storage/#the-filesystemstorage-class +.. _`built-in file system storage`: https://docs.djangoproject.com/en/stable/ref/files/storage/#the-filesystemstorage-class .. note:: @@ -45,7 +40,7 @@ required settings below. :: - DBBACKUP_STORAGE = 'dbbackup.storage.filesystem_storage' + DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage' DBBACKUP_STORAGE_OPTIONS = {'location': '/my/backup/dir/'} @@ -53,25 +48,27 @@ required settings below. Available Settings ~~~~~~~~~~~~~~~~~~ -**location** - Default: Current working directory (``os.getcwd``) +**location** Absolute path to the directory that will hold the files. -.. warning:: +**base_url** - ``settings.DBBACKUP_BACKUP_DIRECTORY`` was used before but is deprecated. - Backup location must no be in ``settings.MEDIA_ROOT``, it will raise an - ``StorageError`` if ``settings.DEBUG`` is ``False`` else a warning. +URL that serves the files stored at this location. -**file_permissions_mode** - Default: ``settings.FILE_UPLOAD_PERMISSIONS`` +**file_permissions_mode** -The file system permissions that the file will receive when it is saved. +The file system permissions that the file will receive when it is saved. + +**directory_permissions_mode** + +The file system permissions that the directory will receive when it is saved. Amazon S3 --------- -Our S3 backend uses Django Storage Redux which uses `boto`_. +We advise to use Django-Storages S3 storage which uses `boto`_. .. _`boto`: http://docs.pythonboto.org/en/latest/# @@ -84,29 +81,27 @@ complete, you can follow the required setup below. :: - pip install boto django-storages-redux + pip install boto django-storages Add the following to your project's settings: :: - DBBACKUP_STORAGE = 'dbbackup.storage.s3_storage' + DBBACKUP_STORAGE = 'storages.storages.backends.s3boto.S3BotoStorageFile' DBBACKUP_STORAGE_OPTIONS = { 'access_key': 'my_id', 'secret_key': 'my_secret', 'bucket_name': 'my_bucket_name' } -Available Settings +Available settings ~~~~~~~~~~~~~~~~~~ .. note:: - More settings are available but without clear official documentation about - it, you can refer to `source code`_ and look at ``S3BotoStorage``'s - attributes. + More settings are available see `official documentation`_ for get more about. -.. _`source code`: https://github.com/jschneier/django-storages/blob/master/storages/backends/s3boto.py#L204 +.. _`official documentation`: https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html **access_key** - Required @@ -115,41 +110,22 @@ Security Credentials page`_. .. _`Amazon Account Security Credentials page`: https://console.aws.amazon.com/iam/home#security_credential -.. note:: - - ``settings.DBBACKUP_S3_ACCESS_KEY`` was used before but is deprecated. - **secret_key** - Required Your Amazon Web Services secret access key, as a string. -.. note:: - - ``settings.DBBACKUP_S3_SECRET_KEY`` was used before but is deprecated. - **bucket_name** - Required Your Amazon Web Services storage bucket name, as a string. This directory must exist before attempting to create your first backup. -.. note:: - - ``settings.DBBACKUP_S3_BUCKET`` was used before but is deprecated. - **host** - Default: ``'s3.amazonaws.com'`` (``boto.s3.connection.S3Connection.DefaultHost``) Specify the Amazon domain to use when transferring the generated backup files. For example, this can be set to ``'s3-eu-west-1.amazonaws.com'``. -.. note:: - - ``settings.DBBACKUP_S3_DOMAIN`` was used before but is deprecated. - **use_ssl** - Default: ``True`` -.. note:: - - ``settings.DBBACKUP_S3_IS_SECURE`` was used before but is deprecated. **default_acl** - Required @@ -180,125 +156,113 @@ Setup Your Dropbox Account importantly the 'App Key' and 'App Secret' values inside. You'll need those later. -Setup Your Django Project -~~~~~~~~~~~~~~~~~~~~~~~~~ +Setup +~~~~~ :: - pip install dropbox + pip install dropbox django-storages ...And make sure you have the following required project settings: :: - DBBACKUP_STORAGE = 'dbbackup.storage.dropbox_storage' - DBBACKUP_TOKENS_FILEPATH = '' - DBBACKUP_DROPBOX_APP_KEY = '' - DBBACKUP_DROPBOX_APP_SECRET = '' + DBBACKUP_STORAGE = 'storages.backends.dropbox.DropBoxStorage + DBBACKUP_STORAGE_OPTIONS = { + 'oauth2_access_token': 'my_token', + } +Available settings +~~~~~~~~~~~~~~~~~~ -FTP ---- +.. note:: -To store your database backups on the remote filesystem via FTP, simply -setup the required settings below. + See `django-storages dropbox official documentation`_ for get more details about. -Setup Your Django Project -~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _`django-storages dropbox official documentation`: https://django-storages.readthedocs.io/en/latest/backends/dropbox.html -.. note:: +**oauth2_access_token** - Required - This storage will be updated for use Django Storage's one. +Your OAuth access token -.. warning:: +**root_path** - This storage doesn't use private connection for communcation, don't use it - if you're not sure about the link between client and server. +Jail storage to this directory -Using FTP does not require any external libraries to be installed, simply -use the below project settings: +FTP +--- + +To store your database backups on a remote filesystem via [a]FTP, simply +setup the required settings below. + +Setup +~~~~~ :: - DBBACKUP_STORAGE = 'dbbackup.storage.ftp_storage' - DBBACKUP_FTP_HOST = 'ftp.host' - DBBACKUP_FTP_USER = 'user, blank if anonymous' - DBBACKUP_FTP_PASSWORD = 'password, can be blank' - DBBACKUP_FTP_PATH = 'path, blank for default' + pip install django-storages -Available Settings -~~~~~~~~~~~~~~~~~~ -**DBBACKUP\_FTP\_HOST** - Required +.. warning:: -Hostname for the server you wish to save your backups. + This storage doesn't use private connection for communcation, don't use it + if you're not sure about the link between client and server. -**DBBACKUP\_FTP\_USER** - Default: ``None`` +:: -Authentication login, do not use if anonymous. + DBBACKUP_STORAGE = 'storages.backends.ftp.FTPStorage + DBBACKUP_STORAGE_OPTIONS = { + 'location': 'ftp://user:pass@server:21' + } -**DBBACKUP\_FTP\_PASSWORD** - Default: ``None`` +Settings +~~~~~~~~ -Authentication password, do not use if there's no password. +**location** - Required -**DBBACKUP\_FTP\_PATH** - Default: ``'.'`` +A FTP URI with optional user, password and port. example: ``'ftp://anonymous@myftp.net'`` -The directory on remote FTP server you wish to save your backups. +**base_url** -.. note:: +URL that serves with HTTP(S) the files stored at this location. - As other updated storages, this settings will be deprecated in favor of - dictionary ``settings.DBBACKUP_STORAGE_OPTIONS``. +SFTP +---- -Django built-in storage API ---------------------------- +To store your database backups on a remote filesystem via SFTP, simply +setup the required settings below. -Django has its own storage API for managing media files. Dbbackup allows -you to use (third-part) Django storage backends. The default backend is -``FileSystemStorage``, which is integrated in Django but we invite you -to take a look at `django-storages-redux`_ which has a great collection of -storage backends. +Setup +~~~~~ -.. _django-storages-redux: https://github.com/jschneier/django-storages +**host** - Required -Setup using built-in storage API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Hostname or adress of the SSH server -To use Django's built-in `FileSystemStorage`_, add the following lines to -your ``settings.py``:: +**root_path** - Default ``~/`` - DBBACKUP_STORAGE = 'dbbackup.storage.builtin_django' - # Default - # DBBACKUP_DJANGO_STORAGE = 'django.core.file.storages.FileSystemStorage' - DBBACKUP_STORAGE_OPTIONS = {'location': '/mybackupdir/'} +Jail storage to this directory -.. _FileSystemStorage: https://docs.djangoproject.com/en/1.8/ref/files/storage/#the-filesystemstorage-class +**params** - Default ``{}`` -``'dbbackup.storage.builtin_django'`` is a wrapper for use the Django storage -defined in ``DBBACKUP_DJANGO_STORAGE`` with the options defined in -``DBBACKUP_STORAGE_OPTIONS``. +Arugment used by meth:`paramikor.SSHClient.connect()`. +See `paramiko SSHClient.connect() documentation`_ for details. -Used settings -~~~~~~~~~~~~~ +.. _`paramiko SSHClient.connect() documentation`: http://docs.paramiko.org/en/latest/api/client.html#paramiko.client.SSHClient.connect -**DBBACKUP_DJANGO_STORAGE** - Default: ``'django.core.file.storages.FileSystemStorage'`` +**interactive** - Default ``False`` -Path to a Django Storage class (in Python dot style). +A boolean indicating whether to prompt for a password if the connection cannot +be made using keys, and there is not already a password in ``params``. -.. warning:: +**file_mode** - Do not use a Django storage backend without configuring its options, - otherwise you will risk mixing media files (with public access) and - backups (strictly private). +UID of the account that should be set as owner of the files on the remote. -**DBBACKUP_STORAGE_OPTIONS** - Default: ``{}`` +**dir_mode** -Dictionary used to instantiate a Django Storage class. For example, the -``location`` key customizes the directory for ``FileSystemStorage``. +GID of the group that should be set on the files on the remote host. -Write your custom storage -------------------------- +**known_host_file** -If you wish to build your own, extend ``dbbackup.storage.base.BaseStorage`` -and point your ``settings.DBBACKUP_STORAGE`` to -``'my_storage.backend.ClassName'``. +Absolute path of know host file, if it isn't set ``"~/.ssh/known_hosts"`` will be used. diff --git a/functional.sh b/functional.sh index d18495c5..aa72a99b 100755 --- a/functional.sh +++ b/functional.sh @@ -49,9 +49,9 @@ main () { export DATABASE_URL="sqlite:///$DATABASE_FILE" fi export PYTHON=${PYTHON:-python} - export STORAGE="dbbackup.storage.filesystem_storage" + export STORAGE="${STORAGE:-django.core.files.storage.FileSystemStorage}" export STORAGE_LOCATION="/tmp/backups/" - export STORAGE_OPTIONS="location=${STORAGE_LOCATION}" + export STORAGE_OPTIONS="${STORAGE_OPTIONS:-location=$STORAGE_LOCATION}" export MEDIA_ROOT="/tmp/media/" make_db_test From 8795ebcd7bf67a1ea5013afb032dd92a5ccac819 Mon Sep 17 00:00:00 2001 From: ZuluPro Date: Thu, 28 Jul 2016 12:03:49 -0400 Subject: [PATCH 2/2] Rebased from master --- dbbackup/management/commands/_base.py | 16 +++++++--------- dbbackup/management/commands/mediarestore.py | 4 ++-- dbbackup/storage.py | 5 ++++- dbbackup/tests/commands/test_base.py | 12 +++++++----- dbbackup/tests/commands/test_dbrestore.py | 7 ++++--- dbbackup/tests/commands/test_mediabackup.py | 6 +++--- dbbackup/tests/utils.py | 5 ++--- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/dbbackup/management/commands/_base.py b/dbbackup/management/commands/_base.py index 1d7bccf6..d895e045 100644 --- a/dbbackup/management/commands/_base.py +++ b/dbbackup/management/commands/_base.py @@ -9,7 +9,7 @@ from django.core.management.base import BaseCommand, LabelCommand, CommandError from django.utils import six -from ...storage.base import StorageError +from ...storage import StorageError input = raw_input if six.PY2 else input # @ReservedAssignment @@ -43,9 +43,7 @@ def read_from_storage(self, path): return self.storage.read_file(path) def write_to_storage(self, file, path): - self.logger.info("Writing file to %s: %s, filename: %s", - self.storage.name, self.storage.backup_dir, - path) + self.logger.info("Writing file to %s", path) self.storage.write_file(file, path) def read_local_file(self, path): @@ -88,8 +86,8 @@ def _cleanup_old_backups(self): DBBACKUP_CLEANUP_KEEP and any backups that occur on first of the month. """ # database = self.database if self.content_type == 'db' else None - file_list = self.storage.clean_old_backups(encrypted=self.encrypt, - compressed=self.compress, - content_type=self.content_type) - # TODO: Make better filter - # database=database) + self.storage.clean_old_backups(encrypted=self.encrypt, + compressed=self.compress, + content_type=self.content_type) + # TODO: Make better filter + # database=database) diff --git a/dbbackup/management/commands/mediarestore.py b/dbbackup/management/commands/mediarestore.py index 13cf3a2e..54d373ed 100644 --- a/dbbackup/management/commands/mediarestore.py +++ b/dbbackup/management/commands/mediarestore.py @@ -8,7 +8,7 @@ from django.core.files.storage import get_storage_class from ._base import BaseDbBackupCommand -from ...storage.base import BaseStorage, StorageError +from ...storage import get_storage, StorageError from ... import utils @@ -39,7 +39,7 @@ def handle(self, *args, **options): self.replace = options.get('replace') self.passphrase = options.get('passphrase') self.interactive = options.get('interactive') - self.storage = BaseStorage.storage_factory() + self.storage = get_storage() self.media_storage = get_storage_class()() self._restore_backup() diff --git a/dbbackup/storage.py b/dbbackup/storage.py index e91cb4b7..ca0cf1e0 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -79,7 +79,10 @@ def write_file(self, filehandle, filename): def read_file(self, filepath): self.logger.debug('Reading file %s', filepath) - return self.storage.open(name=filepath, mode='rb') + file_ = self.storage.open(name=filepath, mode='rb') + if file_.name is None: + file_.name = filepath + return file_ def list_backups(self, encrypted=None, compressed=None, content_type=None, database=None): diff --git a/dbbackup/tests/commands/test_base.py b/dbbackup/tests/commands/test_base.py index 691574eb..196e162d 100644 --- a/dbbackup/tests/commands/test_base.py +++ b/dbbackup/tests/commands/test_base.py @@ -5,8 +5,10 @@ from mock import patch from django.test import TestCase from django.utils import six +from django.core.files import File from dbbackup.management.commands._base import BaseDbBackupCommand -from dbbackup.tests.utils import (FakeStorage, DEV_NULL, HANDLED_FILES) +from dbbackup.storage import get_storage +from dbbackup.tests.utils import DEV_NULL, HANDLED_FILES class BaseDbBackupCommandSetLoggerLevelTest(TestCase): @@ -33,10 +35,10 @@ class BaseDbBackupCommandMethodsTest(TestCase): def setUp(self): HANDLED_FILES.clean() self.command = BaseDbBackupCommand() - self.command.storage = FakeStorage() + self.command.storage = get_storage() def test_read_from_storage(self): - HANDLED_FILES['written_files'].append(['foo', six.BytesIO(b'bar')]) + HANDLED_FILES['written_files'].append(['foo', File(six.BytesIO(b'bar'))]) file_ = self.command.read_from_storage('foo') self.assertEqual(file_.read(), b'bar') @@ -54,7 +56,7 @@ def test_read_local_file(self): os.remove(self.command.path) def test_write_local_file(self): - fd, path = six.BytesIO(b"foo"), '/tmp/foo.bak' + fd, path = File(six.BytesIO(b"foo")), '/tmp/foo.bak' self.command.write_local_file(fd, path) self.assertTrue(os.path.exists(path)) # tearDown @@ -90,7 +92,7 @@ def setUp(self): self.command.encrypt = False self.command.compress = False self.command.servername = 'foo-server' - self.command.storage = FakeStorage() + self.command.storage = get_storage() HANDLED_FILES['written_files'] = [(f, None) for f in [ '2015-02-06-042810.tar', '2015-02-07-042810.tar', diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index f45431bc..40979969 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -7,6 +7,7 @@ from django.test import TestCase from django.core.management.base import CommandError +from django.core.files import File from django.conf import settings from dbbackup import utils @@ -41,7 +42,7 @@ def tearDown(self): def test_no_filename(self, *args): # Prepare backup HANDLED_FILES['written_files'].append( - (utils.filename_generate(TEST_DATABASE), get_dump())) + (utils.filename_generate(TEST_DATABASE), File(get_dump()))) # Check self.command.path = None self.command.filename = None @@ -57,7 +58,7 @@ def test_uncompress(self, *args): self.command.path = None compressed_file, self.command.filename = utils.compress_file(get_dump(), get_dump_name()) HANDLED_FILES['written_files'].append( - (self.command.filename, compressed_file) + (self.command.filename, File(compressed_file)) ) self.command.uncompress = True self.command._restore_backup() @@ -68,7 +69,7 @@ def test_decrypt(self, *args): self.command.decrypt = True encrypted_file, self.command.filename = utils.encrypt_file(get_dump(), get_dump_name()) HANDLED_FILES['written_files'].append( - (self.command.filename, encrypted_file) + (self.command.filename, File(encrypted_file)) ) self.command._restore_backup() diff --git a/dbbackup/tests/commands/test_mediabackup.py b/dbbackup/tests/commands/test_mediabackup.py index bf694574..454aa89f 100644 --- a/dbbackup/tests/commands/test_mediabackup.py +++ b/dbbackup/tests/commands/test_mediabackup.py @@ -4,8 +4,8 @@ from django.test import TestCase from django.core.files.storage import get_storage_class from dbbackup.management.commands.mediabackup import Command as DbbackupCommand -from dbbackup.tests.utils import (FakeStorage, DEV_NULL, HANDLED_FILES, - add_public_gpg) +from dbbackup.storage import get_storage +from dbbackup.tests.utils import DEV_NULL, HANDLED_FILES, add_public_gpg class MediabackupBackupMediafilesTest(TestCase): @@ -13,7 +13,7 @@ def setUp(self): HANDLED_FILES.clean() self.command = DbbackupCommand() self.command.servername = 'foo-server' - self.command.storage = FakeStorage() + self.command.storage = get_storage() self.command.stdout = DEV_NULL self.command.compress = False self.command.encrypt = False diff --git a/dbbackup/tests/utils.py b/dbbackup/tests/utils.py index 6339b407..24b55752 100644 --- a/dbbackup/tests/utils.py +++ b/dbbackup/tests/utils.py @@ -2,6 +2,7 @@ import subprocess from django.conf import settings from django.utils import six, timezone +from django.core.files import File from django.core.files.storage import Storage from dbbackup.db.base import get_connector @@ -62,14 +63,12 @@ def _open(self, name, mode='rb'): return file_ def _save(self, name, content): - HANDLED_FILES['written_files'].append((name, content)) + HANDLED_FILES['written_files'].append((name, File(content))) return name def delete(self, name): HANDLED_FILES['deleted_files'].append(name) -Storage = FakeStorage - def clean_gpg_keys(): try: