Merge branch 'release/v1.0.1'

.travis.yml

@@ -11,8 +11,8 @@ install:
 script:
   - flake8 .
   - coverage run --source goutte -m pytest -v
-  # - coverage report
-  # - coveralls
+  - coverage report
+  - coveralls
 
 before_deploy:
   - poetry build
@@ -32,3 +32,5 @@ deploy:
     on:
       tags: true
       python: 3.6
+after_deploy:
+  - curl -XPOST $DOCKER_HOOK

Dockerfile

@@ -2,11 +2,9 @@ FROM python:3.6-alpine
 
 RUN pip3 install goutte
 
-ENV GOUTTE_CONFIG goutte.yml
+ENV GOUTTE_CONFIG goutte.toml
 ENV GOUTTE_DO_TOKEN ''
 
 WORKDIR /goutte
 
 ENTRYPOINT ["goutte"]
-
-CMD ["$GOUTTE_CONFIG", "$GOUTTE_DO_TOKEN"]

README.md

@@ -1,17 +1,15 @@
-# goutte
+# goutte <a href="https://travis-ci.org/tomochain/goutte"><img align="right" src="https://travis-ci.org/tomochain/goutte.svg?branch=develop"></a>
 DigitalOcean doesn't propose any way of automating snapshots.
 There are [some SaaS](https://snapshooter.io/) that can take care of it but paying to execute some API requests seemed a bit off.
 
-That's why we developed a simple script which you can run with cron jobs or in CI services like Travis for free.
+That's why we developed a simple script which can run with cron jobs or in CI services like Travis for free.
+We use it daily to manage [our backups](https://github.com/tomochain/backups).
 
-## TODO
-- [x] Configuration from a single TOML file
-- [x] Droplets snapshots
-- [x] Droplets snapshots pruning
-- [x] Volume snapshots
-- [x] Volume snapshots pruning
-- [ ] Slack alerting
-- [ ] Add droplets and volumes by tag
+It includes:
+- Snapshoting droplets
+- Snapshoting volumes
+- Retention policy
+- Pruning snapshots
 
 ## Requirements
 - Python ^3.6
@@ -68,7 +66,7 @@ Options:
 Running "snapshot only" for a configuration file containing one droplet and one volume:
 ```bash
 $ goutte goutte.toml $do_token --only snapshot
-13:32:48 - INFO - Starting goutte v1.0.0
+13:32:48 - INFO - Starting goutte v1.0.1
 13:32:52 - INFO - sgp1-website-01 - Snapshot (goutte-sgp1-website-01-20181220-56bde)
 13:32:59 - INFO - sgp1-mariadb-01 - Snapshot (goutte-sgp1-mariadb-01-20181220-3673d)
 ```
@@ -84,10 +82,33 @@ docker run \
   tomochain:goutte
 ```
 
-## Automating
-You can easily automate it via cron job or by leveraging free CI tools like Travis.
-We provided and example travis configuration in `travis.example.yml`.
+## Automating with Travis
+You can easily automate it via cron job but the easiest way would be by leveraging free CI tools like Travis.
 
-You just need to set the environment variables on the Travis website and schedule it with the frequency of your backups.
+1. You can create a repo which contains your `goutte.toml` configuration and the following travis file `.travis.yml` :
 
-TODO
+```yml
+language: python
+python: 3.6
+
+install:
+  - pip install goutte
+
+script:
+  - goutte goutte.toml # Don't forget to set GOUTTE_DO_TOKEN in Travis config
+```
+
+2. Enable the repo in Travis and then go to the configuration
+3. Add the environment variable GOUTTE_DO_TOKEN with the value of your DigitalOcean API key
+4. Enable daily cron job
+5. You're good to go, goutte will run everyday and take care of the snapshots.
+
+**Note**: You can have different retentions for different volumes by having multiple configurations.
+```yml
+# ...
+script:
+  - goutte 10days.toml
+  - goutte 1day.toml
+```
+
+You can see how we set it up for ourself [here](https://github.com/tomochain/backups).

goutte/__init__.py

@@ -1,6 +1,6 @@
 import colorlog
 
-__version__ = '1.0.0'
+__version__ = '1.0.1'
 
 handler = colorlog.StreamHandler()
 handler.setFormatter(colorlog.ColoredFormatter(

goutte/main.py

@@ -12,6 +12,7 @@
 
 log = colorlog.getLogger(__name__)
 token = None
+error = 0
 
 
 @click.command(help='DigitalOcean snapshots automation.')
@@ -35,16 +36,17 @@ def entrypoint(config: click.File, do_token: str, only: str,
         log.debug(f'Will only {only}')
     _process_droplets(conf, only)
     _process_volumes(conf, only)
+    sys.exit(error)
 
 
 def _load_config(config: click.File) -> Dict[str, Dict]:
     """Return a config dict from a toml config file"""
     try:
         # TODO check minimum validity (retention)
         log.debug('Loading config from {}'.format(config.name))
-        config = toml.load(config)
-        assert config['retention']
-        return config
+        conf = toml.load(config)
+        assert conf['retention']
+        return conf
     except TypeError as e:
         log.critical('Could not read conf {}: {}'.format(config.name, e))
         sys.exit(1)
@@ -71,7 +73,7 @@ def _process_droplets(conf: Dict[str, Union[Dict[str, str], str]],
                 if only == 'snapshot' or not only:
                     _snapshot_droplet(droplet)
         else:
-            log.warn('No matching droplet found')
+            log.warning('No matching droplet found')
     except KeyError:
         droplets = None
     except KeyboardInterrupt:
@@ -93,9 +95,9 @@ def _process_volumes(conf: Dict[str, Union[Dict[str, str], str]],
                 if only == 'snapshot' or not only:
                     _snapshot_volume(volume)
         else:
-            log.warn('No matching volume found')
+            log.warning('No matching volume found')
     except KeyError:
-        volumes = None
+        pass
     except KeyboardInterrupt:
         log.critical('Received interuption signal')
         sys.exit(1)
@@ -121,6 +123,7 @@ def _get_droplets(names: List[str]) -> List[digitalocean.Droplet]:
 
 def _snapshot_droplet(droplet: digitalocean.Droplet) -> None:
     """Take a snapshot of a given droplet"""
+    global error
     name = 'goutte-{}-{}-{}'.format(
         droplet.name,
         date.today().strftime('%Y%m%d'),
@@ -129,20 +132,26 @@ def _snapshot_droplet(droplet: digitalocean.Droplet) -> None:
         droplet.take_snapshot(name)
         log.info(f'{droplet.name} - Snapshot ({name})')
     except digitalocean.baseapi.TokenError as e:
-        log.error(f'Token not valid: {e}')
+        log.error(f'Token not valid: {e}.')
+        error = 1
     except digitalocean.baseapi.DataReadError as e:
-        log.error(f'Could not read response: {e}')
+        log.error(f'Could not read response: {e}.')
+        error = 1
     except digitalocean.baseapi.JSONReadError as e:
-        log.error(f'Could not parse json: {e}')
+        log.error(f'Could not parse json: {e}.')
+        error = 1
     except digitalocean.baseapi.NotFoundError as e:
-        log.error(f'Ressource not found: {e}')
+        log.error(f'Ressource not found: {e}.')
+        error = 1
     except Exception as e:
-        log.error(f'Unexpected exception: {e}')
+        log.error(f'Unexpected exception: {e}.')
+        error = 1
 
 
 def _prune_droplet_snapshots(droplet: digitalocean.Droplet,
                              retention: int) -> None:
     """Prune goutte snapshots if tmore than the configured retention time"""
+    global error
     try:
         all_snapshots = _order_snapshots([
             digitalocean.Snapshot.get_object(
@@ -159,14 +168,19 @@ def _prune_droplet_snapshots(droplet: digitalocean.Droplet,
                 snapshot.destroy()
     except digitalocean.baseapi.TokenError as e:
         log.error(f'Token not valid: {e}.')
+        error = 1
     except digitalocean.baseapi.DataReadError as e:
         log.error(f'Could not read response: {e}.')
+        error = 1
     except digitalocean.baseapi.JSONReadError as e:
         log.error(f'Could not parse json: {e}.')
+        error = 1
     except digitalocean.baseapi.NotFoundError as e:
         log.error(f'Ressource not found: {e}.')
+        error = 1
     except Exception as e:
         log.error(f'Unexpected exception: {e}.')
+        error = 1
 
 
 def _get_volumes(names: List[str]) -> List[digitalocean.Volume]:
@@ -189,6 +203,7 @@ def _get_volumes(names: List[str]) -> List[digitalocean.Volume]:
 
 def _snapshot_volume(volume: digitalocean.Volume) -> None:
     """Take a snapshot of a given volume"""
+    global error
     name = 'goutte-{}-{}-{}'.format(
         volume.name,
         date.today().strftime('%Y%m%d'),
@@ -197,20 +212,26 @@ def _snapshot_volume(volume: digitalocean.Volume) -> None:
         volume.snapshot(name)
         log.info(f'{volume.name} - Snapshot ({name})')
     except digitalocean.baseapi.TokenError as e:
-        log.error(f'Token not valid: {e}')
+        log.error(f'Token not valid: {e}.')
+        error = 1
     except digitalocean.baseapi.DataReadError as e:
-        log.error(f'Could not read response: {e}')
+        log.error(f'Could not read response: {e}.')
+        error = 1
     except digitalocean.baseapi.JSONReadError as e:
-        log.error(f'Could not parse json: {e}')
+        log.error(f'Could not parse json: {e}.')
+        error = 1
     except digitalocean.baseapi.NotFoundError as e:
-        log.error(f'Ressource not found: {e}')
+        log.error(f'Ressource not found: {e}.')
+        error = 1
     except Exception as e:
-        log.error(f'Unexpected exception: {e}')
+        log.error(f'Unexpected exception: {e}.')
+        error = 1
 
 
 def _prune_volume_snapshots(volume: digitalocean.Volume,
                             retention: int) -> None:
     """Prune goutte snapshots if tmore than the configured retention time"""
+    global error
     try:
         all_snapshots = _order_snapshots(volume.get_snapshots())
         snapshots = [snapshot for snapshot in all_snapshots
@@ -223,14 +244,19 @@ def _prune_volume_snapshots(volume: digitalocean.Volume,
                 snapshot.destroy()
     except digitalocean.baseapi.TokenError as e:
         log.error(f'Token not valid: {e}.')
+        error = 1
     except digitalocean.baseapi.DataReadError as e:
         log.error(f'Could not read response: {e}.')
+        error = 1
     except digitalocean.baseapi.JSONReadError as e:
         log.error(f'Could not parse json: {e}.')
+        error = 1
     except digitalocean.baseapi.NotFoundError as e:
         log.error(f'Ressource not found: {e}.')
+        error = 1
     except Exception as e:
         log.error(f'Unexpected exception: {e}.')
+        error = 1
 
 
 def _order_snapshots(snapshots: List[digitalocean.Snapshot]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "goutte"
-version = "1.0.0"
+version = "1.0.1"
 description = "DigitalOcean snapshot automation service"
 readme = "README.md"
 license = "GPL-3.0+"

tests/mock.py

@@ -0,0 +1,63 @@
+def nothing(*args, **kwargs):
+    pass
+
+
+class Snapshot:
+    def __init__(self, created_at=None, name=None, id=None):
+        self.created_at = created_at
+        self.name = name
+        self.id = id
+
+    def destroy(self):
+        pass
+
+    @staticmethod
+    def get_object(api_token=None, snapshot_id=None):
+        if snapshot_id == '1337':
+            return Snapshot(name=f'snapshot{snapshot_id}', id=snapshot_id,
+                            created_at=f'{snapshot_id}')
+        else:
+            return Snapshot(name=f'goutte-snapshot{snapshot_id}',
+                            id=snapshot_id, created_at=f'{snapshot_id}')
+
+
+class Volume:
+    def __init__(self, name=None, snapshots=None, throw=None):
+        self.name = name
+        self.snapshots = snapshots
+        self.throw = throw
+
+    def get_snapshots(self):
+        return self.snapshots
+
+    def snapshot(self, name):
+        pass
+
+
+class Droplet:
+    def __init__(self, name=None, snapshot_ids=None):
+        self.name = name
+        self.snapshot_ids = snapshot_ids
+
+    def take_snapshot(self, name):
+        pass
+
+
+class Manager:
+    def __init__(self, token=None):
+        self.token = token
+
+    def get_all_volumes(self):
+        return [
+            Volume(name='testvol')
+        ]
+
+    def get_all_droplets(self):
+        return [
+            Droplet(name='testdroplet')
+        ]
+
+
+class File:
+    def __init__(self, name=None):
+        self.name = name