Replace mdsynthesis with datreant

.travis.yml

@@ -9,13 +9,13 @@ branches:
 cache:
   - pip: true
   - directories:
-    - $PIP_CACHE_DIR
+      - $PIP_CACHE_DIR
 
 env:
   global:
     - MAIN_CMD="pytest"
     - SETUP_CMD="mdbenchmark -v"
-    - CONDA_DEPENDENCIES="mdsynthesis jinja2 click pandas matplotlib xdg<2"
+    - CONDA_DEPENDENCIES="datreant jinja2 click==6.7 pandas matplotlib xdg<2"
     - CONDA_CHANNELS="conda-forge"
     - NUMPY_VERSION=stable
     - PIP_CACHE_DIR=$HOME/.cache/pip
@@ -30,11 +30,9 @@ jobs:
       env:
         - CODECOV="--cov mdbenchmark"
         - PYTEST_COV="pytest-cov codecov"
-    -
-      name: "Python 3.5"
+    - name: "Python 3.5"
       python: "3.5"
-    -
-      name: "Python 2.7"
+    - name: "Python 2.7"
       python: "2.7"
 
     - stage: "Linting and formatting"
@@ -43,26 +41,22 @@ jobs:
         - $UPGRADE_PIP
         - pip install black
       script: make reformat-check
-    -
-      name: "Linting with flake8"
+    - name: "Linting with flake8"
       install:
         - $UPGRADE_PIP
         - pip install flake8
       script: make flake8
-    -
-      name: "Sorting with isort"
+    - name: "Sorting with isort"
       install:
         - $UPGRADE_PIP
         - pip install isort
       script: make isort
-    -
-      name: "Lint reStructuredText"
+    - name: "Lint reStructuredText"
       install:
         - $UPGRADE_PIP
         - pip install restructuredtext_lint
       script: make rst-lint
-    -
-      name: "Try to build documentation"
+    - name: "Try to build documentation"
       install:
         - $UPGRADE_PIP
         - pip install . sphinx-click
@@ -80,7 +74,7 @@ jobs:
 
 install:
   - $UPGRADE_PIP
-  - pip install . mdbenchmark -v $PYTEST_COV
+  - pip install . -v $PYTEST_COV
 
 script:
   - pytest $CODECOV

DEVELOPER.rst

@@ -20,7 +20,7 @@ Create a virtual environment
 
 If you are using ``conda``, you can easily create a `conda environment`_::
 
-    $ conda create -n benchmark -c conda-forge mdsynthesis click
+    $ conda create -n benchmark -c conda-forge python=3
     $ source activate benchmark
 
 Make sure to activate the environment, before trying to install the package.

changelog/110.misc

@@ -0,0 +1 @@
+Replaced ``mdsynthesis`` with ``datreant`` and upgraded to the new ``datreant>=1.0`` format.

docs/docs-requirements.txt

@@ -4,7 +4,7 @@ python-Levenshtein==0.12.0
 matplotlib==2.2.3
 numpy==1.15.1
 pandas==0.23.4
-mdsynthesis==0.6.1
+datreant==1.0.2
 xdg==1.0.7
 jinja2==2.10
 -e .

mdbenchmark/analyze.py

@@ -19,8 +19,8 @@
 # along with MDBenchmark.  If not, see <http://www.gnu.org/licenses/>.
 import click
 
+import datreant as dtr
 import matplotlib.pyplot as plt
-import mdsynthesis as mds
 import numpy as np
 import pandas as pd
 from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
@@ -29,6 +29,7 @@
 from . import console
 from .cli import cli
 from .mdengines import detect_md_engine, utils
+from .migrations import mds_to_dtr
 from .plot import plot_over_group
 from .utils import generate_output_name, DataFrameFromBundle, PrintDataFrame
 
@@ -79,7 +80,11 @@ def analyze(directory, plot, ncores, output_name):
     `--output-name` option and a custom filename. To plot the results use
     `mdbenchmark plot`.
     """
-    bundle = mds.discover(directory)
+
+    # Migrate from MDBenchmark<2 to MDBenchmark=>2
+    mds_to_dtr.migrate_to_datreant(directory)
+
+    bundle = dtr.discover(directory)
 
     df = DataFrameFromBundle(bundle)
 

mdbenchmark/generate.py

@@ -19,13 +19,13 @@
 # along with MDBenchmark.  If not, see <http://www.gnu.org/licenses/>.
 import click
 
-import datreant.core as dtr
-import mdsynthesis as mds
+import datreant as dtr
 import pandas as pd
+
 from . import console, mdengines, utils
 from .cli import cli
 from .mdengines.utils import write_benchmark
-from .utils import DataFrameFromBundle, ConsolidateDataFrame, PrintDataFrame
+from .utils import ConsolidateDataFrame, DataFrameFromBundle, PrintDataFrame
 
 NAMD_WARNING = (
     "NAMD support is experimental. "

mdbenchmark/mdengines/gromacs.py

@@ -24,7 +24,6 @@
 from glob import glob
 from shutil import copyfile
 
-import mdsynthesis as mds
 import numpy as np
 
 from .. import console

mdbenchmark/mdengines/namd.py

@@ -22,7 +22,6 @@
 from glob import glob
 from shutil import copyfile
 
-import mdsynthesis as mds
 import numpy as np
 
 from .. import console

mdbenchmark/mdengines/utils.py

@@ -22,7 +22,7 @@
 from glob import glob
 from shutil import copyfile
 
-import mdsynthesis as mds
+import datreant as dtr
 import numpy as np
 
 from .. import console
@@ -135,7 +135,8 @@ def cleanup_before_restart(engine, sim):
     whitelist = [re.compile(fname) for fname in whitelist]
 
     files_found = []
-    for fname in sim.leaves:
+
+    for fname in sim.leaves():
         keep = False
         for wl in whitelist:
             if wl.match(str(fname)):
@@ -153,8 +154,8 @@ def write_benchmark(
     engine, base_directory, template, nodes, gpu, module, name, host, time
 ):
     """Generate a benchmark folder with the respective Sim object."""
-    # Create the `mds.Sim` object
-    sim = mds.Sim(base_directory["{}/".format(nodes)])
+    # Create the `dtr.Treant` object
+    sim = dtr.Treant(base_directory["{}/".format(nodes)])
 
     # Do MD engine specific things. Here we also format the name.
     name = engine.prepare_benchmark(name=name, sim=sim)

mdbenchmark/migrations/mds_to_dtr.py

@@ -0,0 +1,74 @@
+# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8
+#
+# MDBenchmark
+# Copyright (c) 2017-2018 The MDBenchmark development team and contributors
+# (see the file AUTHORS for the full list of names)
+#
+# MDBenchmark is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# MDBenchmark is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with MDBenchmark.  If not, see <http://www.gnu.org/licenses/>.
+import fnmatch
+import json
+import os
+from glob import glob
+
+import datreant as dtr
+
+from .. import console
+
+
+def search_mdsynthesis_sim_files(folder):
+    """Search for `Sim.*.json` files generated by `mdsynthesis`."""
+    bundles = []
+    for root, dirnames, filenames in os.walk(folder):
+        for filename in fnmatch.filter(filenames, "Sim*"):
+            bundles.append(os.path.join(root, filename))
+
+    return bundles
+
+
+def convert_to_datreant(bundles):
+    """Convert old `mdsynthesis` data structures to vanilla `datreant.
+
+    Converts old `Sim.[...].json` files used in MDSynthesis<1.0 to datreant>1.0
+    `.datreant` folders.
+    """
+    for bundle in bundles:
+        with open(bundle) as fh:
+            sim = json.load(fh)
+
+        folder = "/".join(bundle.split("/")[0:-1])
+        dtr.Treant(folder, categories=sim["categories"], tags=sim["tags"])
+
+        # We want to remove two files, both `Sim.*.json` and `.Sim.*.json.proxy`.
+        files_to_remove = [bundle, glob(os.path.join(folder, ".Sim*"))]
+        for file in files_to_remove:
+            # The proxy file will be a list.
+            # We try to split the globbing result, if we fail, we skip this loop.
+            if isinstance(file, list):
+                try:
+                    file = file[0]
+                except IndexError:
+                    continue
+            os.remove(file)
+
+
+def migrate_to_datreant(directory):
+    """Perform the `mdsynthesis` to `datreant` migration."""
+    bundles = search_mdsynthesis_sim_files(directory)
+    if bundles:
+        console.info("Converting old benchmark metadata to new format!")
+        convert_to_datreant(bundles)
+        console.info("Finished converting old benchmarks to new format!")
+
+    return

mdbenchmark/submit.py

@@ -23,15 +23,16 @@
 
 import click
 
-import mdsynthesis as mds
+import datreant as dtr
 import numpy as np
 import pandas as pd
 
 from . import console
 from .cli import cli
 from .mdengines import detect_md_engine
 from .mdengines.utils import cleanup_before_restart
-from .utils import DataFrameFromBundle, ConsolidateDataFrame, PrintDataFrame
+from .migrations import mds_to_dtr
+from .utils import ConsolidateDataFrame, DataFrameFromBundle, PrintDataFrame
 
 PATHS = os.environ["PATH"].split(":")
 BATCH_SYSTEMS = {"slurm": "sbatch", "sge": "qsub", "Loadleveler": "llsubmit"}
@@ -77,7 +78,10 @@ def submit(directory, force_restart, yes):
     Only runs benchmarks that were not already started. Can be overwritten with
     `--force`.
     """
-    bundle = mds.discover(directory)
+    # Migrate from MDBenchmark<2 to MDBenchmark=>2
+    mds_to_dtr.migrate_to_datreant(directory)
+
+    bundle = dtr.discover(directory)
 
     # Exit if no bundles were found in the current directory.
     if not bundle:

mdbenchmark/tests/data/analyze-files-gromacs/1/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}

mdbenchmark/tests/data/analyze-files-gromacs/2/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}

mdbenchmark/tests/data/analyze-files-gromacs/3/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}

mdbenchmark/tests/data/analyze-files-gromacs/4/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}

mdbenchmark/tests/data/analyze-files-gromacs/5/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}

mdbenchmark/tests/data/analyze-files-namd/1/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 1}

mdbenchmark/tests/data/analyze-files-namd/2/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 2}

mdbenchmark/tests/data/analyze-files-w-errors/1/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}

mdbenchmark/tests/data/analyze-files-w-errors/2/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}

mdbenchmark/tests/data/analyze-files-w-errors/3/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}

mdbenchmark/tests/data/analyze-files-w-errors/4/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}

mdbenchmark/tests/data/analyze-files-w-errors/5/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}

mdbenchmark/tests/data/analyze-files-w-errors/6/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 6}

mdbenchmark/tests/data/analyze-files-w-errors/7/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 7}

mdbenchmark/tests/data/analyze-files-w-errors/8/.datreant/categories.json

@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 8}

mdbenchmark/tests/mdengines/test_gromacs.py

@@ -17,11 +17,11 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with MDBenchmark.  If not, see <http://www.gnu.org/licenses/>.
-import datreant.core as dtr
-import numpy as np
-import pytest
 from six.moves import StringIO
 
+import datreant as dtr
+import numpy as np
+import pytest
 from mdbenchmark.mdengines import gromacs, utils