diff --git a/.travis.yml b/.travis.yml
index 63d65b2d..ab118afa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,13 +9,13 @@ branches:
cache:
- pip: true
- directories:
- - $PIP_CACHE_DIR
+ - $PIP_CACHE_DIR
env:
global:
- MAIN_CMD="pytest"
- SETUP_CMD="mdbenchmark -v"
- - CONDA_DEPENDENCIES="mdsynthesis jinja2 click pandas matplotlib xdg<2"
+ - CONDA_DEPENDENCIES="datreant jinja2 click==6.7 pandas matplotlib xdg<2"
- CONDA_CHANNELS="conda-forge"
- NUMPY_VERSION=stable
- PIP_CACHE_DIR=$HOME/.cache/pip
@@ -30,11 +30,9 @@ jobs:
env:
- CODECOV="--cov mdbenchmark"
- PYTEST_COV="pytest-cov codecov"
- -
- name: "Python 3.5"
+ - name: "Python 3.5"
python: "3.5"
- -
- name: "Python 2.7"
+ - name: "Python 2.7"
python: "2.7"
- stage: "Linting and formatting"
@@ -43,26 +41,22 @@ jobs:
- $UPGRADE_PIP
- pip install black
script: make reformat-check
- -
- name: "Linting with flake8"
+ - name: "Linting with flake8"
install:
- $UPGRADE_PIP
- pip install flake8
script: make flake8
- -
- name: "Sorting with isort"
+ - name: "Sorting with isort"
install:
- $UPGRADE_PIP
- pip install isort
script: make isort
- -
- name: "Lint reStructuredText"
+ - name: "Lint reStructuredText"
install:
- $UPGRADE_PIP
- pip install restructuredtext_lint
script: make rst-lint
- -
- name: "Try to build documentation"
+ - name: "Try to build documentation"
install:
- $UPGRADE_PIP
- pip install . sphinx-click
@@ -80,7 +74,7 @@ jobs:
install:
- $UPGRADE_PIP
- - pip install . mdbenchmark -v $PYTEST_COV
+ - pip install . -v $PYTEST_COV
script:
- pytest $CODECOV
diff --git a/DEVELOPER.rst b/DEVELOPER.rst
index 98fb6e81..ce013fda 100644
--- a/DEVELOPER.rst
+++ b/DEVELOPER.rst
@@ -20,7 +20,7 @@ Create a virtual environment
If you are using ``conda``, you can easily create a `conda environment`_::
- $ conda create -n benchmark -c conda-forge mdsynthesis click
+ $ conda create -n benchmark -c conda-forge python=3
$ source activate benchmark
Make sure to activate the environment, before trying to install the package.
diff --git a/changelog/110.misc b/changelog/110.misc
new file mode 100644
index 00000000..7c1647e7
--- /dev/null
+++ b/changelog/110.misc
@@ -0,0 +1 @@
+Replaced ``mdsynthesis`` with ``datreant`` and upgraded to the new ``datreant>=1.0`` format.
\ No newline at end of file
diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt
index 84734dd2..1fd1f2c5 100644
--- a/docs/docs-requirements.txt
+++ b/docs/docs-requirements.txt
@@ -4,7 +4,7 @@ python-Levenshtein==0.12.0
matplotlib==2.2.3
numpy==1.15.1
pandas==0.23.4
-mdsynthesis==0.6.1
+datreant==1.0.2
xdg==1.0.7
jinja2==2.10
-e .
diff --git a/mdbenchmark/analyze.py b/mdbenchmark/analyze.py
index 070ac246..489131f2 100644
--- a/mdbenchmark/analyze.py
+++ b/mdbenchmark/analyze.py
@@ -19,8 +19,8 @@
# along with MDBenchmark. If not, see .
import click
+import datreant as dtr
import matplotlib.pyplot as plt
-import mdsynthesis as mds
import numpy as np
import pandas as pd
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
@@ -29,6 +29,7 @@
from . import console
from .cli import cli
from .mdengines import detect_md_engine, utils
+from .migrations import mds_to_dtr
from .plot import plot_over_group
from .utils import generate_output_name, DataFrameFromBundle, PrintDataFrame
@@ -79,7 +80,11 @@ def analyze(directory, plot, ncores, output_name):
`--output-name` option and a custom filename. To plot the results use
`mdbenchmark plot`.
"""
- bundle = mds.discover(directory)
+
+ # Migrate from MDBenchmark<2 to MDBenchmark=>2
+ mds_to_dtr.migrate_to_datreant(directory)
+
+ bundle = dtr.discover(directory)
df = DataFrameFromBundle(bundle)
diff --git a/mdbenchmark/generate.py b/mdbenchmark/generate.py
index 37ed0132..216c7c57 100644
--- a/mdbenchmark/generate.py
+++ b/mdbenchmark/generate.py
@@ -19,13 +19,13 @@
# along with MDBenchmark. If not, see .
import click
-import datreant.core as dtr
-import mdsynthesis as mds
+import datreant as dtr
import pandas as pd
+
from . import console, mdengines, utils
from .cli import cli
from .mdengines.utils import write_benchmark
-from .utils import DataFrameFromBundle, ConsolidateDataFrame, PrintDataFrame
+from .utils import ConsolidateDataFrame, DataFrameFromBundle, PrintDataFrame
NAMD_WARNING = (
"NAMD support is experimental. "
diff --git a/mdbenchmark/mdengines/gromacs.py b/mdbenchmark/mdengines/gromacs.py
index f40a80e7..36b219e5 100644
--- a/mdbenchmark/mdengines/gromacs.py
+++ b/mdbenchmark/mdengines/gromacs.py
@@ -24,7 +24,6 @@
from glob import glob
from shutil import copyfile
-import mdsynthesis as mds
import numpy as np
from .. import console
diff --git a/mdbenchmark/mdengines/namd.py b/mdbenchmark/mdengines/namd.py
index 472b3c31..1aff908f 100644
--- a/mdbenchmark/mdengines/namd.py
+++ b/mdbenchmark/mdengines/namd.py
@@ -22,7 +22,6 @@
from glob import glob
from shutil import copyfile
-import mdsynthesis as mds
import numpy as np
from .. import console
diff --git a/mdbenchmark/mdengines/utils.py b/mdbenchmark/mdengines/utils.py
index 0fb506fb..552876bd 100644
--- a/mdbenchmark/mdengines/utils.py
+++ b/mdbenchmark/mdengines/utils.py
@@ -22,7 +22,7 @@
from glob import glob
from shutil import copyfile
-import mdsynthesis as mds
+import datreant as dtr
import numpy as np
from .. import console
@@ -135,7 +135,8 @@ def cleanup_before_restart(engine, sim):
whitelist = [re.compile(fname) for fname in whitelist]
files_found = []
- for fname in sim.leaves:
+
+ for fname in sim.leaves():
keep = False
for wl in whitelist:
if wl.match(str(fname)):
@@ -153,8 +154,8 @@ def write_benchmark(
engine, base_directory, template, nodes, gpu, module, name, host, time
):
"""Generate a benchmark folder with the respective Sim object."""
- # Create the `mds.Sim` object
- sim = mds.Sim(base_directory["{}/".format(nodes)])
+ # Create the `dtr.Treant` object
+ sim = dtr.Treant(base_directory["{}/".format(nodes)])
# Do MD engine specific things. Here we also format the name.
name = engine.prepare_benchmark(name=name, sim=sim)
diff --git a/mdbenchmark/migrations/__init__.py b/mdbenchmark/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/mdbenchmark/migrations/mds_to_dtr.py b/mdbenchmark/migrations/mds_to_dtr.py
new file mode 100644
index 00000000..6eaa9744
--- /dev/null
+++ b/mdbenchmark/migrations/mds_to_dtr.py
@@ -0,0 +1,74 @@
+# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8
+#
+# MDBenchmark
+# Copyright (c) 2017-2018 The MDBenchmark development team and contributors
+# (see the file AUTHORS for the full list of names)
+#
+# MDBenchmark is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# MDBenchmark is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with MDBenchmark. If not, see .
+import fnmatch
+import json
+import os
+from glob import glob
+
+import datreant as dtr
+
+from .. import console
+
+
+def search_mdsynthesis_sim_files(folder):
+ """Search for `Sim.*.json` files generated by `mdsynthesis`."""
+ bundles = []
+ for root, dirnames, filenames in os.walk(folder):
+ for filename in fnmatch.filter(filenames, "Sim*"):
+ bundles.append(os.path.join(root, filename))
+
+ return bundles
+
+
+def convert_to_datreant(bundles):
+ """Convert old `mdsynthesis` data structures to vanilla `datreant.
+
+ Converts old `Sim.[...].json` files used in MDSynthesis<1.0 to datreant>1.0
+ `.datreant` folders.
+ """
+ for bundle in bundles:
+ with open(bundle) as fh:
+ sim = json.load(fh)
+
+ folder = "/".join(bundle.split("/")[0:-1])
+ dtr.Treant(folder, categories=sim["categories"], tags=sim["tags"])
+
+ # We want to remove two files, both `Sim.*.json` and `.Sim.*.json.proxy`.
+ files_to_remove = [bundle, glob(os.path.join(folder, ".Sim*"))]
+ for file in files_to_remove:
+ # The proxy file will be a list.
+ # We try to split the globbing result, if we fail, we skip this loop.
+ if isinstance(file, list):
+ try:
+ file = file[0]
+ except IndexError:
+ continue
+ os.remove(file)
+
+
+def migrate_to_datreant(directory):
+ """Perform the `mdsynthesis` to `datreant` migration."""
+ bundles = search_mdsynthesis_sim_files(directory)
+ if bundles:
+ console.info("Converting old benchmark metadata to new format!")
+ convert_to_datreant(bundles)
+ console.info("Finished converting old benchmarks to new format!")
+
+ return
diff --git a/mdbenchmark/submit.py b/mdbenchmark/submit.py
index 2c0aff0e..e4ec2db6 100644
--- a/mdbenchmark/submit.py
+++ b/mdbenchmark/submit.py
@@ -23,7 +23,7 @@
import click
-import mdsynthesis as mds
+import datreant as dtr
import numpy as np
import pandas as pd
@@ -31,7 +31,8 @@
from .cli import cli
from .mdengines import detect_md_engine
from .mdengines.utils import cleanup_before_restart
-from .utils import DataFrameFromBundle, ConsolidateDataFrame, PrintDataFrame
+from .migrations import mds_to_dtr
+from .utils import ConsolidateDataFrame, DataFrameFromBundle, PrintDataFrame
PATHS = os.environ["PATH"].split(":")
BATCH_SYSTEMS = {"slurm": "sbatch", "sge": "qsub", "Loadleveler": "llsubmit"}
@@ -77,7 +78,10 @@ def submit(directory, force_restart, yes):
Only runs benchmarks that were not already started. Can be overwritten with
`--force`.
"""
- bundle = mds.discover(directory)
+ # Migrate from MDBenchmark<2 to MDBenchmark=>2
+ mds_to_dtr.migrate_to_datreant(directory)
+
+ bundle = dtr.discover(directory)
# Exit if no bundles were found in the current directory.
if not bundle:
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/1/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-gromacs/1/.datreant/categories.json
new file mode 100644
index 00000000..6b1d0f77
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-gromacs/1/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json b/mdbenchmark/tests/data/analyze-files-gromacs/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json
deleted file mode 100644
index 9866f10f..00000000
--- a/mdbenchmark/tests/data/analyze-files-gromacs/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/2/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-gromacs/2/.datreant/categories.json
new file mode 100644
index 00000000..b2cca829
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-gromacs/2/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json b/mdbenchmark/tests/data/analyze-files-gromacs/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json
deleted file mode 100644
index 668360bd..00000000
--- a/mdbenchmark/tests/data/analyze-files-gromacs/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/3/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-gromacs/3/.datreant/categories.json
new file mode 100644
index 00000000..5d9a394b
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-gromacs/3/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json b/mdbenchmark/tests/data/analyze-files-gromacs/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json
deleted file mode 100644
index 7d9af132..00000000
--- a/mdbenchmark/tests/data/analyze-files-gromacs/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/4/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-gromacs/4/.datreant/categories.json
new file mode 100644
index 00000000..c3150a73
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-gromacs/4/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json b/mdbenchmark/tests/data/analyze-files-gromacs/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json
deleted file mode 100644
index 811bb073..00000000
--- a/mdbenchmark/tests/data/analyze-files-gromacs/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/5/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-gromacs/5/.datreant/categories.json
new file mode 100644
index 00000000..b8ef25d3
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-gromacs/5/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-gromacs/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json b/mdbenchmark/tests/data/analyze-files-gromacs/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json
deleted file mode 100644
index 12508654..00000000
--- a/mdbenchmark/tests/data/analyze-files-gromacs/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-namd/1/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-namd/1/.datreant/categories.json
new file mode 100644
index 00000000..00cd1565
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-namd/1/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 1}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-namd/1/Sim.59e83777-6ebb-493e-a478-7195a4261493.json b/mdbenchmark/tests/data/analyze-files-namd/1/Sim.59e83777-6ebb-493e-a478-7195a4261493.json
deleted file mode 100644
index 7b61318d..00000000
--- a/mdbenchmark/tests/data/analyze-files-namd/1/Sim.59e83777-6ebb-493e-a478-7195a4261493.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 1}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-namd/2/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-namd/2/.datreant/categories.json
new file mode 100644
index 00000000..99adbf00
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-namd/2/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 2}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-namd/2/Sim.6b71a7b2-6a3c-4a5a-a740-f4d2f361c337.json b/mdbenchmark/tests/data/analyze-files-namd/2/Sim.6b71a7b2-6a3c-4a5a-a740-f4d2f361c337.json
deleted file mode 100644
index 754590ad..00000000
--- a/mdbenchmark/tests/data/analyze-files-namd/2/Sim.6b71a7b2-6a3c-4a5a-a740-f4d2f361c337.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "apoa1", "started": false, "module": "namd", "host": "draco", "time": 15, "gpu": false, "nodes": 2}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/1/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/1/.datreant/categories.json
new file mode 100644
index 00000000..6b1d0f77
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/1/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json b/mdbenchmark/tests/data/analyze-files-w-errors/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json
deleted file mode 100644
index 9866f10f..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/1/Sim.f2c3a7de-e167-419e-a8b6-2286f27f674c.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 1}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/2/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/2/.datreant/categories.json
new file mode 100644
index 00000000..b2cca829
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/2/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json b/mdbenchmark/tests/data/analyze-files-w-errors/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json
deleted file mode 100644
index 668360bd..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/2/Sim.7c0db0de-70ce-4d55-b53b-d31c904f11cb.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 2}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/3/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/3/.datreant/categories.json
new file mode 100644
index 00000000..5d9a394b
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/3/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json b/mdbenchmark/tests/data/analyze-files-w-errors/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json
deleted file mode 100644
index 7d9af132..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/3/Sim.b4ca19ea-a5ba-49be-8f27-aab19938d888.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 3}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/4/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/4/.datreant/categories.json
new file mode 100644
index 00000000..c3150a73
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/4/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json b/mdbenchmark/tests/data/analyze-files-w-errors/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json
deleted file mode 100644
index 811bb073..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/4/Sim.059d59b5-c86c-4f50-9ff6-6a2061e26bf6.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 4}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/5/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/5/.datreant/categories.json
new file mode 100644
index 00000000..b8ef25d3
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/5/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json b/mdbenchmark/tests/data/analyze-files-w-errors/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json
deleted file mode 100644
index 12508654..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/5/Sim.07ec0ddb-602d-4717-8464-22fc7dd50388.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 5}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/6/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/6/.datreant/categories.json
new file mode 100644
index 00000000..01c025e1
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/6/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 6}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/6/Sim.215509f9-67dd-43d6-b437-99423e68d2d2.json b/mdbenchmark/tests/data/analyze-files-w-errors/6/Sim.215509f9-67dd-43d6-b437-99423e68d2d2.json
deleted file mode 100644
index 3d48a661..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/6/Sim.215509f9-67dd-43d6-b437-99423e68d2d2.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 6}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/7/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/7/.datreant/categories.json
new file mode 100644
index 00000000..096b6c2f
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/7/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 7}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/7/Sim.6a47b788-410a-4b9d-9ab3-9492593b8cae.json b/mdbenchmark/tests/data/analyze-files-w-errors/7/Sim.6a47b788-410a-4b9d-9ab3-9492593b8cae.json
deleted file mode 100644
index 46da7014..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/7/Sim.6a47b788-410a-4b9d-9ab3-9492593b8cae.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 7}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/8/.datreant/categories.json b/mdbenchmark/tests/data/analyze-files-w-errors/8/.datreant/categories.json
new file mode 100644
index 00000000..f45ece8d
--- /dev/null
+++ b/mdbenchmark/tests/data/analyze-files-w-errors/8/.datreant/categories.json
@@ -0,0 +1 @@
+{"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 8}
\ No newline at end of file
diff --git a/mdbenchmark/tests/data/analyze-files-w-errors/8/Sim.dc08e3e4-a081-4bef-9ca1-1a9080669655.json b/mdbenchmark/tests/data/analyze-files-w-errors/8/Sim.dc08e3e4-a081-4bef-9ca1-1a9080669655.json
deleted file mode 100644
index 5801c595..00000000
--- a/mdbenchmark/tests/data/analyze-files-w-errors/8/Sim.dc08e3e4-a081-4bef-9ca1-1a9080669655.json
+++ /dev/null
@@ -1 +0,0 @@
-{"mdsynthesis": {}, "categories": {"name": "bench", "started": true, "module": "gromacs/2016.3", "host": "draco", "time": 15, "gpu": false, "nodes": 8}, "tags": []}
\ No newline at end of file
diff --git a/mdbenchmark/tests/mdengines/test_gromacs.py b/mdbenchmark/tests/mdengines/test_gromacs.py
index 9f9b5a28..5c441990 100644
--- a/mdbenchmark/tests/mdengines/test_gromacs.py
+++ b/mdbenchmark/tests/mdengines/test_gromacs.py
@@ -17,11 +17,11 @@
#
# You should have received a copy of the GNU General Public License
# along with MDBenchmark. If not, see .
-import datreant.core as dtr
-import numpy as np
-import pytest
from six.moves import StringIO
+import datreant as dtr
+import numpy as np
+import pytest
from mdbenchmark.mdengines import gromacs, utils
diff --git a/mdbenchmark/tests/mdengines/test_namd.py b/mdbenchmark/tests/mdengines/test_namd.py
index 28af4139..82e91732 100644
--- a/mdbenchmark/tests/mdengines/test_namd.py
+++ b/mdbenchmark/tests/mdengines/test_namd.py
@@ -17,11 +17,11 @@
#
# You should have received a copy of the GNU General Public License
# along with MDBenchmark. If not, see .
-import datreant.core as dtr
-import numpy as np
-import pytest
from six.moves import StringIO
+import datreant as dtr
+import numpy as np
+import pytest
from mdbenchmark.mdengines import namd, utils
diff --git a/mdbenchmark/tests/mdengines/test_utils.py b/mdbenchmark/tests/mdengines/test_utils.py
index 5de18c23..d0ba896c 100644
--- a/mdbenchmark/tests/mdengines/test_utils.py
+++ b/mdbenchmark/tests/mdengines/test_utils.py
@@ -20,10 +20,8 @@
import os
from glob import glob
-import datreant.core as dtr
-import mdsynthesis as mds
+import datreant as dtr
import pytest
-
from mdbenchmark.mdengines import gromacs, namd, utils
from mdbenchmark.utils import retrieve_host_template
@@ -38,7 +36,7 @@ def test_prepare_benchmark(engine, input_name, extensions, tmpdir):
for ext in extensions:
open("md.{}".format(ext), "a").close()
- sim = mds.Sim("./{}".format(engine))
+ sim = dtr.Treant("./{}".format(engine))
name = engine.prepare_benchmark(input_name, sim=sim)
assert name == "md"
diff --git a/mdbenchmark/tests/migrations/test_mds_to_dtr.py b/mdbenchmark/tests/migrations/test_mds_to_dtr.py
new file mode 100644
index 00000000..50da0e4d
--- /dev/null
+++ b/mdbenchmark/tests/migrations/test_mds_to_dtr.py
@@ -0,0 +1,113 @@
+# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*-
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8
+#
+# MDBenchmark
+# Copyright (c) 2017-2018 The MDBenchmark development team and contributors
+# (see the file AUTHORS for the full list of names)
+#
+# MDBenchmark is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# MDBenchmark is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with MDBenchmark. If not, see .
+import os
+import uuid
+
+import datreant as dtr
+import pytest
+from mdbenchmark.migrations import mds_to_dtr
+
+
+def create_file(bundle, path, filename):
+ data = {
+ "mdsynthesis": {},
+ "categories": {
+ "name": "bench",
+ "started": "true",
+ "module": "gromacs/2016.3",
+ "host": "draco",
+ "time": 15,
+ "gpu": "false",
+ "nodes": 1,
+ },
+ "tags": [],
+ }
+
+ file = bundle.join(filename)
+ data = str(data).replace("'", '"')
+ file.write(str(data))
+
+ return str(file)
+
+
+@pytest.fixture
+def create_sim_files(tmpdir):
+ sim_files = []
+ directory = tmpdir.mkdir("draco_gromacs").mkdir("2018.1")
+ for path in ["1", "2", "3"]:
+ bundle = directory.mkdir(path)
+ filename = "Sim.{uid}.json".format(uid=uuid.uuid4())
+ sim = create_file(bundle, path, filename)
+ sim_files.append(sim)
+
+ proxy_filename = ".{filename}.proxy".format(filename=filename)
+ proxy = create_file(bundle, path, proxy_filename)
+ sim_files.append(proxy)
+
+ tpr = create_file(bundle, path, "md.tpr")
+ sim_files.append(tpr)
+
+ return directory, sim_files
+
+
+def test_search_mdsynthesis_sim_files(create_sim_files):
+ directory, files = create_sim_files
+
+ bundles = mds_to_dtr.search_mdsynthesis_sim_files(str(directory))
+ files = [file for file in files if file.endswith(".json")]
+ assert bundles == files
+
+
+def test_convert_to_datreant(create_sim_files):
+ directory, files = create_sim_files
+
+ sim_files = [file for file in files if file.endswith(".json")]
+ proxy_files = [file for file in files if file.endswith(".proxy")]
+ files_to_keep = [file for file in files if ".json" not in file]
+ mds_to_dtr.convert_to_datreant(sim_files)
+
+ for path in ["1", "2", "3"]:
+ # New directory and files exist
+ treant = os.path.join(str(directory), path, ".datreant")
+ assert os.path.exists(os.path.join(treant))
+ assert os.path.exists(os.path.join(treant, "categories.json"))
+
+ # Old files were deleted correctly, others were kept
+ for file in proxy_files:
+ assert not os.path.exists(file)
+ for file in files_to_keep:
+ assert os.path.exists(file)
+
+
+def test_migrate_to_datreant(tmpdir, capsys, create_sim_files):
+ directory, files = create_sim_files
+
+ with tmpdir.as_cwd():
+ assert mds_to_dtr.migrate_to_datreant(".") is None
+
+ mds_to_dtr.migrate_to_datreant(str(directory))
+ out, err = capsys.readouterr()
+
+ output = (
+ "Converting old benchmark metadata to new format!\n"
+ "Finished converting old benchmarks to new format!\n"
+ )
+
+ assert out == output
diff --git a/mdbenchmark/tests/test_analyze.py b/mdbenchmark/tests/test_analyze.py
index 08c2dbed..c267ee27 100644
--- a/mdbenchmark/tests/test_analyze.py
+++ b/mdbenchmark/tests/test_analyze.py
@@ -19,9 +19,8 @@
# along with MDBenchmark. If not, see .
import os
+import datreant as dtr
import pandas as pd
-import datreant.core as dtr
-import mdsynthesis as mds
from mdbenchmark.utils import PrintDataFrame, ConsolidateDataFrame, DataFrameFromBundle
from mdbenchmark import cli
from mdbenchmark.ext.click_test import cli_runner
@@ -73,7 +72,7 @@ def test_analyze_namd(cli_runner, tmpdir, data):
cli.cli, ["analyze", "--directory={}".format(data["analyze-files-namd"])]
)
- bundle = mds.discover(data["analyze-files-namd"])
+ bundle = dtr.discover(data["analyze-files-namd"])
df = DataFrameFromBundle(bundle)
test_output = PrintDataFrame(df, False) + "\n"
@@ -92,7 +91,7 @@ def test_analyze_with_errors(cli_runner, tmpdir, data):
["analyze", "--directory={}".format(data["analyze-files-w-errors"])],
)
- bundle = mds.discover(data["analyze-files-w-errors"])
+ bundle = dtr.discover(data["analyze-files-w-errors"])
df = DataFrameFromBundle(bundle)
test_output = PrintDataFrame(df, False) + "\n"
@@ -111,7 +110,7 @@ def test_analyze_plot(cli_runner, tmpdir, data):
],
)
- bundle = mds.discover(data["analyze-files-gromacs"])
+ bundle = dtr.discover(data["analyze-files-gromacs"])
df = DataFrameFromBundle(bundle)
test_output = PrintDataFrame(df, False) + "\n"
diff --git a/mdbenchmark/tests/test_generate.py b/mdbenchmark/tests/test_generate.py
index a5fa2b5d..7d96431d 100644
--- a/mdbenchmark/tests/test_generate.py
+++ b/mdbenchmark/tests/test_generate.py
@@ -22,9 +22,8 @@
import pytest
import pandas as pd
from click import exceptions
-import datreant.core as dtr
-import mdsynthesis as mds
+import datreant as dtr
from mdbenchmark import cli
from mdbenchmark.utils import DataFrameFromBundle, PrintDataFrame, ConsolidateDataFrame
from mdbenchmark.ext.click_test import cli_runner
@@ -81,7 +80,7 @@ def _output(gpu=True, n_benchmarks=4, runtime=15):
def generate_output_table():
def _output(short=False):
title = "Benchmark Summary:\n"
- bundle = mds.discover()
+ bundle = dtr.discover()
df = DataFrameFromBundle(bundle)
if short:
df = ConsolidateDataFrame(df)
@@ -365,7 +364,7 @@ def test_generate_odd_number_of_nodes(
output1 = "Creating benchmark system for {} with GPUs.\n".format(module)
- bundle = mds.discover()
+ bundle = dtr.discover()
df = DataFrameFromBundle(bundle)
df = ConsolidateDataFrame(df)
test_output = "Benchmark Summary:\n" + PrintDataFrame(df, False) + "\n"
@@ -503,7 +502,7 @@ def test_generate_namd_experimental_warning(cli_runner, monkeypatch, tmpdir):
"If you use the --gpu option make sure you use the GPU compatible NAMD module!\n"
"Creating benchmark system for namd/123.\n"
)
- bundle = mds.discover()
+ bundle = dtr.discover()
df = DataFrameFromBundle(bundle)
df = ConsolidateDataFrame(df)
test_output = "Benchmark Summary:\n" + PrintDataFrame(df, False) + "\n"
@@ -628,7 +627,7 @@ def test_generate_test_prompt_yes(cli_runner, tmpdir, generate_output):
"WARNING Cannot locate modules available on this host. Not performing module name validation.\n"
"Creating benchmark system for gromacs/2016 with GPUs.\n"
)
- bundle = mds.discover()
+ bundle = dtr.discover()
df = DataFrameFromBundle(bundle)
df = ConsolidateDataFrame(df)
output2 = (
@@ -663,6 +662,6 @@ def test_generate_test_prompt_no(cli_runner, tmpdir, generate_output):
input="n\n",
)
- bundle = mds.discover()
+ bundle = dtr.discover()
assert result.exit_code == 1
assert len(bundle) == 0
diff --git a/mdbenchmark/tests/test_utils.py b/mdbenchmark/tests/test_utils.py
index 8b379461..ea1d06e7 100644
--- a/mdbenchmark/tests/test_utils.py
+++ b/mdbenchmark/tests/test_utils.py
@@ -19,12 +19,11 @@
# along with MDBenchmark. If not, see .
import os
+import datreant as dtr
import jinja2
import numpy as np
import pandas as pd
import tabulate
-import datreant.core as dtr
-import mdsynthesis as mds
from numpy.testing import assert_equal
from pandas.testing import assert_frame_equal
@@ -158,7 +157,7 @@ def test_DataFrameFromBundle(data):
This is used in other tests, therefore everything is hard coded
If changes are made to the layout type this should be changed here.
"""
- bundle = mds.discover(data["analyze-files-gromacs"])
+ bundle = dtr.discover(data["analyze-files-gromacs"])
test_output = utils.DataFrameFromBundle(bundle)
expected_output = pd.read_csv(data["analyze-files-gromacs.csv"])
@@ -173,7 +172,7 @@ def test_ConsolidateDataFrame(data):
This is used in other tests, therefore everyting is hard coded.
If changes are made to the layout type this should be changed here.
"""
- bundle = mds.discover(data["analyze-files-gromacs"])
+ bundle = dtr.discover(data["analyze-files-gromacs"])
df = utils.DataFrameFromBundle(bundle)
test_output = utils.ConsolidateDataFrame(df)
diff --git a/mdbenchmark/utils.py b/mdbenchmark/utils.py
index d19647c1..b7d53dbc 100644
--- a/mdbenchmark/utils.py
+++ b/mdbenchmark/utils.py
@@ -25,9 +25,8 @@
from tabulate import tabulate
import click
+import datreant as dtr
import numpy as np
-import datreant.core as dtr
-import mdsynthesis as mds
import pandas as pd
import xdg
from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PackageLoader
diff --git a/setup.py b/setup.py
index 94a3e462..77650f92 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ def get_property(prop, project):
packages=find_packages(),
install_requires=[
"numpy>=1.8",
- "mdsynthesis==0.6.1",
+ "datreant>=1.0.0",
"click==6.7", # v7.0 introduced some changes that broke our tests.
"jinja2",
"pandas",