Add narrative to Tales

plugin_tests/image_test.py

@@ -91,7 +91,8 @@ def testImageFlow(self):
             self.assertEqual(celeryMock.mock_calls[0][1], ('girder_worker',))
 
             sendTaskCalls = celeryMock.return_value.send_task.mock_calls
-            mock_repo_url = 'https://github.com/{}/archive/{}.tar.gz'.format(GOOD_REPO, GOOD_COMMIT)
+            # mock_repo_url = \
+            #    'https://github.com/{}/archive/{}.tar.gz'.format(GOOD_REPO, GOOD_COMMIT)
             self.assertEqual(len(sendTaskCalls), 1)
             self.assertEqual(sendTaskCalls[0][1], (
                 'gwvolman.tasks.build_image',

plugin_tests/tale_test.py

@@ -1,16 +1,25 @@
-
 import httmock
+import os
 import json
 from tests import base
 from .tests_helpers import \
     GOOD_REPO, GOOD_COMMIT, XPRA_REPO, XPRA_COMMIT, \
     mockOtherRequest, mockCommitRequest, mockReposRequest
+from girder.models.item import Item
+
+
+SCRIPTDIRS_NAME = None
+DATADIRS_NAME = None
 
 
 def setUpModule():
     base.enabledPlugins.append('wholetale')
     base.startServer()
 
+    global SCRIPTDIRS_NAME, DATADIRS_NAME
+    from girder.plugins.wholetale.constants import \
+        SCRIPTDIRS_NAME, DATADIRS_NAME
+
 
 def tearDownModule():
     base.stopServer()
@@ -401,6 +410,96 @@ def testTaleAccess(self):
         self.assertStatusOk(resp)
         self.assertEqual(resp.json['public'], False)
 
+    def testTaleNarrative(self):
+        resp = self.request(
+            path='/resource/lookup', method='GET', user=self.user,
+            params={'path': '/user/{login}/Home'.format(**self.user)})
+        home_dir = resp.json
+        resp = self.request(
+            path='/folder', method='POST', user=self.user, params={
+                'name': 'my_narrative', 'parentId': home_dir['_id']
+            })
+        sub_home_dir = resp.json
+        my_narrative = Item().createItem('notebook.ipynb', self.user, sub_home_dir)
+
+        resp = self.request(
+            path='/tale', method='POST', user=self.user,
+            type='application/json',
+            body=json.dumps({
+                'imageId': str(self.image['_id']),
+                'involatileData': [
+                    {'type': 'folder', 'id': sub_home_dir['_id']}
+                ],
+                'narrative': [str(my_narrative['_id'])]
+            })
+        )
+        self.assertStatusOk(resp)
+        tale = resp.json
+
+        path = os.path.join(
+            '/collection', SCRIPTDIRS_NAME, SCRIPTDIRS_NAME,
+            tale['_id'], 'notebook.ipynb')
+        resp = self.request(
+            path='/resource/lookup', method='GET', user=self.user,
+            params={'path': path})
+        self.assertStatusOk(resp)
+        self.assertEqual(resp.json['name'], my_narrative['name'])
+        self.assertNotEqual(resp.json['_id'], str(my_narrative['_id']))
+
+        resp = self.request(
+            path='/tale/{_id}'.format(**tale), method='DELETE',
+            user=self.admin)
+        self.assertStatusOk(resp)
+
+    def testTaleValidation(self):
+        resp = self.request(
+            path='/resource/lookup', method='GET', user=self.user,
+            params={'path': '/user/{login}/Home'.format(**self.user)})
+        home_dir = resp.json
+        resp = self.request(
+            path='/folder', method='POST', user=self.user, params={
+                'name': 'validate_my_narrative', 'parentId': home_dir['_id']
+            })
+        sub_home_dir = resp.json
+        Item().createItem('notebook.ipynb', self.user, sub_home_dir)
+
+        resp = self.request(
+            path='/resource/lookup', method='GET', user=self.user,
+            params={'path': '/user/{login}/Data'.format(**self.user)})
+        data_dir = resp.json
+        resp = self.request(
+            path='/folder', method='POST', user=self.user, params={
+                'name': 'my_fake_data', 'parentId': data_dir['_id']
+            })
+        sub_data_dir = resp.json
+        Item().createItem('data.dat', self.user, sub_data_dir)
+
+        # Mock old format
+        tale = {
+            "config": None,
+            "creatorId": self.user['_id'],
+            "description": "Fake Tale",
+            "folderId": data_dir['_id'],
+            "imageId": "5873dcdbaec030000144d233",
+            "public": True,
+            "published": False,
+            "title": "Fake Unvalidated Tale"
+        }
+        tale = self.model('tale', 'wholetale').save(tale)  # get's id
+        tale = self.model('tale', 'wholetale').save(tale)  # migrate to new format
+
+        path = os.path.join(
+            '/collection', DATADIRS_NAME, DATADIRS_NAME, str(tale['_id']))
+        resp = self.request(
+            path='/resource/lookup', method='GET', user=self.user,
+            params={'path': path})
+        self.assertStatusOk(resp)
+        new_data_dir = resp.json
+        self.assertEqual(str(tale['folderId']), str(new_data_dir['_id']))
+        self.assertEqual(tale['involatileData'],
+                         [{'id': str(data_dir['_id']), 'type': 'folder'}])
+        self.model('tale', 'wholetale').remove(tale)
+
     def tearDown(self):
         self.model('user').remove(self.user)
         self.model('user').remove(self.admin)

server/constants.py

@@ -8,6 +8,7 @@
 CATALOG_NAME = 'WholeTale Catalog'
 WORKSPACE_NAME = 'WholeTale Workspaces'
 DATADIRS_NAME = 'WholeTale Data Mountpoints'
+SCRIPTDIRS_NAME = 'WholeTale Narrative'
 
 
 class HarvesterType:

server/models/tale.py

@@ -5,7 +5,7 @@
 from bson.objectid import ObjectId
 
 from .image import Image
-from ..constants import WORKSPACE_NAME, DATADIRS_NAME
+from ..constants import WORKSPACE_NAME, DATADIRS_NAME, SCRIPTDIRS_NAME
 from ..utils import getOrCreateRootFolder
 from girder.models.model_base import AccessControlledModel
 from girder.models.item import Item
@@ -19,7 +19,7 @@
 # Whenever the Tale object schema is modified (e.g. fields are added or
 # removed) increase `_currentTaleFormat` to retroactively apply those
 # changes to existing Tales.
-_currentTaleFormat = 2
+_currentTaleFormat = 3
 
 
 class Tale(AccessControlledModel):
@@ -38,14 +38,19 @@ def initialize(self):
         self.exposeFields(
             level=AccessType.READ,
             fields=({'_id', 'folderId', 'imageId', 'creatorId', 'created',
-                     'format', 'involatileData'} | self.modifiableFields))
+                     'format', 'involatileData', 'narrative',
+                     'narrativeId'} | self.modifiableFields))
         self.exposeFields(level=AccessType.ADMIN, fields={'published'})
 
     def validate(self, tale):
         if 'iframe' not in tale:
             tale['iframe'] = False
+
+        if '_id' not in tale:
+            return tale
+
         if tale.get('format', 0) < 2:
-            dataFolder = getOrCreateRootFolder(DATADIRS_NAME)
+            data_folder = getOrCreateRootFolder(DATADIRS_NAME)
             try:
                 origFolder = Folder().load(tale['folderId'], force=True, exc=True)
             except ValidationException:
@@ -60,9 +65,15 @@ def validate(self, tale):
                 {'type': 'folder', 'id': tale.pop('folderId')}
             ]
             newFolder = Folder().copyFolder(
-                origFolder, parent=dataFolder, parentType='folder',
+                origFolder, parent=data_folder, parentType='folder',
                 name=str(tale['_id']), creator=creator, progress=False)
             tale['folderId'] = newFolder['_id']
+        if tale.get('format', 0) < 3:
+            if 'narrativeId' not in tale:
+                default = self.createNarrativeFolder(tale, default=True)
+                tale['narrativeId'] = default['_id']
+            if 'narrative' not in tale:
+                tale['narrative'] = []
         tale['format'] = _currentTaleFormat
         return tale
 
@@ -105,7 +116,8 @@ def list(self, user=None, data=None, image=None, limit=0, offset=0,
 
     def createTale(self, image, data, creator=None, save=True, title=None,
                    description=None, public=None, config=None, published=False,
-                   authors=None, icon=None, category=None, illustration=None):
+                   authors=None, icon=None, category=None, illustration=None,
+                   narrative=None):
         if creator is None:
             creatorId = None
         else:
@@ -122,14 +134,15 @@ def createTale(self, image, data, creator=None, save=True, title=None,
             'category': category,
             'config': config,
             'creatorId': creatorId,
-            'involatileData': data,
+            'involatileData': data or [],
             'description': description,
             'format': _currentTaleFormat,
             'created': now,
             'icon': icon,
             'iframe': image.get('iframe', False),
             'imageId': ObjectId(image['_id']),
             'illustration': illustration,
+            'narrative': narrative or [],
             'title': title,
             'public': public,
             'published': published,
@@ -146,20 +159,38 @@ def createTale(self, image, data, creator=None, save=True, title=None,
         if save:
             tale = self.save(tale)
             self.createWorkspace(tale, creator=creator)
-            parent = self.createDataMountpoint(tale, creator=creator)
-            tale['folderId'] = parent['_id']
-            for obj in data:
+            data_folder = self.createDataMountpoint(tale, creator=creator)
+            tale['folderId'] = data_folder['_id']
+            for obj in tale['involatileData']:
                 if obj['type'] == 'folder':
                     folder = Folder().load(obj['id'], user=creator)
                     Folder().copyFolder(
-                        folder, parent=parent, parentType='folder',
+                        folder, parent=data_folder, parentType='folder',
                         creator=creator, progress=False)
                 elif obj['type'] == 'item':
                     item = Item().load(obj['id'], user=creator)
-                    Item().copyItem(item, creator, folder=parent)
+                    Item().copyItem(item, creator, folder=data_folder)
+
+            narrative_folder = self.createNarrativeFolder(
+                tale, creator=creator, default=not bool(tale['narrative']))
+            for obj_id in tale['narrative']:
+                item = Item().load(obj_id, user=creator)
+                Item().copyItem(item, creator, folder=narrative_folder)
+            tale['narrativeId'] = narrative_folder['_id']
             tale = self.save(tale)
         return tale
 
+    def createNarrativeFolder(self, tale, creator=None, default=False):
+        if default:
+            rootFolder = getOrCreateRootFolder(SCRIPTDIRS_NAME)
+            auxFolder = self.model('folder').createFolder(
+                rootFolder, 'default', parentType='folder',
+                public=True, reuseExisting=True)
+        else:
+            auxFolder = self._createAuxFolder(
+                tale, SCRIPTDIRS_NAME, creator=creator)
+        return auxFolder
+
     def createDataMountpoint(self, tale, creator=None):
         return self._createAuxFolder(tale, DATADIRS_NAME, creator=creator)
 

server/rest/tale.py

@@ -157,7 +157,7 @@ def createTale(self, tale, params):
                                      'images/demo-graph2.jpg')),
                 authors=tale.get('authors', default_author),
                 category=tale.get('category', 'science'),
-                published=False
+                published=False, narrative=tale.get('narrative', [])
             )
 
     @access.user(scope=TokenScope.DATA_OWN)

server/schema/tale.py

@@ -31,13 +31,25 @@
         },
         "folderId": {
             "type": "string",
-            "description": "ID of a folder containing copy of tale['data']"
+            "description": "ID of a folder containing copy of tale['involatileData']"
+        },
+        "narrativeId": {
+            "type": "string",
+            "description": "ID of a folder containing copy of tale['narrative']"
         },
         "involatileData": {
             "type": "array",
             "items": dataResourceSchema,
             "description": "Resources used to create Tale's data folder"
         },
+        "narrative": {
+            "type": "array",
+            "items": {
+                'type': 'string',
+                'description': "Girder Item id"
+            },
+            "description": "List of Girder Items containing Tale's narrative"
+        },
         "format": {
             "type": "integer",
             "description": "Tale format specification"