ucbds-infra · chrispyles · Jan 24, 2025 · Jan 24, 2025 · Jan 24, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 * Update Otter Assign to handle notebooks with an invalid kernelspec by assuming the language is Python per [#895](https://github.com/ucbds-infra/otter-grader/issues/895)
 * Fixed assignment summary in Otter Assign so that manual questions are included per [#886](https://github.com/ucbds-infra/otter-grader/issues/886)
 * Wrap `ModuleNotFoundError` for `otter` in grading image to include debugging instructions per [#907](https://github.com/ucbds-infra/otter-grader/issues/907)
+* Updated Otter Assign to disallow metadata tests for Colab notebooks per [#901](https://github.com/ucbds-infra/otter-grader/issues/901)
 
 **v6.0.5:**
 

diff --git a/otter/assign/notebook_transformer.py b/otter/assign/notebook_transformer.py
@@ -174,6 +174,10 @@ def _get_transformed_cells(self, cells: list[nbf.NotebookNode]) -> list[nbf.Note
                 if config.get("config_file"):
                     self.assignment.load_config_file(config["config_file"])
                 self.assignment.update(config)
+                if self.assignment.runs_on == "colab" and not self.assignment.tests.files:
+                    raise ValueError(
+                        "notebooks that run on Colab must use test files and not notebook metadata tests"
+                    )
                 continue
 
             # check for an end to the current block

diff --git a/test/test_assign/files/generate-gradescope.ipynb b/test/test_assign/files/generate-gradescope.ipynb
@@ -18,6 +18,8 @@
     "    assignment_id: '567'\n",
     "    filtering: true\n",
     "runs_on: colab\n",
+    "tests:\n",
+    "    files: true\n",
     "files:\n",
     "    - data.csv"
    ]

diff --git a/test/test_assign/files/gs-correct/autograder/generate-gradescope.ipynb b/test/test_assign/files/gs-correct/autograder/generate-gradescope.ipynb
@@ -490,103 +490,6 @@
    "pygments_lexer": "ipython3",
    "version": "3.7.7"
   },
-  "otter": {
-   "OK_FORMAT": true,
-   "tests": {
-    "q1": {
-     "name": "q1",
-     "points": 2,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> isinstance(x, int)\nTrue",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> None\n",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> 0 < x < 100\nTrue",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> x\n2",
-         "hidden": true,
-         "locked": false
-        },
-        {
-         "code": ">>> str(print(x))\n2\n'None'",
-         "hidden": true,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    },
-    "q3": {
-     "name": "q3",
-     "points": null,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> nine\n9",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> square(16)\n256",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> square(1)\n1",
-         "hidden": true,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    },
-    "q8": {
-     "name": "q8",
-     "points": null,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> len(z) == 10\nTrue",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> np.allclose(z, [3.07316461, 3.06854049, 4.48392454, 0.17343951, 0.55016433, 2.87542494, 1.97433776, 4.62849467, 2.18395185, 1.1753926])\nFalse",
-         "hidden": true,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    }
-   }
-  },
   "varInspector": {
    "cols": {
     "lenName": 16,

diff --git a/test/test_assign/files/gs-correct/autograder/tests/q1.py b/test/test_assign/files/gs-correct/autograder/tests/q1.py
@@ -0,0 +1,13 @@
+OK_FORMAT = True
+
+test = {   'name': 'q1',
+    'points': 2,
+    'suites': [   {   'cases': [   {'code': '>>> isinstance(x, int)\nTrue', 'hidden': False, 'locked': False},
+                                   {'code': '>>> None\n', 'hidden': False, 'locked': False},
+                                   {'code': '>>> 0 < x < 100\nTrue', 'hidden': False, 'locked': False},
+                                   {'code': '>>> x\n2', 'hidden': True, 'locked': False},
+                                   {'code': ">>> str(print(x))\n2\n'None'", 'hidden': True, 'locked': False}],
+                      'scored': True,
+                      'setup': '',
+                      'teardown': '',
+                      'type': 'doctest'}]}
diff --git a/test/test_assign/files/gs-correct/autograder/tests/q3.py b/test/test_assign/files/gs-correct/autograder/tests/q3.py
@@ -0,0 +1,11 @@
+OK_FORMAT = True
+
+test = {   'name': 'q3',
+    'points': None,
+    'suites': [   {   'cases': [   {'code': '>>> nine\n9', 'hidden': False, 'locked': False},
+                                   {'code': '>>> square(16)\n256', 'hidden': False, 'locked': False},
+                                   {'code': '>>> square(1)\n1', 'hidden': True, 'locked': False}],
+                      'scored': True,
+                      'setup': '',
+                      'teardown': '',
+                      'type': 'doctest'}]}
diff --git a/test/test_assign/files/gs-correct/autograder/tests/q8.py b/test/test_assign/files/gs-correct/autograder/tests/q8.py
@@ -0,0 +1,12 @@
+OK_FORMAT = True
+
+test = {   'name': 'q8',
+    'points': None,
+    'suites': [   {   'cases': [   {'code': '>>> len(z) == 10\nTrue', 'hidden': False, 'locked': False},
+                                   {   'code': '>>> np.allclose(z, [3.07316461, 3.06854049, 4.48392454, 0.17343951, 0.55016433, 2.87542494, 1.97433776, 4.62849467, 2.18395185, 1.1753926])\nFalse',
+                                       'hidden': True,
+                                       'locked': False}],
+                      'scored': True,
+                      'setup': '',
+                      'teardown': '',
+                      'type': 'doctest'}]}
diff --git a/test/test_assign/files/gs-correct/student/generate-gradescope.ipynb b/test/test_assign/files/gs-correct/student/generate-gradescope.ipynb
@@ -388,83 +388,6 @@
    "pygments_lexer": "ipython3",
    "version": "3.7.7"
   },
-  "otter": {
-   "OK_FORMAT": true,
-   "tests": {
-    "q1": {
-     "name": "q1",
-     "points": 2,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> isinstance(x, int)\nTrue",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> None\n",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> 0 < x < 100\nTrue",
-         "hidden": false,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    },
-    "q3": {
-     "name": "q3",
-     "points": null,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> nine\n9",
-         "hidden": false,
-         "locked": false
-        },
-        {
-         "code": ">>> square(16)\n256",
-         "hidden": false,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    },
-    "q8": {
-     "name": "q8",
-     "points": null,
-     "suites": [
-      {
-       "cases": [
-        {
-         "code": ">>> len(z) == 10\nTrue",
-         "hidden": false,
-         "locked": false
-        }
-       ],
-       "scored": true,
-       "setup": "",
-       "teardown": "",
-       "type": "doctest"
-      }
-     ]
-    }
-   }
-  },
   "varInspector": {
    "cols": {
     "lenName": 16,

diff --git a/test/test_assign/files/gs-correct/student/tests/q1.py b/test/test_assign/files/gs-correct/student/tests/q1.py
@@ -0,0 +1,11 @@
+OK_FORMAT = True
+
+test = {   'name': 'q1',
+    'points': 2,
+    'suites': [   {   'cases': [   {'code': '>>> isinstance(x, int)\nTrue', 'hidden': False, 'locked': False},
+                                   {'code': '>>> None\n', 'hidden': False, 'locked': False},
+                                   {'code': '>>> 0 < x < 100\nTrue', 'hidden': False, 'locked': False}],
+                      'scored': True,
+                      'setup': '',
+                      'teardown': '',
+                      'type': 'doctest'}]}
diff --git a/test/test_assign/files/gs-correct/student/tests/q3.py b/test/test_assign/files/gs-correct/student/tests/q3.py
@@ -0,0 +1,9 @@
+OK_FORMAT = True
+
+test = {   'name': 'q3',
+    'points': None,
+    'suites': [   {   'cases': [{'code': '>>> nine\n9', 'hidden': False, 'locked': False}, {'code': '>>> square(16)\n256', 'hidden': False, 'locked': False}],
+                      'scored': True,
+                      'setup': '',
+                      'teardown': '',
+                      'type': 'doctest'}]}
diff --git a/test/test_assign/files/gs-correct/student/tests/q8.py b/test/test_assign/files/gs-correct/student/tests/q8.py
@@ -0,0 +1,3 @@
+OK_FORMAT = True
+
+test = {'name': 'q8', 'points': None, 'suites': [{'cases': [{'code': '>>> len(z) == 10\nTrue', 'hidden': False, 'locked': False}], 'scored': True, 'setup': '', 'teardown': '', 'type': 'doctest'}]}
diff --git a/test/test_assign/test_integration.py b/test/test_assign/test_integration.py
@@ -238,6 +238,22 @@ def test_jupyterlite(generate_master_notebook):
     )
 
 
+def test_colab_no_metadata_tests(generate_master_notebook):
+    """
+    Tests that Otter Assign does not allow metadata tests for Colab notebooks.
+    """
+    master_nb_path = generate_master_notebook({"runs_on": "colab", "tests": {"files": False}})
+    with pytest.raises(
+        ValueError,
+        match="notebooks that run on Colab must use test files and not notebook metadata tests",
+    ):
+        assign_and_check_output(
+            master_nb_path,
+            FILE_MANAGER.get_path("jupyterlite-correct"),
+            assert_dirs_equal_kwargs=dict(variable_path_exts=[".zip"]),
+        )
+
+
 def test_require_no_pdf_ack(generate_master_notebook):
     """
     Tests that Otter Assign produces correct notebooks when configured to require students to
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		OK_FORMAT = True

		test = {'name': 'q8', 'points': None, 'suites': [{'cases': [{'code': '>>> len(z) == 10\nTrue', 'hidden': False, 'locked': False}], 'scored': True, 'setup': '', 'teardown': '', 'type': 'doctest'}]}