[MXNET-707] Add unit test for mxnet to coreml converter (apache#11952)

* Add unittest to coreml converter

* Add unittest to coreml converter

* Add docstring and remove unused method

* updated test and removed unittest folder

* remove unittest

* Add coreml test to CI

* fix lint

* install mxnet-to-coreml for testing

* exclude test that takes too long

* linting to 100 max line width

.travis.yml

@@ -34,7 +34,9 @@ script:
   - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
   - mv make/osx.mk config.mk
   - make -j 2
+
   # We ignore several tests to avoid possible timeouts on large PRs.
   # This lowers our test coverage, but is required for consistent Travis runs.
   # These tests will be tested in a variety of environments in Jenkins based tests.
   - python -m nose --with-timer --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/
+  - python2 -m nose --verbose tools/coreml/test --exclude-test=test_mxnet_image

ci/travis/install.sh

@@ -22,5 +22,5 @@ export HOMEBREW_NO_AUTO_UPDATE=1
 
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
     brew install opencv
-    python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude
+    python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude mxnet-to-coreml
 fi

tools/coreml/test/test_mxnet_converter.py

@@ -18,15 +18,12 @@
 import unittest
 import mxnet as mx
 import numpy as np
-import sys
-import os
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
+
+from converter._mxnet_converter import convert
 from collections import namedtuple
 from converter import utils
 
+
 def _mxnet_remove_batch(input_data):
     for blob in input_data:
         input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:])
@@ -39,7 +36,8 @@ def _get_mxnet_module(net, data_shapes, mode, label_names, input_names=None):
     """
     mx.random.seed(1993)
 
-    mod = utils.create_module(sym=net, data_shapes=data_shapes, label_shapes=input_names, label_names=label_names)
+    mod = utils.create_module(sym=net, data_shapes=data_shapes, label_shapes=input_names,
+                              label_names=label_names)
 
     if mode == 'random':
         mod.init_params(
@@ -62,11 +60,14 @@ def _get_mxnet_module(net, data_shapes, mode, label_names, input_names=None):
 class SingleLayerTest(unittest.TestCase):
     """
     Unit test class for testing where converter is able to convert individual layers or not.
-    In order to do so, it converts model and generates preds on both CoreML and MXNet and check they are the same.
+    In order to do so, it converts model and generates preds on both CoreML and MXNet and check
+    they are the same.
     """
-    def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mode=None, label_names=None, delta=1e-3,
+    def _test_mxnet_model(self, net, input_shape, mode, class_labels=None,
+                          coreml_mode=None, label_names=None, delta=1e-2,
                           pre_processing_args=None, input_name='data'):
-        """ Helper method that convert the CoreML model into CoreML and compares the predictions over random data.
+        """ Helper method that convert the CoreML model into CoreML and compares the predictions
+        over random data.
 
         Parameters
         ----------
@@ -89,7 +90,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
             The name of the input variable to the symbolic graph.
         """
 
-        data_shapes=[(input_name, input_shape)]
+        data_shapes = [(input_name, input_shape)]
 
         mod = _get_mxnet_module(net, data_shapes, mode, label_names)
 
@@ -100,7 +101,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
         mxnet_preds = mod.get_outputs()[0].asnumpy().flatten()
 
         # Get predictions from coreml
-        coreml_model = mxnet_converter.convert(
+        coreml_model = convert(
             model=mod,
             class_labels=class_labels,
             mode=coreml_mode,
@@ -112,7 +113,7 @@ def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mo
         # Check prediction accuracy
         self.assertEquals(len(mxnet_preds), len(coreml_preds))
         for i in range(len(mxnet_preds)):
-            self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta = delta)
+            self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta=delta)
 
     def test_tiny_inner_product_zero_input(self):
         np.random.seed(1988)
@@ -140,7 +141,7 @@ def test_tiny_inner_product_ones_input(self):
         input_shape = (1, 10)
         net = mx.sym.Variable('data')
         net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
-        self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+        self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
 
     def test_tiny_inner_product_random_input(self):
         np.random.seed(1988)
@@ -162,7 +163,8 @@ def test_tiny_softmax_random_input(self):
         net = mx.sym.Variable('data')
         net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
         net = mx.sym.SoftmaxOutput(net, name='softmax')
-        self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'])
+        self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+                               label_names=['softmax_label'])
 
     def test_tiny_relu_activation_random_input(self):
         np.random.seed(1988)
@@ -228,7 +230,7 @@ def test_tiny_conv_ones_input(self):
             pad=pad,
             name='conv_1'
         )
-        self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+        self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
 
     def test_tiny_conv_random_input(self):
         np.random.seed(1988)
@@ -481,7 +483,8 @@ def test_flatten(self):
         net = mx.sym.Flatten(data=net, name='flatten1')
         net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
         net = mx.sym.SoftmaxOutput(net, name='softmax')
-        self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'])
+        self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+                               label_names=['softmax_label'])
 
     def test_transpose(self):
         np.random.seed(1988)
@@ -516,10 +519,8 @@ def test_tiny_synset_random_input(self):
         net = mx.sym.Variable('data')
         net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
         net = mx.sym.SoftmaxOutput(net, name='softmax')
-        mod = _get_mxnet_module(net,
-                                data_shapes=[('data', input_shape)],
-                                mode='random',
-                                label_names=['softmax_label'])
+        mod = _get_mxnet_module(net, data_shapes=[('data', input_shape)],
+                                mode='random', label_names=['softmax_label'])
 
         # Generate some dummy data
         input_data = np.random.uniform(-0.1, 0.1, input_shape)
@@ -529,14 +530,15 @@ def test_tiny_synset_random_input(self):
 
         kwargs = {'input_shape': {'data': input_shape}}
         # Get predictions from coreml
-        coreml_model = mxnet_converter.convert(
+        coreml_model = convert(
             model=mod,
-            class_labels=['Category1','Category2','Category3','Category4','Category5'],
+            class_labels=['Category1', 'Category2', 'Category3', 'Category4', 'Category5'],
             mode='classifier',
             **kwargs
         )
 
-        prediction = coreml_model.predict(_mxnet_remove_batch({'data': input_data}))
+        prediction = coreml_model.predict(
+            _mxnet_remove_batch({'data': input_data}))
         self.assertEqual(prediction['classLabel'], 'Category3')
 
     def test_really_tiny_deconv_random_input(self):
@@ -579,7 +581,7 @@ def test_tiny_deconv_ones_input(self):
             name='deconv_1'
         )
         # Test the mxnet model
-        self._test_mxnet_model(net, input_shape=input_shape, mode='ones')
+        self._test_mxnet_model(net, input_shape=input_shape, mode='ones', delta=0.05)
 
     def test_tiny_deconv_random_input(self):
         np.random.seed(1988)
@@ -915,7 +917,6 @@ def test_batch_norm(self):
 
     def test_batch_norm_no_global_stats(self):
         """ This test should throw an exception since converter doesn't support
-            conversion of MXNet models that use local batch stats (i.e.
             use_global_stats=False). The reason for this is CoreML doesn't support
             local batch stats.
         """
@@ -943,12 +944,17 @@ def test_pre_processing_args(self):
         net = mx.sym.Variable('data')
         net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5)
         net = mx.sym.SoftmaxOutput(net, name='softmax')
-        self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'],
-                               pre_processing_args={'red_bias':0, 'blue_bias':0, 'green_bias':0, 'image_scale':1})
+        self._test_mxnet_model(net, input_shape=input_shape, mode='random',
+                               label_names=['softmax_label'],
+                               pre_processing_args={'red_bias': 0,
+                                                    'blue_bias': 0,
+                                                    'green_bias': 0,
+                                                    'image_scale': 1})
 
     def test_different_input_variables(self):
         """
-        Verifying the behavior when input variable name is different than the standard name - 'data'.
+        Verifying the behavior when input variable name is different than the
+        standard name - 'data'.
         """
         np.random.seed(1988)
         input_shape = (1, 10)
@@ -958,12 +964,13 @@ def test_different_input_variables(self):
 
     def test_really_tiny_conv_optional_params(self):
         """
-        Verifying the behavior of a convolutional layer when stride and pad are not provided.
+        Verifying the behavior of a convolutional layer when stride and pad
+        are not provided.
         """
         np.random.seed(1988)
         input_shape = (1, 1, 10, 10)
         num_filter = 1
-        kernel = (1 ,1)
+        kernel = (1, 1)
 
         # Define a model
         net = mx.sym.Variable('data')

tools/coreml/test/test_mxnet_image.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,19 +16,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import unittest
 import mxnet as mx
 import numpy as np
-import unittest
-import sys
-import os
 
 from six.moves import xrange
-
-current_working_directory = os.getcwd()
-sys.path.append(current_working_directory + "/..")
-sys.path.append(current_working_directory + "/../converter/")
-import _mxnet_converter as mxnet_converter
-from converter.utils import load_model
+from converter._mxnet_converter import convert
+from converter import utils
 
 VAL_DATA = 'data/val-5k-256.rec'
 URL = 'http://data.mxnet.io/data/val-5k-256.rec'
@@ -44,7 +38,7 @@ def read_image(data_val, label_name):
         label_width=1,
         preprocess_threads=4,
         batch_size=32,
-        data_shape=(3,224,224),
+        data_shape=(3, 224, 224),
         label_name=label_name,
         rand_corp=False,
         rand_mirror=False,
@@ -80,11 +74,11 @@ def _test_image_prediction(self, model_name, epoch, label_name):
             epoch_num=epoch,
             data_shapes=data.provide_data,
             label_shapes=data.provide_label,
-            label_names=[label_name,]
+            label_names=[label_name, ]
         )
 
         input_shape = (1, 3, 224, 224)
-        coreml_model = mxnet_converter.convert(mod, input_shape={'data': input_shape})
+        coreml_model = convert(mod, input_shape={'data': input_shape})
 
         mxnet_acc = []
         mxnet_top_5_acc = []
@@ -104,34 +98,43 @@ def _test_image_prediction(self, model_name, epoch, label_name):
                 mxnet_predict = mxnet_preds[i]
                 label = label_numpy[i]
                 mxnet_acc.append(is_correct_top_one(mxnet_predict, label))
-                mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict, label))
+                mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict,
+                                                           label))
                 coreml_acc.append(is_correct_top_one(coreml_predict, label))
-                coreml_top_5_acc.append(is_correct_top_five(coreml_predict, label))
+                coreml_top_5_acc.append(is_correct_top_five(coreml_predict,
+                                                            label))
                 num_batch += 1
-            if (num_batch == 5): break # we only use a subset of the batches.
+            if (num_batch == 5):
+                break  # we only use a subset of the batches.
 
         print("MXNet acc %s" % np.mean(mxnet_acc))
         print("Coreml acc %s" % np.mean(coreml_acc))
         print("MXNet top 5 acc %s" % np.mean(mxnet_top_5_acc))
         print("Coreml top 5 acc %s" % np.mean(coreml_top_5_acc))
         self.assertAlmostEqual(np.mean(mxnet_acc), np.mean(coreml_acc), delta=1e-4)
-        self.assertAlmostEqual(np.mean(mxnet_top_5_acc), np.mean(coreml_top_5_acc), delta=1e-4)
+        self.assertAlmostEqual(np.mean(mxnet_top_5_acc),
+                               np.mean(coreml_top_5_acc),
+                               delta=1e-4)
 
     def test_squeezenet(self):
         print("Testing Image Classification with Squeezenet")
-        self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0, label_name='prob_label')
+        self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0,
+                                    label_name='prob_label')
 
     def test_inception_with_batch_normalization(self):
         print("Testing Image Classification with Inception/BatchNorm")
-        self._test_image_prediction(model_name='Inception-BN', epoch=126, label_name='softmax_label')
+        self._test_image_prediction(model_name='Inception-BN', epoch=126,
+                                    label_name='softmax_label')
 
     def test_resnet18(self):
         print("Testing Image Classification with ResNet18")
-        self._test_image_prediction(model_name='resnet-18', epoch=0, label_name='softmax_label')
+        self._test_image_prediction(model_name='resnet-18', epoch=0,
+                                    label_name='softmax_label')
 
     def test_vgg16(self):
         print("Testing Image Classification with vgg16")
-        self._test_image_prediction(model_name='vgg16', epoch=0, label_name='prob_label')
+        self._test_image_prediction(model_name='vgg16', epoch=0,
+                                    label_name='prob_label')
 
 
 if __name__ == '__main__':