Make pyarrow and msgpack dependency optional

Move pyarrow back for unit tests

Disable msgpack by default

Fix tests

setup.cfg

@@ -45,7 +45,7 @@ combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
 known_first_party = superset
-known_third_party =alembic,backoff,bleach,celery,click,colorama,contextlib2,croniter,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,marshmallow,msgpack,numpy,pandas,parsedatetime,pathlib2,polyline,prison,psycopg2,pyarrow,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml
+known_third_party = alembic,backoff,bleach,celery,click,colorama,contextlib2,croniter,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,marshmallow,numpy,pandas,parsedatetime,pathlib2,polyline,prison,psycopg2,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml
 multi_line_output = 3
 order_by_type = false
 

setup.py

@@ -89,15 +89,13 @@ def get_git_sha():
         "humanize",
         "isodate",
         "markdown>=3.0",
-        "msgpack>=0.6.1, <0.7.0",
         "pandas>=0.24.2, <0.25.0",
         "parsedatetime",
         "pathlib2",
         "polyline",
         "python-dateutil",
         "python-dotenv",
         "python-geohash",
-        "pyarrow>=0.14.1, <0.15.0",
         "pyyaml>=5.1",
         "retry>=0.9.2",
         "selenium>=3.141.0",
@@ -117,6 +115,7 @@ def get_git_sha():
         "presto": ["pyhive[presto]>=0.4.0"],
         "elasticsearch": ["elasticsearch-dbapi>=0.1.0, <0.2.0"],
         "druid": ["pydruid==0.5.7", "requests==2.22.0"],
+        "pyarrow": ["msgpack>=0.6.1, <0.7.0", "pyarrow>=0.14.1, <0.15.0"],
     },
     python_requires="~=3.6",
     author="Apache Software Foundation",

superset/config.py

@@ -486,7 +486,7 @@ class CeleryConfig(object):
 # rather than JSON. This feature requires additional testing from the
 # community before it is fully adopted, so this config option is provided
 # in order to disable should breaking issues be discovered.
-RESULTS_BACKEND_USE_MSGPACK = True
+RESULTS_BACKEND_USE_MSGPACK = False
 
 # The S3 bucket where you want to store your external hive tables created
 # from CSV files. For example, 'companyname-superset'

superset/sql_lab.py

@@ -23,8 +23,6 @@
 from typing import Optional, Tuple, Union
 
 import backoff
-import msgpack
-import pyarrow as pa
 import simplejson as json
 import sqlalchemy
 from celery.exceptions import SoftTimeLimitExceeded
@@ -49,6 +47,10 @@
 from superset.utils.dates import now_as_float
 from superset.utils.decorators import stats_timing
 
+if results_backend_use_msgpack:
+    import msgpack
+    import pyarrow as pa
+
 config = app.config
 stats_logger = config["STATS_LOGGER"]
 SQLLAB_TIMEOUT = config["SQLLAB_ASYNC_TIME_LIMIT_SEC"]

superset/views/core.py

@@ -23,9 +23,7 @@
 from urllib import parse
 
 import backoff
-import msgpack
 import pandas as pd
-import pyarrow as pa
 import simplejson as json
 from flask import (
     abort,
@@ -110,6 +108,10 @@
     get_viz,
 )
 
+if results_backend_use_msgpack:
+    import msgpack
+    import pyarrow as pa
+
 config = app.config
 CACHE_DEFAULT_TIMEOUT = config["CACHE_DEFAULT_TIMEOUT"]
 SQLLAB_QUERY_COST_ESTIMATE_TIMEOUT = config["SQLLAB_QUERY_COST_ESTIMATE_TIMEOUT"]

tests/superset_test_config.py

@@ -57,3 +57,5 @@ class CeleryConfig(object):
 
 
 CELERY_CONFIG = CeleryConfig
+
+RESULTS_BACKEND_USE_MSGPACK = True