From e41681e37837de534b83c271d6bb3f34d402e223 Mon Sep 17 00:00:00 2001 From: Timi Fasubaa Date: Fri, 5 Oct 2018 18:18:29 -0700 Subject: [PATCH] first take --- superset/cli.py | 120 ++++++++++++++++++++---------- tests/access_tests.py | 18 +++-- tests/base_tests.py | 4 +- tests/cache_tests.py | 2 + tests/celery_tests.py | 2 + tests/core_tests.py | 3 +- tests/dashboard_tests.py | 7 +- tests/dataframe_test.py | 2 + tests/datasource_tests.py | 1 + tests/db_engine_specs_test.py | 2 + tests/dict_import_export_tests.py | 2 + tests/druid_tests.py | 1 + tests/email_tests.py | 3 + tests/model_tests.py | 14 ++-- tests/security_tests.py | 2 + tests/sql_parse_tests.py | 3 + tests/sqllab_tests.py | 2 + tests/utils_tests.py | 3 + tests/viz_tests.py | 11 +++ 19 files changed, 143 insertions(+), 59 deletions(-) diff --git a/superset/cli.py b/superset/cli.py index 6183601d02536..05bf07ea43077 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -140,55 +140,95 @@ def version(verbose): def load_examples_run(load_test_data): - print('Loading examples into {}'.format(db)) - - data.load_css_templates() + #raise Exception("Where am I coming from?") + #import sys + #print(sys._getframe(1).f_code.co_name) + #print(sys._getframe(2).f_code.co_name) + #print(sys._getframe(3).f_code.co_name) + #print("^^^^") + #shave the time it spends loading and then parallelize the tests. + + #do we need this? + # parallelize the loading... + + #print('Loading examples into {}'.format(db)) + #start_css_tmp = utils.now_as_float() + #data.load_css_templates() + #print("took {}".format(utils.now_as_float() - start_css_tmp)) print('Loading energy related dataset') + start_energy = utils.now_as_float() data.load_energy() + print("took {}".format(utils.now_as_float() - start_energy)) print("Loading [World Bank's Health Nutrition and Population Stats]") + start_health = utils.now_as_float() data.load_world_bank_health_n_pop() + energy_time = utils.now_as_float() - start_health print('Loading [Birth names]') + start_birth = utils.now_as_float() data.load_birth_names() - - print('Loading [Random time series data]') - data.load_random_time_series_data() - - print('Loading [Random long/lat data]') - data.load_long_lat_data() - - print('Loading [Country Map data]') - data.load_country_map_data() - - print('Loading [Multiformat time series]') - data.load_multiformat_time_series_data() - - print('Loading [Paris GeoJson]') - data.load_paris_iris_geojson() - - print('Loading [San Francisco population polygons]') - data.load_sf_population_polygons() - - print('Loading [Flights data]') - data.load_flights() - - print('Loading [BART lines]') - data.load_bart_lines() - - print('Loading [Multi Line]') - data.load_multi_line() - - print('Loading [Misc Charts] dashboard') - data.load_misc_dashboard() - - if load_test_data: - print('Loading [Unicode test data]') - data.load_unicode_test_data() - - print('Loading DECK.gl demo') - data.load_deck_dash() + print(utils.now_as_float() - start_birth) + + #print('Loading [Random time series data]') + #start_ts = utils.now_as_float() + #data.load_random_time_series_data() + #print("took {}".format(utils.now_as_float() - start_ts)) + + #print('Loading [Random long/lat data]') + #start_lola = utils.now_as_float() + #data.load_long_lat_data() + #print("took {}".format(utils.now_as_float() - start_lola)) + + #print('Loading [Country Map data]') + #start_map = utils.now_as_float() + #data.load_country_map_data() + #print("took {}".format(utils.now_as_float() - start_map)) + + #print('Loading [Multiformat time series]') + #start_mts = utils.now_as_float() + #data.load_multiformat_time_series_data() + #print("took {}".format(utils.now_as_float() - start_mts)) + + #print('Loading [Paris GeoJson]') + #start_geo = utils.now_as_float() + #data.load_paris_iris_geojson() + #print("took {}".format(utils.now_as_float() - start_geo)) + + #print('Loading [San Francisco population polygons]') + #start_polygons = utils.now_as_float() + #data.load_sf_population_polygons() + #print("took {}".format(utils.now_as_float() - start_polygons)) + + #print('Loading [Flights data]') + #start_flights = utils.now_as_float() + #data.load_flights() + #print("took {}".format(utils.now_as_float() - start_flights)) + + #print('Loading [BART lines]') + #start_bart = utils.now_as_float() + #data.load_bart_lines() + #print("took {}".format(utils.now_as_float() - start_bart)) + + #print('Loading [Multi Line]') + #start_ml = utils.now_as_float() + #data.load_multi_line() + #print("took {}".format(utils.now_as_float() - start_ml)) + + #print('Loading [Misc Charts] dashboard') + #start_misc = utils.now_as_float() + #data.load_misc_dashboard() + #print("took {}".format(utils.now_as_float() - start_misc)) + + #if load_test_data: + # print('Loading [Unicode test data]') + # data.load_unicode_test_data() + + #print('Loading DECK.gl demo') + #start_deckgl = utils.now_as_float() + #data.load_deck_dash() + #print("took {}".format(utils.now_as_float() - start_deckgl)) @app.cli.command() diff --git a/tests/access_tests.py b/tests/access_tests.py index a669f5dec5937..6864ccb6e28a3 100644 --- a/tests/access_tests.py +++ b/tests/access_tests.py @@ -87,6 +87,7 @@ def create_access_request(session, ds_type, ds_name, role_name, user_name): class RequestAccessTests(SupersetTestCase): requires_examples = False + #_multiprocess_shared_ = True @classmethod def setUpClass(cls): @@ -170,10 +171,11 @@ def test_override_role_permissions_druid_and_table(self): self.assertEquals(3, len(perms)) def test_override_role_permissions_drops_absent_perms(self): + #use energy usage instead?.... override_me = security_manager.find_role('override_me') override_me.permissions.append( security_manager.find_permission_view_menu( - view_menu_name=self.get_table_by_name('long_lat').perm, + view_menu_name=self.get_table_by_name('energy_usage').perm, permission_name='datasource_access'), ) db.session.flush() @@ -259,9 +261,9 @@ def test_clean_requests_after_db_grant(self): gamma_user = security_manager.find_user(username='gamma') access_request1 = create_access_request( - session, 'table', 'long_lat', TEST_ROLE_1, 'gamma') + session, 'table', 'energy_usage', TEST_ROLE_1, 'gamma') create_access_request( - session, 'table', 'long_lat', TEST_ROLE_2, 'gamma2') + session, 'table', 'energy_usage', TEST_ROLE_2, 'gamma2') ds_1_id = access_request1.datasource_id # gamma gets granted database access database = session.query(models.Database).first() @@ -359,9 +361,9 @@ def test_approve(self, mock_send_mime): # Case 2. Extend the role to have access to the table access_request2 = create_access_request( - session, 'table', 'long_lat', TEST_ROLE_NAME, 'gamma') + session, 'table', 'energy_usage', TEST_ROLE_NAME, 'gamma') ds_2_id = access_request2.datasource_id - long_lat_perm = access_request2.datasource.perm + energy_usage_perm = access_request2.datasource.perm self.client.get(EXTEND_ROLE_REQUEST.format( 'table', access_request2.datasource_id, 'gamma', TEST_ROLE_NAME)) @@ -377,13 +379,13 @@ def test_approve(self, mock_send_mime): '[Superset] Access to the datasource {} was granted'.format( self.get_table(ds_2_id).full_name), call_args[2]['Subject']) self.assertIn(TEST_ROLE_NAME, call_args[2].as_string()) - self.assertIn('long_lat', call_args[2].as_string()) + self.assertIn('energy_usage', call_args[2].as_string()) # request was removed self.assertFalse(access_requests) - # table_role was extended to grant access to the long_lat table/ + # table_role was extended to grant access to the energy_usage table/ perm_view = security_manager.find_permission_view_menu( - 'datasource_access', long_lat_perm) + 'datasource_access', energy_usage_perm) TEST_ROLE = security_manager.find_role(TEST_ROLE_NAME) self.assertIn(perm_view, TEST_ROLE.permissions) diff --git a/tests/base_tests.py b/tests/base_tests.py index f69ab68381465..7c707a496a030 100644 --- a/tests/base_tests.py +++ b/tests/base_tests.py @@ -28,12 +28,14 @@ class SupersetTestCase(unittest.TestCase): requires_examples = False examples_loaded = False + _multiprocess_shared_ = True + def __init__(self, *args, **kwargs): if ( self.requires_examples and not os.environ.get('examples_loaded') ): - logging.info('Loading examples') + logging.info('Loading examples in test case') cli.load_examples_run(load_test_data=True) logging.info('Done loading examples') security_manager.sync_role_definitions() diff --git a/tests/cache_tests.py b/tests/cache_tests.py index 27a2c5f2566e5..c4c0cada8bff4 100644 --- a/tests/cache_tests.py +++ b/tests/cache_tests.py @@ -13,6 +13,8 @@ class CacheTests(SupersetTestCase): + _multiprocess_shared_ = True + def __init__(self, *args, **kwargs): super(CacheTests, self).__init__(*args, **kwargs) diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 06b10031cd1b9..ff16060320f68 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -37,6 +37,8 @@ class CeleryConfig(object): class UtilityFunctionTests(SupersetTestCase): + _multiprocess_shared_ = True + # TODO(bkyryliuk): support more cases in CTA function. def test_create_table_as(self): q = SupersetQuery('SELECT * FROM outer_space;') diff --git a/tests/core_tests.py b/tests/core_tests.py index d0eeacf2296a1..74401305f0d3c 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -36,6 +36,7 @@ class CoreTests(SupersetTestCase): requires_examples = True + _multiprocess_can_split_ = True def __init__(self, *args, **kwargs): super(CoreTests, self).__init__(*args, **kwargs) @@ -378,7 +379,7 @@ def test_warm_up_cache(self): data = self.get_json_resp( '/superset/warm_up_cache?table_name=energy_usage&db_name=main') - assert len(data) == 4 + assert len(data) == 3 def test_shortner(self): self.login(username='admin') diff --git a/tests/dashboard_tests.py b/tests/dashboard_tests.py index bc6c61f5b2d32..2b42bcbf185cd 100644 --- a/tests/dashboard_tests.py +++ b/tests/dashboard_tests.py @@ -19,6 +19,7 @@ class DashboardTests(SupersetTestCase): requires_examples = True + _multiprocess_can_split_ = True def __init__(self, *args, **kwargs): super(DashboardTests, self).__init__(*args, **kwargs) @@ -211,7 +212,7 @@ def test_add_slices(self, username='admin'): dash = db.session.query(models.Dashboard).filter_by( slug='births').first() new_slice = db.session.query(models.Slice).filter_by( - slice_name='Mapbox Long/Lat').first() + slice_name='Energy Force Layout').first() existing_slice = db.session.query(models.Slice).filter_by( slice_name='Name Cloud').first() data = { @@ -225,7 +226,7 @@ def test_add_slices(self, username='admin'): dash = db.session.query(models.Dashboard).filter_by( slug='births').first() new_slice = db.session.query(models.Slice).filter_by( - slice_name='Mapbox Long/Lat').first() + slice_name='Energy Force Layout').first() assert new_slice in dash.slices assert len(set(dash.slices)) == len(dash.slices) @@ -233,7 +234,7 @@ def test_add_slices(self, username='admin'): dash = db.session.query(models.Dashboard).filter_by( slug='births').first() dash.slices = [ - o for o in dash.slices if o.slice_name != 'Mapbox Long/Lat'] + o for o in dash.slices if o.slice_name != 'Energy Force Layout'] db.session.commit() def test_remove_slices(self, username='admin'): diff --git a/tests/dataframe_test.py b/tests/dataframe_test.py index b52b9508abb2e..4fce640b9ea18 100644 --- a/tests/dataframe_test.py +++ b/tests/dataframe_test.py @@ -12,6 +12,8 @@ class SupersetDataFrameTestCase(SupersetTestCase): + _multiprocess_can_split_ = True + def test_dedup(self): self.assertEquals( dedup(['foo', 'bar']), diff --git a/tests/datasource_tests.py b/tests/datasource_tests.py index 7da7d929f5ddd..c7aa602305104 100644 --- a/tests/datasource_tests.py +++ b/tests/datasource_tests.py @@ -14,6 +14,7 @@ class DatasourceTests(SupersetTestCase): requires_examples = True + _multiprocess_can_split_ = True def __init__(self, *args, **kwargs): super(DatasourceTests, self).__init__(*args, **kwargs) diff --git a/tests/db_engine_specs_test.py b/tests/db_engine_specs_test.py index 81709dc67b4ed..62ab713f9f3a3 100644 --- a/tests/db_engine_specs_test.py +++ b/tests/db_engine_specs_test.py @@ -18,6 +18,8 @@ class DbEngineSpecsTestCase(SupersetTestCase): + _multiprocess_can_split_ = True + def test_0_progress(self): log = """ 17/02/07 18:26:27 INFO log.PerfLogger: diff --git a/tests/dict_import_export_tests.py b/tests/dict_import_export_tests.py index c96ce63652c7e..291ec3fcf6a5a 100644 --- a/tests/dict_import_export_tests.py +++ b/tests/dict_import_export_tests.py @@ -26,6 +26,8 @@ class DictImportExportTests(SupersetTestCase): """Testing export import functionality for dashboards""" + _multiprocess_can_split_ = True + def __init__(self, *args, **kwargs): super(DictImportExportTests, self).__init__(*args, **kwargs) diff --git a/tests/druid_tests.py b/tests/druid_tests.py index 00df0f8ea6188..0fd07c9f5f258 100644 --- a/tests/druid_tests.py +++ b/tests/druid_tests.py @@ -82,6 +82,7 @@ def __reduce__(self): class DruidTests(SupersetTestCase): """Testing interactions with Druid""" + _multiprocess_shared_ = True def __init__(self, *args, **kwargs): super(DruidTests, self).__init__(*args, **kwargs) diff --git a/tests/email_tests.py b/tests/email_tests.py index 3ee9f3ebf22d7..4ca689554be99 100644 --- a/tests/email_tests.py +++ b/tests/email_tests.py @@ -19,6 +19,9 @@ class EmailSmtpTest(unittest.TestCase): + + #_multiprocess_shared_ = True + def setUp(self): app.config['smtp_ssl'] = False diff --git a/tests/model_tests.py b/tests/model_tests.py index 1a78567964c27..0cb665931bbd5 100644 --- a/tests/model_tests.py +++ b/tests/model_tests.py @@ -15,6 +15,7 @@ class DatabaseModelTestCase(SupersetTestCase): + _multiprocess_shared_ = True def test_database_schema_presto(self): sqlalchemy_uri = 'presto://presto.airbnb.io:8080/hive/default' @@ -79,7 +80,7 @@ def test_database_impersonate_user(self): def test_select_star(self): main_db = get_main_database(db.session) - table_name = 'bart_lines' + table_name = 'energy_usage' sql = main_db.select_star( table_name, show_cols=False, latest_partition=False) expected = textwrap.dedent("""\ @@ -91,11 +92,10 @@ def test_select_star(self): sql = main_db.select_star( table_name, show_cols=True, latest_partition=False) expected = textwrap.dedent("""\ - SELECT color, - name, - path_json, - polyline - FROM bart_lines + SELECT source, + target, + value + FROM energy_usage LIMIT 100""".format(**locals())) assert sql.startswith(expected) @@ -129,6 +129,8 @@ def test_multi_statement(self): class SqlaTableModelTestCase(SupersetTestCase): + _multiprocess_shared_ = True + def test_get_timestamp_expression(self): tbl = self.get_table_by_name('birth_names') diff --git a/tests/security_tests.py b/tests/security_tests.py index fe7564ed14a69..d9435ba2dca65 100644 --- a/tests/security_tests.py +++ b/tests/security_tests.py @@ -18,6 +18,8 @@ def get_perm_tuples(role_name): class RolePermissionTests(SupersetTestCase): """Testing export import functionality for dashboards""" + _multiprocess_can_split_ = True + def __init__(self, *args, **kwargs): super(RolePermissionTests, self).__init__(*args, **kwargs) diff --git a/tests/sql_parse_tests.py b/tests/sql_parse_tests.py index 3f66fedb434e2..fca1bc610c44b 100644 --- a/tests/sql_parse_tests.py +++ b/tests/sql_parse_tests.py @@ -8,8 +8,11 @@ from superset import sql_parse +#can parallelize + class SupersetTestCase(unittest.TestCase): + _multiprocess_can_split_ = True def extract_tables(self, query): sq = sql_parse.SupersetQuery(query) diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index a39643aa60fb0..7019c97995868 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -22,6 +22,8 @@ class SqlLabTests(SupersetTestCase): """Testings for Sql Lab""" + #_multiprocess_shared_ = True + def __init__(self, *args, **kwargs): super(SqlLabTests, self).__init__(*args, **kwargs) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index eb164b8b1aeac..2ca50bc0848de 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -73,6 +73,9 @@ def mock_to_adhoc(filt, expressionType='SIMPLE', clause='where'): class UtilsTestCase(unittest.TestCase): + + _multiprocess_can_split_ = True + def test_json_int_dttm_ser(self): dttm = datetime(2020, 1, 1) ts = 1577836800000.0 diff --git a/tests/viz_tests.py b/tests/viz_tests.py index da7c02f94094f..48110105c2379 100644 --- a/tests/viz_tests.py +++ b/tests/viz_tests.py @@ -20,6 +20,8 @@ class BaseVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True + def test_constructor_exception_no_datasource(self): form_data = {} datasource = None @@ -172,6 +174,7 @@ def test_cache_timeout(self): class TableVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True def test_get_data_applies_percentage(self): form_data = { @@ -396,6 +399,8 @@ def test_should_be_timeseries_raises_when_no_granularity(self): class PairedTTestTestCase(SupersetTestCase): + _multiprocess_shared_ = True + def test_get_data_transforms_dataframe(self): form_data = { 'groupby': ['groupA', 'groupB', 'groupC'], @@ -528,6 +533,7 @@ def test_get_data_empty_null_keys(self): class PartitionVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True @patch('superset.viz.BaseViz.query_obj') def test_query_obj_time_series_option(self, super_query_obj): @@ -753,6 +759,7 @@ def test_get_data_calls_correct_method(self): class RoseVisTestCase(SupersetTestCase): + _multiprocess_shared_ = True def test_rose_vis_get_data(self): raw = {} @@ -793,6 +800,7 @@ def test_rose_vis_get_data(self): class TimeSeriesTableVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True def test_get_data_metrics(self): form_data = { @@ -873,6 +881,7 @@ def test_query_obj_throws_metrics_and_groupby(self, super_query_obj): class BaseDeckGLVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True def test_get_metrics(self): form_data = load_fixture('deck_path_form_data.json') @@ -1066,6 +1075,8 @@ def test_filter_nulls(self, mock_uuid4): class TimeSeriesVizTestCase(SupersetTestCase): + _multiprocess_shared_ = True + def test_timeseries_unicode_data(self): datasource = self.get_datasource_mock() form_data = {