From 3ebea9ed0dcb5cb974e045b41cf6de9395d89544 Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Tue, 5 Jun 2018 14:16:34 -0500 Subject: [PATCH] Fix stable/pike gate This is a squash of several changes to get the gate to pass again. As a summary, it updates how our pep8 job handles checking that the opts.py file is up to date, and it disables the grenade jobs since those no longer work on this older extended maintenance branch. Move config-generator to tools (cherry picked from commit d3abafdee4affe457e39705fd9b8e27b0f80f657) [DOC BLD FIX] Make opts sections lowercase (cherry picked from commit 8db6335bb5b2bbb19ff5fddca104be257ba795c8) Update generate_cinder_opts path (cherry picked from commit b4485ec09fe327a81c94b44393d1ed2430b6a08f) Add another hack from NOVA_GROUP to generate_cinder_opts.py Closes-Bug: #1717008 (cherry picked from commit db6564bb748446785217e5de852ac9a6946ea481) Remove grenade jobs This branch is in extended maintenance. As part of EM, it was agreed we would run jobs as long as we could keep them passing. The grenade jobs have some issues that would take a lot of work to fix up, so given the stable status and the lack of resources on these older stable branches, just remove them. Change check_uptodate.sh to not call tox Base nodepool images now no longer include tox. This causes a failure in our pep8 jobs because the tools/config/check_uptodate.sh script had been calling "tox -e genopts" to look for config file changes. This is a partial backport of a change that switched our default basepython to python3. That part of the change was removed from tox.ini, so the only other part was an update to the check_uptodate.sh script that refactored things to be more reliable, including the removal of calling tox. The part from the original commit included is: There was also a difference in behavior with the genopts job where it ends up trying to recreate the venv under which it is currently running under, resulting in a corrupted venv and a failure. This cleans up that script and changes it so rather than a tox job calling a tox job it just runs the script. Signed-off-by: Sean McGinnis Change-Id: Ie79528b33070e6c60f9f5e1f2219084a2f0cb039 (cherry picked from commit 37510ac03478704bafba89e8c4f4e0e9041d661d) (cherry picked from commit 5e9e51190dc1751356475c4df2feb785de4089cd) --- .zuul.yaml | 64 ++++----------- cinder/opts.py | 12 +-- doc/source/conf.py | 2 +- {cinder => tools}/config/__init__.py | 0 tools/config/check_uptodate.sh | 79 ++++++++++++------- .../config/cinder-config-generator.conf | 0 .../config/generate_cinder_opts.py | 13 ++- tox.ini | 4 +- 8 files changed, 85 insertions(+), 89 deletions(-) rename {cinder => tools}/config/__init__.py (100%) rename {cinder => tools}/config/cinder-config-generator.conf (100%) rename {cinder => tools}/config/generate_cinder_opts.py (94%) diff --git a/.zuul.yaml b/.zuul.yaml index 0c1c8d0d425..54de9034a66 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -5,15 +5,12 @@ - publish-openstack-sphinx-docs - periodic-stable-jobs - check-requirements - - integrated-gate - - integrated-gate-py35 - release-notes-jobs check: jobs: - - cinder-tempest-dsvm-lvm-lio: - voting: false - branches: ^(?!driverfixes/).*$ + - tempest-full: irrelevant-files: + - ^(test-|)requirements.txt$ - ^.*\.rst$ - ^api-ref/.*$ - ^cinder/hacking/.*$ @@ -25,23 +22,11 @@ - ^releasenotes/.*$ - ^setup.cfg$ - ^tools/.*$ - - - openstack-tox-pylint: - voting: false - timeout: 5400 - irrelevant-files: - - ^.*\.rst$ - - ^api-ref/.*$ - - ^cinder/hacking/.*$ - - ^cinder/locale/.*$ - - ^cinder/tests/unit.*$ - - ^contrib/block-box.*$ - - ^doc/.*$ - - ^releasenotes/.*$ - - devstack-plugin-ceph-tempest: + - ^tox.ini$ + - cinder-tempest-dsvm-lvm-lio: voting: false + branches: ^(?!driverfixes/).*$ irrelevant-files: - - ^(test-|)requirements.txt$ - ^.*\.rst$ - ^api-ref/.*$ - ^cinder/hacking/.*$ @@ -53,9 +38,10 @@ - ^releasenotes/.*$ - ^setup.cfg$ - ^tools/.*$ - - ^tox.ini$ - - openstack-tox-functional: + + - openstack-tox-pylint: voting: false + timeout: 5400 irrelevant-files: - ^.*\.rst$ - ^api-ref/.*$ @@ -65,7 +51,8 @@ - ^contrib/block-box.*$ - ^doc/.*$ - ^releasenotes/.*$ - - legacy-grenade-dsvm-cinder-mn-sub-volbak: + - devstack-plugin-ceph-tempest: + voting: false irrelevant-files: - ^(test-|)requirements.txt$ - ^.*\.rst$ @@ -80,23 +67,18 @@ - ^setup.cfg$ - ^tools/.*$ - ^tox.ini$ - - legacy-tempest-dsvm-lvm-multibackend: + - openstack-tox-functional: voting: false irrelevant-files: - - ^(test-|)requirements.txt$ - ^.*\.rst$ - ^api-ref/.*$ - ^cinder/hacking/.*$ - ^cinder/locale/.*$ - - ^cinder/tests/functional.*$ - ^cinder/tests/unit.*$ - ^contrib/block-box.*$ - ^doc/.*$ - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tools/.*$ - - ^tox.ini$ - - legacy-tempest-dsvm-full-drbd-devstack: + - legacy-tempest-dsvm-lvm-multibackend: voting: false irrelevant-files: - ^(test-|)requirements.txt$ @@ -112,7 +94,7 @@ - ^setup.cfg$ - ^tools/.*$ - ^tox.ini$ - - legacy-tempest-dsvm-full-devstack-plugin-nfs: + - legacy-tempest-dsvm-full-drbd-devstack: voting: false irrelevant-files: - ^(test-|)requirements.txt$ @@ -128,7 +110,8 @@ - ^setup.cfg$ - ^tools/.*$ - ^tox.ini$ - - neutron-grenade: + - legacy-tempest-dsvm-full-devstack-plugin-nfs: + voting: false irrelevant-files: - ^(test-|)requirements.txt$ - ^.*\.rst$ @@ -145,22 +128,7 @@ - ^tox.ini$ gate: jobs: - - legacy-grenade-dsvm-cinder-mn-sub-volbak: - irrelevant-files: - - ^(test-|)requirements.txt$ - - ^.*\.rst$ - - ^api-ref/.*$ - - ^cinder/hacking/.*$ - - ^cinder/locale/.*$ - - ^cinder/tests/functional.*$ - - ^cinder/tests/unit.*$ - - ^contrib/block-box.*$ - - ^doc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tools/.*$ - - ^tox.ini$ - - neutron-grenade: + - tempest-full: irrelevant-files: - ^(test-|)requirements.txt$ - ^.*\.rst$ diff --git a/cinder/opts.py b/cinder/opts.py index 956a4ae2714..37d98dbf730 100644 --- a/cinder/opts.py +++ b/cinder/opts.py @@ -211,21 +211,21 @@ def list_opts(): return [ - ('BACKEND', + ('backend', itertools.chain( [cinder_cmd_volume.host_opt], )), - ('BRCD_FABRIC_EXAMPLE', + ('brcd_fabric_example', itertools.chain( cinder_zonemanager_drivers_brocade_brcdfabricopts. brcd_zone_opts, )), - ('CISCO_FABRIC_EXAMPLE', + ('cisco_fabric_example', itertools.chain( cinder_zonemanager_drivers_cisco_ciscofabricopts. cisco_zone_opts, )), - ('COORDINATION', + ('coordination', itertools.chain( cinder_coordination.coordination_opts, )), @@ -280,13 +280,13 @@ def list_opts(): cinder_volume_manager.volume_manager_opts, cinder_wsgi_eventletserver.socket_opts, )), - ('FC-ZONE-MANAGER', + ('fc-zone-manager', itertools.chain( cinder_zonemanager_drivers_brocade_brcdfczonedriver.brcd_opts, cinder_zonemanager_drivers_cisco_ciscofczonedriver.cisco_opts, cinder_zonemanager_fczonemanager.zone_manager_opts, )), - ('KEY_MANAGER', + ('key_manager', itertools.chain( cinder_keymgr_confkeymgr.key_mgr_opts, )), diff --git a/doc/source/conf.py b/doc/source/conf.py index 960c97cf946..68304f1d5e2 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,7 +63,7 @@ ] config_generator_config_file = ( - '../../cinder/config/cinder-config-generator.conf') + '../../tools/config/cinder-config-generator.conf') sample_config_basename = '_static/cinder' # autodoc generation is a bit aggressive and a nuisance diff --git a/cinder/config/__init__.py b/tools/config/__init__.py similarity index 100% rename from cinder/config/__init__.py rename to tools/config/__init__.py diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh index 77dcce064c3..2fbe29ff2e7 100755 --- a/tools/config/check_uptodate.sh +++ b/tools/config/check_uptodate.sh @@ -1,34 +1,55 @@ #!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is used to check if there have been configuration changes that +# have not been checked in. + +# The opts file needs to be present in order to compare it if [ ! -e cinder/opts.py ]; then - echo -en "\n\n#################################################" - echo -en "\nERROR: cinder/opts.py file is missing." - echo -en "\n#################################################\n" + echo "" + echo "#################################################" + echo "ERROR: cinder/opts.py file is missing." + echo "#################################################" + exit 1 +fi + +# Rename the existing file so we can generate a new one to compare +mv cinder/opts.py cinder/opts.py.orig +python tools/config/generate_cinder_opts.py &> tox-genops.log +if [ $? -ne 0 ]; then + cat tox-genops.log >&2 + echo "" + echo "#################################################" + echo "ERROR: Non-zero exit from generate_cinder_opts.py." + echo " See output above for details." + echo "#################################################" + mv cinder/opts.py.orig cinder/opts.py exit 1 -else - mv cinder/opts.py cinder/opts.py.orig - tox -e genopts &> /dev/null - if [ $? -ne 0 ]; then - echo -en "\n\n#################################################" - echo -en "\nERROR: Non-zero exit from generate_cinder_opts.py." - echo -en "\n See output above for details.\n" - echo -en "#################################################\n" - mv cinder/opts.py.orig cinder/opts.py - exit 1 - else - diff cinder/opts.py.orig cinder/opts.py - if [ $? -ne 0 ]; then - echo -en "\n\n########################################################" - echo -en "\nERROR: Configuration options change detected." - echo -en "\n A new cinder/opts.py file must be generated." - echo -en "\n Run 'tox -e genopts' from the base directory" - echo -en "\n and add the result to your commit." - echo -en "\n########################################################\n\n" - rm cinder/opts.py - mv cinder/opts.py.orig cinder/opts.py - exit 1 - else - rm cinder/opts.py.orig - fi - fi fi + +diff cinder/opts.py.orig cinder/opts.py +if [ $? -ne 0 ]; then + echo "" + echo "########################################################" + echo "ERROR: Configuration options change detected." + echo " A new cinder/opts.py file must be generated." + echo " Run 'tox -e genopts' from the base directory" + echo " and add the result to your commit." + echo "########################################################" + rm cinder/opts.py + mv cinder/opts.py.orig cinder/opts.py + exit 1 +fi + +rm cinder/opts.py.orig diff --git a/cinder/config/cinder-config-generator.conf b/tools/config/cinder-config-generator.conf similarity index 100% rename from cinder/config/cinder-config-generator.conf rename to tools/config/cinder-config-generator.conf diff --git a/cinder/config/generate_cinder_opts.py b/tools/config/generate_cinder_opts.py similarity index 94% rename from cinder/config/generate_cinder_opts.py rename to tools/config/generate_cinder_opts.py index f0b5bc47d46..2132d3afe6b 100644 --- a/cinder/config/generate_cinder_opts.py +++ b/tools/config/generate_cinder_opts.py @@ -19,6 +19,7 @@ from cinder.compute import nova from cinder.volume import configuration +from cinder.compute import nova OrderedDict = collections.OrderedDict @@ -104,7 +105,7 @@ def _check_import(aline): for atree in dir_trees_list: - if atree in ["cinder/config/generate_cinder_opts.py", + if atree in ["tools/config/generate_cinder_opts.py", "cinder/hacking/checks.py", "cinder/volume/configuration.py", "cinder/test.py"]: @@ -238,8 +239,14 @@ def _check_line_length(aline): key = lambda x: x[0])) for key in registered_opts_dict: - section_start_str = (" ('" + key + "',\n" - " itertools.chain(\n") + # NOTE(jsbryant): We need to have 'DEFAULT' in uppercase but any + # other section using uppercase causes a Sphinx warning. + if (key == 'DEFAULT'): + section_start_str = (" ('" + key + "',\n" + " itertools.chain(\n") + else: + section_start_str = (" ('" + key.lower() + "',\n" + " itertools.chain(\n") opt_file.write(section_start_str) for item in registered_opts_dict[key]: _write_item(item) diff --git a/tox.ini b/tox.ini index bf4c6562115..eed2d030376 100644 --- a/tox.ini +++ b/tox.ini @@ -82,12 +82,12 @@ commands = [testenv:genconfig] sitepackages = False envdir = {toxworkdir}/pep8 -commands = oslo-config-generator --config-file=cinder/config/cinder-config-generator.conf +commands = oslo-config-generator --config-file=tools/config/cinder-config-generator.conf [testenv:genopts] sitepackages = False envdir = {toxworkdir}/pep8 -commands = python cinder/config/generate_cinder_opts.py +commands = python tools/config/generate_cinder_opts.py [testenv:venv] commands = {posargs}