setup.py: improvements for pip install and venv

- remove root exception handling - include man pages as data_files - check for config files in sys.prefix/etc/clustershell - if $CLUSTERSHELL_CFGDIR is defined, always try it first This allows clustershell to be installed as user in a venv using pip install or using pip install --user with man pages. Root installation using pip is now discouraged. If done, /usr/local is likely to be used as the install prefix. Fixes #389.
cea-hpc · Dec 7, 2022 · 873120d · 873120d
1 parent 98761ce
commit 873120d
Show file tree

Hide file tree

Showing 6 changed files with 106 additions and 84 deletions.
diff --git a/clustershell.spec.in b/clustershell.spec.in
@@ -1,7 +1,9 @@
+%if 0%{?rhel} < 9
 %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
 %{!?python2_sitelib: %global python2_sitelib %{python_sitelib}}
 %{!?__python: %global __python python}
 %{!?__python2: %global __python2 %{__python}}
+%endif
 
 %if 0%{?fedora} >= 22
 %{!?python2_pkgversion: %global python2_pkgversion 2}
@@ -134,15 +136,6 @@ popd
 ln -s conf/groups.d/local.cfg %{buildroot}/%{_sysconfdir}/clustershell/groups
 %endif
 
-# man pages
-install -d %{buildroot}/%{_mandir}/{man1,man5}
-install -p -m 0644 doc/man/man1/clubak.1 %{buildroot}/%{_mandir}/man1/
-install -p -m 0644 doc/man/man1/cluset.1 %{buildroot}/%{_mandir}/man1/
-install -p -m 0644 doc/man/man1/clush.1 %{buildroot}/%{_mandir}/man1/
-install -p -m 0644 doc/man/man1/nodeset.1 %{buildroot}/%{_mandir}/man1/
-install -p -m 0644 doc/man/man5/clush.conf.5 %{buildroot}/%{_mandir}/man5/
-install -p -m 0644 doc/man/man5/groups.conf.5 %{buildroot}/%{_mandir}/man5/
-
 # vim addons
 %if 0%{?suse_version}
 %define vimdatadir %{_datadir}/vim/site

diff --git a/doc/sphinx/config.rst b/doc/sphinx/config.rst
@@ -11,23 +11,30 @@ clush
 clush.conf
 ^^^^^^^^^^
 
-The following configuration file defines global default values for
-several *clush* tool parameters::
+The following configuration file defines system-wide default values for
+several ``clush`` tool parameters::
 
-    $CLUSTERSHELL_CFGDIR/clush.conf
-    
-If *$CLUSTERSHELL_CFGDIR* is not defined, */etc/clustershell/clush.conf* will
-be used,
+    /etc/clustershell/clush.conf
 
-*clush* settings might then be overridden (globally, or per user) if one of the
-following files is found, in priority order::
+``clush`` settings might then be overridden (globally, or per user) if one of
+the following files is found, in priority order::
 
     $XDG_CONFIG_HOME/clustershell/clush.conf
     $HOME/.config/clustershell/clush.conf (only if $XDG_CONFIG_HOME is not defined)
+    {sys.prefix}/etc/clustershell/clush.conf
     $HOME/.local/etc/clustershell/clush.conf
     $HOME/.clush.conf (deprecated, for 1.6 compatibility only)
 
-The following table describes available *clush* config file settings.
+.. note:: The path using `sys.prefix`_ was added in version 1.9.1 and is
+   useful for Python virtual environments.
+
+In addition, if the environment variable ``$CLUSTERSHELL_CFGDIR`` is defined and
+valid, it will used instead. In such case, the following configuration file
+will be tried first for ``clush``::
+
+    $CLUSTERSHELL_CFGDIR/clush.conf
+
+The following table describes available ``clush`` config file settings.
 
 +-----------------+----------------------------------------------------+
 | Key             | Value                                              |
@@ -36,10 +43,10 @@ The following table describes available *clush* config file settings.
 |                 | number of *ssh(1)* allowed to run at the same      |
 |                 | time).                                             |
 +-----------------+----------------------------------------------------+
-| confdir         | Optional list of directory paths where *clush*     |
+| confdir         | Optional list of directory paths where ``clush``   |
 |                 | should look for **.conf** files which define       |
 |                 | :ref:`run modes <clushmode-config>` that can then  |
-|                 | be activated with `--mode`. All other  *clush*     |
+|                 | be activated with `--mode`. All other ``clush``    |
 |                 | config file settings defined in this table might   |
 |                 | be overriden in a run mode. Each mode section      |
 |                 | should have a name prefixed by "mode:" to clearly  |
@@ -80,25 +87,25 @@ The following table describes available *clush* config file settings.
 |                 | stderr, and cannot be modified.                    |
 +-----------------+----------------------------------------------------+
 | fd_max          | Maximum  number  of  open  file descriptors        |
-|                 | permitted per *clush* process (soft resource limit |
-|                 | for open files). This limit can never exceed the   |
-|                 | system (hard) limit. The *fd_max* (soft) and       |
+|                 | permitted per ``clush`` process (soft resource     |
+|                 | limit for open files). This limit can never exceed |
+|                 | the system (hard) limit. The *fd_max* (soft) and   |
 |                 | system (hard) limits should be high enough to      |
-|                 | run *clush*, although their values depend on       |
+|                 | run ``clush``, although their values depend on     |
 |                 | your fanout value.                                 |
 +-----------------+----------------------------------------------------+
 | history_size    | Set the maximum number of history entries saved in |
 |                 | the GNU readline history list. Negative values     |
 |                 | imply unlimited history file size.                 |
 +-----------------+----------------------------------------------------+
-| node_count      | Should *clush* display additional (node count)     |
+| node_count      | Should ``clush`` display additional (node count)   |
 |                 | information in buffer header? (yes/no)             |
 +-----------------+----------------------------------------------------+
-| maxrc           | Should *clush* return the largest of command       |
+| maxrc           | Should ``clush`` return the largest of command     |
 |                 | return codes? (yes/no)                             |
-|                 | If set to no (the default), *clush* exit status    |
+|                 | If set to no (the default), ``clush`` exit status  |
 |                 | gives no information about command return codes,   |
-|                 | but rather reports on *clush* execution itself     |
+|                 | but rather reports on ``clush`` execution itself   |
 |                 | (zero indicating a successful run).                |
 +-----------------+----------------------------------------------------+
 | password_prompt | Enable password prompt and password forwarding to  |
@@ -143,7 +150,7 @@ The following table describes available *clush* config file settings.
 Run modes
 ^^^^^^^^^
 
-Since version 1.9, *clush* has support for run modes, which are special
+Since version 1.9, ``clush`` has support for run modes, which are special
 :ref:`clush-config` settings with a given name. Two run modes are provided in
 example configuration files that can be copied and modified. They implement
 password-based authentication with *sshpass(1)* and support of interactive
@@ -152,7 +159,7 @@ password-based authentication with *sshpass(1)* and support of interactive
 To use a run mode with ``clush --mode``, install a configuration file in one
 of :ref:`clush-config`'s ``confdir`` (usually ``clush.conf.d``).  Only
 configuration files ending in **.conf** are scanned. If the user running
-*clush* doesn't have read access to a configuration file, is it ignored.
+``clush`` doesn't have read access to a configuration file, it is ignored.
 When ``--mode`` is specified, you can display all available run modes for
 the current user by enabling debug mode (``-d``).
 
@@ -189,21 +196,28 @@ ClusterShell loads *groups.conf* configuration files that define how to
 obtain node groups configuration, ie. the way the library should access
 file-based or external node group **sources**.
 
-The following configuration file defines global default values for
+The following configuration file defines system-wide default values for
 *groups.conf*::
 
-    $CLUSTERSHELL_CFGDIR/groups.conf
-
-If *$CLUSTERSHELL_CFGDIR* is not defined, */etc/clustershell/groups.conf* will
-be used,
+    /etc/clustershell/groups.conf
 
 *groups.conf* settings might then be overridden (globally, or per user) if one
 of the following files is found, in priority order::
 
     $XDG_CONFIG_HOME/clustershell/groups.conf
     $HOME/.config/clustershell/groups.conf (only if $XDG_CONFIG_HOME is not defined)
+    {sys.prefix}/etc/clustershell/groups.conf
     $HOME/.local/etc/clustershell/groups.conf
 
+.. note:: The path using `sys.prefix`_ was added in version 1.9.1 and is
+   useful for Python virtual environments.
+
+In addition, if the environment variable ``$CLUSTERSHELL_CFGDIR`` is defined and
+valid, it will used instead. In such case, the following configuration file
+will be tried first for *groups.conf*::
+
+    $CLUSTERSHELL_CFGDIR/groups.conf
+
 This makes possible for an user to have its own *node groups* configuration.
 If no readable configuration file is found, group support will be disabled but
 other node set operations will still work.
@@ -611,18 +625,22 @@ in *defaults.conf*.
 
 The following configuration file defines ClusterShell system-wide defaults::
 
-    $CLUSTERSHELL_CFGDIR/defaults.conf
-
-If *$CLUSTERSHELL_CFGDIR* is not defined, */etc/clustershell/defaults.conf*
-will be used,
+    /etc/clustershell/defaults.conf
 
 *defaults.conf* settings might then be overridden (globally, or per user) if
 one of the following files is found, in priority order::
 
     $XDG_CONFIG_HOME/clustershell/defaults.conf
     $HOME/.config/clustershell/defaults.conf (only if $XDG_CONFIG_HOME is not defined)
+    {sys.prefix}/etc/clustershell/defaults.conf
     $HOME/.local/etc/clustershell/defaults.conf
 
+In addition, if the environment variable ``$CLUSTERSHELL_CFGDIR`` is defined and
+valid, it will used instead. In such case, the following configuration file
+will be tried first for ClusterShell defaults::
+
+    $CLUSTERSHELL_CFGDIR/defaults.conf
+
 Use case: rsh
 ^^^^^^^^^^^^^^
 
@@ -660,3 +678,4 @@ without error.
 
 .. _ConfigParser: http://docs.python.org/library/configparser.html
 .. _nodeset: https://xcat-docs.readthedocs.io/en/stable/guides/admin-guides/references/man8/nodeset.8.html
+.. _sys.prefix: https://docs.python.org/3/library/sys.html#sys.prefix
diff --git a/doc/sphinx/install.rst b/doc/sphinx/install.rst
@@ -231,45 +231,53 @@ ClusterShell is available since "Natty" release (11.04):
 * http://packages.ubuntu.com/clustershell
 
 
-Installing ClusterShell using PIP
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Installing ClusterShell as root using PIP
-"""""""""""""""""""""""""""""""""""""""""
-
-To install ClusterShell as a standard Python package using PIP [#]_ as root::
-
-    $ pip install ClusterShell
-
-Or alternatively, using the source tarball::
-
-    $ pip install ClusterShell-1.x.tar.gz
+Installing ClusterShell the Python way
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+.. warning:: Installing ClusterShell as root using pip [#]_ is discouraged and
+   can result in conflicting behaviour with the system package manager.  Use
+   packages provided by your OS instead to install ClusterShell system-wide.
 
 .. _install-pip-user:
 
-Installing ClusterShell as user using PIP
+Installing ClusterShell as user using pip
 """""""""""""""""""""""""""""""""""""""""
 
-To install ClusterShell as a standard Python package using PIP as an user::
+To install ClusterShell as a standard Python package using pip as an user::
 
     $ pip install --user ClusterShell
 
 Or alternatively, using the source tarball::
 
     $ pip install --user ClusterShell-1.x.tar.gz
 
-Then, you just need to update your ``PYTHONPATH`` environment variable to be
-able to import the library and ``PATH`` to easily use the :ref:`tools`::
+Then, you might need to update your ``PATH`` to easily use the :ref:`tools`,
+and possibly set the ``PYTHONPATH`` environment variable to be able to import
+the library, and finally ``MANPATH`` for the man pages::
 
-    $ export PYTHONPATH=$PYTHONPATH:~/.local/lib
     $ export PATH=$PATH:~/.local/bin
+    $
+    $ # Might also be needed:
+    $ export PYTHONPATH=$PYTHONPATH:~/.local/lib
+    $ export MANPATH=$MANPATH:$HOME/.local/share/man
 
 Configuration files are installed in ``~/.local/etc/clustershell`` and are
 automatically loaded before system-wide ones (for more info about supported
 user config files, please see the :ref:`clush-config` or :ref:`groups-config`
 config sections).
 
+.. _install-venv-pip:
+
+Isolated environment using virtualenv and pip
+"""""""""""""""""""""""""""""""""""""""""""""
+
+It is possible to use virtual env (`venv`_) and pip to install ClusterShell
+in an isolated environment::
+
+    $ python3 -m venv venv
+    $ source venv/bin/activate
+    $ pip install ClusterShell
+
 .. _install-source:
 
 Source
@@ -291,3 +299,4 @@ the latest development version from the repository::
 .. _EPEL: http://fedoraproject.org/wiki/EPEL
 .. _Alma Linux: https://almalinux.org/
 .. _Rocky Linux: https://rockylinux.org/
+.. _venv: https://docs.python.org/3/tutorial/venv.html
diff --git a/lib/ClusterShell/Defaults.py b/lib/ClusterShell/Defaults.py
@@ -91,15 +91,22 @@ def _distant_workerclass(defaults):
 
 def config_paths(config_name):
     """Return default path list for a ClusterShell config file name."""
-    return [os.path.join(os.environ.get('CLUSTERSHELL_CFGDIR',
-                                        '/etc/clustershell'),
-			             config_name), # global config file
-            # default pip --user config file
-            os.path.expanduser('~/.local/etc/clustershell/%s' % config_name),
-            # per-user config (top override)
-            os.path.join(os.environ.get('XDG_CONFIG_HOME',
-                                        os.path.expanduser('~/.config')),
-                         'clustershell', config_name)]
+
+    paths = [os.path.join('/etc/clustershell', config_name), # system-wide
+             # default pip --user config file
+             os.path.expanduser('~/.local/etc/clustershell/%s' % config_name),
+             # Python installation prefix (for venv)
+             os.path.join(sys.prefix, 'etc/clustershell', config_name),
+             # per-user config (XDG Base Directory Specification)
+             os.path.join(os.environ.get('XDG_CONFIG_HOME',
+                                         os.path.expanduser('~/.config')),
+                          'clustershell', config_name)]
+
+    # $CLUSTERSHELL_CFGDIR has precedence over any other config paths
+    if 'CLUSTERSHELL_CFGDIR' in os.environ:
+        paths.append(os.path.join(os.environ['CLUSTERSHELL_CFGDIR'],
+                                  config_name))
+    return paths
 
 def _converter_integer_tuple(value):
     """ConfigParser converter for tuple of integers"""

diff --git a/setup.py b/setup.py
@@ -25,15 +25,8 @@
 
 VERSION = '1.9'
 
-# Default CFGDIR: in-prefix config install (rpmbuild or pip as user)
 CFGDIR = 'etc/clustershell'
-
-# Use system-wide CFGDIR instead when installing as root on Unix
-try:
-    if os.geteuid() == 0:
-        CFGDIR = '/etc/clustershell'
-except AttributeError:  # Windows?
-    pass
+MANDIR = 'share/man'
 
 # Dependencies (for pip install)
 REQUIRES = ['PyYAML']
@@ -57,7 +50,16 @@
                   (os.path.join(CFGDIR, 'groups.d'),
                    ['conf/groups.d/cluster.yaml.example',
                     'conf/groups.d/local.cfg',
-                    'conf/groups.d/README'])],
+                    'conf/groups.d/README']),
+                  (os.path.join(MANDIR, 'man1'),
+                   ['doc/man/man1/clubak.1',
+                    'doc/man/man1/cluset.1',
+                    'doc/man/man1/clush.1',
+                    'doc/man/man1/nodeset.1']),
+                  (os.path.join(MANDIR, 'man5'),
+                   ['doc/man/man5/clush.conf.5',
+                    'doc/man/man5/groups.conf.5']),
+                    ],
       entry_points={'console_scripts':
                     ['clubak=ClusterShell.CLI.Clubak:main',
                      'cluset=ClusterShell.CLI.Nodeset:main',

diff --git a/tests/CLIConfigTest.py b/tests/CLIConfigTest.py
@@ -320,8 +320,7 @@ def testClushConfigWithInstalledConfig(self):
         config = ClushConfig(options)
 
     def testClushConfigCustomGlobal(self):
-        """test CLI.Config.ClushConfig (CLUSTERSHELL_CFGDIR global custom
-        config)
+        """test CLI.Config.ClushConfig (CLUSTERSHELL_CFGDIR global custom config)
         """
 
         # Save existing environment variable, if it's defined
@@ -375,13 +374,6 @@ def testClushConfigCustomGlobal(self):
     def testClushConfigUserOverride(self):
         """test CLI.Config.ClushConfig (XDG_CONFIG_HOME user config)"""
 
-        # XXX Test should be improved when CLUSTERSHELL_CONFIG is available
-        # Improvement: override CLUSTERSHELL_CONFIG and set a sys clush config
-        # then verify that user config overrides CLUSTERSHELL_CONFIG as
-        # expected...
-        # For now, it has been tested manually. This test only really only
-        # ensures that user config is taken into account.
-
         xdg_config_home_save = os.environ.get('XDG_CONFIG_HOME')
 
         # Create fake XDG_CONFIG_HOME