diff --git a/docker/images/yoda_portal/yoda_portal_init.sh b/docker/images/yoda_portal/yoda_portal_init.sh index 02658f511..2105eb801 100644 --- a/docker/images/yoda_portal/yoda_portal_init.sh +++ b/docker/images/yoda_portal/yoda_portal_init.sh @@ -198,6 +198,10 @@ UPLOAD_PART_FILES = True # Text file extensions configuration TEXT_FILE_EXTENSIONS = ['bash', 'csv', 'c', 'cpp', 'csharp', 'css', 'diff', 'fortran', 'gams', 'gauss', 'go', 'graphql', 'ini', 'irpf90', 'java', 'js', 'json', 'julia', 'julia-repl', 'kotlin', 'less', 'lua', 'makefile', 'markdown', 'md', 'mathematica', 'matlab', 'maxima', 'mizar', 'objectivec', 'openscad', 'perl', 'php', 'php-template', 'plaintext', 'txt', 'python', 'py', 'python-repl', 'r', 'ruby', 'rust', 'sas', 'scilab', 'scss', 'shell', 'sh', 'sql', 'stan', 'stata', 'swift', 'typescript', 'ts', 'vbnet', 'wasm', 'xml', 'yaml', 'html'] + +# Monitor thread configuration +MONITOR_OUTPUT_DIR = "/tmp" +MONITOR_SIGNAL_FILE = "/var/www/yoda/show-tech.sig" FLASKCFG progress_update "Portal configured" diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md index 89cea6a88..4f56cecc7 100644 --- a/docs/administration/configuring-yoda.md +++ b/docs/administration/configuring-yoda.md @@ -159,6 +159,8 @@ yoda_theme_path | Path where themes for the Yoda Portal are re portal_session_cookie_samesite | Samesite setting for session cookies Yoda Portal. Should be 'Lax' if OIDC is enabled and identity provider is in different domain. Otherwise it should be 'Strict'. Default value: 'Strict'. yoda_portal_wsgi_daemon_threads | The number of threads to be created to handle requests in each daemon process (default: 15) yoda_portal_upload_part_files | Whether the portal uploader function should upload multi-chunk files as .part files initially and rename them to their final name later (boolean value, default: true). It is generally recommended to keep this enabled, so that users can easily see when an upload failed and the result is partial. However, on storage systems where renaming data objects takes much time, such as S3 object storage in consistent mode, it may be necessary to switch use of .part files off. +yoda_portal_monitor_signal_file | Path to the signal file for the portal monitoring thread. If this file is present, the monitor thread will start logging technical support information to the directory configured in `yoda_portal_monitor_output_dir`. Set to empty string to disable the monitoring thread. Default value: /var/www/yoda/show-tech.sig +yoda_portal_monitor_output_dir | Path for the portal monitoring thread to log technical support information to if the signal file is present. Default value: /tmp. Note that the portal uses a private /tmp directory, rather than the global one. ### Generic logging configuration diff --git a/docs/administration/index.md b/docs/administration/index.md index 2cd77d25a..e3594205c 100644 --- a/docs/administration/index.md +++ b/docs/administration/index.md @@ -34,6 +34,7 @@ Specific information: ## Troubleshooting +- [Collecting portal technical support information](portal-tech-support-info.md) - [Restoring a collection](restore-collection.md) - [Troubleshooting email](troubleshooting-email.md) - [Troubleshooting data package archival in the vault](troubleshooting-copy-to-vault.md) diff --git a/docs/administration/portal-tech-support-info.md b/docs/administration/portal-tech-support-info.md new file mode 100644 index 000000000..99404c19a --- /dev/null +++ b/docs/administration/portal-tech-support-info.md @@ -0,0 +1,81 @@ +--- +parent: Administration Tasks +title: Collecting portal technical support information +nav_order: 17 +--- +# Collecting portal technical support information + +In case there is a problem with the Yoda portal and the application logs don't provide enough +information about what is going wrong, you may need to collect additional information. The portal +has a monitoring thread to collect information for technical support purposes. + +You can start information collection by creating a signal file for the monitoring thread. By default, +this is `/var/www/yoda/show-tech.sig`: + +```bash +$ sudo touch /var/www/yoda/show-tech.sig +``` + +When the signal file is present, the portal monitoring thread will start logging technical support information. +By default, the support information files are written to the Apache private `tmp` directory within the +global `/tmp` directory. These information files contain stack traces for each portal thread, as well +as some other information about the system and the portal. + +Remove the signal file after you have collected enough information: + +```bash +$ sudo rm /var/www/yoda/show-tech.sig +``` + +## Example output + +``` +Portal tech support info for combined.yoda.test, collected on 30/07/2024 at 16:42:46.926656 +Yoda version, as per portal config: development (ae4de73e4fe931423caf27f7537a57b72a606764) + +System-wide CPU percent: 2.5% +Memory: global total: 4.1 GB +Memory: global available: 2.0 GB +Memory: global buffers: 136.7 MB +Memory: global cached: 674.2 MB +Memory: process RSS: 55.9 MB +Memory: process VMS: 740.7 MB +Memory: process shared: 17.3 MB + +Thread ID: 140688039364352 + /usr/lib/python3.8/threading.py:890 [_bootstrap] + self._bootstrap_inner() + /usr/lib/python3.8/threading.py:932 [_bootstrap_inner] + self.run() + /usr/lib/python3.8/threading.py:1254 [run] + self.function(*self.args, **self.kwargs) + /var/www/yoda/monitor.py:47 [record_info_if_needed] + tshoot_info: StringIO = self.get_tshoot_info() + /var/www/yoda/monitor.py:80 [get_tshoot_info] + for filename, line_number, function_name, line in traceback.extract_stack(stack): + +Thread ID: 140688058500864 + /usr/lib/python3.8/threading.py:890 [_bootstrap] + self._bootstrap_inner() + /usr/lib/python3.8/threading.py:932 [_bootstrap_inner] + self.run() + /usr/lib/python3.8/threading.py:870 [run] + self._target(*self._args, **self._kwargs) + /var/www/yoda/research/research.py:48 [irods_writer] + chunk = q.get() + /usr/lib/python3.8/queue.py:170 [get] + self.not_empty.wait() + /usr/lib/python3.8/threading.py:302 [wait] + waiter.acquire() + +Thread ID: 140688071923456 + /usr/lib/python3.8/threading.py:890 [_bootstrap] + self._bootstrap_inner() + /usr/lib/python3.8/threading.py:932 [_bootstrap_inner] + self.run() + /usr/lib/python3.8/threading.py:870 [run] + self._target(*self._args, **self._kwargs) + /var/www/yoda/connman.py:50 [gc] + time.sleep(1) +``` + diff --git a/roles/apache/templates/usr.sbin.apache2.j2 b/roles/apache/templates/usr.sbin.apache2.j2 index 96863bfb9..1d57bd669 100644 --- a/roles/apache/templates/usr.sbin.apache2.j2 +++ b/roles/apache/templates/usr.sbin.apache2.j2 @@ -32,7 +32,7 @@ /etc/gss/mech.d/ r, /etc/magic r, /etc/mime.types r, - /proc/*/fd/ r, + /proc/** r, /run/apache2/wsgi.*.sock rw, /usr/bin/dash ix, /usr/bin/uname mrix, diff --git a/roles/yoda_portal/defaults/main.yml b/roles/yoda_portal/defaults/main.yml index 73fa81755..5fa6c1a35 100644 --- a/roles/yoda_portal/defaults/main.yml +++ b/roles/yoda_portal/defaults/main.yml @@ -85,3 +85,7 @@ token_lifetime: 72 # Lifetime of data access tok # SRAM Configuration enable_sram: false # Enable SRAM configuration + +# Monitor thread configuration for extracting tech support information +yoda_portal_monitor_output_dir: /tmp +yoda_portal_monitor_signal_file: /var/www/yoda/show-tech.sig diff --git a/roles/yoda_portal/templates/flask.cfg.j2 b/roles/yoda_portal/templates/flask.cfg.j2 index 85bf22689..7725b492b 100644 --- a/roles/yoda_portal/templates/flask.cfg.j2 +++ b/roles/yoda_portal/templates/flask.cfg.j2 @@ -113,3 +113,7 @@ UPLOAD_PART_FILES = {{ yoda_portal_upload_part_files }} # Text file extensions configuration TEXT_FILE_EXTENSIONS = {{ text_file_extensions | default(['txt'])}} + +# Monitor thread configuration +MONITOR_OUTPUT_DIR = "{{ yoda_portal_monitor_output_dir }}" +MONITOR_SIGNAL_FILE = "{{ yoda_portal_monitor_signal_file }}"