Merge branch 'master_test' into feature/new_distr

vitodb · Feb 11, 2025 · e357e85 · e357e85
2 parents 244f858 + 5577e9e
commit e357e85
Show file tree

Hide file tree

Showing 24 changed files with 5,630 additions and 141 deletions.
diff --git a/.Jenkins/workflows/Jenkinsfile_EL9 b/.Jenkins/workflows/Jenkinsfile_EL9
@@ -2,14 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pipeline {
-    agent { label 'docker' }
+    agent { label 'podman' }
     stages{
         stage('DE modules tests') {
             parallel {
                 stage('flake8') {
                     agent {
                         node {
-                            label 'docker'
+                            label 'podman'
                             customWorkspace "${WORKSPACE}/${STAGE_NAME}"
                         }
                     }
@@ -18,7 +18,8 @@ pipeline {
                     }
                     steps {
                         script {
-                            // DOCKER_IMAGE is defined through Jenkins project
+                            // 
+                            _IMAGE is defined through Jenkins project
                             flake8StageDockerImage="${DOCKER_IMAGE}_${BUILD_NUMBER}_${STAGE_NAME}"
                             // Set custom Build Name
                             if (params.GITHUB_PR_NUMBER) {
@@ -47,23 +48,23 @@ pipeline {
                             fi
                             cd ..
                         '''
-                        echo "prepare docker image ${flake8StageDockerImage}"
-                        sh "docker build --pull --tag ${flake8StageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
+                        echo "prepare podman image ${flake8StageDockerImage}"
+                        sh "podman build --pull --tag ${flake8StageDockerImage} --build-arg BASEIMAGE=docker.io/hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
                         echo "Run ${STAGE_NAME} tests"
-                        sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${flake8StageDockerImage} \"-m pytest -m flake8 --flake8\" \"flake8.log\" \"${BRANCH}\""
+                        sh "podman run --userns keep-id:uid=\$(id -u),gid=\$(id -g) --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${flake8StageDockerImage} \"-m flake8\" \"flake8.log\" \"${BRANCH}\""
                     }
                     post {
                         always {
                             archiveArtifacts artifacts: "decisionengine_modules/flake8.log"
-                            echo "cleanup docker image ${flake8StageDockerImage}"
-                            sh "docker rmi ${flake8StageDockerImage}"
+                            echo "cleanup podman image ${flake8StageDockerImage}"
+                            sh "podman rmi ${flake8StageDockerImage}"
                         }
                     }
                 }
                 stage('unit_tests') {
                     agent {
                         node {
-                            label 'docker'
+                            label 'podman'
                             customWorkspace "${WORKSPACE}/${STAGE_NAME}"
                         }
                     }
@@ -91,23 +92,23 @@ pipeline {
                             fi
                             cd ..
                         '''
-                        echo "prepare docker image ${unit_testsStageDockerImage}"
-                        sh "docker build --pull --tag ${unit_testsStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
+                        echo "prepare podman image ${unit_testsStageDockerImage}"
+                        sh "podman build --pull --tag ${unit_testsStageDockerImage} --build-arg BASEIMAGE=docker.io/hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
                         echo "Run ${STAGE_NAME} tests"
-                        sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${unit_testsStageDockerImage} \"-m pytest --cov-report term --cov=decisionengine_modules --no-cov-on-fail\" \"pytest.log\" \"${BRANCH}\""
+                        sh "podman run --userns keep-id:uid=\$(id -u),gid=\$(id -g) --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${unit_testsStageDockerImage} \"-m pytest --cov-report term --cov=decisionengine_modules --no-cov-on-fail\" \"pytest.log\" \"${BRANCH}\""
                     }
                     post {
                         always {
                             archiveArtifacts artifacts: "decisionengine_modules/pytest.log"
-                            echo "cleanup docker image ${unit_testsStageDockerImage}"
-                            sh "docker rmi ${unit_testsStageDockerImage}"
+                            echo "cleanup podman image ${unit_testsStageDockerImage}"
+                            sh "podman rmi ${unit_testsStageDockerImage}"
                             }
                     }
                 }
                 stage('build_whl') {
                     agent {
                         node {
-                            label 'docker'
+                            label 'podman'
                             customWorkspace "${WORKSPACE}/${STAGE_NAME}"
                         }
                     }
@@ -135,16 +136,16 @@ pipeline {
                             fi
                             cd ..
                         '''
-                        echo "prepare docker image ${buildwhlStageDockerImage}"
-                        sh "docker build --pull --tag ${buildwhlStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
+                        echo "prepare podman image ${buildwhlStageDockerImage}"
+                        sh "podman build --pull --tag ${buildwhlStageDockerImage} --build-arg BASEIMAGE=docker.io/hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/"
                         echo "Run ${STAGE_NAME} tests"
-                        sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${buildwhlStageDockerImage} \"setup.py bdist_wheel\" \"build_whl.log\" \"${BRANCH}\""
+                        sh "podman run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${buildwhlStageDockerImage} \"setup.py bdist_wheel\" \"build_whl.log\" \"${BRANCH}\""
                     }
                     post {
                         always {
                             archiveArtifacts artifacts: "decisionengine_modules/build_whl.log,decisionengine_modules/dist/*.whl"
-                            echo "cleanup docker image ${buildwhlStageDockerImage}"
-                            sh "docker rmi ${buildwhlStageDockerImage}"
+                            echo "cleanup podman image ${buildwhlStageDockerImage}"
+                            sh "podman rmi ${buildwhlStageDockerImage}"
                         }
                     }
                 }

diff --git a/.flake8 b/.flake8
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
+# SPDX-License-Identifier: Apache-2.0
+[flake8]
+  max-line-length = 120
+  # Do not test external dependencies
+  extend-exclude = ./glideinwms,./decisionengine,./build
+  # Errors and warnings to ignore
+  extend-ignore =
+     # line too long (90 > 79 characters)
+     E501,
+     # too many blank lines (2)
+     E303,
+     # expected 2 blank lines, found 1
+     E302,
+     # at least two spaces before inline comment
+     E261,
+     # block comment should start with '# '
+     E265,
+     # whitespace before ':'
+     E203,
+     # permit line breaks in binary operators
+     W503,
+     # permit line breaks in binary operators
+     W504
+  exclude: []
+  count: True
+  show-source: True
+  statistics: True
diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml
@@ -14,8 +14,7 @@ on:
 
 jobs:
   run_flake8:
-    if: ${{ github.ref == 'refs/heads/1.7' }}
-    name: Run pytest-flake8 against code tree
+    name: Run flake8 against code tree
     runs-on: ubuntu-latest
     steps:
       - name: checkout DE Modules
@@ -92,7 +91,7 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/../:${{ github.workspace }}
         run: |
           cd ${{ github.workspace }}/decisionengine_modules
-          pytest -m flake8 --flake8 --log-level=WARNING
+          python3 -m flake8
 
   run_pylint:
     name: Run pylint against code tree

diff --git a/.reuse/dep5 b/.reuse/dep5
@@ -11,7 +11,7 @@ Source: https://github.com/HEPCloud/decisionengine_modules
 # SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC
 # SPDX-License-Identifier: Apache-2.0
 
-Files: src/*.jsonnet src/*/readme src/*.conf src/*.fixture src/*.csv src/*/AWS/scratch src/*/gce_limits_factory_entries.test src/*/data/monitoring.json src/*/data/passwd
+Files: src/*.jsonnet src/*/readme src/*.conf src/*.fixture src/*.csv src/*/AWS/scratch src/*/gce_limits_factory_entries.test src/*/data/monitoring.json src/*/data/passwd src/*/data/expected_metric_values.json
 Copyright: 2017 Fermi Research Alliance, LLC
 License: Apache-2.0
 
@@ -22,3 +22,7 @@ License: Apache-2.0
 Files: package/rpm/* package/systemd/*
 Copyright: 2017 Fermi Research Alliance, LLC
 License: Apache-2.0
+
+Files: dashboards/*.json
+Copyright: 2017 Fermi Research Alliance, LLC
+License: Apache-2.0