Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Jenkins files for Test PYPI check #219

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

Jenkins files for Test PYPI check #219

Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
8decea9
install from testpyi -pre
Christian-B Jul 17, 2023
061872b
remove DataSpecification
Christian-B Jul 18, 2023
5d81599
remove more DataSpecification
Christian-B Jul 18, 2023
e015210
merged in main
Christian-B Oct 9, 2023
529f864
merge
Christian-B Nov 10, 2023
406ab85
merged in main
Christian-B Dec 20, 2023
b386e5c
merged in main
Christian-B Jan 8, 2024
5793328
merged in main
Christian-B Jan 15, 2024
f2846ca
merged in main
Christian-B Jan 19, 2024
56ea415
merge
Christian-B May 2, 2024
12e9319
merged in main
Christian-B May 17, 2024
7b820db
merge
Christian-B Jun 25, 2024
c1cfd34
dont do make
Christian-B Aug 12, 2024
0230fdf
remove none pypi stuff
Christian-B Aug 12, 2024
69d1e00
sh 'mkdir ${WORKSPACE}/global_reports'
Christian-B Aug 12, 2024
fdd601b
remove GFE tests as they need built ocde not on pypi
Christian-B Aug 13, 2024
38d5219
merged in main
Christian-B Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 9 additions & 156 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,19 +63,11 @@ pipeline {
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/spinn_common.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/SpiNNFrontEndCommon.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/sPyNNaker.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/Visualiser.git'
// Java dependencies
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/JavaSpiNNaker'
// scripts
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/PyNNExamples.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/sPyNNakerNewModelTemplate.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/microcircuit_model.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/SpiNNGym.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/MarkovChainMonteCarlo.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/TestBase.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/SpiNNaker_PDP2.git'
sh 'support/gitclone.sh https://github.com/SpiNNakerManchester/SpiNNakerJupyterExamples.git'
sh 'support/gitclone.sh https://$GITHUB_TOKEN@github.com/SpiNNakerManchester/TSPonSpiNNaker.git'
}
}
stage('Install') {
Expand All @@ -89,40 +81,14 @@ pipeline {
run_in_pyenv('pip3 install --upgrade setuptools wheel')
run_in_pyenv('pip install --upgrade pip')

// Install SpiNNUtils first as needed for C build
run_in_pyenv('pip install ./SpiNNUtils[test]')
// C Build next as builds files to be installed in Python
run_in_pyenv('make -C $SPINN_DIRS')
run_in_pyenv('make -C spinn_common install')
run_in_pyenv('make -C SpiNNFrontEndCommon/c_common')
run_in_pyenv('make -C SpiNNFrontEndCommon/c_common install')
run_in_pyenv('make -C sPyNNaker/neural_modelling')
run_in_pyenv('make -C sPyNNakerNewModelTemplate/c_models')
run_in_pyenv('make -C SpiNNakerGraphFrontEnd/gfe_examples')
run_in_pyenv('make -C SpiNNakerGraphFrontEnd/gfe_integration_tests/')
run_in_pyenv('make -C SpiNNGym/c_code')
run_in_pyenv('make -C MarkovChainMonteCarlo/c_models')
run_in_pyenv('make -C SpiNNaker_PDP2/c_code')
run_in_pyenv('make -C Visualiser')
run_in_pyenv('make -C TSPonSpiNNaker/spinnaker_c')
// Python install
run_in_pyenv('pip install ./SpiNNMachine[test]')
run_in_pyenv('pip install ./SpiNNMan[test]')
run_in_pyenv('pip install ./PACMAN[test]')
run_in_pyenv('pip install ./spalloc[test]')
run_in_pyenv('pip install ./SpiNNFrontEndCommon[test]')
run_in_pyenv('pip install ./TestBase[test]')
run_in_pyenv('pip install ./sPyNNaker[test]')
run_in_pyenv('pip install ./sPyNNakerNewModelTemplate[test]')
run_in_pyenv('pip install ./SpiNNakerGraphFrontEnd[test]')
run_in_pyenv('pip install ./SpiNNGym[test]')
run_in_pyenv('pip install ./MarkovChainMonteCarlo[test]')
// Due to the binaries being outside of the package
run_in_pyenv('pip install -e ./SpiNNaker_PDP2[test]')
run_in_pyenv('pip install ./Visualiser[test]')
run_in_pyenv('pip install ./TSPonSpiNNaker[test]')
// no install SpiNNakerJupyterExamples
// Python install from testpypi
run_in_pyenv('pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ sPyNNaker --pre')
run_in_pyenv('pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ SpiNNakerGraphFrontEnd --pre')
run_in_pyenv('pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ SpiNNakerTestBase --pre')

run_in_pyenv('python -m spynnaker.pyNN.setup_pynn')
// Stuff normally installed by [test] installs
run_in_pyenv('pip install pytest-cov testfixtures "httpretty != 1.0.0" pylint testfixtures mock graphviz')
// Additional requirements for testing here
// coverage version capped due to https://github.com/nedbat/coveragepy/issues/883
run_in_pyenv('pip install nbmake python-coveralls "coverage>=5.0.0" pytest-instafail pytest-xdist pytest-progress pytest-forked pytest-timeout')
Expand All @@ -135,41 +101,17 @@ pipeline {
steps {
sh 'rm -r spinn_common'
sh 'rm -r SpiNNUtils/spinn_utilities'
sh 'rm -r SpiNNUtils/build'
sh 'rm -r SpiNNMachine/spinn_machine'
sh 'rm -r SpiNNMachine/build'
sh 'rm -r SpiNNMan/spinnman'
sh 'rm -r SpiNNMan/build'
sh 'rm -r PACMAN/pacman'
sh 'rm -r PACMAN/pacman_test_objects'
sh 'rm -r PACMAN/build'
sh 'rm -r spalloc/spalloc_client'
sh 'rm -r spalloc/build'
sh 'rm -r SpiNNFrontEndCommon/spinn_front_end_common'
sh 'rm -r SpiNNFrontEndCommon/c_common'
sh 'rm -r SpiNNFrontEndCommon/build'
sh 'rm -r TestBase/spinnaker_testbase'
sh 'rm -r TestBase/build'
sh 'rm -r sPyNNaker/spynnaker'
sh 'rm -r sPyNNaker/neural_modelling'
sh 'rm -r sPyNNaker/build'
sh 'rm -r sPyNNakerNewModelTemplate/python_models8'
sh 'rm -r sPyNNakerNewModelTemplate/build'
sh 'rm -r SpiNNakerGraphFrontEnd/spinnaker_graph_front_end'
sh 'rm -r SpiNNakerGraphFrontEnd/build'
sh 'rm -r SpiNNGym/spinn_gym'
sh 'rm -r SpiNNGym/c_code'
sh 'rm -r SpiNNGym/build'
sh 'rm -r MarkovChainMonteCarlo/mcmc'
sh 'rm -r MarkovChainMonteCarlo/build'
// Due to the binaries being outside of the package
//NO remove SpiNNaker_PDP2
sh 'rm -r Visualiser/visualiser_example_binaries'
sh 'rm -r Visualiser/build'
// No remove SpiNNakerJupyterExamples
sh 'rm -r TSPonSpiNNaker/spinnaker_c'
sh 'rm -r TSPonSpiNNaker/build'
}
}
}
stage('Before Script') {
steps {
Expand Down Expand Up @@ -197,12 +139,7 @@ pipeline {
run_pytest('sPyNNaker/unittests', 1200, 'sPyNNaker', 'unit', 'auto')
run_pytest('SpiNNakerGraphFrontEnd/unittests', 1200, 'SpiNNakerGraphFrontEnd', 'unit', 'auto')
run_pytest('PyNNExamples/unittests', 1200, 'PyNNExamples', 'unit', 'auto')
run_pytest('SpiNNGym/unittests', 1200, 'SpiNNGym', 'unit', 'auto')
run_pytest('MarkovChainMonteCarlo/unittests', 1200, 'MarkovChainMonteCarlo', 'unit', 'auto')
run_pytest('SpiNNaker_PDP2/unittests', 1200, 'SpiNNaker_PDP2', 'unit', 'auto')
run_pytest('TSPonSpiNNaker/unittests', 1200, 'TSPonSpiNNaker', 'unit', 'auto')
run_in_pyenv('python -m spinn_utilities.executable_finder')
// no SpiNNakerJupyterExamples
}
}
stage('Run sPyNNaker Integration Tests') {
Expand All @@ -213,15 +150,6 @@ pipeline {
}
}
}
stage('Run GFE Integeration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_gfe_config()
run_in_pyenv('python SpiNNakerGraphFrontEnd/gfe_integration_tests/script_builder.py')
run_pytest('SpiNNakerGraphFrontEnd/gfe_integration_tests/', 3600, 'GFE_Integration', 'integration', 'auto')
}
}
}
stage('Run PyNNExamples Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
Expand All @@ -231,15 +159,6 @@ pipeline {
}
}
}
stage('Run sPyNNakerNewModelTemplate Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_spynnaker_config()
run_in_pyenv('python sPyNNakerNewModelTemplate/nmt_integration_tests/script_builder.py')
run_pytest('sPyNNakerNewModelTemplate/nmt_integration_tests', 3600, 'sPyNNakerNewModelTemplate_Integration', 'integration', 'auto')
}
}
}
stage('Run microcircuit_model Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
Expand All @@ -248,76 +167,10 @@ pipeline {
}
}
}
stage('Run SpiNNGym Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_spynnaker_config()
run_in_pyenv('python SpiNNGym/integration_tests/script_builder.py')
run_pytest('SpiNNGym/integration_tests', 3600, 'SpiNNGym_Integration', 'integration', 'auto')
}
}
}
stage('Run MarkovChainMonteCarlo Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_gfe_config()
run_in_pyenv('python MarkovChainMonteCarlo/mcmc_integration_tests/script_builder.py')
run_pytest('MarkovChainMonteCarlo/mcmc_integration_tests', 3600, 'MarkovChainMonteCarlo_Integration', 'integration', 'auto')
}
}
}
stage('Run SpiNNaker_PDP2 Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_gfe_config()
run_in_pyenv('python SpiNNaker_PDP2/integration_tests/script_builder.py')
run_pytest('SpiNNaker_PDP2/integration_tests', 3600, 'SpiNNaker_PDP2_Integration', 'integration', 'auto')
}
}
}
stage('Run Visualiser Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_spynnaker_config()
run_pytest('Visualiser/visualiser_integration_tests', 12000, 'visualiser_Integration', 'integration', 'auto')
}
}
}
stage("SpiNNakerJupyterExamples") {
steps {
create_spynnaker_config()
run_in_pyenv("pytest -n auto --nbmake SpiNNakerJupyterExamples/**/*.ipynb SpiNNakerJupyterExamples/**/**/*.ipynb")
}
}
stage('Run TSPonSpiNNaker Integration Tests') {
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_gfe_config()
run_in_pyenv('python TSPonSpiNNaker/integration_tests/script_builder.py')
run_pytest('TSPonSpiNNaker/integration_tests', 3600, 'TSPonSpiNNaker', 'integration', 'auto')
}
}
}
/*
stage('Run Whole Machine Tests') {
when {
environment name: 'THE_JOB', value: 'Integration_Tests_Cron_Job'
}
environment {
SPALLOC_PASSWORD = credentials('spalloc-password')
}
steps {
catchError(stageResult: 'FAILURE', catchInterruptions: false) {
create_spynnaker_config()
run_pytest('sPyNNaker/test_whole_board', 12000, 'test_whole_machine', 'integration', '16')
}
}
}
*/
stage('Reports') {
steps {
sh 'find . -maxdepth 3 -type f -wholename "*/global_reports/*" -print -exec cat \\{\\} \\;'
run_in_pyenv('python -m spinn_utilities.executable_finder')
sh 'find ${WORKSPACE}/global_reports -print -exec cat \\{\\} \\;'
}
}
stage('Check Destroyed') {
Expand Down