Skip to content

Commit fa7a95d

Browse files
zbeekmanzbeekman
committed
Unit test cleanup as discussed in jacobwilliams#30
-Fixes jacobwilliams#30: CMake tests 3 and 8 fail -Fixes jacobwilliams#49: tests dir should be under src -Maybe addresses jacobwilliams#27... I'll leave that up to @jacobwilliams -maybe concludes jacobwilliams#42... -SCons build works and runs test with `scons test` -Probably breaks visual studio build, @jacobwilliams: I can't work on this, I don't have access to visual studio -Tests return the number of errors encountered -CMake and build.sh scripts work fine, VS build likely broken -CMake flags now distinguished between compiler flags (for all compilers) and fortran specific compiler flags -Small CMake bug fixed where CMake wasn't passing the requested GNU specific flags -CMake and build.sh have added options to compile for code coverage analysis with gcov when using GNU compiler
1 parent dc416f2 commit fa7a95d

16 files changed

+1954
-1171
lines changed

‎.travis.yml

+6-5
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,26 @@ cache: apt
88
# Build matrix: Run the three build systems and tests in parallel
99
env:
1010
global:
11-
- DEPENDS="gfortran-4.9"
11+
- DEPENDS="gfortran-4.9 binutils"
1212
matrix:
1313
# CMake build with unit tests, no documentation
1414
# Allow to fail for now until tests are fixed
15-
- BUILD_SCRIPT="mkdir cmake-build && cd cmake-build && cmake -DSKIP_DOC_GEN:BOOL=TRUE .. && make -j 3 && (make test || true)"
15+
- BUILD_SCRIPT="mkdir cmake-build && cd cmake-build && cmake -DENABLE_CODE_COVERAGE:BOOL=TRUE -DSKIP_DOC_GEN:BOOL=TRUE .. && make -j 4 && make test"
1616
SPECIFIC_DEPENDS="cmake nodejs"
1717
JLINT="yes"
1818
DOCS="no"
1919
FoBiS="no"
2020

2121
# build with build.sh, make documentation, and do sniff test
22-
- BUILD_SCRIPT="./build.sh && cd bin && ./json"
22+
- BUILD_SCRIPT="./build.sh"
2323
SPECIFIC_DEPENDS=""
2424
JLINT="no"
2525
DOCS="yes"
2626
FoBiS="yes"
27+
CODE_COVERAGE="yes"
2728

2829
# test scons build, no documentation or jsonlint, sniff test
29-
- BUILD_SCRIPT="scons && cd bin && ./json"
30+
- BUILD_SCRIPT="scons && scons test"
3031
SPECIFIC_DEPENDS=""
3132
JLINT="no"
3233
DOCS="no"
@@ -43,7 +44,7 @@ install:
4344
- sudo apt-get install -y $SPECIFIC_DEPENDS $DEPENDS
4445
- if [[ $JLINT == [yY]* ]]; then sudo npm install -g jsonlint; fi
4546
- sudo ln -fs /usr/bin/gfortran-4.9 /usr/bin/gfortran && gfortran --version
46-
- if [[ $FoBiS == [yY]* ]]; then sudo -H pip install FoBiS.py && FoBiS.py --version; fi
47+
- if [[ $FoBiS == [yY]* ]]; then sudo -H pip install FoBiS.py==1.5.4 && FoBiS.py --version; fi
4748
- if [[ $DOCS == [yY]* ]]; then sudo dpkg -i robodoc_4.99.41-1_amd64.deb && robodoc --version; fi
4849

4950
script:

‎CMakeLists.txt

+117-146
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ set ( CMAKE_BUILD_TYPE "Release"
1717
CACHE STRING "Select which configuration to build." )
1818
set_property ( CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS ${CMAKE_CONFIGURATION_TYPES} )
1919

20-
2120
enable_language ( Fortran )
2221
include ( "cmake/pickFortranCompilerFlags.cmake" )
2322

@@ -42,14 +41,14 @@ set ( VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" )
4241
#-------------------------------------
4342
# Collect source files for the library
4443
#-------------------------------------
45-
set ( JF_LIB_SRCS src/json_module.f90 )
46-
set ( JF_TEST_SRCS tests/json_example.f90 )
44+
set ( JF_LIB_SRCS src/json_module.f90 )
45+
file ( GLOB JF_TEST_SRCS "src/tests/jf_test_*.f90" )
4746

4847
#-----------------------------------------
4948
# Collect all the mod files into their own
5049
# directory to ease installation issues
5150
#-----------------------------------------
52-
set ( CMAKE_Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/lib" )
51+
set ( MODULE_DIR "${CMAKE_BINARY_DIR}/lib" )
5352

5453
#-------------------------------------
5554
# Define where our files get installed
@@ -75,14 +74,14 @@ if ( "${CMAKE_SYSTEM_NAME}" MATCHES "Darwin" )
7574
set ( ENABLE_DYLIBS_USE_RPATH TRUE CACHE BOOL
7675
"Enable @rpath install name for dylibs" )
7776
mark_as_advanced ( ENABLE_DYLIBS_USE_RPATH )
78-
endif ( "${CMAKE_SYSTEM_NAME}" MATCHES "Darwin" )
77+
endif ()
7978

8079
if ( ENABLE_DYLIBS_USE_RPATH )
8180
set ( CMAKE_MACOSX_RPATH TRUE )
82-
else ( ENABLE_DYLIBS_USE_RPATH )
81+
else ()
8382
set ( CMAKE_INSTALL_NAME_DIR
8483
"${ABS_LIB_INSTALL_DIR}" )
85-
endif ( ENABLE_DYLIBS_USE_RPATH )
84+
endif ()
8685

8786
#---------------------------------------------
8887
# Build a shared and static library by default
@@ -96,7 +95,8 @@ set_target_properties ( ${LIB_NAME}-static
9695
OUTPUT_NAME ${LIB_NAME}
9796
PREFIX lib
9897
VERSION ${VERSION}
99-
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib )
98+
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
99+
Fortran_MODULE_DIRECTORY ${MODULE_DIR} )
100100
set_target_properties ( ${LIB_NAME}
101101
PROPERTIES
102102
OUTPUT_NAME ${LIB_NAME}
@@ -106,17 +106,6 @@ set_target_properties ( ${LIB_NAME}
106106
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib
107107
Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR} )
108108

109-
#--------------------------
110-
# Build the test executable
111-
#--------------------------
112-
add_executable ( test-${CMAKE_PROJECT_NAME} ${JF_TEST_SRCS} )
113-
target_link_libraries ( test-${CMAKE_PROJECT_NAME} ${LIB_NAME} )
114-
add_executable ( test-${CMAKE_PROJECT_NAME}-static ${JF_TEST_SRCS} )
115-
target_link_libraries ( test-${CMAKE_PROJECT_NAME}-static ${LIB_NAME}-static )
116-
set_target_properties ( test-${CMAKE_PROJECT_NAME} test-${CMAKE_PROJECT_NAME}-static
117-
PROPERTIES
118-
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin )
119-
120109
#-------------------------------------
121110
# Build the documentation with ROBODoc
122111
#-------------------------------------
@@ -142,143 +131,125 @@ if ( NOT SKIP_DOC_GEN )
142131
execute_process ( COMMAND ${CMAKE_COMMAND} -E remove_directory ${DOC_DIR} )
143132
message ( STATUS "Done dynamically computing ROBODoc outputs." )
144133

134+
foreach ( SRC_FILE ${JF_LIB_SRCS} ${JF_TEST_SRCS} )
135+
list ( APPEND ROBO_DEPENDS "${SRC_FILE}" )
136+
endforeach ( SRC_FILE )
145137
add_custom_command ( OUTPUT ${ROBODOC_OUTPUTS}
146138
COMMAND "${CMAKE_COMMAND}" -E make_directory "${DOC_DIR}" # Ensure DOC_DIR exists at build time
147139
COMMAND "${ROBODOC}" ${REQUIRED_ROBODOC_OPTIONS} ${ROBODOC_OPTIONS}
148-
DEPENDS "${CMAKE_SOURCE_DIR}/${JF_LIB_SRCS}" "${CMAKE_SOURCE_DIR}/${JF_TEST_SRCS}"
140+
DEPENDS ${ROBO_DEPENDS}
149141
COMMENT "Building HTML documentation for ${CMAKE_PROJECT_NAME} using ROBODoc" )
150142
add_custom_target ( documentation ALL
151143
DEPENDS ${ROBODOC_OUTPUTS} )
152-
else ( ROBODOC ) # Not found
144+
else () # Not found
153145
message ( WARNING
154146
"ROBODoc not found! Please set the CMake cache variable ROBODOC to point to the installed ROBODoc binary, and reconfigure or disable building the documentation. ROBODoc can be installed from: http://www.xs4all.nl/~rfsber/Robo/ If you do not wish to install ROBODoc and build the json-fortran documentation, then please set the CMake cache variable SKIP_DOC_GEN to TRUE." )
155-
endif ( ROBODOC )
156-
endif ( NOT SKIP_DOC_GEN )
147+
endif ()
148+
endif ()
149+
150+
#--------------------------
151+
# Handle test related stuff
152+
#--------------------------
153+
set ( ENABLE_TESTS TRUE CACHE BOOL
154+
"Enable the json-fortran tests. If true this will force build of example program too" )
157155

158156
#---------------------------------------------------------------------
159157
# Add some tests to ensure that the software is performing as expected
160158
#---------------------------------------------------------------------
161-
enable_testing()
162-
find_program ( JSONLINT jsonlint )
163-
find_program ( DIFF diff )
164-
set ( DATA_DIR ${CMAKE_BINARY_DIR}/files )
165-
configure_file ( ${CMAKE_SOURCE_DIR}/files/test1.json ${DATA_DIR}/test1.json COPYONLY )
166-
configure_file ( ${CMAKE_SOURCE_DIR}/files/test5.json ${DATA_DIR}/test5.json COPYONLY )
167-
configure_file ( ${CMAKE_SOURCE_DIR}/files/invalid.json ${DATA_DIR}/invalid.json COPYONLY )
168-
configure_file ( ${CMAKE_SOURCE_DIR}/files/invalid2.json ${DATA_DIR}/invalid2.json COPYONLY )
169-
170-
# Validate input
171-
if ( JSONLINT )
172-
add_test ( NAME validate-input1
173-
WORKING_DIRECTORY ${DATA_DIR}
174-
COMMAND ${JSONLINT} test1.json )
175-
add_test ( NAME validate-input5
176-
WORKING_DIRECTORY ${DATA_DIR}
177-
COMMAND ${JSONLINT} test5.json )
178-
endif ( JSONLINT )
179-
# As of now these are kind of sniff tests... Need to modify test program to indicate failure with `stop` codes
180-
# or more easily parsed results, and be able to validate json sent to stdout
181-
182-
# Dynamic lib
183-
add_test ( NAME test-${CMAKE_PROJECT_NAME}
184-
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin/
185-
COMMAND test-${CMAKE_PROJECT_NAME} )
186-
# Validate output
187-
if ( JSONLINT )
188-
add_test ( NAME validate-test2
189-
WORKING_DIRECTORY ${DATA_DIR}
190-
COMMAND ${JSONLINT} test2.json )
191-
set ( OPTIONAL_TESTS2 ${OPTIONAL_TESTS2} validate-test2 )
192-
add_test ( NAME validate-test4
193-
WORKING_DIRECTORY ${DATA_DIR}
194-
COMMAND ${JSONLINT} test4.json )
195-
set ( OPTIONAL_TESTS4 ${OPTIONAL_TESTS4} validate-test4 )
196-
endif ( JSONLINT )
197-
# Check output for differences
198-
if ( DIFF )
199-
add_test ( NAME test2-regression
200-
WORKING_DIRECTORY ${DATA_DIR}
201-
COMMAND ${DIFF} -q test2.json ${CMAKE_SOURCE_DIR}/files/test2.json )
202-
set ( OPTIONAL_TESTS2 ${OPTIONAL_TESTS2} test2-regression )
203-
add_test ( NAME test4-regression
204-
WORKING_DIRECTORY ${DATA_DIR}
205-
COMMAND ${DIFF} -q test4.json ${CMAKE_SOURCE_DIR}/files/test4.json )
206-
set ( OPTIONAL_TESTS4 ${OPTIONAL_TESTS4} test4-regression )
207-
else ( DIFF )
208-
message ( WARNING
209-
"For full test coverage diff, or a similar tool must be present on your system" )
210-
endif ( DIFF )
211-
if ( DIFF OR JSONLINT )
212-
set_tests_properties ( ${OPTIONAL_TESTS2}
213-
PROPERTIES
214-
DEPENDS test-${CMAKE_PROJECT_NAME}
215-
REQUIRED_FILES "${DATA_DIR}/test2.json"
216-
RESOURCE_LOCK "${DATA_DIR}/test2.json" )
217-
set_tests_properties ( ${OPTIONAL_TESTS4}
218-
PROPERTIES
219-
DEPENDS test-${CMAKE_PROJECT_NAME}
220-
REQUIRED_FILES "${DATA_DIR}/test4.json"
221-
RESOURCE_LOCK "${DATA_DIR}/test4.json" )
222-
endif ( DIFF OR JSONLINT )
223-
# Static lib
224-
add_test ( NAME test-${CMAKE_PROJECT_NAME}-static
225-
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/bin/"
226-
COMMAND test-${CMAKE_PROJECT_NAME}-static )
227-
unset ( OPTIONAL_TESTS2 )
228-
unset ( OPTIONAL_TESTS4 )
229-
# Validate output
230-
if ( JSONLINT )
231-
add_test ( NAME validate-test2-static
232-
WORKING_DIRECTORY ${DATA_DIR}
233-
COMMAND ${JSONLINT} test2.json )
234-
set ( OPTIONAL_TESTS2 ${OPTIONAL_TESTS2} validate-test2-static )
235-
add_test ( NAME validate-test4-static
236-
WORKING_DIRECTORY ${DATA_DIR}
237-
COMMAND ${JSONLINT} test4.json )
238-
set ( OPTIONAL_TESTS4 ${OPTIONAL_TESTS4} validate-test4-static )
239-
endif ( JSONLINT )
240-
# Check output for differences
241-
if ( DIFF )
242-
add_test ( NAME test2-regression-static
243-
WORKING_DIRECTORY ${DATA_DIR}
244-
COMMAND ${DIFF} -q test2.json ${CMAKE_SOURCE_DIR}/files/test2.json )
245-
set ( OPTIONAL_TESTS2 ${OPTIONAL_TESTS2} test2-regression-static )
246-
add_test ( NAME test4-regression-static
247-
WORKING_DIRECTORY ${DATA_DIR}
248-
COMMAND ${DIFF} -q test4.json ${CMAKE_SOURCE_DIR}/files/test4.json )
249-
set ( OPTIONAL_TESTS4 ${OPTIONAL_TESTS4} test4-regression-static )
250-
endif ( DIFF )
251-
if ( JSONLINT OR DIFF )
252-
set_tests_properties ( ${OPTIONAL_TESTS2}
253-
PROPERTIES
254-
DEPENDS test-${CMAKE_PROJECT_NAME}-static
255-
REQUIRED_FILES "${DATA_DIR}/test2.json"
256-
RESOURCE_LOCK "${DATA_DIR}/test2.json" )
257-
set_tests_properties ( ${OPTIONAL_TESTS4}
258-
PROPERTIES
259-
DEPENDS test-${CMAKE_PROJECT_NAME}-static
260-
REQUIRED_FILES "${DATA_DIR}/test4.json"
261-
RESOURCE_LOCK "${DATA_DIR}/test4.json" )
262-
endif ( JSONLINT OR DIFF )
263-
264-
if ( JSONLINT )
265-
set_tests_properties ( test-${CMAKE_PROJECT_NAME} test-${CMAKE_PROJECT_NAME}-static
266-
PROPERTIES
267-
FAIL_REGULAR_EXPRESSION "Error;ERROR;error"
268-
REQUIRED_FILES "${DATA_DIR}/test1.json;${DATA_DIR}/test5.json"
269-
RESOURCE_LOCK "${DATA_DIR}/test2.json;${DATA_DIR}/test4.json"
270-
DEPENDS "validate-input1;validate-input5" )
271-
else ( JSONLINT ) # Don't force validation of input if no JSONLINT
272-
message ( WARNING
273-
"For full test coverage please download and install jsonlint, a NODEJS package <http://nodejs.org>" )
274-
set_tests_properties ( test-${CMAKE_PROJECT_NAME} test-${CMAKE_PROJECT_NAME}-static
275-
PROPERTIES
276-
FAIL_REGULAR_EXPRESSION "Error;ERROR;error"
277-
REQUIRED_FILES "${DATA_DIR}/test1.json;${DATA_DIR}/test5.json"
278-
RESOURCE_LOCK "${DATA_DIR}/test2.json;${DATA_DIR}/test4.json" )
279-
endif ( JSONLINT )
280-
set_directory_properties ( PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES
281-
"${DATA_DIR}/test2.json;${DATA_DIR}/test4.json" )
159+
if ( ENABLE_TESTS ) # BUILD_EXAMPLE_PROGRAM will be true too
160+
enable_testing()
161+
162+
find_program ( JSONLINT jsonlint )
163+
find_program ( DIFF diff )
164+
set ( DATA_DIR ${CMAKE_BINARY_DIR}/files )
165+
configure_file ( ${CMAKE_SOURCE_DIR}/files/test1.json ${DATA_DIR}/test1.json COPYONLY )
166+
configure_file ( ${CMAKE_SOURCE_DIR}/files/test5.json ${DATA_DIR}/test5.json COPYONLY )
167+
configure_file ( ${CMAKE_SOURCE_DIR}/files/invalid.json ${DATA_DIR}/invalid.json COPYONLY )
168+
configure_file ( ${CMAKE_SOURCE_DIR}/files/invalid2.json ${DATA_DIR}/invalid2.json COPYONLY )
169+
170+
set_directory_properties ( PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES
171+
"${DATA_DIR}/test2.json;${DATA_DIR}/test4.json" )
172+
173+
# Validate input
174+
if ( JSONLINT )
175+
add_test ( NAME validate-input1
176+
WORKING_DIRECTORY ${DATA_DIR}
177+
COMMAND ${JSONLINT} test1.json )
178+
add_test ( NAME validate-input5
179+
WORKING_DIRECTORY ${DATA_DIR}
180+
COMMAND ${JSONLINT} test5.json )
181+
add_test ( NAME validate-invalid
182+
WORKING_DIRECTORY ${DATA_DIR}
183+
COMMAND ${JSONLINT} invalid.json )
184+
set_property ( TEST validate-invalid
185+
PROPERTY
186+
WILL_FAIL TRUE)
187+
add_test ( NAME validate-invalid2
188+
WORKING_DIRECTORY ${DATA_DIR}
189+
COMMAND ${JSONLINT} invalid2.json )
190+
set_property ( TEST validate-invalid2
191+
PROPERTY
192+
WILL_FAIL TRUE)
193+
endif ()
194+
195+
set ( UNIT_TESTS '' )
196+
foreach ( UNIT_TEST ${JF_TEST_SRCS} )
197+
get_filename_component ( TEST ${UNIT_TEST} NAME_WE )
198+
add_executable ( ${TEST} ${UNIT_TEST} )
199+
target_link_libraries ( ${TEST} ${LIB_NAME} )
200+
set_target_properties ( ${TEST}
201+
PROPERTIES
202+
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin )
203+
add_test( NAME ${TEST}
204+
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin
205+
COMMAND ./${TEST})
206+
list ( APPEND UNIT_TESTS ${TEST} )
207+
if ( JSONLINT )
208+
set_property ( TEST ${TEST}
209+
APPEND
210+
PROPERTY DEPENDS validate-input1 validate-input2 )
211+
endif()
212+
endforeach ( UNIT_TEST )
213+
214+
set_property ( TEST jf_test_3
215+
APPEND
216+
PROPERTY DEPENDS jf_test_2 )
217+
218+
# Validate output
219+
if ( JSONLINT )
220+
file ( GLOB JSON_FILES "${DATA_DIR}/test*.json" )
221+
foreach ( JSON_FILE ${JSON_FILES} )
222+
get_filename_component ( JSON_STEM ${JSON_FILE} NAME_WE )
223+
add_test ( NAME validate-output-${JSON_STEM}.json
224+
WORKING_DIRECTORY ${DATA_DIR}
225+
COMMAND ${JSONLINT} ${JSON_STEM}.json )
226+
set_property ( TEST validate-output-${JSON_STEM}.json
227+
APPEND
228+
PROPERTY
229+
DEPENDS ${UNIT_TESTS}
230+
REQUIRED_FILES ${JSON_FILES} )
231+
endforeach ( JSON_FILE )
232+
endif ()
233+
# Check output for differences
234+
if ( DIFF )
235+
file ( GLOB JSON_FILES "${DATA_DIR}/test*.json" )
236+
foreach ( JSON_FILE ${JSON_FILES} )
237+
get_filename_component ( JSON_STEM ${JSON_FILE} NAME_WE )
238+
add_test ( NAME regression-${JSON_STEM}.json
239+
WORKING_DIRECTORY ${DATA_DIR}
240+
COMMAND ${DIFF} -q ${JSON_STEM}.json ${CMAKE_SOURCE_DIR}/files/${JSON_STEM}.json )
241+
set_property ( TEST regression-${JSON_STEM}.json
242+
APPEND
243+
PROPERTY
244+
DEPENDS ${UNIT_TESTS}
245+
REQUIRED_FILES ${JSON_FILES} )
246+
endforeach ( JSON_FILE )
247+
else ()
248+
message ( WARNING
249+
"For full test coverage diff, or a similar tool must be present on your system" )
250+
endif ()
251+
252+
endif ()
282253

283254
#-------------------------
284255
# Perform the installation
@@ -292,7 +263,7 @@ install ( TARGETS ${LIB_NAME} ${LIB_NAME}-static
292263
# Code to fix the dylib install name on Mac.
293264
include ( cmake/fixupInstallNameDir.cmake )
294265

295-
install ( DIRECTORY "${CMAKE_Fortran_MODULE_DIRECTORY}/" DESTINATION "${INSTALL_MOD_DIR}" )
266+
install ( DIRECTORY "${MODULE_DIR}/" DESTINATION "${INSTALL_MOD_DIR}" )
296267

297268
#------------------------------------------
298269
# Add portable unistall command to makefile
@@ -345,4 +316,4 @@ set ( ENABLE_BUILD_TREE_EXPORT FALSE CACHE BOOL
345316
"Add the ${PACKAGE_NAME} build tree to the CMake package registry?" )
346317
if ( ENABLE_BUILD_TREE_EXPORT )
347318
export ( PACKAGE ${PACKAGE_NAME} )
348-
endif ( ENABLE_BUILD_TREE_EXPORT )
319+
endif ()

0 commit comments

Comments
 (0)