Skip to content

Commit

Permalink
Merge pull request LLNL#528 from LLNL/feature/goxberry/use-regex-for-…
Browse files Browse the repository at this point in the history
…language-detection

Use regexes for source file language detection instead of `get_filename_component(... ... EXT)`
  • Loading branch information
white238 authored Nov 10, 2021
2 parents 1720b66 + c34269f commit cfd35c4
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 18 deletions.
5 changes: 5 additions & 0 deletions .gitlab/build_quartz.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ quartz-inteloneapi_2021_3:
HOST_CONFIG: "intel-oneapi@2021.3.cmake"
extends: [.build_on_quartz]

quartz-inteloneapi_2021_3_cxx17:
variables:
HOST_CONFIG: "intel-oneapi@2021.3-c++17.cmake"
extends: [.build_on_quartz]

quartz-pgi_20_1:
variables:
HOST_CONFIG: "pgi@20.1.cmake"
Expand Down
10 changes: 8 additions & 2 deletions RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ The project release numbers follow [Semantic Versioning](http://semver.org/spec/
### Changed
- `BLT_C_FILE_EXTS` updated to include `.cuh`
- Fold `BLT_CLANG_HIP_ARCH` into the `CMAKE_HIP_ARCHITECTURES` variable
- When using `ENABLE_ALL_WARNINGS`, append the flag to the beginning of `CMAKE_{C,CXX}_FLAGS` instead of the end
- When using `ENABLE_ALL_WARNINGS`, append the flag to the beginning of `CMAKE_{C,CXX}_FLAGS` instead
of the end

### Fixed
- Source code filename extension filtering now uses regular expressions to allow
for more user customization and to improve handling of file names with multiple
periods, e.g. `1d.cube.order2.c` is considered a `.c` file.

## [Version 0.4.1] - Release date 2021-07-20

Expand Down Expand Up @@ -86,7 +92,7 @@ The project release numbers follow [Semantic Versioning](http://semver.org/spec/
- ``blt_patch_target`` no longer attempts to set system include directories when a target
has no include directories
- Header-only libraries now can have dependencies via DEPENDS_ON in ``blt_add_library``
- Added a workaround for include directories of imported targets on PGI. CMake was
- Added a workaround for include directories of imported targets on PGI. CMake was
erroneously marking them as SYSTEM but this is not supported by PGI.
- Check added to make sure that if HIP is enabled with fortran, the LINKER LANGUAGE
is not changed back to Fortran.
Expand Down
9 changes: 7 additions & 2 deletions SetupBLT.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -172,13 +172,18 @@ if (NOT BLT_LOADED)
# Global variables needed by BLT
#
################################

# File extension lists used to filter sources based on languages for code checks
# and filtering Fortran sources out of cuda/hip source lists
# Note: this filtering is case-insensitive
set(BLT_C_FILE_EXTS ".cpp" ".hpp" ".cxx" ".hxx" ".c" ".h" ".cc" ".hh" ".inl" ".cu" ".cuh"
CACHE STRING "List of known file extensions used for C/CXX sources")
set(BLT_Fortran_FILE_EXTS ".F" ".f" ".f90" ".F90"
set(BLT_Fortran_FILE_EXTS ".f" ".f90"
CACHE STRING "List of known file extensions used for Fortran sources")
set(BLT_Python_FILE_EXTS ".py"
CACHE STRING "List of known file extensions used for Python sources")
set(BLT_CMAKE_FILE_EXTS ".cmake" # NOTE: CMakeLists.txt handled elsewhere
# NOTE: CMakeLists.txt handled in `blt_split_source_list_by_language`
set(BLT_CMAKE_FILE_EXTS ".cmake"
CACHE STRING "List of known file extensions used for CMake sources")

################################
Expand Down
4 changes: 2 additions & 2 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Copyright (c) 2017-2021, Lawrence Livermore National Security, LLC and
# other BLT Project Developers. See the top-level LICENSE file for details
#
#
# SPDX-License-Identifier: (BSD-3-Clause)

variables:
Expand Down Expand Up @@ -53,7 +53,7 @@ steps:
displayName: 'OSX/Windows CMake'
condition: or( eq( variables['Agent.OS'], 'Windows_NT'), eq( variables['Agent.OS'], 'Darwin'))
- script: |
cmake --build build --config Release
cmake --build build --config Release
displayName: 'OSX/Windows Build'
condition: or( eq( variables['Agent.OS'], 'Windows_NT'), eq( variables['Agent.OS'], 'Darwin'))
- script: |
Expand Down
81 changes: 70 additions & 11 deletions cmake/BLTPrivateMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -754,19 +754,63 @@ macro(blt_add_hip_executable)

endmacro(blt_add_hip_executable)


##-----------------------------------------------------------------------------
## blt_make_file_ext_regex( EXTENSIONS [ext1 [ext2 ...]]
## OUTPUT_REGEX <regex variable name>)
##
## This function converts the list of extensions in EXTENSIONS and
## fills the variable, given in OUTPUT_REGEX, with a joined, with '|',
## regular expression. This regex should match any file name starting with
## a string and ending with any one of the extensions in EXTENSIONS.
## This also lower cases all extensions because we do not care about file casing.
## -----------------------------------------------------------------------------
macro(blt_make_file_ext_regex)

set(options)
set(singleValueArgs OUTPUT_REGEX)
set(multiValueArgs EXTENSIONS)

# Parse the arguments
cmake_parse_arguments(arg "${options}" "${singleValueArgs}"
"${multiValueArgs}" ${ARGN} )

# Check arguments
if ( NOT DEFINED arg_EXTENSIONS )
message( FATAL_ERROR "Must provide a EXTENSIONS argument to the 'blt_make_file_ext_regex' macro" )
endif()

if ( NOT DEFINED arg_OUTPUT_REGEX )
message( FATAL_ERROR "Must provide a OUTPUT_REGEX argument to the 'blt_make_file_ext_regex' macro" )
endif()

# Join with 'or', and escape periods
list(JOIN arg_EXTENSIONS "|" ${arg_OUTPUT_REGEX})
# Lower-case because we do case-insensitive checks
string(TOLOWER "${BLT_C_FILE_REGEX}" BLT_C_FILE_REGEX)
# Escape periods before adding
string(REPLACE "." "\\." ${arg_OUTPUT_REGEX} "${${arg_OUTPUT_REGEX}}")
# Regex for by any set of characters followed by any of the given
# file extensions at the end of the string
set(${arg_OUTPUT_REGEX} "^.*(${${arg_OUTPUT_REGEX}})$")

endmacro(blt_make_file_ext_regex)


##------------------------------------------------------------------------------
## blt_split_source_list_by_language( SOURCES <sources>
## C_LIST <list name>
## Fortran_LIST <list name>
## Python_LIST <list name>)
## Python_LIST <list name>
## CMAKE_LIST <list name>)
##
## Filters source list by file extension into C/C++, Fortran, Python, and
## CMake source lists based on BLT_C_FILE_EXTS, BLT_Fortran_FILE_EXTS,
## and BLT_CMAKE_FILE_EXTS (global BLT variables). Files named
## "CMakeLists.txt" are also filtered here. Files with no extension
## or generator expressions that are not object libraries (of the form
## "$<TARGET_OBJECTS:nameofobjectlibrary>") will throw fatal errors.
## and BLT_CMAKE_FILE_EXTS (global BLT variables). This filtering is
## case-insensitive. Files named "CMakeLists.txt" are also filtered here.
## Files with no extension or generator expressions that are not object
## libraries (of the form "$<TARGET_OBJECTS:nameofobjectlibrary>") will
## throw fatal errors.
## ------------------------------------------------------------------------------
macro(blt_split_source_list_by_language)

Expand All @@ -783,6 +827,20 @@ macro(blt_split_source_list_by_language)
message( FATAL_ERROR "Must provide a SOURCES argument to the 'blt_split_source_list_by_language' macro" )
endif()

# Convert extensions lists to regexes
set(BLT_C_FILE_REGEX)
blt_make_file_ext_regex(EXTENSIONS ${BLT_C_FILE_EXTS}
OUTPUT_REGEX BLT_C_FILE_REGEX)
set(BLT_Fortran_FILE_REGEX)
blt_make_file_ext_regex(EXTENSIONS ${BLT_Fortran_FILE_EXTS}
OUTPUT_REGEX BLT_Fortran_FILE_REGEX)
set(BLT_Python_FILE_REGEX)
blt_make_file_ext_regex(EXTENSIONS ${BLT_Python_FILE_EXTS}
OUTPUT_REGEX BLT_Python_FILE_REGEX)
set(BLT_CMAKE_FILE_REGEX)
blt_make_file_ext_regex(EXTENSIONS ${BLT_CMAKE_FILE_EXTS}
OUTPUT_REGEX BLT_CMAKE_FILE_REGEX)

# Generate source lists based on language
foreach(_file ${arg_SOURCES})
# Allow CMake object libraries but disallow generator expressions
Expand All @@ -798,29 +856,30 @@ macro(blt_split_source_list_by_language)
message(FATAL_ERROR "blt_split_source_list_by_language given source file with no extension: ${_file}")
endif()

get_filename_component(_name "${_file}" NAME)
get_filename_component(_name "${_file}" NAME)

string(TOLOWER "${_ext}" _ext_lower)
string(TOLOWER "${_file}" _lower_file)

if("${_ext_lower}" IN_LIST BLT_C_FILE_EXTS)
if("${_lower_file}" MATCHES "${BLT_C_FILE_REGEX}")
if (DEFINED arg_C_LIST)
list(APPEND ${arg_C_LIST} "${_file}")
endif()
elseif("${_ext_lower}" IN_LIST BLT_Fortran_FILE_EXTS)
elseif("${_lower_file}" MATCHES "${BLT_Fortran_FILE_REGEX}")
if (DEFINED arg_Fortran_LIST)
list(APPEND ${arg_Fortran_LIST} "${_file}")
endif()
elseif("${_ext_lower}" IN_LIST BLT_Python_FILE_EXTS)
elseif("${_lower_file}" MATCHES "${BLT_Python_FILE_EXTS}")
if (DEFINED arg_Python_LIST)
list(APPEND ${arg_Python_LIST} "${_file}")
endif()
elseif("${_ext_lower}" IN_LIST BLT_CMAKE_FILE_EXTS OR "${_name}" STREQUAL "CMakeLists.txt")
elseif("${_lower_file}" MATCHES "${BLT_CMAKE_EXTS}" OR "${_name}" STREQUAL "CMakeLists.txt")
if (DEFINED arg_CMAKE_LIST)
list(APPEND ${arg_CMAKE_LIST} "${_file}")
endif()
else()
message(FATAL_ERROR "blt_split_source_list_by_language given source file with unknown file extension. Add the missing extension to the corresponding list (BLT_C_FILE_EXTS, BLT_Fortran_FILE_EXTS, BLT_Python_FILE_EXTS, or BLT_CMAKE_FILE_EXTS).\n Unknown file: ${_file}")
endif()

endforeach()

endmacro(blt_split_source_list_by_language)
Expand Down
3 changes: 2 additions & 1 deletion tests/internal/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

cmake_minimum_required(VERSION 3.8)

project(blt-example LANGUAGES C CXX)
project(blt-internal-tests LANGUAGES C CXX)

#------------------------------------------------------------------------------
# Setup BLT
Expand Down Expand Up @@ -269,6 +269,7 @@ if(ENABLE_HIP)
add_subdirectory(src/hip_defines_test)
endif()

add_subdirectory(unit)
#------------------------------------------------------------------------------
# Format the testing code using ClangFormat
#------------------------------------------------------------------------------
Expand Down
152 changes: 152 additions & 0 deletions tests/internal/unit/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
# Copyright (c) 2017-2021, Lawrence Livermore National Security, LLC and
# other BLT Project Developers. See the top-level LICENSE file for details
#
# SPDX-License-Identifier: (BSD-3-Clause)

#------------------------------------------------------------------------------
# BLT Internal Testing Project
#------------------------------------------------------------------------------

cmake_minimum_required(VERSION 3.8)

project(blt-unit-tests LANGUAGES C CXX)

#------------------------------------------------------------------------------
# Setup BLT
#------------------------------------------------------------------------------
# Set BLT_SOURCE_DIR to default location, if not set by user
if(NOT BLT_SOURCE_DIR)
set(BLT_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../..")
endif()

if (NOT BLT_CXX_STD)
set(BLT_CXX_STD "c++11" CACHE STRING "")
endif()

include(${BLT_SOURCE_DIR}/SetupBLT.cmake)

#------------------------------------------------------------------------------
# Test blt_split_source_list_by_language
#------------------------------------------------------------------------------

message(
"*****************************************************\n"
"Testing `blt_split_source_list_by_language`...\n"
"*****************************************************")

# create list of source paths and sorted, correct list of answers
set(original_srcs
src/Example.cpp
src/Example.hpp
multiple.exts.c
quadrature_rule.order2.c
source/.f90/with/extension/in/middle.c
fortran.F
fortran2.f
fortran3.f90
fortran4.F90
python.py
some_other.multi.py
CMakeLists.txt
long/path/CMakeLists.txt
macro_file.cmake
)

set(correct_c_srcs
src/Example.cpp
src/Example.hpp
multiple.exts.c
quadrature_rule.order2.c
source/.f90/with/extension/in/middle.c
)

set(correct_fortran_srcs
fortran.F
fortran2.f
fortran3.f90
fortran4.F90
)

set(correct_python_srcs
python.py
some_other.multi.py
)

set(correct_cmake_srcs
CMakeLists.txt
long/path/CMakeLists.txt
macro_file.cmake
)

# Split sources
set(c_srcs)
set(fortran_srcs)
set(python_srcs)
set(cmake_srcs)
blt_split_source_list_by_language( SOURCES ${original_srcs}
C_LIST c_srcs
Fortran_LIST fortran_srcs
Python_LIST python_srcs
CMAKE_LIST cmake_srcs)

# Macro to see if lists have the same items and errors on non-equality
macro(compare_source_lists)

set(options)
set(singleValueArgs A_LIST B_LIST)
set(multiValueArgs )

# Parse the arguments
cmake_parse_arguments(arg "${options}" "${singleValueArgs}"
"${multiValueArgs}" ${ARGN} )

message(STATUS "Comparing lists for equality:\n"
" List A: ${${arg_A_LIST}}\n"
" List B: ${${arg_B_LIST}}")

set(a_len)
set(b_len)
list(LENGTH ${arg_A_LIST} a_len)
list(LENGTH ${arg_B_LIST} b_len)
if (NOT a_len EQUAL b_len)
message(FATAL_ERROR "Split source test failed. Lists had differing lengths.")
endif()

set(sorted_a_list ${${arg_A_LIST}})
set(sorted_b_list ${${arg_B_LIST}})
list(SORT sorted_a_list)
list(SORT sorted_b_list)

foreach(i RANGE ${a_len})
set(a_item)
set(b_item)
list(GET sorted_a_list i a_item)
list(GET sorted_b_list i b_item)

if(NOT a_item STREQUAL b_item)
message(FATAL_ERROR "Split source test failed. ${a_item} != ${b_item}")
endif()
endforeach()

# Success if we reached here
endmacro(compare_source_lists)

message(STATUS "Full mixed source list: ${original_srcs}")

message(STATUS "Checking C/CXX filtering...")
compare_source_lists(A_LIST c_srcs B_LIST correct_c_srcs)

message(STATUS "Checking Fortran filtering...")
compare_source_lists(A_LIST fortran_srcs B_LIST correct_fortran_srcs)

message(STATUS "Checking Python filtering...")
compare_source_lists(A_LIST python_srcs B_LIST correct_python_srcs)

message(STATUS "Checking CMake filtering...")
compare_source_lists(A_LIST cmake_srcs B_LIST correct_cmake_srcs)

message(
"*****************************************************\n"
"Tests passed for `blt_split_source_list_by_language`.\n"
"*****************************************************")

0 comments on commit cfd35c4

Please sign in to comment.