Switched to CMake for building tests.

To run tests with the system compiler, execute

    make

To run tests with the system compiler in all configurations,

    make default-all

To re-generate the examples from the documentation Markdown, and then
test all configurations,

    make default-thorough

Other Makefile targets are for exhaustive testing with multiple
locally-installed compilers, or for use on CI servers, and are not for
general use.

Python and CxxTest are still required. On Windows, it is helpful to
have a Cygwin environment with a non-Cygwin CMake, and MSBuild.exe
should be in PATH.

Better Enums is now tested in fewer configurations. C++11 features are
no longer tested on clang 3.3, gcc 4.3-4.6, because CMake claims
(apparently falsely, in some cases) that those compilers don't support
constexpr and enum class.
This commit is contained in:
Anton Bachin 2015-06-30 11:45:34 -05:00
parent 535f7f151b
commit ba9139e075
5 changed files with 280 additions and 288 deletions

2
.gitignore vendored
View File

@ -5,4 +5,4 @@ scratch/
doc/html/
doc-publish/
test/cxxtest/*.cc
test/platform/
test/build

View File

@ -87,9 +87,6 @@ clang++34 -std=c++11
clang++34 -std=c++11 -DBETTER_ENUMS_STRICT_CONVERSION
clang++34 -std=c++11 -DBETTER_ENUMS_CONSTEXPR_TO_STRING
clang++34 -std=c++98
clang++33 -std=c++11
clang++33 -std=c++11 -DBETTER_ENUMS_STRICT_CONVERSION
clang++33 -std=c++11 -DBETTER_ENUMS_CONSTEXPR_TO_STRING
clang++33 -std=c++98
g++51 -std=c++11
g++51 -std=c++11 -DBETTER_ENUMS_STRICT_CONVERSION
@ -107,16 +104,9 @@ g++47 -std=c++11
g++47 -std=c++11 -DBETTER_ENUMS_STRICT_CONVERSION
g++47 -std=c++11 -DBETTER_ENUMS_CONSTEXPR_TO_STRING
g++47 -std=c++98
g++46 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR
g++46 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR -DBETTER_ENUMS_STRICT_CONVERSION
g++46 -std=c++98
g++45 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR
g++45 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR -DBETTER_ENUMS_STRICT_CONVERSION
g++45 -std=c++98
g++44 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR
g++44 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR -DBETTER_ENUMS_STRICT_CONVERSION
g++44 -std=c++98
g++43 -std=c++0x -DBETTER_ENUMS_NO_CONSTEXPR
g++43 -std=c++98
~~~

132
test/CMakeLists.txt Normal file
View File

@ -0,0 +1,132 @@
cmake_minimum_required(VERSION 3.1.0 FATAL_ERROR)
project("Better Enums Testing" CXX)
# Detect compiler feature support.
list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_constexpr CONSTEXPR_INDEX)
if(CONSTEXPR_INDEX EQUAL -1)
set(SUPPORTS_CONSTEXPR 0)
else()
set(SUPPORTS_CONSTEXPR 1)
endif()
list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_strong_enums ENUM_CLASS_INDEX)
if(ENUM_CLASS_INDEX EQUAL -1)
set(SUPPORTS_ENUM_CLASS 0)
else()
set(SUPPORTS_ENUM_CLASS 1)
endif()
# Not supporting C++11 usage on g++46 due to buggy constexpr.
if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.7)
set(SUPPORTS_CONSTEXPR 0)
set(SUPPORTS_ENUM_CLASS 0)
endif()
# Select standard based on the requested configuration. If the compiler does not
# support the requested configuration, write a message and generate a no-op
# build. This condition is not a failure.
#
# If no configuration is explicitly requested, default to compiling with no
# special flags, with the latest standard supported by the compiler.
set(DO_NOT_TEST_FILE "${CMAKE_BINARY_DIR}/do-not-test")
if(CONFIGURATION STREQUAL CONSTEXPR)
if(SUPPORTS_CONSTEXPR)
set(CMAKE_CXX_STANDARD 11)
else()
message(WARNING "This compiler does not support constexpr")
file(WRITE "${DO_NOT_TEST_FILE}")
return()
endif()
elseif(CONFIGURATION STREQUAL FULL_CONSTEXPR)
if(SUPPORTS_CONSTEXPR)
set(CMAKE_CXX_STANDARD 11)
add_definitions(-DBETTER_ENUMS_CONSTEXPR_TO_STRING)
else()
message(WARNING "This compiler does not support constexpr")
file(WRITE "${DO_NOT_TEST_FILE}")
return()
endif()
elseif(CONFIGURATION STREQUAL STRICT_CONVERSION)
if(SUPPORTS_ENUM_CLASS)
set(CMAKE_CXX_STANDARD 11)
add_definitions(-DBETTER_ENUMS_STRICT_CONVERSION)
else()
message(WARNING "This compiler does not support enum class")
file(WRITE "${DO_NOT_TEST_FILE}")
return()
endif()
elseif(CONFIGURATION STREQUAL CXX98)
set(CMAKE_CXX_STANDARD 98)
else()
set(CMAKE_CXX_STANDARD 11)
endif()
# Basic tests.
add_executable(cxxtest cxxtest/tests.cc)
add_executable(link link/helper.cc link/main.cc)
set(PERFORMANCE_TESTS
1-simple 2-include_empty 3-only_include_enum 4-declare_enums 5-iostream)
foreach(TEST ${PERFORMANCE_TESTS})
add_executable(performance-${TEST} performance/${TEST}.cc)
endforeach(TEST)
# Select examples to build.
set(EXAMPLES
1-hello-world 2-conversions 3-iterate 4-switch 5-iostreams 6-safety
7-representation 8-constexpr 101-special-values 102-any-underlying
103-bitset 104-quine)
set(SKIPPED_FOR_CXX98
8-constexpr 101-special-values 102-any-underlying 103-bitset 104-quine)
set(SKIPPED_FOR_STRICT_CONVERSION 4-switch 102-any-underlying)
if(CONFIGURATION STREQUAL CXX98 OR NOT SUPPORTS_CONSTEXPR)
list(REMOVE_ITEM EXAMPLES ${SKIPPED_FOR_CXX98})
endif()
if(CONFIGURATION STREQUAL STRICT_CONVERSION)
list(REMOVE_ITEM EXAMPLES ${SKIPPED_FOR_STRICT_CONVERSION})
endif()
foreach(EXAMPLE ${EXAMPLES})
add_executable(example-${EXAMPLE} ../example/${EXAMPLE}.cc)
endforeach(EXAMPLE)
# Add compiler flags.
include_directories(..)
if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES Clang)
include(CheckCXXCompilerFlag)
macro(add_cxx_flag_if_supported FLAG)
string(REPLACE "=" "_equals_" ESCAPED ${FLAG})
string(REPLACE "+" "_plus_" ESCAPED ${ESCAPED})
check_cxx_compiler_flag("${FLAG}" HAVE_FLAG_${ESCAPED})
if(HAVE_FLAG_${ESCAPED})
add_definitions(${FLAG})
endif()
endmacro()
add_cxx_flag_if_supported("-Wpedantic")
add_cxx_flag_if_supported("-Wall")
add_cxx_flag_if_supported("-Wextra")
add_cxx_flag_if_supported("-Wno-variadic-macros")
add_definitions("-Werror")
endif()

View File

@ -1,15 +1,157 @@
CXXTEST_GENERATED := cxxtest/tests.cc
UNIX_MAKE_COMMAND := make
WINDOWS_MAKE_COMMAND := "MSBuild.exe \"Better Enums Testing.sln\""
UNIX_OUTPUT_DIRECTORY := .
WINDOWS_OUTPUT_DIRECTORY := Debug
ifdef COMSPEC
WIN32 := true
endif
ifdef ComSpec
WIN32 := true
endif
ifndef WIN32
DEFAULT_MAKE_COMMAND := $(UNIX_MAKE_COMMAND)
DEFAULT_OUTPUT_DIRECTORY := $(UNIX_OUTPUT_DIRECTORY)
define PATH_FIX
@true
endef
SUFFIX :=
else
DEFAULT_MAKE_COMMAND := $(WINDOWS_MAKE_COMMAND)
DEFAULT_OUTPUT_DIRECTORY := $(WINDOWS_OUTPUT_DIRECTORY)
define PATH_FIX
sed 's!include "/!include "C:/cygwin/!g' $1 > $$$$ && mv $$$$ $1
endef
SUFFIX := .exe
endif
DEFAULTS := \
TITLE=default \
MAKE_COMMAND=$(DEFAULT_MAKE_COMMAND) \
OUTPUT_DIRECTORY=$(DEFAULT_OUTPUT_DIRECTORY)
# Builds one configuration with the system compiler. This will be either a
# regular C++11 or C++98 build (no constexpr to_string and no strict
# conversions).
.PHONY : default
default :
python test.py
make $(DEFAULTS) one-configuration
.PHONY : platform
platform : examples
python test.py --all
# Builds all configurations with the system compiler.
.PHONY : default-all
default-all :
make $(DEFAULTS) all-configurations
# Re-generates the examples from the Markdown, and then builds all
# configurations with the system compiler. Should be done before a pull request,
# but not practical for regular use because it touches dependencies and causes
# full rebuilds.
.PHONY : default-thorough
default-thorough : examples default-all
.PHONY : examples
examples :
make -C ../doc examples
# Example: make COMPILER=clang++36 unix
.PHONY : unix
unix :
make TITLE=$(COMPILER) CMAKE_OPTIONS="-DCMAKE_CXX_COMPILER=$(COMPILER)" \
MAKE_COMMAND=$(UNIX_MAKE_COMMAND) \
OUTPUT_DIRECTORY=$(UNIX_OUTPUT_DIRECTORY) all-configurations
# Example: make TITLE=vc2013 COMPILER="Visual Studio 12 2013" ms
.PHONY : ms
ms :
make TITLE=$(TITLE) CMAKE_OPTIONS="-G \\\"$(COMPILER)\\\"" \
MAKE_COMMAND=$(WINDOWS_MAKE_COMMAND) \
OUTPUT_DIRECTORY=$(WINDOWS_OUTPUT_DIRECTORY) all-configurations
# Expects three variables to be defined:
# CMAKE_OPTIONS:
# First, the compiler:
# - Empty for a "default" build.
# - -G "Visual Studio XX YYYY" to select a specific Microsoft compiler.
# - -DCMAKE_CXX_COMPILER=AAA to select a specific Unix compiler binary.
# Configuration selection (e.g. -DCONFIGURATION=CXX98) also go here.
# TITLE:
# The build title (subdirectory). Some combination of compiler/configuration.
# MAKE_COMMAND:
# Either make or msbuild "Better Enums Testing.sln"
# OUTPUT_DIRECTORY:
# Path to generated binaries relative to build directory. Either "." or
# "Debug".
.PHONY : one-configuration
one-configuration : $(CXXTEST_GENERATED)
mkdir -p build/$(TITLE)
cd build/$(TITLE) && cmake $(CMAKE_OPTIONS) ../.. && $(MAKE_COMMAND)
rm -rf build/$(TITLE)/bin
ln -s $(OUTPUT_DIRECTORY) build/$(TITLE)/bin
[ -f build/$(TITLE)/do-not-test ] || make BIN=build/$(TITLE)/bin run-tests
.PHONY : run-tests
run-tests :
$(BIN)/cxxtest
@for FILE in $(BIN)/example-*$(SUFFIX) ; \
do \
EXAMPLE=$$(basename $$FILE | sed s/\.exe$$// | sed s/^example-//) ; \
$$FILE | sed 's/\r$$//' | cmp - expect/$$EXAMPLE ; \
RESULT=$$? ; \
if (( $$RESULT )) ; \
then \
echo \'$$FILE\' produced bad output ; \
exit $$RESULT ; \
fi ; \
done
@echo Example program output matches expected output
.PHONY : all-configurations
all-configurations :
make TITLE=$(TITLE)-c++11 \
CMAKE_OPTIONS="$(CMAKE_OPTIONS) -DCONFIGURATION=CONSTEXPR" \
one-configuration
make TITLE=$(TITLE)-full-constexpr \
CMAKE_OPTIONS="$(CMAKE_OPTIONS) -DCONFIGURATION=FULL_CONSTEXPR" \
one-configuration
make TITLE=$(TITLE)-enum-class \
CMAKE_OPTIONS="$(CMAKE_OPTIONS) -DCONFIGURATION=STRICT_CONVERSION" \
one-configuration
make TITLE=$(TITLE)-c++98 \
CMAKE_OPTIONS="$(CMAKE_OPTIONS) -DCONFIGURATION=CXX98" \
one-configuration
.PHONY : all-unix
all-unix :
make COMPILER=clang++36 unix
make COMPILER=clang++35 unix
make COMPILER=clang++34 unix
make COMPILER=clang++33 unix
make COMPILER=g++51 unix
make COMPILER=g++49 unix
make COMPILER=g++48 unix
make COMPILER=g++47 unix
make COMPILER=g++46 unix
make COMPILER=g++45 unix
make COMPILER=g++44 unix
make COMPILER=g++43 unix
.PHONY : all-ms
all-ms :
make TITLE=vc2013 COMPILER="Visual Studio 12 2013" ms
make TITLE=vc2015 COMPILER="Visual Studio 14 2015" ms
$(CXXTEST_GENERATED) : cxxtest/*.h
cxxtestgen --error-printer -o $@ $^
$(call PATH_FIX,$@)
.PHONY : clean
clean :
rm -rf platform *.obj cxxtest/*.cc cxxtest/*.exe
rm -rf build $(CXXTEST_GENERATED)

View File

@ -1,272 +0,0 @@
#! /usr/bin/env python
import glob
import os
import os.path
import platform
import re
import shutil
import subprocess
import sys
BASE_DIRECTORY = "platform"
CXXTEST_GENERATED = "cxxtest/tests.cc"
quiet = True
windows = False
def file_title(path):
return os.path.splitext(os.path.basename(path))[0]
expected_example_outputs = {}
def load_expected_outputs():
files = glob.glob("expect/*")
for file in files:
stream = open(file)
try:
content = stream.read()
finally:
stream.close()
expected_example_outputs[file_title(file)] = content
def run(command, catch_warnings = False):
if not quiet:
print command
try:
output = subprocess.check_output(command.split(),
stderr = subprocess.STDOUT)
if not catch_warnings:
return output
else:
if re.search("warning", output) != None:
raise subprocess.CalledProcessError(0, command, output)
return output
except subprocess.CalledProcessError as e:
print e.output
print command, "failed"
sys.exit(1)
class Configuration(object):
def __init__(self, subdirectory, command, skip_examples = []):
self._subdirectory = subdirectory
self._command = command
self._skip_examples = skip_examples
def elaborate(self, include, output, source):
command = self._command
if re.match("clang|[gc]\+\+", command) != None:
return "%s -I%s -Wall -o %s %s" % (command, include, output, source)
elif re.match("vc|cl", command) != None:
return "%s /I%s /Fe%s %s" % (command, include, output, source)
else:
raise Exception("unrecognized compiler in '%s'" % command)
def make_directories(self):
base = self.directory()
directories = \
[base,
os.path.join(base, "example"),
os.path.join(base, "cxxtest"),
os.path.join(base, "link"),
os.path.join(base, "performance")]
for directory in directories:
if not os.path.lexists(directory):
os.makedirs(directory)
def make_all(self):
print self._command
self.make_directories()
base = self.directory()
examples = glob.glob("../example/*.cc")
example_directory = os.path.join(base, "example")
for file in examples:
title = file_title(file)
if title in self._skip_examples:
continue
if title not in expected_example_outputs:
print "no expected output for example", title
sys.exit(1)
binary = os.path.join(example_directory, title) + ".exe"
command = self.elaborate("..", binary, file)
run(command, True)
output = run(binary)
output = re.sub("\r\n", "\n", output)
if output != expected_example_outputs[title]:
print output
print "output does not match expected output"
sys.exit(1)
cxxtest_binary = os.path.join(base, "cxxtest", "tests.exe")
command = self.elaborate("..", cxxtest_binary, CXXTEST_GENERATED)
run(command, True)
run(cxxtest_binary)
link_sources = glob.glob("link/*.cc")
link_binary = os.path.join(base, "link", "link.exe")
command = self.elaborate("..", link_binary, " ".join(link_sources))
run(command, True)
performance_sources = glob.glob("performance/*.cc")
performance_directory = os.path.join(base, "performance")
for file in performance_sources:
title = file_title(file)
binary = os.path.join(performance_directory, title) + ".exe"
command = "time " + self.elaborate("..", binary, file)
output = run(command, True)
output_file = os.path.join(performance_directory, title) + ".times"
stream = open(output_file, "w")
try:
stream.write(output)
finally:
stream.close()
def directory(self):
return os.path.join(BASE_DIRECTORY, self._subdirectory)
skip_cxx98 = [
"101-special-values", "102-any-underlying", "103-bitset", "104-quine",
"8-constexpr"
]
skip_strict = ["4-switch", "102-any-underlying"]
def modern_gnu(command):
return [
Configuration(command + "-c++11", command + " -std=c++11"),
Configuration(command + "-strict-conversion",
command + " -std=c++11 " +
"-DBETTER_ENUMS_STRICT_CONVERSION", skip_strict),
Configuration(command + "-all-constexpr",
command + " -std=c++11 " +
"-DBETTER_ENUMS_CONSTEXPR_TO_STRING"),
Configuration(command + "-c++98", command + " -std=c++98", skip_cxx98)
]
# g++46 should be one of these, but g++46 c++11 mode currently not supported due
# to the presence of this bug and lack of a workaround in enum.h.
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54086
def older_gnu(command):
return [
Configuration(command + "-c++0x", command + " -std=c++0x"),
Configuration(command + "-strict-conversion",
command + " -std=c++0x " +
"-DBETTER_ENUMS_STRICT_CONVERSION", skip_strict),
Configuration(command + "-all-constexpr",
command + " -std=c++0x " +
"-DBETTER_ENUMS_CONSTEXPR_TO_STRING"),
Configuration(command + "-c++98", command + " -std=c++98", skip_cxx98)
]
def gnu_pre_constexpr(command):
return [
Configuration(command + "-c++0x-noconstexpr-war",
command + " -std=c++0x " +
"-DBETTER_ENUMS_NO_CONSTEXPR", skip_cxx98),
Configuration(command + "-strict-conversion",
command + " -std=c++0x " +
"-DBETTER_ENUMS_NO_CONSTEXPR " +
"-DBETTER_ENUMS_STRICT_CONVERSION",
skip_cxx98 + skip_strict),
Configuration(command + "-c++98", command + " -std=c++98", skip_cxx98)
]
def gnu_pre_enum_class(command):
return [
Configuration(command + "-c++0x-noconstexpr-war",
command + " -std=c++0x " +
"-DBETTER_ENUMS_NO_CONSTEXPR", skip_cxx98),
Configuration(command + "-c++98", command + " -std=c++98", skip_cxx98)
]
def vc(command):
return [
Configuration(command, command + " /EHsc", skip_cxx98),
Configuration(command + "-strict-conversion",
command + " /EHsc /DBETTER_ENUMS_STRICT_CONVERSION",
skip_cxx98)
]
unix_configurations = \
modern_gnu("clang++36") + \
modern_gnu("clang++35") + \
modern_gnu("clang++34") + \
modern_gnu("clang++33") + \
modern_gnu("g++51") + \
modern_gnu("g++49") + \
modern_gnu("g++48") + \
modern_gnu("g++47") + \
gnu_pre_constexpr("g++46") + \
gnu_pre_constexpr("g++45") + \
gnu_pre_constexpr("g++44") + \
gnu_pre_enum_class("g++43")
windows_configurations = vc("vc2015") + vc("vc2013")
unix_default = Configuration("default", "c++ --std=c++11")
windows_default = Configuration("default", "cl /EHsc")
def main():
global quiet
global windows
load_expected_outputs()
cxxtest_headers = " ".join(glob.glob(os.path.join("cxxtest", "*.h")))
run("cxxtestgen --error-printer -o %s %s" %
(CXXTEST_GENERATED, cxxtest_headers))
if re.search("Windows|CYGWIN", platform.system()) != None:
full = windows_configurations
default = windows_default
windows = True
cygwin_fix_command = \
("sed 's#\"/home#\"C:/cygwin/home#g' %s > $$$$ ; " + \
"mv $$$$ %s") % (CXXTEST_GENERATED, CXXTEST_GENERATED)
os.system(cygwin_fix_command)
else:
full = unix_configurations
default = unix_default
if len(sys.argv) > 1 and sys.argv[1] == "--all":
configurations = full
quiet = True
else:
configurations = [default]
quiet = False
for configuration in configurations:
configuration.make_all()
if __name__ == "__main__":
main()