From 73f43c98d0e734b2309fd49a6587171af3296471 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 15 May 2017 21:13:14 +0100 Subject: [PATCH 01/31] Normalize all the line endings --- .../cmake/build/bin_bbackupd.vcxproj.user | 14 +- .../cmake/build/bin_bbstored.vcxproj.user | 14 +- .../cmake/build/test_backupstore.vcxproj.user | 16 +- .../build/test_backupstorefix.vcxproj.user | 12 +- .../cmake/build/test_bbackupd.vcxproj.user | 14 +- .../cmake/build/test_common.vcxproj.user | 12 +- .../cmake/build/test_httpserver.vcxproj.user | 12 +- .../cmake/build/test_raidfile.vcxproj.user | 12 +- lib/win32/bsd_getopt.h | 210 +++++++++--------- lib/win32/messages.h | 114 +++++----- 10 files changed, 215 insertions(+), 215 deletions(-) diff --git a/infrastructure/cmake/build/bin_bbackupd.vcxproj.user b/infrastructure/cmake/build/bin_bbackupd.vcxproj.user index 51928554f..fa1f3d341 100755 --- a/infrastructure/cmake/build/bin_bbackupd.vcxproj.user +++ b/infrastructure/cmake/build/bin_bbackupd.vcxproj.user @@ -1,8 +1,8 @@ - - - - testfiles\bbackupd.conf - $(ProjectDir)\..\..\..\debug\test\bbackupd - WindowsLocalDebugger - + + + + testfiles\bbackupd.conf + $(ProjectDir)\..\..\..\debug\test\bbackupd + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/bin_bbstored.vcxproj.user b/infrastructure/cmake/build/bin_bbstored.vcxproj.user index e48b8383d..339cddee6 100755 --- a/infrastructure/cmake/build/bin_bbstored.vcxproj.user +++ b/infrastructure/cmake/build/bin_bbstored.vcxproj.user @@ -1,8 +1,8 @@ - - - - testfiles/bbstored.conf - $(ProjectDir)\..\..\..\debug\test\backupstorefix - WindowsLocalDebugger - + + + + testfiles/bbstored.conf + $(ProjectDir)\..\..\..\debug\test\backupstorefix + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstore.vcxproj.user b/infrastructure/cmake/build/test_backupstore.vcxproj.user index 79ef75713..7d7b31586 100755 --- a/infrastructure/cmake/build/test_backupstore.vcxproj.user +++ b/infrastructure/cmake/build/test_backupstore.vcxproj.user @@ -1,9 +1,9 @@ - - - - $(ProjectDir)\..\..\..\debug\test\backupstore - WindowsLocalDebugger - - - + + + + $(ProjectDir)\..\..\..\debug\test\backupstore + WindowsLocalDebugger + + + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstorefix.vcxproj.user b/infrastructure/cmake/build/test_backupstorefix.vcxproj.user index c08951916..170fb4967 100755 --- a/infrastructure/cmake/build/test_backupstorefix.vcxproj.user +++ b/infrastructure/cmake/build/test_backupstorefix.vcxproj.user @@ -1,7 +1,7 @@ - - - - $(ProjectDir)\..\..\..\debug\test\backupstorefix - WindowsLocalDebugger - + + + + $(ProjectDir)\..\..\..\debug\test\backupstorefix + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_bbackupd.vcxproj.user b/infrastructure/cmake/build/test_bbackupd.vcxproj.user index 9ca2b5e9a..ebf8c6a3c 100755 --- a/infrastructure/cmake/build/test_bbackupd.vcxproj.user +++ b/infrastructure/cmake/build/test_bbackupd.vcxproj.user @@ -1,8 +1,8 @@ - - - - -e test_basics - $(ProjectDir)\..\..\..\debug\test\bbackupd - WindowsLocalDebugger - + + + + -e test_basics + $(ProjectDir)\..\..\..\debug\test\bbackupd + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_common.vcxproj.user b/infrastructure/cmake/build/test_common.vcxproj.user index 7a375f104..e5854a80b 100755 --- a/infrastructure/cmake/build/test_common.vcxproj.user +++ b/infrastructure/cmake/build/test_common.vcxproj.user @@ -1,7 +1,7 @@ - - - - $(ProjectDir)\..\..\..\debug\test\common - WindowsLocalDebugger - + + + + $(ProjectDir)\..\..\..\debug\test\common + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_httpserver.vcxproj.user b/infrastructure/cmake/build/test_httpserver.vcxproj.user index b2da015c1..ac1512a84 100755 --- a/infrastructure/cmake/build/test_httpserver.vcxproj.user +++ b/infrastructure/cmake/build/test_httpserver.vcxproj.user @@ -1,7 +1,7 @@ - - - - $(ProjectDir)\..\..\..\debug\test\httpserver - WindowsLocalDebugger - + + + + $(ProjectDir)\..\..\..\debug\test\httpserver + WindowsLocalDebugger + \ No newline at end of file diff --git a/infrastructure/cmake/build/test_raidfile.vcxproj.user b/infrastructure/cmake/build/test_raidfile.vcxproj.user index d2c2fc34e..620aa4bb6 100755 --- a/infrastructure/cmake/build/test_raidfile.vcxproj.user +++ b/infrastructure/cmake/build/test_raidfile.vcxproj.user @@ -1,7 +1,7 @@ - - - - $(ProjectDir)\..\..\..\debug\test\raidfile - WindowsLocalDebugger - + + + + $(ProjectDir)\..\..\..\debug\test\raidfile + WindowsLocalDebugger + \ No newline at end of file diff --git a/lib/win32/bsd_getopt.h b/lib/win32/bsd_getopt.h index 9cfdd32eb..3e2441ca8 100755 --- a/lib/win32/bsd_getopt.h +++ b/lib/win32/bsd_getopt.h @@ -1,105 +1,105 @@ -/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */ -/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */ - -/*- - * Copyright (c) 2000 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Dieter Baron and Thomas Klausner. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef REPLACE_GETOPT -#error You must include box_getopt.h, not bsd_getopt.h -#endif - -#if REPLACE_GETOPT // defined in box_getopt.h; until end of file - -#ifndef _GETOPT_H_ -#define _GETOPT_H_ - -// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html - -/* __BEGIN_DECLS should be used at the beginning of your declarations, - so that C++ compilers don't mangle their names. Use __END_DECLS at - the end of C declarations. */ -#undef __BEGIN_DECLS -#undef __END_DECLS -#ifdef __cplusplus -# define __BEGIN_DECLS extern "C" { -# define __END_DECLS } -#else -# define __BEGIN_DECLS /* empty */ -# define __END_DECLS /* empty */ -#endif - -/* - * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions - */ -#define no_argument 0 -#define required_argument 1 -#define optional_argument 2 - -struct option { - /* name of long option */ - const char *name; - /* - * one of no_argument, required_argument, and optional_argument: - * whether option takes an argument - */ - int has_arg; - /* if not NULL, set *flag to val when option found */ - int *flag; - /* if flag not NULL, value to set *flag to; else return value */ - int val; -}; - -__BEGIN_DECLS -int getopt_long(int, char * const *, const char *, - const struct option *, int *); -int getopt_long_only(int, char * const *, const char *, - const struct option *, int *); -#ifndef _GETOPT_DEFINED_ -#define _GETOPT_DEFINED_ -int getopt(int, char * const *, const char *); -int getsubopt(char **, char * const *, char **); - -extern char *optarg; /* getopt(3) external variables */ -extern int opterr; -extern int optind; -extern int optopt; -extern int optreset; -extern char *suboptarg; /* getsubopt(3) external variable */ -#endif -__END_DECLS - -#endif /* !_GETOPT_H_ */ -#endif // REPLACE_GETOPT +/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */ +/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */ + +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef REPLACE_GETOPT +#error You must include box_getopt.h, not bsd_getopt.h +#endif + +#if REPLACE_GETOPT // defined in box_getopt.h; until end of file + +#ifndef _GETOPT_H_ +#define _GETOPT_H_ + +// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html + +/* __BEGIN_DECLS should be used at the beginning of your declarations, + so that C++ compilers don't mangle their names. Use __END_DECLS at + the end of C declarations. */ +#undef __BEGIN_DECLS +#undef __END_DECLS +#ifdef __cplusplus +# define __BEGIN_DECLS extern "C" { +# define __END_DECLS } +#else +# define __BEGIN_DECLS /* empty */ +# define __END_DECLS /* empty */ +#endif + +/* + * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions + */ +#define no_argument 0 +#define required_argument 1 +#define optional_argument 2 + +struct option { + /* name of long option */ + const char *name; + /* + * one of no_argument, required_argument, and optional_argument: + * whether option takes an argument + */ + int has_arg; + /* if not NULL, set *flag to val when option found */ + int *flag; + /* if flag not NULL, value to set *flag to; else return value */ + int val; +}; + +__BEGIN_DECLS +int getopt_long(int, char * const *, const char *, + const struct option *, int *); +int getopt_long_only(int, char * const *, const char *, + const struct option *, int *); +#ifndef _GETOPT_DEFINED_ +#define _GETOPT_DEFINED_ +int getopt(int, char * const *, const char *); +int getsubopt(char **, char * const *, char **); + +extern char *optarg; /* getopt(3) external variables */ +extern int opterr; +extern int optind; +extern int optopt; +extern int optreset; +extern char *suboptarg; /* getsubopt(3) external variable */ +#endif +__END_DECLS + +#endif /* !_GETOPT_H_ */ +#endif // REPLACE_GETOPT diff --git a/lib/win32/messages.h b/lib/win32/messages.h index 6959591bc..222902267 100755 --- a/lib/win32/messages.h +++ b/lib/win32/messages.h @@ -1,57 +1,57 @@ - // Message source file, to be compiled to a resource file with - // Microsoft Message Compiler (MC), to an object file with a Resource - // Compiler, and linked into the application. - - // The main reason for this file is to work around Windows' stupid - // messages in the Event Log, which say: - - // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) ) - // cannot be found. The local computer may not have the necessary - // registry information or message DLL files to display messages from a - // remote computer. The following information is part of the event: - // Message definitions follow -// -// Values are 32 bit values layed out as follows: -// -// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 -// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 -// +---+-+-+-----------------------+-------------------------------+ -// |Sev|C|R| Facility | Code | -// +---+-+-+-----------------------+-------------------------------+ -// -// where -// -// Sev - is the severity code -// -// 00 - Success -// 01 - Informational -// 10 - Warning -// 11 - Error -// -// C - is the Customer code flag -// -// R - is a reserved bit -// -// Facility - is the facility code -// -// Code - is the facility's status code -// -// -// Define the facility codes -// - - -// -// Define the severity codes -// - - -// -// MessageId: MSG_ERR -// -// MessageText: -// -// %1 -// -#define MSG_ERR ((DWORD)0x40000001L) - + // Message source file, to be compiled to a resource file with + // Microsoft Message Compiler (MC), to an object file with a Resource + // Compiler, and linked into the application. + + // The main reason for this file is to work around Windows' stupid + // messages in the Event Log, which say: + + // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) ) + // cannot be found. The local computer may not have the necessary + // registry information or message DLL files to display messages from a + // remote computer. The following information is part of the event: + // Message definitions follow +// +// Values are 32 bit values layed out as follows: +// +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---+-+-+-----------------------+-------------------------------+ +// |Sev|C|R| Facility | Code | +// +---+-+-+-----------------------+-------------------------------+ +// +// where +// +// Sev - is the severity code +// +// 00 - Success +// 01 - Informational +// 10 - Warning +// 11 - Error +// +// C - is the Customer code flag +// +// R - is a reserved bit +// +// Facility - is the facility code +// +// Code - is the facility's status code +// +// +// Define the facility codes +// + + +// +// Define the severity codes +// + + +// +// MessageId: MSG_ERR +// +// MessageText: +// +// %1 +// +#define MSG_ERR ((DWORD)0x40000001L) + From 7b8a668f00e61c742146cf7b944f3f3b900ea331 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 15 May 2017 21:13:58 +0100 Subject: [PATCH 02/31] Tell Git to treat .cpp files as text too. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 0e35d1c9c..ae83a8942 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,6 +4,7 @@ # Explicitly declare text files you want to always be normalized and converted # to native line endings on checkout. +*.cpp text *.c text *.h text From c7c2b9345dd6a4d8805fd097361736a3746f7392 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 15 May 2017 21:14:39 +0100 Subject: [PATCH 03/31] Tell Git to ignore some more generated files --- .gitignore | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 869daf8cd..ad9bfba29 100644 --- a/.gitignore +++ b/.gitignore @@ -62,8 +62,8 @@ test/*/*.memleaks # built by makebuildenv.pl test/*/_main.cpp -test/*/_t -test/*/_t-gdb +test/*/t +test/*/t-gdb bin/*/autogen_*.cpp bin/*/autogen_*.h lib/*/autogen_*.cpp @@ -77,9 +77,35 @@ qdbm/libqdbm.a qdbm/qdbm.pc qdbm/qdbm.spec qdbm/LTmakefile +qdbm/cbcodec +qdbm/cbtest +qdbm/crmgr +qdbm/crtest +qdbm/crtsv +qdbm/dpmgr +qdbm/dptest +qdbm/dptsv +qdbm/hvmgr +qdbm/hvtest +qdbm/libqdbm.so +qdbm/libqdbm.so.14 +qdbm/libqdbm.so.14.13.0 +qdbm/odidx +qdbm/odmgr +qdbm/odtest +qdbm/qmttest +qdbm/rlmgr +qdbm/rltest +qdbm/vlmgr +qdbm/vltest +qdbm/vltsv # development *.orig *.patch .*.swo .*.swp + +# generated documentation +docs/htmlguide +docs/man From a3e328ef8f83d81f74b6a334ff6ad66d7dcbd534 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 18 May 2017 21:41:15 +0100 Subject: [PATCH 04/31] Update versions of dependencies for Windows unibuild, add Boost * Add a variable in appveyor.yml for the CMake out-of-source build tree. * Update the directory name in commands in appveyor.yml to match new CMake unibuild tree structure. * Increase build verbosity slightly. * Disable GUI crash dialogs that cause test hangs. * Disable the AppVeyor cache that we don't use. --- appveyor.yml | 23 ++----- infrastructure/cmake/windows/CMakeLists.txt | 69 ++++++++++++++++----- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1def8a7ab..a0ccf95ce 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,17 +21,7 @@ environment: Generator: Visual Studio 10 OPENSSL_VERSION: 1.0.2f PCRE_VERSION: 8.38 - -cache: - - '..\zlib-1.2.8' - - '..\zlib-%PLATFORM%' - - '..\openssl-%OPENSSL_VERSION%.tar.gz' - - '..\openssl-%OPENSSL_VERSION%' - - '..\openssl-%PLATFORM%' - - '..\pcre-%PCRE_VERSION%.zip' - - '..\pcre-%PCRE_VERSION%' - - '..\pcre-%PLATFORM%' - - 'infrastructure\cmake\build' + CMAKE_UNIBUILD_DIR: '%APPVEYOR_BUILD_FOLDER%\..\cmake' init: # Uncomment the following two lines to enable RDP access to the virtual machine for debugging. @@ -41,7 +31,7 @@ init: build: parallel: true project: ..\cmake\BoxBackup_Windows.sln - verbosity: quiet + verbosity: minimal install: # test_bbackupd needs 7zip (or cmake -E tar) to extract tar archives on Windows: @@ -56,9 +46,8 @@ install: # cinst -y cmake strawberryperl git vim visualstudio2012wdx - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86' - - cd %APPVEYOR_BUILD_FOLDER%\.. - - if not exist cmake md cmake - - cd cmake + - if not exist %CMAKE_UNIBUILD_DIR% md %CMAKE_UNIBUILD_DIR% + - cd %CMAKE_UNIBUILD_DIR% # We need to specify the generator here, in case the user has more than one installed. # CMake always seems to default to the latest version of Visual Studio, not the one on # the current PATH. @@ -67,10 +56,10 @@ install: # Leave the current directory in the correct place to find the solution file using its relative path above. test_script: - - cd %APPVEYOR_BUILD_FOLDER%\..\cmake\src\boxbackup-build + - cd %CMAKE_UNIBUILD_DIR%\Build\boxbackup # - dir # - dir bin_bbackupd.dir # - dir %PLATFORM% # - dir %PLATFORM%\%CONFIGURATION% - - ctest -C %CONFIGURATION% -V + - ctest -C %CONFIGURATION% -V --interactive-debug-mode 0 diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index 421527140..a6d93ee80 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -3,26 +3,36 @@ cmake_minimum_required(VERSION 2.6) project(BoxBackup_Windows) set(boxbackup_dir ${CMAKE_SOURCE_DIR}/../../..) -set_property(DIRECTORY PROPERTY EP_PREFIX .) +set_property(DIRECTORY PROPERTY EP_BASE .) set(install_dir ${CMAKE_BINARY_DIR}/install) # Automate the process of downloading, building and "installing" dependencies on Windows, # as used by AppVeyor. -set(ZLIB_VERSION 1.2.8 CACHE STRING "Version of zlib to download, build, and compile Box Backup against") -set(ZLIB_HASH MD5=126f8676442ffbd97884eb4d6f32afb4 - CACHE STRING "Hash of the zlib download file, to be verified after download") -set(OPENSSL_VERSION 1.0.2h CACHE STRING "Version of OpenSSL to download, build, and compile Box Backup against") -set(OPENSSL_HASH SHA256=1d4007e53aad94a5b2002fe045ee7bb0b3d98f1a47f8b2bc851dcd1c74332919 - CACHE STRING "Hash of the OpenSSL download file, to be verified after download") -set(PCRE_VERSION 8.38 CACHE STRING "Version of PCRE to download, build, and compile Box Backup against") -set(OPENSSL_HASH SHA256=dbef7cf80258c29396d435804cd5dba34006a77548850bca8bad6db6a6eac110 - CACHE STRING "Hash of the PCRE download file, to be verified after download") + +# Version of zlib to download, build, and compile Box Backup against: +set(ZLIB_VERSION 1.2.11) +# Hash of zlib-${ZLIB_VERSION}.tar.gz, to be verified after download: +set(ZLIB_HASH SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1) + +# Version of OpenSSL to download, build, and compile Box Backup against: +set(OPENSSL_VERSION 1.0.2j) +# Hash of openssl-${OPENSSL_VERSION}.tar.gz, to be verified after download: +set(OPENSSL_HASH SHA256=e7aff292be21c259c6af26469c7a9b3ba26e9abaaffd325e3dccc9785256c431) + +# Version of PCRE to download, build, and compile Box Backup against: +set(PCRE_VERSION 8.39) +# Hash of pcre-${PCRE_VERSION}.tar.gz, to be verified after download: +set(PCRE_HASH SHA256=ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7) + +# Version of Boost to download, unpack, and compile Box Backup against: +set(BOOST_VERSION 1.62.0) +# Hash of the Boost download file, to be verified after download: +set(BOOST_HASH SHA1=5fd97433c3f859d8cbab1eaed4156d3068ae3648) include(ExternalProject) -string(REPLACE "." "" zlib_version_nodots ${ZLIB_VERSION}) ExternalProject_Add(zlib - URL "http://zlib.net/zlib${zlib_version_nodots}.zip" + URL "http://zlib.net/zlib-${ZLIB_VERSION}.tar.gz" URL_HASH ${ZLIB_HASH} DOWNLOAD_NO_PROGRESS 1 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} @@ -71,19 +81,46 @@ else() endif() ExternalProject_Add(pcre - URL "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-${PCRE_VERSION}.zip" - URL_HASH ${PCRE_HASH} + # Temporarily use SVN repo until the PCRE_STATIC issue in 8.40 is fixed: + # https://vcs.pcre.org/pcre?view=revision&revision=1677 + SVN_REPOSITORY svn://vcs.exim.org/pcre/code/trunk + SVN_REVISION -r 1677 DOWNLOAD_NO_PROGRESS 1 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} + -DPCRE_SUPPORT_LIBREADLINE=OFF + -DPCRE_SUPPORT_LIBBZ2=OFF # We need to build both versions, debug and release, because cmake requires both to be # present to generate its multi-configuration project files for Visual Studio/MSBuild. INSTALL_COMMAND ${CMAKE_COMMAND} --build --target install --config Debug COMMAND ${CMAKE_COMMAND} --build --target install --config Release ) +string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) +ExternalProject_Add(boost + URL "http://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION}/boost_${BOOST_VERSION_UNDERSCORES}.tar.bz2" + URL_HASH ${BOOST_HASH} + # DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" # none needed + BUILD_COMMAND ${CMAKE_COMMAND} -E echo "No build step needed" # none needed + INSTALL_COMMAND "" # none needed +) + +if(BOXBACKUP_VERSION) + list(APPEND boxbackup_cmake_args "-DBOXBACKUP_VERSION=${BOXBACKUP_VERSION}") +endif() +string(REPLACE ";" " " boxbackup_cmake_args "${boxbackup_cmake_args}") + ExternalProject_Add(boxbackup - DEPENDS zlib openssl pcre + DEPENDS zlib openssl pcre boost SOURCE_DIR ${boxbackup_dir}/infrastructure/cmake - CMAKE_ARGS -DZLIB_ROOT=${install_dir} -DOPENSSL_ROOT_DIR=${install_dir} -DPCRE_ROOT=${install_dir} -DAPPVEYOR_MODE=1 + CMAKE_ARGS + -DZLIB_ROOT=${install_dir} + -DOPENSSL_ROOT_DIR=${install_dir} + -DPCRE_ROOT=${install_dir} + -DBOOST_ROOT=${CMAKE_BINARY_DIR}/Source/boost + -DBOX_SUPPORT_READLINE=OFF + -DAPPVEYOR_MODE=1 + -DDEBUG=${DEBUG} + ${boxbackup_cmake_args} STEP_TARGETS configure build install ) From 14e48b9851920cb2beb000610b36391879dcb201 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 15 May 2017 21:16:13 +0100 Subject: [PATCH 05/31] Add Boost to Travis build environment, and OSX builds --- .travis.yml | 14 +++++++++++++- infrastructure/travis-build.sh | 34 +++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96e8bf26c..57bd36f34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,10 +19,12 @@ addons: packages: - cmake - cmake-data + - libboost-dev - libdb-dev - libreadline-dev - libssl-dev - libwww-perl + - time - xsltproc - zlib1g-dev @@ -32,5 +34,15 @@ env: - BUILD=cmake TEST_TARGET=debug - BUILD=cmake TEST_TARGET=release +os: + - linux + - osx + +matrix: + exclude: + # No point testing both GCC and Clang on Mac OS X, as they're the same thing. + - os: osx + compiler: gcc + script: - - ./infrastructure/travis-build.sh + - env EXTRA_MAKE_ARGS=-j2 ./infrastructure/travis-build.sh diff --git a/infrastructure/travis-build.sh b/infrastructure/travis-build.sh index 72a187b39..74b58a3d8 100755 --- a/infrastructure/travis-build.sh +++ b/infrastructure/travis-build.sh @@ -3,23 +3,47 @@ set -e set -x +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + brew update + # Travis appears to have Boost and OpenSSL installed already: + # brew install boost ccache openssl + ls /usr/local /usr/local/opt /usr/local/opt/openssl + brew install ccache +fi + ccache -s if [ "$BUILD" = 'cmake' ]; then + if [ -z "$TEST_TARGET" ]; then + echo "TEST_TARGET must be set to 'release' or 'debug' for CMake builds" + exit 2 + fi + + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + EXTRA_ARGS="-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DBOOST_ROOT=/usr/local/opt/boost" + fi + cd `dirname $0` mkdir -p cmake/build cd cmake/build cmake --version - cmake -DCMAKE_BUILD_TYPE:STRING=$TEST_TARGET .. - make install + cmake -DCMAKE_BUILD_TYPE:STRING=$TEST_TARGET $EXTRA_ARGS "$@" .. + make + [ "$TEST" = "n" ] || ctest -C $TEST_TARGET -V else + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + EXTRA_ARGS="--with-ssl-lib=/usr/local/opt/openssl/lib --with-ssl-headers=/usr/local/opt/openssl/include --with-boost=/usr/local/opt/boost" + fi + cd `dirname $0`/.. ./bootstrap - ./configure CC="ccache $CC" CXX="ccache $CXX" "$@" + ./configure CC="ccache $CC" CXX="ccache $CXX" $EXTRA_ARGS "$@" grep CXX config.status - make V=1 - ./runtest.pl ALL $TEST_TARGET + make V=1 $EXTRA_MAKE_ARGS + + [ "$TEST" = "n" ] || ./runtest.pl ALL $TEST_TARGET + if [ "$TEST_TARGET" = "release" ]; then make make parcels From ab617bdce7280a59333da695f44507cc36c35d25 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 18 May 2017 21:54:27 +0100 Subject: [PATCH 06/31] Install executables in test tree after building, not during "make install" Build buildenv-testmain-template.cpp in-place, instead of copying, modifying and compiling it. Increase timeouts for some tests that sometimes run out of time on AppVeyor. --- infrastructure/cmake/CMakeLists.txt | 108 ++++++++++++++-------------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/infrastructure/cmake/CMakeLists.txt b/infrastructure/cmake/CMakeLists.txt index 22c215532..cba7c7391 100644 --- a/infrastructure/cmake/CMakeLists.txt +++ b/infrastructure/cmake/CMakeLists.txt @@ -101,8 +101,6 @@ if(WIN32) add_definitions(-DWIN32) endif() -file(READ "${base_dir}/infrastructure/buildenv-testmain-template.cpp" test_template) - # Parsing Makefile.extra files in CMake script is a pain, so the relevant rules for # code-generating Perl scripts are hard-coded here. @@ -176,6 +174,13 @@ foreach(documentation_file ${documentation_files}) set(${module_name}_extra_files ${${module_name}_extra_files} ${output_file}) endforeach() +set(testmain_template_cpp + "${base_dir}/infrastructure/buildenv-testmain-template.cpp" +) +set(release_or_debug_dir + "${base_dir}/$<$:debug>$<$:release>$<$:release>" +) + file(STRINGS ${base_dir}/modules.txt module_deps REGEX "^[^#]") foreach(module_dep ${module_deps}) @@ -211,19 +216,26 @@ foreach(module_dep ${module_deps}) if(DEBUG) message(STATUS "add executable '${module_name}': '${module_files}'") endif() - add_executable(${module_name} ${module_files}) - # Unfortunately we have to use install(PROGRAMS) instead of - # install(TARGETS) because TARGETS doesn't allow us to change - # the executable name. - install(PROGRAMS "$" - CONFIGURATIONS Debug - DESTINATION "${base_dir}/debug/${module_dir}" - RENAME "${bin_name}${CMAKE_EXECUTABLE_SUFFIX}") - install(PROGRAMS "$" - CONFIGURATIONS Release - DESTINATION "${base_dir}/release/${module_dir}" - RENAME "${bin_name}${CMAKE_EXECUTABLE_SUFFIX}") + add_executable(${module_name} ${module_files}) + # Rename the output executable from bin_bbackupd(.exe) to bbackupd(.exe): + set_target_properties(${module_name} PROPERTIES + OUTPUT_NAME "${bin_name}") + + # Use a custom post-build command instead of install(...) to install + # binaries ready for running tests, because we don't want "make install" to + # do this too, and absolute-path installations break the CPack generators. + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + "$" + "${release_or_debug_dir}/${module_dir}/${bin_name}${CMAKE_EXECUTABLE_SUFFIX}" + VERBATIM) + + # For "make install" and CPack generators: + install(TARGETS ${module_name} RUNTIME + CONFIGURATIONS Debug;Release + DESTINATION "." + COMPONENT Applications) elseif(module_name MATCHES "^test_") string(REGEX MATCH "^test_(.*)" valid_test ${module_name}) set(test_name ${CMAKE_MATCH_1}) @@ -233,36 +245,17 @@ foreach(module_dep ${module_deps}) message(STATUS "add test '${module_name}': '${module_files}'") endif() - string(REPLACE "TEST_NAME" ${test_name} test_main "${test_template}") - file(WRITE "${module_path}/_main.cpp.new" "${test_main}") - replace_file_if_different( - "${module_path}/_main.cpp" - "${module_path}/_main.cpp.new") - add_executable(${module_name} ${module_files} - "${module_path}/_main.cpp") + set(module_files ${module_files} "${testmain_template_cpp}") + add_executable(${module_name} ${module_files}) - if(WIN32) - install(PROGRAMS "$" - CONFIGURATIONS Debug - DESTINATION "${base_dir}/debug/${module_dir}") - install(PROGRAMS "$" - CONFIGURATIONS Release - DESTINATION "${base_dir}/release/${module_dir}") - set(test_executable "$") - else() - # Unfortunately we have to use install(PROGRAMS) instead of - # install(TARGETS) because TARGETS doesn't allow us to change - # the executable name. - install(PROGRAMS "$" - CONFIGURATIONS Debug - DESTINATION "${base_dir}/debug/${module_dir}" - RENAME "_test") - install(PROGRAMS "$" - CONFIGURATIONS Release - DESTINATION "${base_dir}/release/${module_dir}" - RENAME "_test") - set(test_executable "./_test") - endif() + # Use a custom post-build command instead of install(...) to install + # binaries ready for running tests, because we don't want "make install" to + # do this too, and absolute-path installations break the CPack generators. + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + "$" + "${release_or_debug_dir}/${module_dir}/${bin_name}${CMAKE_EXECUTABLE_SUFFIX}" + VERBATIM) if(${APPVEYOR_MODE}) set(appveyor_runtest_pl_switch -a) @@ -270,12 +263,19 @@ foreach(module_dep ${module_deps}) set(appveyor_runtest_pl_switch) endif() + if(WIN32) + set(test_command_internal "$") + else() + set(test_command_internal "./$") + endif() + target_compile_definitions(${module_name} PRIVATE - -DTEST_EXECUTABLE="${test_executable}") + -DTEST_EXECUTABLE="${test_command_internal}") add_test(NAME ${test_name} COMMAND ${PERL_EXECUTABLE} ${base_dir}/runtest.pl ${appveyor_runtest_pl_switch} -c ${test_name} - $ "$" "${test_executable}" + $<$:DEBUG>$<$:RELEASE>$<$:RELEASE> + "$" "${test_command_internal}" WORKING_DIRECTORY ${base_dir}) if(${APPVEYOR_MODE}) @@ -288,7 +288,8 @@ foreach(module_dep ${module_deps}) add_custom_target(${module_name}-prepare COMMAND ${PERL_EXECUTABLE} ${base_dir}/runtest.pl -n -c ${test_name} - $ "$" "${test_executable}" + $<$:DEBUG>$<$:RELEASE>$<$:RELEASE> + "$" "${test_command_internal}" WORKING_DIRECTORY ${base_dir}) elseif(module_name MATCHES "^(lib_.*|qdbm)$") if(DEBUG) @@ -337,11 +338,12 @@ target_compile_definitions(lib_common PUBLIC $<$:BOX_RELEASE_BUI # Detect platform features and write BoxConfig.h.in. Reuse code from # infrastructure/m4/boxbackup_tests.m4 where possible -include(CheckIncludeFiles) +include(CheckCXXCompilerFlag) +include(CheckCXXSourceCompiles) include(CheckFunctionExists) -include(CheckSymbolExists) +include(CheckIncludeFiles) include(CheckLibraryExists) -include(CheckCXXSourceCompiles) +include(CheckSymbolExists) set(boxconfig_h_file "${CMAKE_BINARY_DIR}/BoxConfig.h.in") file(REMOVE "${boxconfig_h_file}") @@ -662,7 +664,7 @@ target_compile_definitions(qdbm PUBLIC -DQDBM_STATIC) # Silence some less-useful warnings if(MSVC) - add_definitions(/wd4996 /wd4291) + add_definitions(/wd4291 /wd4710 /wd4820 /wd4996) set_property(TARGET qdbm PROPERTY CMAKE_STATIC_LINKER_FLAGS /IGNORE:LNK4006) endif(MSVC) @@ -675,12 +677,12 @@ target_compile_definitions(test_backupstorefix PRIVATE -DPERL_EXECUTABLE="${perl # I've set the timeout to 4 times as long as it took to run on a particular run on Appveyor: # https://ci.appveyor.com/project/qris/boxbackup/build/job/xm10itascygtu93j set_tests_properties(common PROPERTIES TIMEOUT 20) -set_tests_properties(crypto PROPERTIES TIMEOUT 4) +set_tests_properties(crypto PROPERTIES TIMEOUT 10) set_tests_properties(compress PROPERTIES TIMEOUT 80) set_tests_properties(raidfile PROPERTIES TIMEOUT 32) set_tests_properties(basicserver PROPERTIES TIMEOUT 80) -set_tests_properties(backupstore PROPERTIES TIMEOUT 560) -set_tests_properties(backupstorefix PROPERTIES TIMEOUT 140) +set_tests_properties(backupstore PROPERTIES TIMEOUT 1320) +set_tests_properties(backupstorefix PROPERTIES TIMEOUT 180) set_tests_properties(backupstorepatch PROPERTIES TIMEOUT 320) set_tests_properties(backupdiff PROPERTIES TIMEOUT 32) set_tests_properties(bbackupd PROPERTIES TIMEOUT 1200) From 910840de529d1566924ea938883969307b91c209 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 23 Jan 2017 21:18:27 +0000 Subject: [PATCH 07/31] Fix build on SmartOS by switching back to GNU Make I have no idea where I got the idea that there was a "bmake" command on SmartOS that is BSD make, or why that would be better than gmake (GNU make). I can't find any reference to the existence of such a thing. So I've just switched back to using gmake unconditionally. --- infrastructure/BoxPlatform.pm.in | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/infrastructure/BoxPlatform.pm.in b/infrastructure/BoxPlatform.pm.in index bdcca279a..325e56c3a 100644 --- a/infrastructure/BoxPlatform.pm.in +++ b/infrastructure/BoxPlatform.pm.in @@ -47,16 +47,8 @@ BEGIN } elsif ($build_os eq 'SunOS') { - if ($build_os_ver <= 5.10) - { - $make_command = 'gmake'; - $bsd_make = 0; - } - else - { - $make_command = 'bmake'; - $bsd_make = 1; - } + $make_command = 'gmake'; + $bsd_make = 0; } else { From 81e9aa6545f7f19124c9f5e88982b867d8732965 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 26 Jan 2017 07:26:34 +0000 Subject: [PATCH 08/31] Hopefully fix test/raidfile on Solaris. 64-bit Solaris has _FILE_OFFSET_BITS defined (to 64), which was wrongly causing the substitute lseek in lib/intercept to enter the 32-bit branch. --- lib/intercept/intercept.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 88ea0d6e5..ac8c44310 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -389,7 +389,7 @@ lseek(int fildes, off_t offset, int whence) #else #ifdef HAVE_LSEEK_DUMMY_PARAM off_t r = syscall(SYS_lseek, fildes, 0 /* extra 0 required here! */, offset, whence); - #elif defined(_FILE_OFFSET_BITS) + #elif defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 32 // Don't bother trying to call SYS__llseek on 32 bit since it is // fiddly and not needed for the tests off_t r = syscall(SYS_lseek, fildes, (uint32_t)offset, whence); From 3a3fc72cd3b3c98870028f6d968b9cde64341ccd Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 22 May 2017 21:54:06 +0100 Subject: [PATCH 09/31] Relax timings on test_changing_client_store_marker_pauses_daemon Also increase test verbosity to help debug the test if it fails again. (cherry picked from commit 00c2127e9832591c248fa6eea05dfaf785b8380d) --- test/bbackupd/testbbackupd.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index 60ae4e897..7113880d3 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -361,8 +361,8 @@ bool configure_bbackupd(BackupDaemon& bbackupd, const std::string& config_file) // Stop bbackupd initialisation from changing the console logging level // and the program name tag. Logger& console(Logging::GetConsole()); - Logger::LevelGuard guard(console, console.GetLevel()); - Logging::Tagger(); + Logger::LevelGuard undo_log_level_change(console, console.GetLevel()); + Logging::Tagger undo_program_name_change; std::vector args; size_t last_arg_start = 0; @@ -3748,35 +3748,35 @@ bool test_changing_client_store_marker_pauses_daemon() // Test that there *are* differences still, i.e. that bbackupd // didn't successfully run a backup during that time. - BOX_TRACE("Compare starting, expecting differences"); + BOX_INFO("Compare starting, expecting differences"); TEST_COMPARE(Compare_Different); BOX_TRACE("Compare finished, expected differences"); // Wait out the expected delay in bbackupd. This is quite // time-sensitive, so we use sub-second precision. - box_time_t wait = + box_time_t wait = SecondsToBoxTime(BACKUP_ERROR_DELAY_SHORTENED - 1) - compare_time * 2; - BOX_TRACE("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << - " (plus another compare taking " << - BOX_FORMAT_MICROSECONDS(compare_time) << ") until " - "just before bbackupd recovers"); + BOX_INFO("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << " " + "until just before bbackupd recovers"); ShortSleep(wait, true); // bbackupd should not have recovered yet, so there should // still be differences. - BOX_TRACE("Compare starting, expecting differences"); + BOX_INFO("Compare starting, expecting differences"); TEST_COMPARE(Compare_Different); BOX_TRACE("Compare finished, expected differences"); - // Now wait for it to recover and finish a sync, and check - // that the differences are gone (successful backup). - wait = sync_time + SecondsToBoxTime(2); - BOX_TRACE("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << + // Now wait for it to recover and finish a sync, and check that + // the differences are gone (successful backup). Wait until ~2 + // seconds after we expect the sync to have finished, to reduce + // the risk of random failure on AppVeyor when heavily loaded. + wait = sync_time + SecondsToBoxTime(6); + BOX_INFO("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << " until just after bbackupd recovers and finishes a sync"); ShortSleep(wait, true); - BOX_TRACE("Compare starting, expecting no differences"); + BOX_INFO("Compare starting, expecting no differences"); TEST_COMPARE(Compare_Same); BOX_TRACE("Compare finished, expected no differences"); } From 25e445a6848a870aabd89a3b8a013265aa3cf17b Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 19 Jun 2017 21:11:09 +0100 Subject: [PATCH 10/31] Stop Git from normalising line endings Although recommended, "text" mode causes constant and unfixable conficts that I give up fighting with. --- .gitattributes | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitattributes b/.gitattributes index ae83a8942..11b61ab9d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,12 +2,6 @@ # Set the default behavior, in case people don't have core.autocrlf set. * text=auto -# Explicitly declare text files you want to always be normalized and converted -# to native line endings on checkout. -*.cpp text -*.c text -*.h text - # Declare files that will always have CRLF line endings on checkout. *.sln text eol=crlf *.vcxproj.user text eol=crlf From d8422a8b053b62c67e5fe2e861a09d2c792b657a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 18 Jun 2017 19:42:43 +0100 Subject: [PATCH 11/31] Use AX_CHECK_COMPILE_FLAG, add more checks Disable pointless deprecation warnings for std::auto_ptr. --- infrastructure/m4/ax_check_compile_flag.m4 | 72 ++++++++++++++++++++++ infrastructure/m4/boxbackup_tests.m4 | 39 +++++++----- 2 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 infrastructure/m4/ax_check_compile_flag.m4 diff --git a/infrastructure/m4/ax_check_compile_flag.m4 b/infrastructure/m4/ax_check_compile_flag.m4 new file mode 100644 index 000000000..c3a8d695a --- /dev/null +++ b/infrastructure/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/infrastructure/m4/boxbackup_tests.m4 b/infrastructure/m4/boxbackup_tests.m4 index c7838a7fd..849ffd7ae 100644 --- a/infrastructure/m4/boxbackup_tests.m4 +++ b/infrastructure/m4/boxbackup_tests.m4 @@ -10,22 +10,30 @@ solaris*) ;; esac -if test "x$GXX" = "xyes"; then - # Use -Wall if we have gcc. This gives better warnings - CXXFLAGS_STRICT='-Wall -Wundef' - - # Check whether gcc accepts -Werror=return-type, and if so add it to CXXFLAGS_STRICT - my_save_cflags="$CXXFLAGS" - CXXFLAGS="-Werror=return-type" - AC_MSG_CHECKING([whether $CXX accepts $CXXFLAGS]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])], - [AC_MSG_RESULT([yes])] - [CXXFLAGS_STRICT="$CXXFLAGS_STRICT $CXXFLAGS"], - [AC_MSG_RESULT([no])] - ) - CXXFLAGS="$my_save_cflags" - AC_SUBST([CXXFLAGS_STRICT]) +# Enable some compiler flags if the compiler supports them. This gives better warnings +# and detects some problems early. +AX_CHECK_COMPILE_FLAG(-Wall, [cxxflags_strict="$cxxflags_strict -Wall"]) +# -Wundef would be a good idea, but Boost is full of undefined variable use, so we need +# to disable it for now so that we can concentrate on real errors: +dnl AX_CHECK_COMPILE_FLAG(-Wundef, [cxxflags_strict="$cxxflags_strict -Wundef"]) +AX_CHECK_COMPILE_FLAG(-Werror=return-type, + [cxxflags_strict="$cxxflags_strict -Werror=return-type"]) +AX_CHECK_COMPILE_FLAG(-Werror=delete-non-virtual-dtor, + [cxxflags_strict="$cxxflags_strict -Werror=delete-non-virtual-dtor"]) +AX_CHECK_COMPILE_FLAG(-Werror=undefined-bool-conversion, + [cxxflags_strict="$cxxflags_strict -Werror=undefined-bool-conversion"]) +# We should really enable -Werror=sometimes-uninitialized, but QDBM violates it: +dnl AX_CHECK_COMPILE_FLAG(-Werror=sometimes-uninitialized, +dnl [cxxflags_strict="$cxxflags_strict -Werror=sometimes-uninitialized"]) +# This error is detected by MSVC, but not usually by GCC/Clang: +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58114 +AX_CHECK_COMPILE_FLAG(-Werror=delete-incomplete, + [cxxflags_strict="$cxxflags_strict -Werror=delete-incomplete"]) +AX_CHECK_COMPILE_FLAG(-Wno-deprecated-declarations, + [cxxflags_strict="$cxxflags_strict -Wno-deprecated-declarations"]) +AC_SUBST([CXXFLAGS_STRICT], [$cxxflags_strict]) +if test "x$GXX" = "xyes"; then # Don't check for gcc -rdynamic on Solaris as it's broken, but returns 0. # On Cygwin it does nothing except cause gcc to emit a warning message. case $build_os in @@ -49,7 +57,6 @@ if test "x$GXX" = "xyes"; then LDFLAGS=$save_LDFLAGS ;; esac - fi AC_PATH_PROG([PERL], [perl], [AC_MSG_ERROR([[perl executable was not found]])]) From 6a04b0abd728da5211e6702b1d42aef95c02d8da Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 18 Jun 2017 18:56:08 +0000 Subject: [PATCH 12/31] Fix intercept of SYS_open on platforms that only have SYS_openat (e.g. arm64) --- infrastructure/m4/boxbackup_tests.m4 | 1 + lib/intercept/intercept.cpp | 6 ++++++ lib/intercept/intercept.h | 9 +++++++++ test/raidfile/testraidfile.cpp | 9 +-------- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/infrastructure/m4/boxbackup_tests.m4 b/infrastructure/m4/boxbackup_tests.m4 index 849ffd7ae..7410159a5 100644 --- a/infrastructure/m4/boxbackup_tests.m4 +++ b/infrastructure/m4/boxbackup_tests.m4 @@ -224,6 +224,7 @@ AC_CHECK_DECLS([INFTIM],,, [[#include ]]) AC_CHECK_DECLS([SO_PEERCRED],,, [[#include ]]) AC_CHECK_DECLS([SOL_TCP],,, [[#include ]]) AC_CHECK_DECLS([TCP_INFO],,, [[#include ]]) +AC_CHECK_DECLS([SYS_open, SYS_openat],,, [[#include ]]) if test -n "$have_sys_socket_h"; then AC_CHECK_DECLS([SO_SNDBUF],,, [[#include ]]) diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index ac8c44310..72bd8d4e9 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -242,6 +242,10 @@ extern "C" int open(const char *path, int flags, ...) #endif // DEFINE_ONLY_OPEN64 { + // Some newer architectures don't have an open() syscall, but use openat() instead. + // In these cases we will need to call sys_openat() instead of sys_open(). + // https://chromium.googlesource.com/linux-syscall-support/ + if(intercept_count > 0) { if(intercept_filename != NULL && @@ -264,6 +268,8 @@ extern "C" int #ifdef PLATFORM_NO_SYSCALL int r = TEST_open(path, flags, mode); +#elif HAVE_DECL_SYS_OPENAT && !HAVE_DECL_SYS_OPEN + int r = syscall(SYS_openat, AT_FDCWD, path, flags, mode); #else int r = syscall(SYS_open, path, flags, mode); #endif diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index c0d616385..4de5f9f26 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -62,5 +62,14 @@ void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); void intercept_clear_setup(); +// Some newer architectures don't have an open() syscall, but use openat() instead. +// In these cases we define SYS_open (which is otherwise undefined) to equal SYS_openat +// (which is defined) so that everywhere else we can call intercept_setup_error(SYS_open) +// without caring about the difference. +// https://chromium.googlesource.com/linux-syscall-support/ +#if !HAVE_DECL_SYS_OPEN && HAVE_DECL_SYS_OPENAT +# define SYS_open SYS_openat +#endif + #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif // !INTERCEPT_H diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp index c81503871..d771f23dc 100644 --- a/test/raidfile/testraidfile.cpp +++ b/test/raidfile/testraidfile.cpp @@ -25,6 +25,7 @@ #include "RaidFileException.h" #include "RaidFileRead.h" #include "Guards.h" +#include "intercept.h" #include "MemLeakFindOn.h" @@ -37,14 +38,6 @@ #define TRF_CAN_INTERCEPT #endif - -#ifdef TRF_CAN_INTERCEPT -// function in intercept.cpp for setting up errors -void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror); -bool intercept_triggered(); -void intercept_clear_setup(); -#endif - // Nice random data for testing written files class R250 { public: From 105f2620b98bfd4e5d7ed576b5cc4b2317dcf634 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 20 Jun 2017 21:02:42 +0100 Subject: [PATCH 13/31] Reduce precision of common timer test Ideally timers would be perfectly accurate and we could sleep for 1.0 seconds, but on OSX in particular they could fire 50-100 ms late (I've seen 4 ms in practice) and we don't want the tests to fail because of this, because we don't really need that kind of precision in practice. So we reduce the timer intervals by 100ms to be safe. Increase logging level and add timestamps in testcommon. Should help to debug frequent test failures such as Condition [t2.HasExpired()] on OSX hosts on Travis. Add a SettingsGuard to the Console logger for its specific settings, such as showing times and microseconds. (cherry picked from commit a5e6cff1d435329b0121417ed9509e315ce0edd5) (cherry picked from commit 8d02eebce553ed822e0fcd60d6e319384e15ba4b) --- lib/common/Logging.h | 26 +++++++++++ test/common/testcommon.cpp | 95 ++++++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 41 deletions(-) diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 01358617b..3dc3e69ca 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -295,6 +295,32 @@ class Console : public Logger static void SetShowTimeMicros(bool enabled); static void SetShowPID(bool enabled); static bool GetShowTag() { return sShowTag; } + + class SettingsGuard + { + private: + bool mShowTag; + bool mShowTime; + bool mShowTimeMicros; + bool mShowPID; + std::string mTag; + public: + SettingsGuard() + : mShowTag(Console::sShowTag), + mShowTime(Console::sShowTime), + mShowTimeMicros(Console::sShowTimeMicros), + mShowPID(Console::sShowPID), + mTag(Console::sTag) + { } + ~SettingsGuard() + { + Console::SetShowTag(mShowTag); + Console::SetShowTime(mShowTime); + Console::SetShowTimeMicros(mShowTimeMicros); + Console::SetShowPID(mShowPID); + Console::sTag = mTag; + } + }; }; // -------------------------------------------------------------------------- diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index fbdf8d9c2..bd75ac37a 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -344,47 +344,60 @@ int test(int argc, const char *argv[]) Timers::Init(); - Timer t0(0, "t0"); // should never expire - Timer t1(1000, "t1"); - Timer t2(2000, "t2"); - Timer t3(3000, "t3"); - - TEST_THAT(!t0.HasExpired()); - TEST_THAT(!t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - // Try both ways of resetting an existing timer. - t1 = Timer(1000, "t1a"); - t2.Reset(2000); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(!t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(t2.HasExpired()); - TEST_THAT(t3.HasExpired()); + // Ideally timers would be perfectly accurate and we could sleep for 1.0 seconds, but + // on OSX in particular they could fire 50-100 ms late (I've seen 4 ms in practice) + // and we don't want the tests to fail because of this, because we don't really need + // that kind of precision in practice. So we reduce the timer intervals by 100ms to + // be safe. + + { + Logger::LevelGuard temporary_verbosity(Logging::GetConsole(), Log::TRACE); + Console::SettingsGuard save_old_settings; + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + + Timer t0(0, "t0"); // should never expire + Timer t1(900, "t1"); + Timer t2(1900, "t2"); + Timer t3(2900, "t3"); + + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + safe_sleep(1); + + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + // Try both ways of resetting an existing timer. + t1 = Timer(900, "t1a"); + t2.Reset(1900); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + } // Leave timers initialised for rest of test. // Test main() will cleanup after test finishes. From 809d9ebacc409c4a86edb6077f489da67a2653d4 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 11 May 2017 20:50:26 +0100 Subject: [PATCH 14/31] Increase precision of timing on Windows. Should hopefully reduce random failures of timing-dependent tests which rely on subsecond precision for accuracy. (cherry picked from commit c845b2e39fffeb560983a301d810616a6495469d) --- lib/common/BoxTime.cpp | 39 +++++++++++++++++++++------------- test/bbackupd/testbbackupd.cpp | 3 ++- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index 78269def2..77daae6dc 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -35,21 +35,30 @@ // -------------------------------------------------------------------------- box_time_t GetCurrentBoxTime() { - #ifdef HAVE_GETTIMEOFDAY - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) - { - BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " - "dropping precision"); - } - else - { - box_time_t timeNow = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) - + tv.tv_usec; - return timeNow; - } - #endif - +#ifdef HAVE_GETTIMEOFDAY + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) + { + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); + } + else + { + box_time_t time_now = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) + tv.tv_usec; + return time_now; + } +#elif WIN32 + // There's no Win32 API function that returns the current time as a UNIX timestamp with + // sub-second precision. So we use time(0) and add the fractional part from + // GetSystemTime() in the hope that the difference between these two (if any) is a whole + // number of seconds. + box_time_t time_now = SecondsToBoxTime(time(0)); + SYSTEMTIME system_time; + GetSystemTime(&system_time); + time_now += MilliSecondsToBoxTime(system_time.wMilliseconds); + return time_now; +#endif + return SecondsToBoxTime(time(0)); } diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index 7113880d3..cc602f228 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -3675,6 +3675,7 @@ bool test_changing_client_store_marker_pauses_daemon() box_time_t sync_start_time = GetCurrentBoxTime(); sync_and_wait(); box_time_t sync_time = GetCurrentBoxTime() - sync_start_time; + BOX_INFO("Sync takes " << BOX_FORMAT_MICROSECONDS(sync_time)); // Time how long a compare takes. On NetBSD it's 3 seconds, and that // interferes with test timing unless we account for it. @@ -3682,7 +3683,7 @@ bool test_changing_client_store_marker_pauses_daemon() // There should be no differences right now (yet). TEST_COMPARE(Compare_Same); box_time_t compare_time = GetCurrentBoxTime() - compare_start_time; - BOX_TRACE("Compare takes " << BOX_FORMAT_MICROSECONDS(compare_time)); + BOX_INFO("Compare takes " << BOX_FORMAT_MICROSECONDS(compare_time)); // Wait for the end of another sync, to give us ~3 seconds to change // the client store marker. From 4b7ab2833ebec9c026c10a761d64ada1ce809438 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 26 Jun 2017 20:52:57 +0000 Subject: [PATCH 15/31] Fix build on ARM64 with unsigned char Thanks to Reinhard Tartler for pointing out the problem and suggesting the fix! --- lib/httpserver/cdecode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp index e632f1827..11c59d629 100644 --- a/lib/httpserver/cdecode.cpp +++ b/lib/httpserver/cdecode.cpp @@ -12,7 +12,7 @@ extern "C" int base64_decode_value(char value_in) { - static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static signed const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; static const char decoding_size = sizeof(decoding); value_in -= 43; if (value_in < 0 || value_in > decoding_size) return -1; From c0ff77b8a73e0b7804a9eac26b59603e06d1cbc0 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 3 Jul 2017 08:33:36 +0100 Subject: [PATCH 16/31] Build Win32/64 binary packages automatically Every AppVeyor build on the specified branches that passes will be packaged into a ZIP file and installer using CPack and uploaded to GitHub Releases. Generate version number entirely in AppVeyor config, and build it into the binary, and use it in the names of the created binary packages, and compile it into the binaries. (cherry picked from commit c5d8173bf03ab9fa93d3b3ffd871b378b0819567) --- appveyor.yml | 59 ++++++++++++++++++++----- infrastructure/cmake/CMakeLists.txt | 67 ++++++++++++++++++++++------- infrastructure/cmake/getversion.pl | 2 +- 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a0ccf95ce..0ca5d8ee8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 1.0.{build}-{branch} +version: 0.12.{branch}.appveyor_{build} clone_depth: 1 @@ -8,25 +8,34 @@ skip_tags: true os: Windows Server 2012 platform: -# - x86 -# - x64 - - Win32 + - Win32 # CMake sucks + - x64 configuration: - Debug - Release environment: - VisualStudioVersion: 10.0 - Generator: Visual Studio 10 + VisualStudioVersion: 11.0 + Generator_Base: Visual Studio 11 2012 OPENSSL_VERSION: 1.0.2f PCRE_VERSION: 8.38 CMAKE_UNIBUILD_DIR: '%APPVEYOR_BUILD_FOLDER%\..\cmake' + BOXBACKUP_VERSION_BASE: 0.12 init: # Uncomment the following two lines to enable RDP access to the virtual machine for debugging. # - reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp" /v UserAuthentication /t REG_DWORD /d 0 /f # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + - ps: $env:date_string = Get-Date -Format "yyMMdd" + - ps: $env:sane_platform = $($env:PLATFORM.replace("Win32", "x86")) + # http://help.appveyor.com/discussions/problems/2874-how-can-i-add-commit-id-to-build-version + - ps: $env:boxbackup_version_full="$env:BOXBACKUP_VERSION_BASE.$env:APPVEYOR_REPO_BRANCH.$env:date_string.$($env:APPVEYOR_REPO_COMMIT.substring(0,7))" + - ps: Update-AppveyorBuild -Version "$env:boxbackup_version_full" + - ps: $env:compiled_version="$($env:boxbackup_version_full)_Win_$($env:sane_platform)_$($env:CONFIGURATION)" + # The only way to switch between 32-bit and 64-bit compilers appears to be to append " Win64" + # to the generator name if you want a 64-bit build (x64 platform): + - ps: $env:generator_name="$($env:Generator_Base)$(if ($env:PLATFORM.equals('x64')) {' Win64'})" build: parallel: true @@ -35,15 +44,15 @@ build: install: # test_bbackupd needs 7zip (or cmake -E tar) to extract tar archives on Windows: - - cinst -y --limit-output 7zip.commandline - # Install cmake.portable instead of cmake, to get it on the path again: - # http://disq.us/p/xdknrt + - cinst -y --limit-output 7zip.commandline nsis.portable - dir "c:\Program Files" - dir "c:\Program Files (x86)" # We don't need strawberryperl on AppVeyor because there is already a Perl in c:\Perl. # If you are doing this on a fresh box for development, you would probably want to # install Chocolatey and then run: - # cinst -y cmake strawberryperl git vim visualstudio2012wdx + # cinst -y cmake.portable strawberryperl git vim visualstudio2012wdx + # We install cmake.portable instead of cmake, to get it on the path again + # . - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86' - if not exist %CMAKE_UNIBUILD_DIR% md %CMAKE_UNIBUILD_DIR% @@ -51,7 +60,12 @@ install: # We need to specify the generator here, in case the user has more than one installed. # CMake always seems to default to the latest version of Visual Studio, not the one on # the current PATH. - - cmake -G "%Generator%" -A %PLATFORM% %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows + - echo cmake -G "%generator_name%" -DDEBUG=1 + -DBOXBACKUP_VERSION=%compiled_version% + %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows + - cmake -G "%generator_name%" -DDEBUG=1 + -DBOXBACKUP_VERSION=%compiled_version% + %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows # Leave the current directory in the correct place to find the solution file using its relative path above. @@ -63,3 +77,26 @@ test_script: # - dir %PLATFORM%\%CONFIGURATION% - ctest -C %CONFIGURATION% -V --interactive-debug-mode 0 + # After running tests successfully, and before collecting artifacts, build them and copy them + # to the right place: + - cmake --build . --config %CONFIGURATION% --target package + + # AppVeyor refuses to package files outside of the project directory, so we need to push them: + - dir + - appveyor PushArtifact -path BoxBackup-%compiled_version%.zip -DeploymentName BoxBackup-%compiled_version% + - appveyor PushArtifact -path BoxBackup-%compiled_version%.exe -DeploymentName BoxBackup-%compiled_version% + +deploy: + - provider: GitHub + release: BoxBackup-$(boxbackup_version_full) + artifact: BoxBackup-$(compiled_version) + description: "Windows client binaries auto-built by AppVeyor" + draft: false + prerelease: true + auth_token: + secure: WZi3MJGA5zIIAAij0if4auYeltJlyWUOePTYlCGvrNrgEVjYRkqILHzvVKDnLn43 + on: + branch: + - master + - s3_support_merge + diff --git a/infrastructure/cmake/CMakeLists.txt b/infrastructure/cmake/CMakeLists.txt index cba7c7391..250154525 100644 --- a/infrastructure/cmake/CMakeLists.txt +++ b/infrastructure/cmake/CMakeLists.txt @@ -78,25 +78,38 @@ foreach(file_to_configure ${files_to_configure}) "${base_dir}/${file_to_configure}.out") endforeach() -# Work out the current Box version (requires Perl) and update lib/common/BoxVersion.h, -# but only if it has changed, to avoid unnecessary complete rebuilds due to timestamps. -execute_process( - COMMAND ${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl - WORKING_DIRECTORY ${base_dir}/infrastructure - RESULT_VARIABLE status - OUTPUT_VARIABLE new_version_define - ERROR_VARIABLE command_output) -if(NOT status EQUAL 0) - message(FATAL_ERROR "Failed to execute: " - "${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl: " - "status ${status}: ${command_output}") +# If BOXBACKUP_VERSION is defined when running CMake (as the AppVeyor config does), use it +# as-is, since it contains the full version number, branch, and platform (Win32/Win64): +if(BOXBACKUP_VERSION) + set(boxbackup_version ${BOXBACKUP_VERSION}) + # Remove CPACK_SYSTEM_NAME from the default CPACK_PACKAGE_NAME, because it's already + # included in the CPACK_PACKAGE_VERSION: + set(CPACK_PACKAGE_FILE_NAME ${CMAKE_PROJECT_NAME}-${boxbackup_version}) +else() + # Work out the current Box version (requires Perl) and update lib/common/BoxVersion.h, + # but only if it has changed, to avoid unnecessary complete rebuilds due to timestamps. + execute_process( + COMMAND ${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl + WORKING_DIRECTORY ${base_dir}/infrastructure + RESULT_VARIABLE status + OUTPUT_VARIABLE boxbackup_version + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_VARIABLE command_output) + if(NOT status EQUAL 0) + message(FATAL_ERROR "Failed to execute: " + "${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl: " + "status ${status}: ${command_output}") + endif() endif() -file(WRITE "${base_dir}/lib/common/BoxVersion.h.new" "${new_version_define}") + +file(WRITE "${base_dir}/lib/common/BoxVersion.h.new" + "#define BOX_VERSION \"${boxbackup_version}\"\n") replace_file_if_different( "${base_dir}/lib/common/BoxVersion.h" "${base_dir}/lib/common/BoxVersion.h.new") add_definitions(-DBOX_CMAKE -DNEED_BOX_VERSION_H) + if(WIN32) add_definitions(-DWIN32) endif() @@ -331,6 +344,16 @@ foreach(module_dep ${module_deps}) endif() endforeach() +if(WIN32) + install(FILES ${base_dir}/bin/bbackupd/win32/NotifySysAdmin.vbs + DESTINATION "." COMPONENT Extras) + install(FILES ${base_dir}/bin/bbackupd/win32/bbackupd.conf + DESTINATION "." COMPONENT Extras) +else() + install(FILES ${base_dir}/bin/bbackupd/bbackupd-config + DESTINATION "." COMPONENT Extras) +endif() + # We can't do anything conditional on CMAKE_BUILD_TYPE because that's not valid for multi-configuration # generators such as MSVC. We need to use a generator expression instead. target_compile_definitions(lib_common PUBLIC $<$:BOX_RELEASE_BUILD>) @@ -353,9 +376,6 @@ if(WIN32) target_link_libraries(lib_common PUBLIC ws2_32 gdi32) endif() -# Link to ZLib -# http://stackoverflow.com/a/6174604/648162 -include_directories(${base_dir}/../zlib-win32/include) # On Windows we want to statically link zlib to make debugging and distribution easier, # but FindZLIB.cmake doesn't offer that as an option, so we have to go through some # contortions to "find" the correct library. ZLIB_ROOT is required in this case. @@ -688,3 +708,18 @@ set_tests_properties(backupdiff PROPERTIES TIMEOUT 32) set_tests_properties(bbackupd PROPERTIES TIMEOUT 1200) set_tests_properties(s3store PROPERTIES TIMEOUT 20) set_tests_properties(httpserver PROPERTIES TIMEOUT 40) + +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Box Backup is an open source, completely automatic, on-line backup system") +set(CPACK_PACKAGE_VENDOR "www.BoxBackup.org") +set(CPACK_PACKAGE_DESCRIPTION_FILE "${base_dir}/README.md") +set(CPACK_RESOURCE_FILE_LICENSE "${base_dir}/LICENSE.txt") +set(CPACK_PACKAGE_VERSION ${boxbackup_version}) +set(CPACK_PACKAGE_INSTALL_DIRECTORY "Box Backup") +set(CPACK_COMPONENTS_ALL Applications Extras) +set(CPACK_GENERATOR "ZIP;NSIS") +set(CPACK_NSIS_DISPLAY_NAME "Box Backup") +set(CPACK_NSIS_HELP_LINK "http://www.boxbackup.org/") +set(CPACK_NSIS_URL_INFO_ABOUT "http://www.boxbackup.org/") +set(CPACK_NSIS_CONTACT "boxbackup@boxbackup.org") +set(CPACK_NSIS_MODIFY_PATH ON) +include(CPack) diff --git a/infrastructure/cmake/getversion.pl b/infrastructure/cmake/getversion.pl index 67e1f8a51..d7ca03331 100755 --- a/infrastructure/cmake/getversion.pl +++ b/infrastructure/cmake/getversion.pl @@ -8,6 +8,6 @@ require "$basedir/infrastructure/BoxPlatform.pm.in"; -print "#define BOX_VERSION \"$BoxPlatform::product_version\"\n"; +print "$BoxPlatform::product_version\n"; exit 0; From 94ec413235f38419bf70a084a37808b6ae8bc09f Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 3 Jul 2017 18:10:34 +0100 Subject: [PATCH 17/31] Remove Xamarin to remove 500 lines of junk from build logs See http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored for details. (cherry picked from commit 019025deceff49485b03593f7a77a570d960546a) --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 0ca5d8ee8..0ec97f651 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,6 +42,11 @@ build: project: ..\cmake\BoxBackup_Windows.sln verbosity: minimal +# Remove Xamarin to remove 500 lines of junk from build logs +# http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored +before_build: + - del "C:\Program Files (x86)\MSBuild\4.0\Microsoft.Common.Targets\ImportAfter\Xamarin.Common.targets" + install: # test_bbackupd needs 7zip (or cmake -E tar) to extract tar archives on Windows: - cinst -y --limit-output 7zip.commandline nsis.portable From 75190f42eac277e5b936c60fde6ed4274a9e740c Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 3 Jul 2017 18:30:21 +0100 Subject: [PATCH 18/31] Reorder lines in appveyor.yml to match actual execution order (cherry picked from commit ea4f1728b529aff1dfc912d8f51567f8cd58ea77) --- appveyor.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0ec97f651..f4dac271c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,16 +37,7 @@ init: # to the generator name if you want a 64-bit build (x64 platform): - ps: $env:generator_name="$($env:Generator_Base)$(if ($env:PLATFORM.equals('x64')) {' Win64'})" -build: - parallel: true - project: ..\cmake\BoxBackup_Windows.sln - verbosity: minimal - -# Remove Xamarin to remove 500 lines of junk from build logs -# http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored -before_build: - - del "C:\Program Files (x86)\MSBuild\4.0\Microsoft.Common.Targets\ImportAfter\Xamarin.Common.targets" - +# scripts that run after cloning repository (before the build step, not after!) install: # test_bbackupd needs 7zip (or cmake -E tar) to extract tar archives on Windows: - cinst -y --limit-output 7zip.commandline nsis.portable @@ -74,6 +65,16 @@ install: # Leave the current directory in the correct place to find the solution file using its relative path above. +# Remove Xamarin to remove 500 lines of junk from build logs +# http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored +before_build: + - del "C:\Program Files (x86)\MSBuild\4.0\Microsoft.Common.Targets\ImportAfter\Xamarin.Common.targets" + +build: + parallel: true + project: ..\cmake\BoxBackup_Windows.sln + verbosity: minimal + test_script: - cd %CMAKE_UNIBUILD_DIR%\Build\boxbackup # - dir @@ -82,8 +83,7 @@ test_script: # - dir %PLATFORM%\%CONFIGURATION% - ctest -C %CONFIGURATION% -V --interactive-debug-mode 0 - # After running tests successfully, and before collecting artifacts, build them and copy them - # to the right place: + # After running tests successfully, build the artifacts that we want to upload: - cmake --build . --config %CONFIGURATION% --target package # AppVeyor refuses to package files outside of the project directory, so we need to push them: From 796d758eef318c012fde12c8aacac671934fb8e9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 3 Jul 2017 18:29:53 +0100 Subject: [PATCH 19/31] Reduce AppVeyor build verbosity by passing extra args to sub-CMake (cherry picked from commit 4876a5573a8bcd3312e9804c27fe9f30f3369747) --- appveyor.yml | 2 ++ infrastructure/cmake/windows/CMakeLists.txt | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f4dac271c..89651f6d5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -58,9 +58,11 @@ install: # the current PATH. - echo cmake -G "%generator_name%" -DDEBUG=1 -DBOXBACKUP_VERSION=%compiled_version% + -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows - cmake -G "%generator_name%" -DDEBUG=1 -DBOXBACKUP_VERSION=%compiled_version% + -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows # Leave the current directory in the correct place to find the solution file using its relative path above. diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index a6d93ee80..1c88472e1 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -35,7 +35,7 @@ ExternalProject_Add(zlib URL "http://zlib.net/zlib-${ZLIB_VERSION}.tar.gz" URL_HASH ${ZLIB_HASH} DOWNLOAD_NO_PROGRESS 1 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} ${SUB_CMAKE_EXTRA_ARGS} # We need to build both versions, debug and release, because cmake requires both to be # present to generate its multi-configuration project files for Visual Studio/MSBuild. INSTALL_COMMAND ${CMAKE_COMMAND} --build --target install --config Debug @@ -86,7 +86,7 @@ ExternalProject_Add(pcre SVN_REPOSITORY svn://vcs.exim.org/pcre/code/trunk SVN_REVISION -r 1677 DOWNLOAD_NO_PROGRESS 1 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} ${SUB_CMAKE_EXTRA_ARGS} -DPCRE_SUPPORT_LIBREADLINE=OFF -DPCRE_SUPPORT_LIBBZ2=OFF # We need to build both versions, debug and release, because cmake requires both to be @@ -122,5 +122,6 @@ ExternalProject_Add(boxbackup -DAPPVEYOR_MODE=1 -DDEBUG=${DEBUG} ${boxbackup_cmake_args} + ${SUB_CMAKE_EXTRA_ARGS} STEP_TARGETS configure build install ) From 606b42ef0b265098bb107a4a36039c1bc83b2651 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 3 Jul 2017 18:47:20 +0100 Subject: [PATCH 20/31] Disable DEBUG to reduce CMake build verbosity on AppVeyor (cherry picked from commit 793d5f57f5f99f50bffadea674f58be79f9cb6ff) --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 89651f6d5..0042373b0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -56,11 +56,11 @@ install: # We need to specify the generator here, in case the user has more than one installed. # CMake always seems to default to the latest version of Visual Studio, not the one on # the current PATH. - - echo cmake -G "%generator_name%" -DDEBUG=1 + - echo cmake -G "%generator_name%" -DBOXBACKUP_VERSION=%compiled_version% -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows - - cmake -G "%generator_name%" -DDEBUG=1 + - cmake -G "%generator_name%" -DBOXBACKUP_VERSION=%compiled_version% -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows From 69e19233045b484b2378e3568bc7dd711d104122 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 16 Sep 2017 20:52:25 +0100 Subject: [PATCH 21/31] Change branches for which Windows packages are built --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 0042373b0..093113f73 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -105,5 +105,5 @@ deploy: on: branch: - master - - s3_support_merge + - windows_binary_packages From b7c6a133b8868223bc39084968e0cac332fd4dfe Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 18 Sep 2017 06:09:25 +0100 Subject: [PATCH 22/31] Install Box Backup in a temporary directory in Windows unibuilds Otherwise the unibuild fails at the end for non-administrator users when it tries to write to C:\Program Files\BoxBackup. --- infrastructure/cmake/windows/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index 1c88472e1..80d1369b2 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -100,9 +100,9 @@ ExternalProject_Add(boost URL "http://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION}/boost_${BOOST_VERSION_UNDERSCORES}.tar.bz2" URL_HASH ${BOOST_HASH} # DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" # none needed - BUILD_COMMAND ${CMAKE_COMMAND} -E echo "No build step needed" # none needed - INSTALL_COMMAND "" # none needed + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "No configure step needed" + BUILD_COMMAND ${CMAKE_COMMAND} -E echo "No build step needed" + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step needed" ) if(BOXBACKUP_VERSION) @@ -119,6 +119,7 @@ ExternalProject_Add(boxbackup -DPCRE_ROOT=${install_dir} -DBOOST_ROOT=${CMAKE_BINARY_DIR}/Source/boost -DBOX_SUPPORT_READLINE=OFF + -DCMAKE_INSTALL_PREFIX=${install_dir} -DAPPVEYOR_MODE=1 -DDEBUG=${DEBUG} ${boxbackup_cmake_args} From fea4efbf5648dddb63f51b4b076baceb872be87b Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 16 Oct 2017 20:47:58 +0100 Subject: [PATCH 23/31] Fix parsing of OpenSSL 1.1 output in bbstored-certs Thanks to Dean Hamstead (@djzort on GitHub) for the bug report and the patch! --- bin/bbstored/bbstored-certs.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bbstored/bbstored-certs.in b/bin/bbstored/bbstored-certs.in index 855607488..00085662d 100755 --- a/bin/bbstored/bbstored-certs.in +++ b/bin/bbstored/bbstored-certs.in @@ -288,7 +288,7 @@ sub get_csr_common_name my $subject; while() { - $subject = $1 if m/Subject:.+?CN=([-\.\w]+)/ + $subject = $1 if m/Subject:.+?CN\s?=\s?([-\.\w]+)/; } close CSRTEXT; From a631c82882b039f8467a2ef9abeb343f2ec5b3da Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 12 Oct 2017 21:27:54 +0100 Subject: [PATCH 24/31] Fix -Wundefined-bool-conversion compile failure of master branch Master build fails with: lib/common/Timer.cpp:171:10: error: reference cannot be bound to dereferenced null pointer in well-defined C++ code; pointer may be assumed to always convert to true [-Werror,-Wundefined-bool-conversion]. Cannot get regtests to pass on this feature branch without fixing this. Also remove unused code from Timers. --- lib/bbackupd/BackupDaemon.cpp | 2 -- lib/common/Timer.cpp | 24 ------------------------ lib/common/Timer.h | 1 - test/common/testcommon.cpp | 10 ++-------- 4 files changed, 2 insertions(+), 35 deletions(-) diff --git a/lib/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp index 3427a7226..996c1919c 100644 --- a/lib/bbackupd/BackupDaemon.cpp +++ b/lib/bbackupd/BackupDaemon.cpp @@ -922,8 +922,6 @@ std::auto_ptr BackupDaemon::GetNewContext // it, let it be destroyed and close the connection. std::auto_ptr BackupDaemon::RunSyncNow() { - Timers::AssertInitialised(); - // Delete the serialised store object file, // so that we don't try to reload it after a // partially completed backup diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index 6ce84b7d1..4f8c989e6 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -120,26 +120,6 @@ void Timers::Cleanup(bool throw_exception_if_not_initialised) spTimers = NULL; } -// -------------------------------------------------------------------------- -// -// Function -// Name: static void Timers::AssertInitialised() -// Purpose: Throw an assertion error if timers are not ready -// NOW. It's a common mistake (for me) when writing -// tests to forget to initialise timers first. -// Created: 15/05/2014 -// -// -------------------------------------------------------------------------- - -void Timers::AssertInitialised() -{ - if (!spTimers) - { - THROW_EXCEPTION(CommonException, TimersNotInitialised); - } - ASSERT(spTimers); -} - // -------------------------------------------------------------------------- // // Function @@ -151,7 +131,6 @@ void Timers::AssertInitialised() void Timers::Add(Timer& rTimer) { ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); spTimers->push_back(&rTimer); Reschedule(); @@ -168,8 +147,6 @@ void Timers::Add(Timer& rTimer) // -------------------------------------------------------------------------- void Timers::Remove(Timer& rTimer) { - ASSERT(&rTimer); - if(!spTimers) { BOX_WARNING(TIMER_ID_OF(rTimer) " was still active after " @@ -343,7 +320,6 @@ void Timers::Reschedule() // -------------------------------------------------------------------------- void Timers::SignalHandler(int unused) { - // ASSERT(spTimers); Timers::RequestReschedule(); } diff --git a/lib/common/Timer.h b/lib/common/Timer.h index 68592aaa8..172332034 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -44,7 +44,6 @@ class Timers public: static void Init(); static void Cleanup(bool throw_exception_if_not_initialised = true); - static void AssertInitialised(); static void Add (Timer& rTimer); static void Remove(Timer& rTimer); static void RequestReschedule(); diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index bd75ac37a..bcfc92e15 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -312,14 +312,8 @@ int test(int argc, const char *argv[]) // Check that using timer methods without initialisation // throws an assertion failure. Can only do this in debug mode #ifndef BOX_RELEASE_BUILD - TEST_CHECK_THROWS(Timers::Add(*(Timer*)NULL), - CommonException, AssertFailed); - TEST_CHECK_THROWS(Timers::Remove(*(Timer*)NULL), - CommonException, AssertFailed); - #endif - - // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); - #ifndef BOX_RELEASE_BUILD + TEST_CHECK_THROWS(Timer t1(900, "t1"), CommonException, + AssertFailed); TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, AssertFailed); #endif From 447c2cd3d6884b8383884dbbfe65db845ea5c04d Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 20 Nov 2017 21:56:37 +0000 Subject: [PATCH 25/31] Remove invalid use of null references (undefined behaviour) Enable the relevant compiler warning as an error, if supported (-Werror=undefined-bool-conversion). http://www.gotw.ca/conv/002.htm http://stackoverflow.com/questions/2165078/a-reference-can-not-be-null-or-it-can-be-null (cherry picked from commit f2911acac0c8375a08ecc0a55f853a5a59c4d511) (cherry picked from commit 5a50b98401302a5ba89366e4c0f8cccdd88d8722) --- lib/common/Timer.cpp | 3 --- test/common/testcommon.cpp | 9 +++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index 6ce84b7d1..78fbeadcc 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -151,7 +151,6 @@ void Timers::AssertInitialised() void Timers::Add(Timer& rTimer) { ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); spTimers->push_back(&rTimer); Reschedule(); @@ -168,8 +167,6 @@ void Timers::Add(Timer& rTimer) // -------------------------------------------------------------------------- void Timers::Remove(Timer& rTimer) { - ASSERT(&rTimer); - if(!spTimers) { BOX_WARNING(TIMER_ID_OF(rTimer) " was still active after " diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index bd75ac37a..3de7da1d7 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -312,10 +312,11 @@ int test(int argc, const char *argv[]) // Check that using timer methods without initialisation // throws an assertion failure. Can only do this in debug mode #ifndef BOX_RELEASE_BUILD - TEST_CHECK_THROWS(Timers::Add(*(Timer*)NULL), - CommonException, AssertFailed); - TEST_CHECK_THROWS(Timers::Remove(*(Timer*)NULL), - CommonException, AssertFailed); + { + Timer tim(0, "tim"); + TEST_CHECK_THROWS(Timers::Add(tim), CommonException, AssertFailed); + Timers::Remove(tim); + } #endif // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); From 971fbbce13186199b3ac67f35a7fb5acfc777295 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 1 Jan 2018 11:12:16 +0000 Subject: [PATCH 26/31] Speed up superbuild rebuild by not extracting Boost again (cherry picked from commit 8976906a563ab04eb3d1310c59d4c13cdc3b525e) --- infrastructure/cmake/windows/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index 80d1369b2..d78faeccd 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -100,6 +100,8 @@ ExternalProject_Add(boost URL "http://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION}/boost_${BOOST_VERSION_UNDERSCORES}.tar.bz2" URL_HASH ${BOOST_HASH} # DOWNLOAD_NO_PROGRESS 1 + # Disable automatic updating (untarring) as it's slow and not necessary + UPDATE_DISCONNECTED 1 CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "No configure step needed" BUILD_COMMAND ${CMAKE_COMMAND} -E echo "No build step needed" INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step needed" From d432128c6f9390a3749af1430c8a76fbc52badf7 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 5 Aug 2017 11:37:51 +0100 Subject: [PATCH 27/31] Remove BoxBackup install step from Windows unibuild We don't need to install anything in C:\Program Files\BoxBackup, and often don't have permission to do that anyway. (cherry picked from commit 2d31a3178ba5a39af594c1a0188748f5c8045b18) (cherry picked from commit dab8ccc4b0b9eef06c9e063fc510f9952a0757f7) --- infrastructure/cmake/windows/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index d78faeccd..0fbe35e34 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -126,5 +126,6 @@ ExternalProject_Add(boxbackup -DDEBUG=${DEBUG} ${boxbackup_cmake_args} ${SUB_CMAKE_EXTRA_ARGS} - STEP_TARGETS configure build install + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step needed" + STEP_TARGETS configure build ) From d22ea1891f8930b105665d896ad6c7d5034769cf Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 2 Dec 2016 22:25:15 +0000 Subject: [PATCH 28/31] Detect which debugger to use in t-gdb scripts: lldb or gdb Should make debugging on Mac and BSD less painful. Make t-gdb fail with an error when no debugger was detected. (cherry picked from commit da98a8183737fb1fcb658cebd9612791a4750c4c) (cherry picked from commit ea6bc722131222d1fad08521e61f552f16941b01) (cherry picked from commit 38ba7510f2cad9accf445d5633c64e98a637a139) (cherry picked from commit 02bfd7e5bdd2abc185b12f37e1f536269db6817a) --- infrastructure/m4/boxbackup_tests.m4 | 8 +++++++- infrastructure/makebuildenv.pl.in | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/infrastructure/m4/boxbackup_tests.m4 b/infrastructure/m4/boxbackup_tests.m4 index 7410159a5..59467e66a 100644 --- a/infrastructure/m4/boxbackup_tests.m4 +++ b/infrastructure/m4/boxbackup_tests.m4 @@ -357,4 +357,10 @@ fi ;; esac - +AC_CHECK_PROGS(default_debugger, [lldb gdb]) +AC_ARG_WITH([debugger], + [AS_HELP_STRING([--with-debugger=], + [use this debugger in t-gdb scripts to debug tests @<:@default=lldb if present, otherwise gdb@:>@])], + [], + [with_debugger=$default_debugger]) +AC_SUBST([with_debugger]) diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index 597b6f82c..e43798247 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -379,6 +379,7 @@ $default_cflags =~ s/ -O2//g; $default_cxxflags =~ s/ -O2//g; my $debug_base_dir = 'debug'; my $release_base_dir = 'release'; +my $debugger = '@with_debugger@'; my $release_flags = "-O2"; if ($target_windows) @@ -614,9 +615,18 @@ __E writetestfile("$mod/t", "GLIBCXX_FORCE_NEW=1 ". './_test' . $platform_exe_ext . ' "$@"', $mod); - writetestfile("$mod/t-gdb", "GLIBCXX_FORCE_NEW=1 ". - 'gdb ./_test' . $platform_exe_ext . ' "$@"', $mod); - + + if($debugger) + { + writetestfile("$mod/t-gdb", "GLIBCXX_FORCE_NEW=1 ". + $debugger . ' ./_test' . $platform_exe_ext . ' "$@"', $mod); + } + else + { + writetestfile("$mod/t-gdb", + "echo 'No debugger was detected by configure script'\n". + "exit 2"); + } } my @all_deps_for_module; From 01755b62f6542d25a885a4fe26ce9fc1ac6decaf Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 2 Jun 2017 21:59:21 +0100 Subject: [PATCH 29/31] Fix quoting of CXXFLAGS and LDFLAGS in Makefiles Previously, it was impossible to include an @ sign in these flags (for example, to link to support --with-ssl-headers=/usr/local/opt/openssl@1.1/include/) because it was interpreted as a list variable name by Perl. Now the variables are evaluated single-quoted before being substituted into the Makefile output. (cherry picked from commit 2e9c90d3f32c69cc89d01fb64ced9fca80aae985) --- infrastructure/makebuildenv.pl.in | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index e43798247..d5ac9f2f4 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -391,6 +391,9 @@ if ($target_windows) print "done\n\nGenerating Makefiles...\n"; my $makefile_ifdef_prefix = $bsd_make ? "." : ""; +my $autoconf_cppflags = '@CPPFLAGS@'; +my $autoconf_cxxflags = '@CXXFLAGS_STRICT@'; +my $autoconf_ldflags = '@LDFLAGS@'; open MASTER_MAKEFILE, ">Makefile" or die "Makefile: $!"; print MASTER_MAKEFILE <<__E; @@ -412,13 +415,13 @@ WINDRES = @WINDRES@ # Work around a mistake in QDBM (using includes for a file not in the # system path) by adding it to the include path with -I. -DEFAULT_CFLAGS = \@CPPFLAGS@ $default_cflags \@CXXFLAGS_STRICT@ \\ +DEFAULT_CFLAGS = $autoconf_cppflags $default_cflags $autoconf_cxxflags \\ $extra_platform_defines $platform_compile_line_extra \\ -DBOX_VERSION="\\"$product_version\\"" -Iqdbm -DEFAULT_CXXFLAGS = \@CPPFLAGS@ $default_cxxflags \@CXXFLAGS_STRICT@ \\ +DEFAULT_CXXFLAGS = $autoconf_cppflags $default_cxxflags $autoconf_cxxflags \\ $extra_platform_defines $platform_compile_line_extra \\ -DBOX_VERSION="\\"$product_version\\"" -LDFLAGS += \@LDFLAGS@ \@LDADD_RDYNAMIC@ $platform_link_line_extra +LDFLAGS += $autoconf_ldflags \@LDADD_RDYNAMIC@ $platform_link_line_extra RELEASE_CFLAGS = \$(DEFAULT_CFLAGS) -DBOX_RELEASE_BUILD $release_flags RELEASE_CXXFLAGS = \$(DEFAULT_CXXFLAGS) -DBOX_RELEASE_BUILD $release_flags From 3db0438644b10370293d9b2f3bd8457561e2c9f8 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 3 Jun 2017 12:06:35 +0100 Subject: [PATCH 30/31] Replace RAND_pseudo_bytes with RAND_bytes RAND_pseudo_bytes is less secure and is now deprecated. (cherry picked from commit f449986b68bd12c18d742c91b625779ae75b11a2) --- lib/crypto/Random.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp index 1d6a07f01..c34a6eea5 100644 --- a/lib/crypto/Random.cpp +++ b/lib/crypto/Random.cpp @@ -50,7 +50,7 @@ void Random::Initialise() // -------------------------------------------------------------------------- void Random::Generate(void *pOutput, int Length) { - if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1) + if(RAND_bytes((uint8_t*)pOutput, Length) == -1) { THROW_EXCEPTION(CipherException, PseudoRandNotAvailable) } From 85e7efc3fa0477f60318d2cd2144503a9ea8feb9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 3 Jun 2017 12:13:42 +0100 Subject: [PATCH 31/31] Add support for OpenSSL 1.1 and replace deprecated function calls See https://github.com/boxbackup/boxbackup/issues/16 for details. Thanks to Chris West @FauxFaux for the initial patch! Use OpenSSL 1.1.0g for Windows superbuild. Fix Windows build by removing calls to obsolete do_ms.bat . Workaround for incorrect library suffixes searched by FindOpenSSL . Link OpenSSL statically and fix missing dependency on crypt32.lib. (cherry picked from commit edd3687f067c68b131822e0064cdeff5bf7a3835) (cherry picked from commit b003e009d1fccadf55a0f59a01c66f7b80f9b3e5) (cherry picked from commit cc6e20466b857f7e20d816a2c20cb36f112398ab) (cherry picked from commit 303c6405e3bc1e673dbc691792e0633742c01f4e) (cherry picked from commit 4e24006f75142a254f1a7db76cf9da1e752d9283) (cherry picked from commit 448ac48a89538f67f508f8d21931c5e6e90472c0) (cherry picked from commit 00b3bb0697219ef8c742698dd43d53b538e362b1) --- appveyor.yml | 2 +- infrastructure/cmake/CMakeLists.txt | 21 ++- infrastructure/cmake/windows/CMakeLists.txt | 18 ++- lib/crypto/CipherBlowfish.cpp | 2 +- lib/crypto/CipherContext.cpp | 157 +++++++++----------- lib/crypto/CipherContext.h | 24 ++- lib/crypto/CipherException.txt | 1 + lib/server/TLSContext.cpp | 13 +- test/crypto/testcrypto.cpp | 2 +- 9 files changed, 141 insertions(+), 99 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 093113f73..d75eff77e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,7 +18,7 @@ configuration: environment: VisualStudioVersion: 11.0 Generator_Base: Visual Studio 11 2012 - OPENSSL_VERSION: 1.0.2f + OPENSSL_VERSION: 1.1.0f PCRE_VERSION: 8.38 CMAKE_UNIBUILD_DIR: '%APPVEYOR_BUILD_FOLDER%\..\cmake' BOXBACKUP_VERSION_BASE: 0.12 diff --git a/infrastructure/cmake/CMakeLists.txt b/infrastructure/cmake/CMakeLists.txt index 250154525..65f59eb86 100644 --- a/infrastructure/cmake/CMakeLists.txt +++ b/infrastructure/cmake/CMakeLists.txt @@ -407,7 +407,26 @@ else() endif() # Link to OpenSSL -find_package(OpenSSL REQUIRED) +# Workaround for incorrect library suffixes searched by FindOpenSSL: +# https://gitlab.kitware.com/cmake/cmake/issues/17604 +if(WIN32 AND MSVC) + find_package(OpenSSL) + set(OPENSSL_SSL_LIBRARY ${SSL_EAY_RELEASE}) + set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY_RELEASE}) + set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY} crypt32) + find_package_handle_standard_args(OpenSSL + REQUIRED_VARS + OPENSSL_SSL_LIBRARY + OPENSSL_CRYPTO_LIBRARY + OPENSSL_INCLUDE_DIR + VERSION_VAR + OPENSSL_VERSION + FAIL_MESSAGE + "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR" + ) +else() + find_package(OpenSSL REQUIRED) +endif() include_directories(${OPENSSL_INCLUDE_DIR}) target_link_libraries(lib_crypto PUBLIC ${OPENSSL_LIBRARIES}) diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index 0fbe35e34..49a1ea4d4 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -15,9 +15,9 @@ set(ZLIB_VERSION 1.2.11) set(ZLIB_HASH SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1) # Version of OpenSSL to download, build, and compile Box Backup against: -set(OPENSSL_VERSION 1.0.2j) +set(OPENSSL_VERSION 1.1.0g) # Hash of openssl-${OPENSSL_VERSION}.tar.gz, to be verified after download: -set(OPENSSL_HASH SHA256=e7aff292be21c259c6af26469c7a9b3ba26e9abaaffd325e3dccc9785256c431) +set(OPENSSL_HASH SHA256=de4d501267da39310905cb6dc8c6121f7a2cad45a7707f76df828fe1b85073af) # Version of PCRE to download, build, and compile Box Backup against: set(PCRE_VERSION 8.39) @@ -49,15 +49,21 @@ if(WIN32) URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" URL_HASH ${OPENSSL_HASH} DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND perl Configure debug-VC-WIN32 no-asm --prefix=${install_dir} - COMMAND cmd /c ms\\do_ms.bat + CONFIGURE_COMMAND perl Configure debug-VC-WIN32 no-asm no-shared + --prefix=${install_dir} + --openssldir=etc + # Run tests before install, but don't make the main target depend on them, so that + # we don't have to run them whenever we build manually on Windows. + TEST_BEFORE_INSTALL 1 + TEST_EXCLUDE_FROM_MAIN 1 # You would expect us to use nt.mak to compile a static library here, but mk1mf.pl uses the /MT[d] # CRT in that case, which is incompatible with our dynamic runtime, /MD[d]. It seems that the libs # built by ntdll.mak, which are compiled with /MD[d], are full libraries and not import libs, # so we can link statically against them and still get a dynamic runtime. BUILD_IN_SOURCE 1 - BUILD_COMMAND nmake /s /f ms\\nt.mak - INSTALL_COMMAND nmake /s /f ms\\nt.mak install + BUILD_COMMAND nmake /s + TEST_COMMAND nmake /s test + INSTALL_COMMAND nmake /s install ) elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ExternalProject_Add(openssl diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp index e16cc6ed2..4c75b1de7 100644 --- a/lib/crypto/CipherBlowfish.cpp +++ b/lib/crypto/CipherBlowfish.cpp @@ -206,7 +206,7 @@ void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const } // Set key #ifndef HAVE_OLD_SSL - if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) + if(EVP_CipherInit_ex(pCipherContext, GetCipher(), NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) #else if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1) #endif diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index fd1493951..3de88c642 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -2,7 +2,7 @@ // // File // Name: CipherContext.cpp -// Purpose: Context for symmetric encryption / descryption +// Purpose: Context for symmetric encryption / decryption // Created: 1/12/03 // // -------------------------------------------------------------------------- @@ -50,7 +50,7 @@ CipherContext::~CipherContext() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -98,7 +98,7 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes // Check for bad usage if(mInitialised) { - THROW_EXCEPTION(CipherException, AlreadyInitialised) + THROW_EXCEPTION(CipherException, AlreadyInitialised); } if(Function != Decrypt && Function != Encrypt) { @@ -109,43 +109,45 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes mFunction = Function; // Initialise the cipher -#ifndef HAVE_OLD_SSL - EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does - - if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, - (mFunction == Encrypt) ? 1 : 0) != 1) -#else +#ifdef HAVE_OLD_SSL // Use old version of init call if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, (mFunction == Encrypt) ? 1 : 0) != 1) +#else + BOX_OPENSSL_INIT_CTX(ctx); + + // Don't set key or IV yet, because we will modify the parameters: + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), rDescription.GetCipher(), NULL, NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) #endif { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to initialise " << rDescription.GetFullName() - << "cipher: " << LogError("initialising cipher")); + << ": " << LogError("initialising cipher")); } + UsePadding(mPaddingOn); try { mCipherName = rDescription.GetFullName(); #ifndef HAVE_OLD_SSL // Let the description set up everything else - rDescription.SetupParameters(&ctx); + mpDescription = &rDescription; #else // With the old version, a copy needs to be taken first. mpDescription = rDescription.Clone(); // Mark it as not a leak, otherwise static cipher contexts // cause spurious memory leaks to be reported MEMLEAKFINDER_NOT_A_LEAK(mpDescription); - mpDescription->SetupParameters(&ctx); #endif + mpDescription->SetupParameters(BOX_OPENSSL_CTX(ctx)); } catch(...) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to configure " << mCipherName << " cipher: " << + "Failed to configure " << mCipherName << ": " << LogError("configuring cipher")); - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); throw; } @@ -166,7 +168,7 @@ void CipherContext::Reset() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_cleanup(BOX_OPENSSL_CTX(ctx)); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -177,6 +179,7 @@ void CipherContext::Reset() } #endif mWithinTransform = false; + mIV.clear(); } @@ -192,24 +195,22 @@ void CipherContext::Begin() { if(!mInitialised) { - THROW_EXCEPTION(CipherException, NotInitialised) + THROW_EXCEPTION(CipherException, NotInitialised); } - // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) if(mWithinTransform) { - BOX_WARNING("CipherContext::Begin called when context " - "flagged as within a transform"); + THROW_EXCEPTION(CipherException, AlreadyInTransform); } - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), + -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to reset " << mCipherName << " cipher: " << - LogError("resetting cipher")); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } - + // Mark as being within a transform mWithinTransform = true; } @@ -251,18 +252,18 @@ int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuf } // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } // Do the transform int outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } return outLength; @@ -300,7 +301,7 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) } // Check output buffer size - if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (2 * EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } @@ -308,12 +309,11 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) // Do the transform int outLength = OutLength; #ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength) != 1) { mWithinTransform = false; THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } #else OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); @@ -340,11 +340,11 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) // Old version needs to use a different form, and then set up the cipher again for next time around int outLength = rOutLengthOut; // Have to emulate padding off... - int blockSize = EVP_CIPHER_CTX_block_size(&ctx); + int blockSize = EVP_CIPHER_CTX_block_size(ctx); if(mPaddingOn) { // Just use normal final call - if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + if(EVP_CipherFinal(ctx, Buffer, &outLength) != 1) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } @@ -357,13 +357,13 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) { // NASTY -- fiddling around with internals like this is bad. // But only way to get this working on old versions of OpenSSL. - if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0) + if(!EVP_EncryptUpdate(ctx,Buffer,&outLength,ctx.buf,0) || outLength != blockSize) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_free(ctx); } else { @@ -391,12 +391,14 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) } } // Reinitialise the cipher for the next time around - if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, + if(EVP_CipherInit_ex(&ctx, mpDescription->GetCipher(), NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), (mFunction == Encrypt) ? 1 : 0) != 1) { THROW_EXCEPTION(CipherException, EVPInitFailure) } mpDescription->SetupParameters(&ctx); + UsePadding(mPaddingOn); // Update length for caller rOutLengthOut = outLength; @@ -421,7 +423,7 @@ int CipherContext::InSizeForOutBufferSize(int OutLength) // Strictly speaking, the *2 is unnecessary. However... // Final() is paranoid, and requires two input blocks of space to work. - return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2); + return OutLength - (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 2); } // -------------------------------------------------------------------------- @@ -442,7 +444,7 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // Final() is paranoid, and requires two input blocks of space to work, and so we need to add // three blocks on to be absolutely sure. - return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3); + return InLength + (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 3); } @@ -456,20 +458,8 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // -------------------------------------------------------------------------- int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) { - if(!mInitialised) - { - THROW_EXCEPTION(CipherException, NotInitialised) - } - - // Warn if in a transformation - if(mWithinTransform) - { - BOX_WARNING("CipherContext::TransformBlock called when " - "context flagged as within a transform"); - } - // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { // Check if padding is off, in which case the buffer can be smaller if(!mPaddingOn && OutLength <= InLength) @@ -481,40 +471,36 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } } - - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) - { - THROW_EXCEPTION(CipherException, EVPInitFailure) - } + + Begin(); // Do the entire block - int outLength = 0; + int output_space_used = OutLength; // Update - outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &output_space_used, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } // Finalise - int outLength2 = OutLength - outLength; -#ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + int output_space_remain = OutLength - output_space_used; + +#ifdef HAVE_OLD_SSL + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + output_space_used, output_space_remain); +#else + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), ((unsigned char*)pOutBuffer) + output_space_used, + &output_space_remain) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } -#else - OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); #endif - outLength += outLength2; - return outLength; + mWithinTransform = false; + return output_space_used + output_space_remain; } @@ -533,7 +519,7 @@ int CipherContext::GetIVLength() THROW_EXCEPTION(CipherException, NotInitialised) } - return EVP_CIPHER_CTX_iv_length(&ctx); + return EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); } @@ -559,12 +545,14 @@ void CipherContext::SetIV(const void *pIV) "flagged as within a transform"); } + mIV = std::string((const char *)pIV, GetIVLength()); + // Set IV - if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)mIV.c_str(), -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to " << GetFunction() << " (set IV) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } #ifdef HAVE_OLD_SSL @@ -601,19 +589,20 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) } // Get length of IV - unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx); - if(ivLen > sizeof(mGeneratedIV)) + uint8_t generated_iv[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; + unsigned int ivLen = EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); + if(ivLen > sizeof(generated_iv)) { THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded) } // Generate some random data - Random::Generate(mGeneratedIV, ivLen); - SetIV(mGeneratedIV); + Random::Generate(generated_iv, ivLen); + SetIV(generated_iv); // Return the IV and it's length rLengthOut = ivLen; - return mGeneratedIV; + return mIV.c_str(); } @@ -628,9 +617,11 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) void CipherContext::UsePadding(bool Padding) { #ifndef HAVE_OLD_SSL - if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1) + if(EVP_CIPHER_CTX_set_padding(BOX_OPENSSL_CTX(ctx), Padding) != 1) { - THROW_EXCEPTION(CipherException, EVPSetPaddingFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPSetPaddingFailure, + "Failed to set padding for " << mCipherName << ": " << + LogError(GetFunction())); } #endif mPaddingOn = Padding; diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h index 93c889d6c..b6e97b4e3 100644 --- a/lib/crypto/CipherContext.h +++ b/lib/crypto/CipherContext.h @@ -19,6 +19,22 @@ class CipherDescription; #define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH 32 +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) ctx = EVP_CIPHER_CTX_new(); +# define BOX_OPENSSL_CTX(ctx) ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_free(ctx) +typedef EVP_CIPHER_CTX* BOX_EVP_CIPHER_CTX; +#else // OpenSSL < 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does +# define BOX_OPENSSL_CTX(ctx) &ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_cleanup(&ctx) +typedef EVP_CIPHER_CTX BOX_EVP_CIPHER_CTX; +#endif + + // -------------------------------------------------------------------------- // // Class @@ -74,16 +90,14 @@ class CipherContext #endif private: - EVP_CIPHER_CTX ctx; + BOX_EVP_CIPHER_CTX ctx; bool mInitialised; bool mWithinTransform; bool mPaddingOn; - uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; CipherFunction mFunction; std::string mCipherName; -#ifdef HAVE_OLD_SSL - CipherDescription *mpDescription; -#endif + const CipherDescription *mpDescription; + std::string mIV; }; diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt index abdbac872..494ed3cc3 100644 --- a/lib/crypto/CipherException.txt +++ b/lib/crypto/CipherException.txt @@ -16,3 +16,4 @@ PseudoRandNotAvailable 12 EVPSetPaddingFailure 13 RandomInitFailed 14 Failed to read from random device LengthRequestedTooLongForRandomHex 15 +AlreadyInTransform 16 Tried to initialise crypto when already in a transform diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 35e254fdb..1a6d4a534 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -23,6 +23,17 @@ #define MAX_VERIFICATION_DEPTH 2 #define CIPHER_LIST "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH" +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_TLS_SERVER_METHOD TLS_server_method +# define BOX_TLS_CLIENT_METHOD TLS_client_method +#else // OpenSSL < 1.1 +# define BOX_TLS_SERVER_METHOD TLSv1_server_method +# define BOX_TLS_CLIENT_METHOD TLSv1_client_method +#endif + // -------------------------------------------------------------------------- // // Function @@ -67,7 +78,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c ::SSL_CTX_free(mpContext); } - mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); + mpContext = ::SSL_CTX_new(AsServer ? BOX_TLS_SERVER_METHOD() : BOX_TLS_CLIENT_METHOD()); if(mpContext == NULL) { THROW_EXCEPTION(ServerException, TLSAllocationFailed) diff --git a/test/crypto/testcrypto.cpp b/test/crypto/testcrypto.cpp index 4e623cc28..32d2efb8e 100644 --- a/test/crypto/testcrypto.cpp +++ b/test/crypto/testcrypto.cpp @@ -266,7 +266,7 @@ int test(int argc, const char *argv[]) // Check rolling checksums uint8_t *checkdata_blk = (uint8_t *)malloc(CHECKSUM_DATA_SIZE); uint8_t *checkdata = checkdata_blk; - RAND_pseudo_bytes(checkdata, CHECKSUM_DATA_SIZE); + RAND_bytes(checkdata, CHECKSUM_DATA_SIZE); for(int size = CHECKSUM_BLOCK_SIZE_BASE; size <= CHECKSUM_BLOCK_SIZE_LAST; ++size) { // Test skip-roll code