diff --git a/.travis.yml b/.travis.yml index 57bd36f34..19b8515aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ addons: - time - xsltproc - zlib1g-dev + - libboost-dev env: - TEST_TARGET=debug @@ -44,5 +45,10 @@ matrix: - os: osx compiler: gcc +branches: + except: + # Exclude tags created by AppVeyor for releases (binary builds) + - /^BoxBackup-/ + script: - env EXTRA_MAKE_ARGS=-j2 ./infrastructure/travis-build.sh diff --git a/appveyor.yml b/appveyor.yml index d75eff77e..b4857fdc6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,10 +59,12 @@ install: - echo cmake -G "%generator_name%" -DBOXBACKUP_VERSION=%compiled_version% -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" + -DPLATFORM=%sane_platform% %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows - cmake -G "%generator_name%" -DBOXBACKUP_VERSION=%compiled_version% -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" + -DPLATFORM=%sane_platform% %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows # Leave the current directory in the correct place to find the solution file using its relative path above. @@ -99,11 +101,11 @@ deploy: artifact: BoxBackup-$(compiled_version) description: "Windows client binaries auto-built by AppVeyor" draft: false - prerelease: true + # Master branch builds are full releases (not pre-releases), all others are pre-releases + prerelease: $(if ($env:APPVEYOR_REPO_BRANCH.equals('master')) {'false'} else {'true'})" auth_token: secure: WZi3MJGA5zIIAAij0if4auYeltJlyWUOePTYlCGvrNrgEVjYRkqILHzvVKDnLn43 on: branch: - master - - windows_binary_packages diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index 0e0c1e9c8..16c8d1f7e 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -173,19 +173,33 @@ int main(int argc, const char *argv[]) // Wait for the configuration summary std::string configSummary; - if(!getLine.GetLine(configSummary, false, PROTOCOL_DEFAULT_TIMEOUT)) + while(true) { - BOX_ERROR("Failed to receive configuration summary " - "from daemon"); - return 1; - } + try + { + configSummary = getLine.GetLine(false, PROTOCOL_DEFAULT_TIMEOUT); + break; + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + BOX_ERROR("Server rejected the connection. Are you running " + "bbackupctl as the same user as the daemon?"); + } + else + { + BOX_ERROR("Failed to receive configuration summary " + "from daemon: " << e.what()); + } - // Was the connection rejected by the server? - if(getLine.IsEOF()) - { - BOX_ERROR("Server rejected the connection. Are you running " - "bbackupctl as the same user as the daemon?"); - return 1; + return 1; + } } // Decode it @@ -206,10 +220,28 @@ int main(int argc, const char *argv[]) " MaxUploadWait = " << maxUploadWait << " seconds"); std::string stateLine; - if(!getLine.GetLine(stateLine, false, PROTOCOL_DEFAULT_TIMEOUT) || getLine.IsEOF()) + while(true) { - BOX_ERROR("Failed to receive state line from daemon"); - return 1; + try + { + stateLine = getLine.GetLine(false, PROTOCOL_DEFAULT_TIMEOUT); + break; + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else + { + BOX_ERROR("Failed to receive configuration summary " + "from daemon: " << e.what()); + } + + return 1; + } } // Decode it @@ -297,13 +329,37 @@ int main(int argc, const char *argv[]) } // Read the response - std::string line; bool syncIsRunning = false; bool finished = false; - while(command != NoCommand && !finished && !getLine.IsEOF() && - getLine.GetLine(line, false, PROTOCOL_DEFAULT_TIMEOUT)) + while(command != NoCommand && !finished) { + std::string line; + try + { + line = getLine.GetLine(false, PROTOCOL_DEFAULT_TIMEOUT); + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + BOX_ERROR("Server disconnected unexpectedly. It may have shut " + "down."); + } + else + { + BOX_ERROR("Failed to receive configuration summary " + "from daemon: " << e.what()); + } + + break; + } + BOX_TRACE("Received line: " << line); if(line.substr(0, 6) == "state ") diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp index e10c48fe1..e50e2d337 100644 --- a/bin/bbackupquery/bbackupquery.cpp +++ b/bin/bbackupquery/bbackupquery.cpp @@ -189,24 +189,6 @@ int main(int argc, const char *argv[]) "bbackupquery") MAINHELPER_START -#ifdef WIN32 - WSADATA info; - - // Under Win32 we must initialise the Winsock library - // before using it. - - if (WSAStartup(0x0101, &info) == SOCKET_ERROR) - { - // throw error? perhaps give it its own id in the future - THROW_EXCEPTION(BackupStoreException, Internal) - } -#endif - - // Really don't want trace statements happening, even in debug mode - #ifndef BOX_RELEASE_BUILD - BoxDebugTraceOn = false; - #endif - FILE *logFile = 0; // Filename for configuration file? @@ -313,7 +295,7 @@ int main(int argc, const char *argv[]) true)); // open in append mode } - BOX_NOTICE(BANNER_TEXT("Backup Query Tool")); + BOX_NOTICE(BANNER_TEXT("query tool (bbackupquery)")); #ifdef WIN32 if (unicodeConsole) @@ -355,7 +337,7 @@ int main(int argc, const char *argv[]) } // Easier coding const Configuration &conf(*config); - + // Setup and connect // 1. TLS context SSLLib::Initialise(); @@ -485,7 +467,7 @@ int main(int argc, const char *argv[]) try { - cmd_str = apGetLine->GetLine(); + cmd_str = apGetLine->GetLine(false); } catch(CommonException &e) { @@ -530,12 +512,6 @@ int main(int argc, const char *argv[]) // Let everything be cleaned up on exit. -#ifdef WIN32 - // Clean up our sockets - // FIXME we should do this, but I get an abort() when I try - // WSACleanup(); -#endif - MAINHELPER_END return returnCode; diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp index 6a173680d..80de8d67c 100644 --- a/bin/bbstoreaccounts/bbstoreaccounts.cpp +++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp @@ -18,12 +18,17 @@ #include +#include + #include "box_getopt.h" +#include "BackupAccountControl.h" +#include "BackupDaemonConfigVerify.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" #include "BackupStoreConfigVerify.h" #include "BackupStoreInfo.h" +#include "BannerText.h" #include "BoxPortsAndFiles.h" #include "HousekeepStoreAccount.h" #include "MainHelper.h" @@ -36,44 +41,54 @@ #include -void PrintUsageAndExit() +int PrintUsage() { - printf( -"Usage: bbstoreaccounts [-c config_file] action account_id [args]\n" -"Account ID is integer specified in hex\n" -"\n" -"Commands (and arguments):\n" -" create \n" -" Creates the specified account number (in hex with no 0x) on the\n" -" specified raidfile disc set number (see raidfile.conf for valid\n" -" set numbers) with the specified soft and hard limits (in blocks\n" -" if suffixed with B, MB with M, GB with G)\n" -" info [-m] \n" -" Prints information about the specified account including number\n" -" of blocks used. The -m option enable machine-readable output.\n" -" enabled \n" -" Sets the account as enabled or disabled for new logins.\n" -" setlimit \n" -" Changes the limits of the account as specified. Numbers are\n" -" interpreted as for the 'create' command (suffixed with B, M or G)\n" -" delete [yes]\n" -" Deletes the specified account. Prompts for confirmation unless\n" -" the optional 'yes' parameter is provided.\n" -" check [fix] [quiet]\n" -" Checks the specified account for errors. If the 'fix' option is\n" -" provided, any errors discovered that can be fixed automatically\n" -" will be fixed. If the 'quiet' option is provided, less output is\n" -" produced.\n" -" name \n" -" Changes the \"name\" of the account to the specified string.\n" -" The name is purely cosmetic and intended to make it easier to\n" -" identify your accounts.\n" -" housekeep \n" -" Runs housekeeping immediately on the account. If it cannot be locked,\n" -" bbstoreaccounts returns an error status code (1), otherwise success\n" -" (0) even if any errors were fixed by housekeeping.\n" - ); - exit(2); + std::string configFilename = BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE; + + std::cout << + BANNER_TEXT("account management utility (bbstoreaccounts)") << "\n" + "\n" + "Usage: bbstoreaccounts [-3] [-c config_file] [args]\n" + "Account ID is integer specified in hex, with no 0x prefix.\n" + "\n" + "Options:\n" + " -3 Amazon S3 mode. Not all commands are supported yet. Use account\n" + " name for , and bbackupd.conf for .\n" + " -c Use an alternate configuration file instead of\n" + " " << configFilename << ".\n" + << Logging::OptionParser::GetUsageString() << + "\n" + "Commands (and arguments):\n" + " create \n" + " Creates a RaidFile account with the specified account number, on the\n" + " specified RaidFile disc set number (see raidfile.conf for valid set\n" + " numbers) with the specified soft and hard limits (in blocks if\n" + " suffixed with B, MB with M, GB with G).\n" + " info [-m] \n" + " Prints information about the specified account including number\n" + " of blocks used. The -m option enables machine-readable output.\n" + " enabled \n" + " Sets the account as enabled or disabled for new logins.\n" + " setlimit \n" + " Changes the limits of the account as specified. Numbers are\n" + " interpreted as for the 'create' command (suffixed with B, M or G).\n" + " delete [yes]\n" + " Deletes the specified account. Prompts for confirmation unless\n" + " the optional 'yes' parameter is provided.\n" + " check [fix] [quiet]\n" + " Checks the specified account for errors. If the 'fix' option is\n" + " provided, any errors discovered that can be fixed automatically\n" + " will be fixed. If the 'quiet' option is provided, less output is\n" + " produced.\n" + " name \n" + " Changes the \"name\" of the account to the specified string.\n" + " The name is purely cosmetic and intended to make it easier to\n" + " identify your accounts.\n" + " housekeep \n" + " Runs housekeeping immediately on the account. If it cannot be locked,\n" + " bbstoreaccounts returns an error status code (1), otherwise success\n" + " (0) even if any errors were fixed by housekeeping.\n"; + return 2; } int main(int argc, const char *argv[]) @@ -87,28 +102,25 @@ int main(int argc, const char *argv[]) // Filename for configuration file? std::string configFilename = BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE; - int logLevel = Log::EVERYTHING; + Logging::OptionParser log_level; bool machineReadableOutput = false; - + bool amazon_S3_mode = false; + // See if there's another entry on the command line int c; - while((c = getopt(argc, (char * const *)argv, "c:W:m")) != -1) + std::string options = Logging::OptionParser::GetOptionString() + "3c:m"; + while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1) { switch(c) { + case '3': + amazon_S3_mode = true; + break; + case 'c': // store argument configFilename = optarg; break; - - case 'W': - logLevel = Logging::GetNamedLevel(optarg); - if(logLevel == Log::INVALID) - { - BOX_FATAL("Invalid logging level: " << optarg); - return 2; - } - break; case 'm': // enable machine readable output @@ -116,23 +128,55 @@ int main(int argc, const char *argv[]) break; case '?': + return PrintUsage(); + break; + default: - PrintUsageAndExit(); + if(log_level.ProcessOption(c) != 0) + { + return PrintUsage(); + } } } - Logging::FilterConsole((Log::Level) logLevel); + Logging::FilterConsole(log_level.GetCurrentLevel()); Logging::FilterSyslog (Log::NOTHING); // Adjust arguments argc -= optind; argv += optind; + // We should have at least one argument at this point. + if(argc < 1) + { + return PrintUsage(); + } + std::string command = argv[0]; + argv++; + argc--; + // Read in the configuration file std::string errs; - std::auto_ptr config( - Configuration::LoadAndVerify - (configFilename, &BackupConfigFileVerify, errs)); + std::auto_ptr config; + if(amazon_S3_mode) + { + // Read a bbackupd configuration file, instead of a bbstored one. + if(configFilename.empty()) + { + configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + } + config = Configuration::LoadAndVerify + (configFilename, &BackupDaemonConfigVerify, errs); + } + else + { + if(configFilename.empty()) + { + configFilename = BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE; + } + config = Configuration::LoadAndVerify + (configFilename, &BackupConfigFileVerify, errs); + } if(config.get() == 0 || !errs.empty()) { @@ -140,150 +184,203 @@ int main(int argc, const char *argv[]) ":" << errs); } - // Initialise the raid file controller - RaidFileController &rcontroller(RaidFileController::GetController()); - rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); + std::auto_ptr apS3Control; + std::auto_ptr apStoreControl; + BackupAccountControl* pControl; - // Then... check we have two arguments - if(argc < 2) - { - PrintUsageAndExit(); +#define STORE_ONLY \ + if(amazon_S3_mode) \ + { \ + BOX_ERROR("The '" << command << "' command only applies to bbstored " \ + "backends"); \ + return 2; \ } - - // Get the id - int32_t id; - if(::sscanf(argv[1], "%x", &id) != 1) + + if(amazon_S3_mode) { - PrintUsageAndExit(); + apS3Control.reset(new S3BackupAccountControl(*config, + machineReadableOutput)); + pControl = apS3Control.get(); } - - std::string command = argv[0]; - BackupStoreAccountsControl control(*config, machineReadableOutput); - - // Now do the command. - if(command == "create") + else { - // which disc? - int32_t discnum; - int32_t softlimit; - int32_t hardlimit; - if(argc < 5 - || ::sscanf(argv[2], "%d", &discnum) != 1) + // Initialise the raid file controller. Not needed in Amazon S3 mode. + RaidFileController &rcontroller(RaidFileController::GetController()); + rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); + + // Get the Account ID (in hex without the leading 0x). + int32_t id; + if(argc == 0 || ::sscanf(argv[0], "%x", &id) != 1) { - BOX_ERROR("create requires raid file disc number, " - "soft and hard limits."); - return 1; + BOX_FATAL("All commands require an account ID, in hex without 0x"); + return PrintUsage(); } - - // Decode limits - int blocksize = control.BlockSizeOfDiscSet(discnum); - softlimit = control.SizeStringToBlocks(argv[3], blocksize); - hardlimit = control.SizeStringToBlocks(argv[4], blocksize); - control.CheckSoftHardLimits(softlimit, hardlimit); - - // Create the account... - return control.CreateAccount(id, discnum, softlimit, hardlimit); - } - else if(command == "info") - { - // Print information on this account - return control.PrintAccountInfo(id); + argv++; + argc--; + + apStoreControl.reset(new BackupStoreAccountControl(*config, id, + machineReadableOutput)); + pControl = apStoreControl.get(); } - else if(command == "enabled") + + BackupAccountControl& control(*pControl); + + // Now do the command. + try { - // Change the AccountEnabled flag on this account - if(argc != 3) - { - PrintUsageAndExit(); - } - - bool enabled = true; - std::string enabled_string = argv[2]; - if(enabled_string == "yes") + if(command == "create") { - enabled = true; - } - else if(enabled_string == "no") - { - enabled = false; + // which disc? + int32_t discnum; + + if(amazon_S3_mode) + { + if(argc != 3) + { + BOX_ERROR("create requires an account name/label, " + "soft and hard limits."); + return 2; + } + } + else + { + if(argc != 3 || ::sscanf(argv[0], "%d", &discnum) != 1) + { + BOX_ERROR("create requires raid file disc number, " + "soft and hard limits."); + return 2; + } + } + + // Create the account... + if(amazon_S3_mode) + { + int blocksize = apS3Control->GetBlockSize(); + // Decode limits + int32_t softlimit = pControl->SizeStringToBlocks(argv[1], blocksize); + int32_t hardlimit = pControl->SizeStringToBlocks(argv[2], blocksize); + return apS3Control->CreateAccount(argv[0], softlimit, hardlimit); + } + else + { + int blocksize = apStoreControl->BlockSizeOfDiscSet(discnum); + // Decode limits + int32_t softlimit = pControl->SizeStringToBlocks(argv[1], blocksize); + int32_t hardlimit = pControl->SizeStringToBlocks(argv[2], blocksize); + return apStoreControl->CreateAccount(discnum, softlimit, hardlimit); + } } - else + else if(command == "info") { - PrintUsageAndExit(); - } - - return control.SetAccountEnabled(id, enabled); - } - else if(command == "setlimit") - { - // Change the limits on this account - if(argc < 4) - { - BOX_ERROR("setlimit requires soft and hard limits."); - return 1; + // Print information on this account + return control.PrintAccountInfo(); } - - return control.SetLimit(id, argv[2], argv[3]); - } - else if(command == "name") - { - // Change the limits on this account - if(argc != 3) + else if(command == "enabled") { - BOX_ERROR("name command requires a new name."); - return 1; + // Change the AccountEnabled flag on this account + if(argc != 1) + { + return PrintUsage(); + } + + bool enabled = true; + std::string enabled_string = argv[0]; + if(enabled_string == "yes") + { + enabled = true; + } + else if(enabled_string == "no") + { + enabled = false; + } + else + { + return PrintUsage(); + } + + return control.SetAccountEnabled(enabled); } - - return control.SetAccountName(id, argv[2]); - } - else if(command == "delete") - { - // Delete an account - bool askForConfirmation = true; - if(argc >= 3 && (::strcmp(argv[2], "yes") == 0)) + else if(command == "setlimit") { - askForConfirmation = false; + // Change the limits on this account + if(argc != 2) + { + BOX_ERROR("setlimit requires soft and hard limits."); + return 2; + } + + return control.SetLimit(argv[0], argv[1]); } - return control.DeleteAccount(id, askForConfirmation); - } - else if(command == "check") - { - bool fixErrors = false; - bool quiet = false; - - // Look at other options - for(int o = 2; o < argc; ++o) + else if(command == "name") { - if(::strcmp(argv[o], "fix") == 0) + // Change the limits on this account + if(argc != 1) { - fixErrors = true; + BOX_ERROR("name command requires a new name."); + return 1; } - else if(::strcmp(argv[o], "quiet") == 0) + + return control.SetAccountName(argv[0]); + } + else if(command == "delete") + { + // Delete an account + STORE_ONLY; + + bool askForConfirmation = true; + if(argc >= 1 && (::strcmp(argv[0], "yes") == 0)) { - quiet = true; + askForConfirmation = false; } - else + return apStoreControl->DeleteAccount(askForConfirmation); + } + else if(command == "check") + { + STORE_ONLY; + + bool fixErrors = false; + bool quiet = false; + + // Look at other options + for(int o = 0; o < argc; ++o) { - BOX_ERROR("Unknown option " << argv[o] << "."); - return 2; + if(::strcmp(argv[o], "fix") == 0) + { + fixErrors = true; + } + else if(::strcmp(argv[o], "quiet") == 0) + { + quiet = true; + } + else + { + BOX_ERROR("Unknown option " << argv[o] << "."); + return 2; + } } + + // Check the account + return apStoreControl->CheckAccount(fixErrors, quiet); + } + else if(command == "housekeep") + { + STORE_ONLY; + return apStoreControl->HousekeepAccountNow(); + } + else + { + BOX_ERROR("Unknown command '" << command << "'."); + return 2; } - - // Check the account - return control.CheckAccount(id, fixErrors, quiet); - } - else if(command == "housekeep") - { - return control.HousekeepAccountNow(id); } - else + catch(BoxException &e) { - BOX_ERROR("Unknown command '" << command << "'."); + BOX_ERROR("Failed command: " << command << ": " << e.what()); return 1; } return 0; - + MAINHELPER_END } diff --git a/infrastructure/BoxPlatform.pm.in b/infrastructure/BoxPlatform.pm.in index 325e56c3a..4d475a6d1 100644 --- a/infrastructure/BoxPlatform.pm.in +++ b/infrastructure/BoxPlatform.pm.in @@ -81,6 +81,7 @@ BEGIN # for developers, use Git version (SVN is no more): my $gitversion = `git rev-parse HEAD`; chomp $gitversion; + $gitversion = substr($gitversion, 0, 7); $product_version =~ s/USE_SVN_VERSION/git_$gitversion/; } diff --git a/infrastructure/buildenv-testmain-template.cpp b/infrastructure/buildenv-testmain-template.cpp index d946c25d2..bf4ae21bc 100644 --- a/infrastructure/buildenv-testmain-template.cpp +++ b/infrastructure/buildenv-testmain-template.cpp @@ -47,6 +47,7 @@ #include "box_getopt.h" #include "depot.h" #include "Logging.h" +#include "MainHelper.h" #include "Test.h" #include "Timer.h" @@ -70,6 +71,7 @@ std::string bbackupd_args = QUIET_PROCESS, bbstored_args = QUIET_PROCESS, bbackupquery_args, test_args; +bool bbackupd_args_overridden = false, bbstored_args_overridden = false; bool filedes_initialised = false; @@ -252,9 +254,10 @@ int Usage(const std::string& ProgramName) int main(int argc, char * const * argv) { - // Start memory leak testing - MEMLEAKFINDER_START + // Initialise sockets, start memory leak monitoring, catch exceptions: + MAINHELPER_START + Logging::ToSyslog(false); Logging::SetProgramName(BOX_MODULE); struct option longopts[] = @@ -284,6 +287,7 @@ int main(int argc, char * const * argv) bbackupd_args += " "; } bbackupd_args += optarg; + bbackupd_args_overridden = true; } break; @@ -313,6 +317,7 @@ int main(int argc, char * const * argv) { bbstored_args += " "; bbstored_args += optarg; + bbstored_args_overridden = true; } break; @@ -329,7 +334,6 @@ int main(int argc, char * const * argv) } } - Logging::FilterSyslog(Log::NOTHING); Logging::FilterConsole(LogLevel.GetCurrentLevel()); argc -= optind - 1; @@ -341,7 +345,9 @@ int main(int argc, char * const * argv) if(fulltestmode) { // banner - BOX_NOTICE("Running test TEST_NAME in " MODE_TEXT " mode..."); + std::string box_module = BOX_MODULE; + std::string test_name = box_module.substr(5); + BOX_NOTICE("Running test " << test_name << " in " MODE_TEXT " mode..."); // Count open file descriptors for a very crude "files left open" test Logging::GetSyslog().Shutdown(); @@ -353,21 +359,40 @@ int main(int argc, char * const * argv) ::gethostbyname("nonexistent"); check_filedes(false); - - #ifdef WIN32 - // Under win32 we must initialise the Winsock library - // before using sockets - - WSADATA info; - TEST_THAT(WSAStartup(0x0101, &info) != SOCKET_ERROR) - #endif } try { - #ifdef BOX_MEMORY_LEAK_TESTING - memleakfinder_init(); - #endif +#ifdef WIN32 + // Create a Windows "Job Object" for Test.cpp to use as a + // container for all our child processes (daemons). We will + // close the handle (killing any running daemons) when we exit. + // This is the best way to avoid daemons hanging around and + // causing subsequent tests to fail, and/or the test runner to + // hang waiting for a daemon that will never terminate. + + sTestChildDaemonJobObject = CreateJobObject(NULL, NULL); + if(sTestChildDaemonJobObject == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_WARNING("Failed to create job object " + "to contain child daemons"); + } + else + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits; + limits.BasicLimitInformation.LimitFlags = + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if(!SetInformationJobObject( + sTestChildDaemonJobObject, + JobObjectExtendedLimitInformation, + &limits, + sizeof(limits))) + { + BOX_LOG_WIN_WARNING("Failed to set limits on " + "job object for child daemons"); + } + } +#endif // WIN32 Timers::Init(); int returncode = test(argc, (const char **)argv); @@ -414,7 +439,14 @@ int main(int argc, char * const * argv) printf("PASSED\n"); } } - + + // If test() returns 0 but there have been some failures, we still want + // to return nonzero to the OS. + if(returncode == 0 && num_failures) + { + returncode = 1; + } + return returncode; } catch(std::exception &e) @@ -434,5 +466,11 @@ int main(int argc, char * const * argv) printf("WARNING: Files were left open\n"); } } + +#ifdef WIN32 + CloseHandle(sTestChildDaemonJobObject); +#endif + + MAINHELPER_END } diff --git a/infrastructure/cmake/CMakeLists.txt b/infrastructure/cmake/CMakeLists.txt index 65f59eb86..36550ef8c 100644 --- a/infrastructure/cmake/CMakeLists.txt +++ b/infrastructure/cmake/CMakeLists.txt @@ -139,6 +139,7 @@ foreach(exception_file ${exception_files}) set(output_file "${base_dir}/${CMAKE_MATCH_1}/autogen_${CMAKE_MATCH_2}.cpp") add_custom_command(OUTPUT "${output_file}" MAIN_DEPENDENCY "${base_dir}/${exception_file}" + DEPENDS "${base_dir}/lib/common/makeexception.pl" COMMAND ${PERL_EXECUTABLE} "${base_dir}/lib/common/makeexception.pl" "${CMAKE_MATCH_2}.txt" WORKING_DIRECTORY "${base_dir}/${CMAKE_MATCH_1}") @@ -160,6 +161,7 @@ foreach(protocol_file ${protocol_files}) set(output_file "${base_dir}/${CMAKE_MATCH_1}/autogen_${CMAKE_MATCH_2}.cpp") add_custom_command(OUTPUT "${output_file}" MAIN_DEPENDENCY "${base_dir}/${protocol_file}" + DEPENDS "${base_dir}/lib/server/makeprotocol.pl.in" COMMAND ${PERL_EXECUTABLE} "${base_dir}/lib/server/makeprotocol.pl" "${CMAKE_MATCH_2}.txt" WORKING_DIRECTORY "${base_dir}/${CMAKE_MATCH_1}") @@ -246,7 +248,7 @@ foreach(module_dep ${module_deps}) # For "make install" and CPack generators: install(TARGETS ${module_name} RUNTIME - CONFIGURATIONS Debug;Release + CONFIGURATIONS Debug;Release;RelWithDebInfo DESTINATION "." COMPONENT Applications) elseif(module_name MATCHES "^test_") @@ -286,7 +288,8 @@ foreach(module_dep ${module_deps}) -DTEST_EXECUTABLE="${test_command_internal}") add_test(NAME ${test_name} COMMAND ${PERL_EXECUTABLE} ${base_dir}/runtest.pl - ${appveyor_runtest_pl_switch} -c ${test_name} + ${appveyor_runtest_pl_switch} + -cT$<$:O-LBackupFileSystem.cpp=trace> ${test_name} $<$:DEBUG>$<$:RELEASE>$<$:RELEASE> "$" "${test_command_internal}" WORKING_DIRECTORY ${base_dir}) @@ -356,7 +359,10 @@ endif() # We can't do anything conditional on CMAKE_BUILD_TYPE because that's not valid for multi-configuration # generators such as MSVC. We need to use a generator expression instead. -target_compile_definitions(lib_common PUBLIC $<$:BOX_RELEASE_BUILD>) +target_compile_definitions(lib_common PUBLIC + $<$:BOX_RELEASE_BUILD> + $<$:BOX_RELEASE_BUILD> +) # Detect platform features and write BoxConfig.h.in. Reuse code from # infrastructure/m4/boxbackup_tests.m4 where possible @@ -373,7 +379,7 @@ file(REMOVE "${boxconfig_h_file}") file(WRITE "${boxconfig_h_file}" "// Auto-generated by CMake. Do not edit.\n") if(WIN32) - target_link_libraries(lib_common PUBLIC ws2_32 gdi32) + target_link_libraries(lib_common PUBLIC dbghelp ws2_32 gdi32) endif() # On Windows we want to statically link zlib to make debugging and distribution easier, @@ -450,18 +456,34 @@ else() include_directories(${PCRE_INCLUDE_DIRS}) target_link_libraries(lib_common PUBLIC ${PCRE_LIBRARIES}) + if(NOT PCRE_FOUND) + # You must use pkg-config. PCRE_ROOT is not used except on Win32. + message(FATAL_ERROR "Please install libpcre and ensure that pkg-config --list-all can find it") + endif() + if(DEBUG) message(STATUS "Linking PCRE libraries from ${PCRE_LIBRARY_DIRS}: ${PCRE_LIBRARIES}") endif() endif() +# Enable the readline option (BOX_SUPPORT_READLINE), on by default, if readline is detected. list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}") find_package(Readline) if(READLINE_FOUND) + option(BOX_SUPPORT_READLINE "Enable support for linking with libreadline." ON) +endif() +if(BOX_SUPPORT_READLINE) include_directories(${Readline_INCLUDE_DIR}) target_link_libraries(lib_common PUBLIC ${Readline_LIBRARY}) endif() +find_package(Boost) +if(NOT Boost_FOUND) + message(FATAL_ERROR "Cannot find Boost libraries, which are required. Please set BOOST_ROOT") +endif() +include_directories(${Boost_INCLUDE_DIRS}) +target_link_libraries(lib_httpserver PUBLIC ${Boost_LIBRARIES}) + set(boxconfig_cmake_h_dir "${base_dir}/lib/common") # Get the values of all directories added to the INCLUDE_DIRECTORIES property # by include_directory() statements, and save it in CMAKE_REQUIRED_INCLUDES @@ -477,7 +499,7 @@ move_file_if_exists( "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h.bak") foreach(m4_filename boxbackup_tests.m4 ax_check_mount_point.m4 ax_func_syscall.m4) - file(STRINGS "${base_dir}/infrastructure/m4/${m4_filename}" m4_functions REGEX "^ *AC[_A-Z]+\\(.*\\)$") + file(STRINGS "${base_dir}/infrastructure/m4/${m4_filename}" m4_functions REGEX "^ *(AC|AX|BOX)_[A-Z_]+\\(.*\\)$") foreach(m4_function ${m4_functions}) if(DEBUG) message(STATUS "Processing m4_function: ${m4_function}") @@ -495,7 +517,7 @@ foreach(m4_filename boxbackup_tests.m4 ax_check_mount_point.m4 ax_func_syscall.m foreach(header_file ${header_files}) list(APPEND detect_header_files ${header_file}) endforeach() - elseif(m4_function MATCHES "^ *AC_CHECK_FUNCS\\(\\[([a-z./_ ]+)\\](.*)\\)$") + elseif(m4_function MATCHES "^ *AC_CHECK_FUNCS\\(\\[([a-z0-9./_ ]+)\\](.*)\\)$") if(DEBUG) message(STATUS "Processing ac_check_funcs: ${CMAKE_MATCH_1}") endif() @@ -506,13 +528,15 @@ foreach(m4_filename boxbackup_tests.m4 ax_check_mount_point.m4 ax_func_syscall.m foreach(function_name ${function_names}) list(APPEND detect_functions ${function_name}) endforeach() - elseif(m4_function MATCHES "^ *AC_CHECK_DECLS\\(\\[([A-Za-z._/ ]+)\\](,,, ..#include <([^>]+)>..)?\\)$") + elseif(m4_function MATCHES "^ *AC_CHECK_DECLS\\(\\[([A-Za-z._/ ,]+)\\](,,, ..#include <([^>]+)>..)?\\)$") if(DEBUG) message(STATUS "Processing ac_check_decls: ${CMAKE_MATCH_1} in ${CMAKE_MATCH_3}") endif() + # Multiple declarations to check are comma-separated, which is unusual. # http://stackoverflow.com/questions/5272781/what-is-common-way-to-split-string-into-list-with-cmake - string(REPLACE " " ";" decl_names "${CMAKE_MATCH_1}") + string(REPLACE "," ";" decl_names "${CMAKE_MATCH_1}") + string(REPLACE " " "" decl_names "${decl_names}") string(REPLACE " " ";" header_files "${CMAKE_MATCH_3}") foreach(decl_name ${decl_names}) @@ -559,6 +583,28 @@ foreach(m4_filename boxbackup_tests.m4 ax_check_mount_point.m4 ax_func_syscall.m } ]=] "HAVE_${platform_var_name}") file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_${platform_var_name}\n") + elseif(m4_function MATCHES "^ *BOX_CHECK_CXX_FLAG\\((-[A-Za-z_,=-]+)(, *(-[A-Za-z_,=-]+))?\\)") + if(DEBUG) + message(STATUS "Processing BOX_CHECK_CXX_FLAG: ${CMAKE_MATCH_1} ${CMAKE_MATCH_3}") + endif() + + if(NOT CMAKE_MATCH_1 STREQUAL "-Wall") + set(flag_to_check "${CMAKE_MATCH_3}") + set(flag_to_add "${CMAKE_MATCH_1}") + + if("${flag_to_check}" STREQUAL "") + set(flag_to_check "${flag_to_add}") + endif() + + string(TOLOWER "have_flag_${flag_to_add}" have_flag_var_name) + string(REGEX REPLACE "[^a-z_]" "_" have_flag_var_name ${have_flag_var_name}) + string(REGEX REPLACE "__+" "_" have_flag_var_name ${have_flag_var_name}) + + CHECK_CXX_COMPILER_FLAG(${flag_to_check} ${have_flag_var_name}) + if(${have_flag_var_name}) + add_definitions("${flag_to_add}") + endif() + endif() endif() endforeach() @@ -577,7 +623,7 @@ foreach(header_file ${detect_header_files}) endforeach() if(NOT HAVE_PCREPOSIX_H) - message(FATAL_ERROR "pcreposix.h not found at PCRE_ROOT/include: ${PCRE_ROOT}/include") + message(FATAL_ERROR "pcreposix.h not found even though PCRE was apparently detected?!") endif() # PCRE is required, so unconditionally define this: @@ -594,6 +640,11 @@ endforeach() check_symbol_exists(dirfd "dirent.h" HAVE_DECL_DIRFD) file(APPEND "${boxconfig_h_file}" "#cmakedefine01 HAVE_DECL_DIRFD\n") +if(WIN32) + # We emulate this in lib/win32, so we know it's present: + file(APPEND "${boxconfig_h_file}" "#define HAVE_VALID_DIRENT_D_TYPE 1\n") +endif() + # Emulate ax_check_mount_point.m4 # These checks are run by multi-line M4 commands which are harder to parse/fake using # regexps above, so we hard-code them here: @@ -646,6 +697,24 @@ endif() file(APPEND "${boxconfig_h_file}" "#cmakedefine RANDOM_DEVICE \"${RANDOM_DEVICE}\"\n") file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_RANDOM_DEVICE\n") +# Emulate check for HAVE___SYSCALL_NEED_DEFN in ax_func_syscall.m4: +if(HAVE___SYSCALL) + CHECK_CXX_SOURCE_COMPILES([=[ + #include "BoxConfig.cmake.h" + #ifdef HAVE_SYS_SYSCALL_H + # include + #endif + int main() + { + return __syscall(SYS_exit, 0); + } + ]=] "HAVE___SYSCALL_DONT_NEED_DEFN") + if(NOT HAVE___SYSCALL_DONT_NEED_DEFN) + set(HAVE___SYSCALL_NEED_DEFN 1) + endif() + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE___SYSCALL_NEED_DEFN\n") +endif() + # Build an intermediate version of BoxConfig.cmake.h for use in the following tests: configure_file("${boxconfig_h_file}" "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h") @@ -712,6 +781,10 @@ file(TO_NATIVE_PATH "${PERL_EXECUTABLE}" perl_executable_native) string(REPLACE "\\" "\\\\" perl_path_escaped ${perl_executable_native}) target_compile_definitions(test_backupstorefix PRIVATE -DPERL_EXECUTABLE="${perl_path_escaped}") +target_compile_features(lib_common PRIVATE cxx_auto_type) +target_compile_features(lib_httpserver PRIVATE cxx_auto_type) +target_compile_features(lib_backupstore PRIVATE cxx_auto_type) + # Configure test timeouts: # I've set the timeout to 4 times as long as it took to run on a particular run on Appveyor: # https://ci.appveyor.com/project/qris/boxbackup/build/job/xm10itascygtu93j diff --git a/infrastructure/cmake/build/bin_bbackupd.vcxproj.user b/infrastructure/cmake/build/bin_bbackupd.vcxproj.user deleted file mode 100755 index fa1f3d341..000000000 --- a/infrastructure/cmake/build/bin_bbackupd.vcxproj.user +++ /dev/null @@ -1,8 +0,0 @@ - - - - testfiles\bbackupd.conf - $(ProjectDir)\..\..\..\debug\test\bbackupd - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/bin_bbstored.vcxproj.user b/infrastructure/cmake/build/bin_bbstored.vcxproj.user deleted file mode 100755 index 339cddee6..000000000 --- a/infrastructure/cmake/build/bin_bbstored.vcxproj.user +++ /dev/null @@ -1,8 +0,0 @@ - - - - testfiles/bbstored.conf - $(ProjectDir)\..\..\..\debug\test\backupstorefix - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstore.vcxproj.user b/infrastructure/cmake/build/test_backupstore.vcxproj.user deleted file mode 100755 index 7d7b31586..000000000 --- a/infrastructure/cmake/build/test_backupstore.vcxproj.user +++ /dev/null @@ -1,9 +0,0 @@ - - - - $(ProjectDir)\..\..\..\debug\test\backupstore - WindowsLocalDebugger - - - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstorefix.vcxproj.user b/infrastructure/cmake/build/test_backupstorefix.vcxproj.user deleted file mode 100755 index 170fb4967..000000000 --- a/infrastructure/cmake/build/test_backupstorefix.vcxproj.user +++ /dev/null @@ -1,7 +0,0 @@ - - - - $(ProjectDir)\..\..\..\debug\test\backupstorefix - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_bbackupd.vcxproj.user b/infrastructure/cmake/build/test_bbackupd.vcxproj.user deleted file mode 100755 index ebf8c6a3c..000000000 --- a/infrastructure/cmake/build/test_bbackupd.vcxproj.user +++ /dev/null @@ -1,8 +0,0 @@ - - - - -e test_basics - $(ProjectDir)\..\..\..\debug\test\bbackupd - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_common.vcxproj.user b/infrastructure/cmake/build/test_common.vcxproj.user deleted file mode 100755 index e5854a80b..000000000 --- a/infrastructure/cmake/build/test_common.vcxproj.user +++ /dev/null @@ -1,7 +0,0 @@ - - - - $(ProjectDir)\..\..\..\debug\test\common - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_httpserver.vcxproj.user b/infrastructure/cmake/build/test_httpserver.vcxproj.user deleted file mode 100755 index ac1512a84..000000000 --- a/infrastructure/cmake/build/test_httpserver.vcxproj.user +++ /dev/null @@ -1,7 +0,0 @@ - - - - $(ProjectDir)\..\..\..\debug\test\httpserver - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/build/test_raidfile.vcxproj.user b/infrastructure/cmake/build/test_raidfile.vcxproj.user deleted file mode 100755 index 620aa4bb6..000000000 --- a/infrastructure/cmake/build/test_raidfile.vcxproj.user +++ /dev/null @@ -1,7 +0,0 @@ - - - - $(ProjectDir)\..\..\..\debug\test\raidfile - WindowsLocalDebugger - - \ No newline at end of file diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt index 49a1ea4d4..1539323ae 100644 --- a/infrastructure/cmake/windows/CMakeLists.txt +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -20,9 +20,9 @@ set(OPENSSL_VERSION 1.1.0g) set(OPENSSL_HASH SHA256=de4d501267da39310905cb6dc8c6121f7a2cad45a7707f76df828fe1b85073af) # Version of PCRE to download, build, and compile Box Backup against: -set(PCRE_VERSION 8.39) -# Hash of pcre-${PCRE_VERSION}.tar.gz, to be verified after download: -set(PCRE_HASH SHA256=ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7) +set(PCRE_VERSION 8.41) +# Hash of pcre-${PCRE_VERSION}.tar.bz2, to be verified after download: +set(PCRE_HASH SHA256=e62c7eac5ae7c0e7286db61ff82912e1c0b7a0c13706616e94a7dd729321b530) # Version of Boost to download, unpack, and compile Box Backup against: set(BOOST_VERSION 1.62.0) @@ -44,26 +44,30 @@ ExternalProject_Add(zlib ) if(WIN32) + if(PLATFORM STREQUAL "x64") + set(openssl_config debug-VC-WIN64A) + set(batch_config ms\\do_win64a.bat) + else() + set(openssl_config debug-VC-WIN32) + set(batch_config ms\\do_ms.bat) + endif() ExternalProject_Add(openssl DEPENDS zlib URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" URL_HASH ${OPENSSL_HASH} DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND perl Configure debug-VC-WIN32 no-asm no-shared + CONFIGURE_COMMAND perl Configure ${openssl_config} no-asm no-shared --prefix=${install_dir} --openssldir=etc # Run tests before install, but don't make the main target depend on them, so that # we don't have to run them whenever we build manually on Windows. TEST_BEFORE_INSTALL 1 TEST_EXCLUDE_FROM_MAIN 1 - # You would expect us to use nt.mak to compile a static library here, but mk1mf.pl uses the /MT[d] - # CRT in that case, which is incompatible with our dynamic runtime, /MD[d]. It seems that the libs - # built by ntdll.mak, which are compiled with /MD[d], are full libraries and not import libs, - # so we can link statically against them and still get a dynamic runtime. BUILD_IN_SOURCE 1 BUILD_COMMAND nmake /s TEST_COMMAND nmake /s test INSTALL_COMMAND nmake /s install + STEP_TARGETS test ) elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ExternalProject_Add(openssl @@ -87,10 +91,8 @@ else() endif() ExternalProject_Add(pcre - # Temporarily use SVN repo until the PCRE_STATIC issue in 8.40 is fixed: - # https://vcs.pcre.org/pcre?view=revision&revision=1677 - SVN_REPOSITORY svn://vcs.exim.org/pcre/code/trunk - SVN_REVISION -r 1677 + URL "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-${PCRE_VERSION}.tar.bz2" + URL_HASH ${PCRE_HASH} DOWNLOAD_NO_PROGRESS 1 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} ${SUB_CMAKE_EXTRA_ARGS} -DPCRE_SUPPORT_LIBREADLINE=OFF diff --git a/infrastructure/m4/ax_boost_base.m4 b/infrastructure/m4/ax_boost_base.m4 new file mode 100644 index 000000000..f3279f2b7 --- /dev/null +++ b/infrastructure/m4/ax_boost_base.m4 @@ -0,0 +1,285 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_base.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_BASE([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# +# DESCRIPTION +# +# Test for the Boost C++ libraries of a particular version (or newer) +# +# If no path to the installed boost library is given the macro searchs +# under /usr, /usr/local, /opt and /opt/local and evaluates the +# $BOOST_ROOT environment variable. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg +# Copyright (c) 2009 Peter Adolphs +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 26 + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + [AS_HELP_STRING([--with-boost@<:@=ARG@:>@], + [use Boost library from a standard location (ARG=yes), + from the specified location (ARG=), + or disable it (ARG=no) + @<:@ARG=yes@:>@ ])], + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + + +AC_ARG_WITH([boost-libdir], + AS_HELP_STRING([--with-boost-libdir=LIB_DIR], + [Force given directory for boost libraries. Note that this will override library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]), + [ + if test -d "$withval" + then + ac_boost_lib_path="$withval" + else + AC_MSG_ERROR(--with-boost-libdir expected directory name) + fi + ], + [ac_boost_lib_path=""] +) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl On 64-bit systems check for system libraries in both lib64 and lib. + dnl The former is specified by FHS, but e.g. Debian does not adhere to + dnl this (as it rises problems for generic multi-arch support). + dnl The last entry in the list is chosen by default when no libraries + dnl are found, e.g. when only header-only libraries are installed! + libsubdirs="lib" + ax_arch=`uname -m` + case $ax_arch in + x86_64) + libsubdirs="lib64 libx32 lib lib64" + ;; + ppc64|s390x|sparc64|aarch64|ppc64le) + libsubdirs="lib64 lib lib64 ppc64le" + ;; + esac + + dnl allow for real multi-arch paths e.g. /usr/lib/x86_64-linux-gnu. Give + dnl them priority over the other paths since, if libs are found there, they + dnl are almost assuredly the ones desired. + AC_REQUIRE([AC_CANONICAL_HOST]) + libsubdirs="lib/${host_cpu}-${host_os} $libsubdirs" + + case ${host_cpu} in + i?86) + libsubdirs="lib/i386-${host_os} $libsubdirs" + ;; + esac + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_CPPFLAGS="-I$ac_boost_path/include" + for ac_boost_path_tmp in $libsubdirs; do + if test -d "$ac_boost_path"/"$ac_boost_path_tmp" ; then + BOOST_LDFLAGS="-L$ac_boost_path/$ac_boost_path_tmp" + break + fi + done + elif test "$cross_compiling" != yes; then + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + for libsubdir in $libsubdirs ; do + if ls "$ac_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$ac_boost_path_tmp/$libsubdir" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + dnl overwrite ld flags if we have required special directory with + dnl --with-boost-libdir parameter + if test "$ac_boost_lib_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_lib_path" + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_REQUIRE([AC_PROG_CXX]) + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + BOOST_CPPFLAGS= + BOOST_LDFLAGS= + _version=0 + if test "$ac_boost_path" != ""; then + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + dnl if nothing found search for layout used in Windows distributions + if test -z "$BOOST_CPPFLAGS"; then + if test -d "$ac_boost_path/boost" && test -r "$ac_boost_path/boost"; then + BOOST_CPPFLAGS="-I$ac_boost_path" + fi + fi + fi + else + if test "$cross_compiling" != yes; then + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + if test "$ac_boost_lib_path" = ""; then + for libsubdir in $libsubdirs ; do + if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$best_path/$libsubdir" + fi + fi + + if test "x$BOOST_ROOT" != "x"; then + for libsubdir in $libsubdirs ; do + if ls "$BOOST_ROOT/stage/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/$libsubdir" && test -r "$BOOST_ROOT/stage/$libsubdir"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_NOTICE([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + # execute ACTION-IF-NOT-FOUND (if present): + ifelse([$3], , :, [$3]) + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + # execute ACTION-IF-FOUND (if present): + ifelse([$2], , :, [$2]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) diff --git a/infrastructure/m4/ax_check_dirent_d_type.m4 b/infrastructure/m4/ax_check_dirent_d_type.m4 index 078a39ee3..ec38dbd14 100644 --- a/infrastructure/m4/ax_check_dirent_d_type.m4 +++ b/infrastructure/m4/ax_check_dirent_d_type.m4 @@ -25,7 +25,7 @@ AC_DEFUN([AX_CHECK_DIRENT_D_TYPE], [ DIR* dir = opendir("."); struct dirent* res = NULL; if(dir) res = readdir(dir); - return res ? (res->d_type != DT_FILE && res->d_type != DT_DIR) : 1; + return res ? (res->d_type != DT_REG && res->d_type != DT_DIR) : 1; } ], [box_cv_have_valid_dirent_d_type=yes], diff --git a/infrastructure/m4/boxbackup_tests.m4 b/infrastructure/m4/boxbackup_tests.m4 index 59467e66a..9c3e6d8e1 100644 --- a/infrastructure/m4/boxbackup_tests.m4 +++ b/infrastructure/m4/boxbackup_tests.m4 @@ -10,27 +10,46 @@ solaris*) ;; esac +# If the compiler supports it, force errors on unknown flags, so that detection works: +AX_CHECK_COMPILE_FLAG(-Werror=unknown-warning-option, + [cxxflags_force_error="-Werror=unknown-warning-option"]) + +# Reduce compiler flag checking to a one-liner, needed for CMake to parse them +AC_DEFUN([BOX_CHECK_CXX_FLAG], + # GCC suppresses warnings for -Wno-* flags unless another error occurs, + # so we need to use the opposite flag for detection purpose + _ac_detect_flag="$2" + _ac_detect_flag=${_ac_detect_flag:-$1} + AX_CHECK_COMPILE_FLAG($_ac_detect_flag, + [cxxflags_strict="$cxxflags_strict $1"],, + $cxxflags_force_error) +) + # Enable some compiler flags if the compiler supports them. This gives better warnings # and detects some problems early. -AX_CHECK_COMPILE_FLAG(-Wall, [cxxflags_strict="$cxxflags_strict -Wall"]) -# -Wundef would be a good idea, but Boost is full of undefined variable use, so we need -# to disable it for now so that we can concentrate on real errors: -dnl AX_CHECK_COMPILE_FLAG(-Wundef, [cxxflags_strict="$cxxflags_strict -Wundef"]) -AX_CHECK_COMPILE_FLAG(-Werror=return-type, - [cxxflags_strict="$cxxflags_strict -Werror=return-type"]) -AX_CHECK_COMPILE_FLAG(-Werror=delete-non-virtual-dtor, - [cxxflags_strict="$cxxflags_strict -Werror=delete-non-virtual-dtor"]) -AX_CHECK_COMPILE_FLAG(-Werror=undefined-bool-conversion, - [cxxflags_strict="$cxxflags_strict -Werror=undefined-bool-conversion"]) -# We should really enable -Werror=sometimes-uninitialized, but QDBM violates it: -dnl AX_CHECK_COMPILE_FLAG(-Werror=sometimes-uninitialized, -dnl [cxxflags_strict="$cxxflags_strict -Werror=sometimes-uninitialized"]) +BOX_CHECK_CXX_FLAG(-Wall) +BOX_CHECK_CXX_FLAG(-Werror=return-type) +BOX_CHECK_CXX_FLAG(-Werror=non-virtual-dtor) +BOX_CHECK_CXX_FLAG(-Werror=delete-non-virtual-dtor) +BOX_CHECK_CXX_FLAG(-Werror=narrowing) +BOX_CHECK_CXX_FLAG(-Werror=parentheses) +BOX_CHECK_CXX_FLAG(-Werror=undefined-bool-conversion) +BOX_CHECK_CXX_FLAG(-Werror=unused-private-field) +BOX_CHECK_CXX_FLAG(-Werror=overloaded-virtual) +BOX_CHECK_CXX_FLAG(-Werror=writable-strings) # This error is detected by MSVC, but not usually by GCC/Clang: # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58114 -AX_CHECK_COMPILE_FLAG(-Werror=delete-incomplete, - [cxxflags_strict="$cxxflags_strict -Werror=delete-incomplete"]) -AX_CHECK_COMPILE_FLAG(-Wno-deprecated-declarations, - [cxxflags_strict="$cxxflags_strict -Wno-deprecated-declarations"]) +BOX_CHECK_CXX_FLAG(-Werror=delete-incomplete) +# Using Boost properly seems to need C++0x, or at least the "auto" type. +# We don't need CMake to parse this, because it has built-in feature detecting macros +# which we use instead: +AX_CHECK_COMPILE_FLAG(-std=c++0x, + [cxxflags_strict="$cxxflags_strict -std=c++0x"]) +# And if we're going to do that, we don't want warnings about std::auto_ptr being deprecated: +BOX_CHECK_CXX_FLAG(-Wno-deprecated-declarations, -Wdeprecated-declarations) +# We also get copious warnings from the 'register' storage class specifier in the system +# headers (ntohl and friends) which we can't do much about, so disable it: +BOX_CHECK_CXX_FLAG(-Wno-deprecated-register, -Wdeprecated-register) AC_SUBST([CXXFLAGS_STRICT], [$cxxflags_strict]) if test "x$GXX" = "xyes"; then @@ -147,8 +166,8 @@ Upgrade or read the documentation for alternatives]]) AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS([cxxabi.h dirent.h dlfcn.h fcntl.h getopt.h netdb.h process.h pwd.h signal.h]) -AC_CHECK_HEADERS([syslog.h time.h unistd.h]) +AC_CHECK_HEADERS([cxxabi.h dirent.h dlfcn.h fcntl.h getopt.h lmcons.h netdb.h process.h pwd.h]) +AC_CHECK_HEADERS([signal.h syslog.h time.h unistd.h]) AC_CHECK_HEADERS([netinet/in.h netinet/tcp.h]) AC_CHECK_HEADERS([sys/file.h sys/param.h sys/poll.h sys/socket.h sys/stat.h sys/time.h]) AC_CHECK_HEADERS([sys/types.h sys/uio.h sys/un.h sys/wait.h sys/xattr.h]) @@ -190,6 +209,21 @@ else have_regex_support=no fi +# Check for Boost PropertyTree (XML and JSON support for lib/httpserver) +AX_BOOST_BASE(, + # ax_check_boost.m4 thwarts our attempts to modify CPPFLAGS and + # LDFLAGS by restoring them AFTER running ACTION-IF-FOUND. But we + # can fight back by updating the _SAVED variables instead, and use + # the fact that we know that CPPFLAGS and LDFLAGS are still set with + # the correct values for Boost, to preserve them by overwriting + # CPPFLAGS_SAVED and LDFLAGS_SAVED. + [CPPFLAGS_SAVED="$CPPFLAGS" + LDFLAGS_SAVED="$LDFLAGS"], + [AC_MSG_ERROR([[cannot find Boost, try installing libboost-dev]])]) + +AC_CHECK_HEADER([boost/property_tree/ptree.hpp],, + [AC_MSG_ERROR([[cannot find Boost::PropertyTree, try installing libboost-dev]])]) + ### Checks for typedefs, structures, and compiler characteristics. AC_CHECK_TYPES([u_int8_t, u_int16_t, u_int32_t, u_int64_t]) @@ -222,8 +256,8 @@ AC_CHECK_DECLS([O_BINARY],,, [[#include ]]) AC_CHECK_DECLS([ENOTSUP],,, [[#include ]]) AC_CHECK_DECLS([INFTIM],,, [[#include ]]) AC_CHECK_DECLS([SO_PEERCRED],,, [[#include ]]) -AC_CHECK_DECLS([SOL_TCP],,, [[#include ]]) -AC_CHECK_DECLS([TCP_INFO],,, [[#include ]]) +AC_CHECK_DECLS([IPPROTO_TCP],,, [[#include ]]) +AC_CHECK_DECLS([SOL_TCP,TCP_INFO,TCP_CONNECTION_INFO],,, [[#include ]]) AC_CHECK_DECLS([SYS_open, SYS_openat],,, [[#include ]]) if test -n "$have_sys_socket_h"; then @@ -294,9 +328,8 @@ AX_CHECK_MALLOC_WORKAROUND AC_FUNC_CLOSEDIR_VOID AC_FUNC_ERROR_AT_LINE AC_TYPE_SIGNAL -AC_FUNC_STAT -AC_CHECK_FUNCS([ftruncate getpeereid getpeername getpid gettimeofday lchown]) -AC_CHECK_FUNCS([setproctitle utimensat]) +AC_CHECK_FUNCS([ftruncate getpeereid getpeername getpid getpwuid gettimeofday lchown]) +AC_CHECK_FUNCS([setproctitle utimensat stat64 lstat64 __lxstat __lxstat64]) AC_SEARCH_LIBS([setproctitle], [bsd]) # NetBSD implements kqueue too differently for us to get it fixed by 0.10 @@ -343,6 +376,7 @@ fi AC_CHECK_FUNCS([flock fcntl]) AC_CHECK_DECLS([O_EXLOCK],,, [[#include ]]) AC_CHECK_DECLS([F_SETLK],,, [[#include ]]) +AC_CHECK_DECLS([F_OFD_SETLK],,, [[#include ]]) case $target_os in mingw32*) ;; @@ -357,6 +391,8 @@ fi ;; esac +AC_CHECK_DECLS([GetUserNameA],,, [[#include ]]) + AC_CHECK_PROGS(default_debugger, [lldb gdb]) AC_ARG_WITH([debugger], [AS_HELP_STRING([--with-debugger=], diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index d5ac9f2f4..38d18c362 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -1,4 +1,6 @@ #!@PERL@ + +use warnings; use strict; use Symbol; @@ -41,19 +43,42 @@ my %env_flags; # print "Flag: $_\n" for(keys %env_flags); +sub substitute_make_vars($$) +{ + my ($input, $vars) = @_; + $vars->{_PERL} = '@PERL@'; + + # substitute variables, repeatedly + while(1) + { + my $did_subst = 0; + + for my $k (keys %$vars) + { + $did_subst = 1 if $input =~ s/\$\($k\)/$vars->{$k}/g; + } + + last unless $did_subst; + } + + return $input; +} + # seed autogen code print "Seeding autogen code...\n"; open FINDAUTOGEN,"find . -follow -name Makefile.extra |" or die "Can't use find for locating files"; while() { chomp; + my $file = $_; $file =~ m~\A(.+)/[^/]+\Z~; my $dir = $1; open FL,$file or die "Can't open $_ for reading"; + my %vars; - $vars{_PERL} = "@PERL@"; my $do_cmds = 0; + while() { chomp; @@ -78,22 +103,11 @@ while() if($do_cmds) { $do_cmds = 2; # flag something has been done - - # subsitute variables, repeatedly - my $c = $_; + + # substitute variables, repeatedly + my $c = substitute_make_vars($_, \%vars); $c =~ s/\A\s+//; - while(1) - { - my $did_subst = 0; - - for my $k (keys %vars) - { - $did_subst = 1 if $c =~ s/\$\($k\)/$vars{$k}/g; - } - - last unless $did_subst; - } - + # run command unless (0 == system("(cd $dir; $c)")) { @@ -111,11 +125,6 @@ print "done\n\n"; # open test mail program template file my $test_template_file = 'infrastructure/buildenv-testmain-template.cpp'; -open FL,$test_template_file or die "Can't open test template file\n"; -my $test_template; -read FL,$test_template,-s $test_template_file; -close FL; - # extra platform defines my $extra_platform_defines = ''; @@ -149,7 +158,7 @@ mkdir "debug",0755; # is the library code in another directory? my $external_lib = readlink('lib'); -if($external_lib ne '') +if(defined $external_lib and $external_lib ne '') { # adjust to root of the library distribution $external_lib =~ s!/lib\Z!!; @@ -196,7 +205,7 @@ for(@modules_files) # clean up line chomp; s/\A\s+//; s/#.*\Z//; s/\s+\Z//; s/\s+/ /g; next unless m/\S/; - + # omit bits on some platforms? if(m/\AEND-OMIT/) { @@ -215,21 +224,24 @@ for(@modules_files) } next; } - + # split up... my ($mod, @deps_i) = split / /; - + # ignore this module? next if ignore_module($mod); - + # deps for this platform my @deps; for(@deps_i) { my ($dep,$exclude_from) = split /!/; + $exclude_from = '' if not defined $exclude_from; + # generic library translation $dep = $env_flags{'LIBTRANS_'.$dep} if exists($env_flags{'LIBTRANS_'.$dep}); next if $dep eq ''; + if($exclude_from =~ m/\A\+(.+)\Z/) { $exclude_from = $1; @@ -250,10 +262,10 @@ for(@modules_files) push @deps,$dep if $inc } } - + # check directory exists die "Module $mod can't be found\n" unless -d $mod; - + # and put in lists push @modules,$mod; my @md; # module dependencies @@ -271,15 +283,18 @@ for(@modules_files) } $module_dependency{$mod} = [@implicit_deps,@md]; $module_library_link_opts{$mod} = [@lo]; - + # make directories, but not if we're using an external library and this a library module my ($s,$d) = split /\//,$mod; - if ($s ne 'lib' or $external_lib eq '') + if (defined $s and ($s ne 'lib' or not defined $external_lib)) { mkdir "release/$s",0755; - mkdir "release/$s/$d",0755; mkdir "debug/$s",0755; - mkdir "debug/$s/$d",0755; + if (defined $d) + { + mkdir "release/$s/$d",0755; + mkdir "debug/$s/$d",0755; + } } } @@ -320,7 +335,7 @@ for my $mod (@modules, @implicit_deps) # add in items from autogen directories, and create output directories { my @autogen_items; - + for my $di (@items) { if($di =~ m/\Aautogen/ && -d "$mod/$di") @@ -364,8 +379,6 @@ for my $mod (@modules, @implicit_deps) while($f =~ m/\#include\s+"([^"]+?)"/g) { my $i = $1; - # ignore autogen exceptions - next if $i =~ m/\Aautogen_.+?Exception.h\Z/; # record dependency ${$header_dependency{$h}}{$i} = 1 if exists $hfiles{$i}; } @@ -380,6 +393,7 @@ $default_cxxflags =~ s/ -O2//g; my $debug_base_dir = 'debug'; my $release_base_dir = 'release'; my $debugger = '@with_debugger@'; +my $perl = '@PERL@'; my $release_flags = "-O2"; if ($target_windows) @@ -415,7 +429,7 @@ WINDRES = @WINDRES@ # Work around a mistake in QDBM (using includes for a file not in the # system path) by adding it to the include path with -I. -DEFAULT_CFLAGS = $autoconf_cppflags $default_cflags $autoconf_cxxflags \\ +DEFAULT_CFLAGS = $autoconf_cppflags $default_cflags \\ $extra_platform_defines $platform_compile_line_extra \\ -DBOX_VERSION="\\"$product_version\\"" -Iqdbm DEFAULT_CXXFLAGS = $autoconf_cppflags $default_cxxflags $autoconf_cxxflags \\ @@ -488,7 +502,7 @@ my %library_targets; for my $mod (@implicit_deps, @modules) { print $mod,"\n"; - + my ($type,$name) = split /\//,$mod; if (not $name) { @@ -529,16 +543,10 @@ for my $mod (@implicit_deps, @modules) # add additional files for tests if($type eq 'test') { - my $testmain = $test_template; - $testmain =~ s/TEST_NAME/$name/g; - open TESTMAIN,">$mod/_main.cpp" or die "Can't open test main file for $mod for writing\n"; - print TESTMAIN $testmain; - close TESTMAIN; - # test file... sub writetestfile { - my ($filename,$runcmd,$module) = @_; + my ($filename,$runcmd,$module) = @_; open TESTFILE,">$filename" or die "Can't open " . "test script file for $module for writing\n"; @@ -615,7 +623,7 @@ __E print TESTFILE "exit \$exit_status\n"; close TESTFILE; } - + writetestfile("$mod/t", "GLIBCXX_FORCE_NEW=1 ". './_test' . $platform_exe_ext . ' "$@"', $mod); @@ -628,10 +636,10 @@ __E { writetestfile("$mod/t-gdb", "echo 'No debugger was detected by configure script'\n". - "exit 2"); + "exit 2", "#"); } } - + my @all_deps_for_module; { @@ -662,7 +670,7 @@ __E $d_done{$dep} = 1; } } - } + } # get the list of library things to add -- in order of dependency # so things link properly @@ -708,14 +716,31 @@ __E # ${makefile_ifdef_prefix}ifdef RELEASE TARGET = $release_base_dir/$mod/$end_target_file +TEST_DEST = ../../$release_base_dir/$mod +TEST_MODE = RELEASE ${makefile_ifdef_prefix}else TARGET = $debug_base_dir/$mod/$end_target_file +TEST_DEST = ../../$debug_base_dir/$mod +TEST_MODE = DEBUG ${makefile_ifdef_prefix}endif +TESTFILES_DEST = \$(TEST_DEST)/testfiles + .PHONY: default default: \$(MAKE) -C ../.. \$(TARGET) +.PHONY: prepare +prepare: default + ../../runtest.pl -n ${name} \$(TEST_MODE) + test ! -r \$(TESTFILES_DEST) || chmod -R a+rwx \$(TESTFILES_DEST) + rm -rf \$(TESTFILES_DEST) + cp -p -R testfiles \$(TEST_DEST) + +.PHONY: test +test: default + ../../runtest.pl ${name} \$(TEST_MODE) + .PHONY: clean clean: \$(MAKE) -C ../.. clean_${type}_${name} @@ -727,9 +752,15 @@ __E close MINI_MODULE_MAKEFILE; opendir DIR, $mod; - my @items = readdir DIR; + my @files = readdir DIR; closedir DIR; - + my @items = map {"$mod/$_"} @files; + + if($type eq 'test') + { + push @items, $test_template_file; + } + # add in items from autogen directories, and create output directories { my @autogen_items; @@ -747,7 +778,7 @@ __E next if m/\A\./; push @autogen_items,"$di/$_" } - + # output directories mkdir "release/$mod/$di",0755; mkdir "debug/$mod/$di",0755; @@ -755,14 +786,14 @@ __E } @items = (@items, @autogen_items); } - + # first, obtain a list of dependencies within the .h files my %headers; for my $h (grep /\.h\Z/i, @items) { - open FL,"$mod/$h"; + open FL,"$h" or die "$h: $!"; my $f; - read FL,$f,-s "$mod/$h"; + read FL,$f,-s "$h"; close FL; while($f =~ m/\#include\s+"([^"]+?)"/g) @@ -770,44 +801,106 @@ __E ${$headers{$h}}{$1} = 1 if exists $hfiles{$1}; } } - + + # need to see if the extra makefile fragments require extra object files + # or include any more makefiles + my @objs_extra; + my @makefile_includes; + my %autogen_targets_global; + my %autogen_targets_platform; + + my %makefile_to_autogen = ( + "$mod/Makefile.extra" => \%autogen_targets_global, + "$mod/Makefile.extra.$build_os" => \%autogen_targets_platform, + ); + + while (my ($makefile, $autogens) = each %makefile_to_autogen) + { + additional_objects_from_make_fragment($makefile, \@objs_extra, + \@makefile_includes, $autogens); + } + + while (my ($makefile, $autogens) = each %makefile_to_autogen) + { + $makefile =~ s|^$mod/||g or die "$makefile should start with $mod"; + while (my ($outputs, $inputs) = each %$autogens) + { + my @prefixed_inputs = map {"$mod/$_"} (split /\s+/, $inputs); + my @prefixed_outputs = map {"$mod/$_"} (split /\s+/, $outputs); + + # If any inputs contain .., it will break Make's dependency tracking, so + # convert paths to canonical form: + sub canonicalise($) + { + my ($current_value) = @_; + my $previous_value = ''; + while($current_value ne $previous_value) + { + $previous_value = $current_value; + $current_value =~ s@([^/]+)/\.\./@@; + } + return $current_value; + } + @prefixed_inputs = map {canonicalise($_)} (@prefixed_inputs); + + # Delegate to the extra sub-makefile: + print MASTER_MAKEFILE "\n"; + print MASTER_MAKEFILE "@prefixed_outputs: @prefixed_inputs\n"; + # Since the command produces all $outputs, to avoid a spurious "target + # is up to date" message for the 2nd+ targets, we only make the first: + my $first_output = $outputs; + $first_output =~ s/\s+.*//; + print MASTER_MAKEFILE "\t\$(MAKE) _PERL='$perl' -C $mod -f $makefile $first_output\n"; + } + } + + print MASTER_MAKEFILE "\n"; + # write the recipes for debug and release builds of each file foreach my $var_prefix ('DEBUG', 'RELEASE') { my $make; - # then... do the cpp files... - my @obj_base; + # Then, do the cpp files. Ensure that @objs_extra (from Makefile.extra) are listed + # first in the output dependencies (@objs), since any changed to their own + # dependencies (e.g. a Protocol.txt file) must cause them to be rebuilt before we + # attempt to compile any other source files, otherwise those other files might not + # compile (due to newly defined Commands not existing in the autogen_Protocol.h + # file yet) which could cause us to fail to even try to build the later dependencies + # that would fix the problem. + my @objs = @objs_extra; + for my $file (@items) { - my $is_c = $file =~ m/\A(.+)\.c\Z/i; - my $is_cpp = $file =~ m/\A(.+)\.cpp\Z/i; - my $is_rc = $file =~ m/\A(.+)\.rc\Z/i; - my $base = $1; + my $is_c = $file =~ m@(([^/]+)\.c)\Z@i; + my $is_cpp = $file =~ m@(([^/]+)\.cpp)\Z@i; + my $is_rc = $file =~ m@(([^/]+)\.rc)\Z@i; + my $basename = $1; + my $base = $2; # Don't try to compile .rc files except on Windows: next if not $is_c and not $is_cpp and not ($is_rc and $target_windows); next if $file =~ /\A\._/; # Temp Mac OS Resource hack # store for later - push @obj_base, $base; - + push @objs, $base; + # get the file... - open FL, "$mod/$file"; + open FL, $file or die "$file: $!"; my $f; - read FL, $f, -s "$mod/$file"; + read FL, $f, -s $file; close FL; - + my %dep; while($f =~ m/\#include\s+"([^"]+?)"/g) { insert_dep($1, \%dep) if exists $hfiles{$1}; } - + # output filename my $out_name = "\$(${var_prefix}_OUTBASE)/$mod/$base.o"; - + # write the line for this cpp file my @dep_paths = map { @@ -817,19 +910,19 @@ __E } keys %dep; - $make .= "$out_name: $mod/$file @dep_paths\n"; + $make .= "$out_name: $file @dep_paths\n"; if ($is_c) { $make .= "\t\$(_CC) \$(${var_prefix}_CFLAGS) ". "\$(${type}_${name}_includes) -DBOX_MODULE=\"\\\"$mod\\\"\" " . - "-c $mod/$file -o $out_name\n\n"; + "-c $file -o $out_name\n\n"; } - if ($is_cpp) + elsif ($is_cpp) { $make .= "\t\$(_CXX) \$(${var_prefix}_CXXFLAGS) ". "\$(${type}_${name}_includes) -DBOX_MODULE=\"\\\"$mod\\\"\" " . - "-c $mod/$file -o $out_name\n\n"; + "-c $file -o $out_name\n\n"; } elsif ($is_rc) { @@ -841,23 +934,15 @@ __E } } - # need to see if the extra makefile fragments require extra object files - # or include any more makefiles - my @objs = @obj_base; - my @makefile_includes; - - additional_objects_from_make_fragment("$mod/Makefile.extra", \@objs, \@makefile_includes); - additional_objects_from_make_fragment("$mod/Makefile.extra.$build_os", \@objs, \@makefile_includes); - my $prefixed_end_target = "\$(${var_prefix}_OUTBASE)/$mod/$end_target_file"; - my $o_file_list = join(' ',map {"\$(${var_prefix}_OUTBASE)/$mod/$_.o"} sort @objs); + my $o_file_list = join(' ',map {"\$(${var_prefix}_OUTBASE)/$mod/$_.o"} @objs); my @prefixed_lib_files = map {"\$(${var_prefix}_OUTBASE)/$_"} @lib_files; my @prefixed_dep_targets = map {"\$(${var_prefix}_OUTBASE)/$_"} @dep_targets; print MASTER_MAKEFILE "$prefixed_end_target: $o_file_list"; print MASTER_MAKEFILE " @prefixed_dep_targets" unless $target_is_library; print MASTER_MAKEFILE "\n"; - + if ($target_windows) { foreach my $dep (@all_deps_for_module) @@ -882,26 +967,26 @@ __E # work out library options # need to be... least used first, in absolute order they appear in the modules.txt file my @libops; - + sub libops_fill { my ($module, $libops_ref) = @_; - + my $library_link_opts = $module_library_link_opts{$module}; if ($library_link_opts) { push @$libops_ref, @$library_link_opts; } - + my $deps = $module_dependency{$module}; foreach my $dep (@$deps) { libops_fill($dep, $libops_ref); } } - + libops_fill($mod,\@libops); - + my $lo = ''; my %ldone; for(@libops) @@ -910,7 +995,7 @@ __E $lo .= ' '.$_; $ldone{$_} = 1; } - + # link line... print MASTER_MAKEFILE "\t\$(_LINK) \$(LDFLAGS) " . "-o $prefixed_end_target $o_file_list " . @@ -947,27 +1032,6 @@ realclean_${type}_${name}: clean_${type}_${name} __E push @all_clean_targets, "clean_${type}_${name}"; push @all_realclean_targets, "realclean_${type}_${name}"; - - my $includes = ""; - - if(-e "$mod/Makefile.extra") - { - $includes .= ".include <$mod/Makefile.extra>\n\n"; - } - if(-e "$mod/Makefile.extra.$build_os") - { - $includes .= ".include <$mod/Makefile.extra.$build_os>\n\n"; - } - - if(!$bsd_make) - { - # need to post process this into a GNU makefile - $includes =~ s/\A\.\s*(ifdef|else|endif|ifndef)/$1/; - $includes =~ s/\A\.\s*include\s+<(.+?)>/include $1/; - $includes =~ s/-D\s+(\w+)/$1=1/g; - } - - print MASTER_MAKEFILE $includes; } my @parcels; @@ -1003,7 +1067,7 @@ open PARCELS,"parcels.txt" or die "Can't open parcels file"; { chomp; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g; next unless m/\S/; - + # omit bits on some platforms? next if m/\AEND-OMIT/; if(m/\AOMIT:(.+)/) @@ -1012,7 +1076,7 @@ open PARCELS,"parcels.txt" or die "Can't open parcels file"; { while() { - last if m/\AEND-OMIT/; + last if m/\AEND-OMIT/; } } next; @@ -1043,7 +1107,7 @@ open PARCELS,"parcels.txt" or die "Can't open parcels file"; next; } next if (m'\AEND-EXCEPT'); - + # new parcel, or a new parcel definition? if(m/\A\s+(.+)\Z/) { @@ -1218,7 +1282,7 @@ $make_target: @parcel_deps EOF push @parcel_targets, "build-$parcel"; - + for(@{$parcel_contents{$parcel}}) { my @args = split /\s+/; @@ -1284,11 +1348,11 @@ EOF } unless ($target_windows) - { + { close SCRIPT; chmod 0755,"parcels/scripts/install-$parcel"; } - + my $root = BoxPlatform::parcel_root($parcel); unless ($target_windows) @@ -1297,16 +1361,44 @@ EOF } print MASTER_MAKEFILE "\t(cd parcels; tar cf - $root | gzip -9 - > $root.tgz )\n"; - + print MASTER_MAKEFILE "\n"; unless ($target_windows) - { + { print MASTER_MAKEFILE "install-$parcel:\n"; print MASTER_MAKEFILE "\t(cd $dir; ./install-$parcel)\n\n"; } } +# There doesn't seem to be an easy way to get the list of files configured with AC_CONFIG_FILES +# out of Autoconf, so we open config.status and parse it ourselves. +my @configured_files; +open CONFIG_STATUS, "< config.status" or die "config.status: $!"; +while (my $line = ) +{ + if($line =~ m/^config_files=" +([^"]*)"/) + { + @configured_files = split / +/, $1; + } +} +close CONFIG_STATUS; +if (not @configured_files) +{ + die "Failed to find value of config_files in config.status"; +} + +# Output targets to regenerate the output if the inputs change +foreach my $output (@configured_files) +{ + my $input = $output . ".in"; + print MASTER_MAKEFILE <) { chomp; - if(m/link-extra:\s*(.+)\Z/) + if(m/\A(.+)\s+=\s+(.+)\Z/) + { + # is a variable + $vars{$1} = $2; + next; + } + elsif(m/link-extra:\s*(.+)\Z/) { my $extra = $1; do @@ -1404,13 +1503,20 @@ sub additional_objects_from_make_fragment { push @$include_r,$1 } + elsif(m/^(\S.*\S)\s*:\s*(\S.*\S)\s*$/) + { + my $outputs = $1; + my $inputs = $2; + $outputs = substitute_make_vars($outputs, \%vars); + $inputs = substitute_make_vars($inputs, \%vars); + $autogen_targets_r->{$outputs} = $inputs; + } } - + close FL; } } - sub ignore_module { exists $env_flags{'IGNORE_'.$_[0]} diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index d3300604d..52dd2789e 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -30,7 +30,7 @@ #include "BackupStoreFile.h" #include "CollectInBufferStream.h" #include "FileStream.h" -#include "Utils.h" +#include "Utils.h" // for ObjectExists_* (object_exists_t) #include "MemLeakFindOn.h" @@ -238,7 +238,7 @@ static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, // Create the local directory, if not already done. // Path and owner set later, just use restrictive owner mode. - int exists; + object_exists_t exists; try { @@ -277,7 +277,7 @@ static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, "out of the way of restored directory. " "Use specific restore with ID to " "restore this object."); - if(::unlink(rLocalDirectoryName.c_str()) != 0) + if(EMU_UNLINK(rLocalDirectoryName.c_str()) != 0) { BOX_LOG_SYS_ERROR("Failed to delete " "file '" << @@ -334,7 +334,7 @@ static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, } #endif - int parentExists; + object_exists_t parentExists; try { @@ -379,9 +379,9 @@ static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, return Restore_TargetPathNotFound; default: - BOX_ERROR("Failed to restore: unknown " - "result from ObjectExists('" << - parentDirectoryName << "')"); + BOX_ERROR("Failed to restore: unexpected result from " + "ObjectExists('" << parentDirectoryName << "'): " << + parentExists); return Restore_UnknownError; } } @@ -513,11 +513,10 @@ static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, // files already there. if(ObjectExists(localFilename) != ObjectExists_NoObject && - ::unlink(localFilename.c_str()) != 0) + EMU_UNLINK(localFilename.c_str()) != 0) { - BOX_LOG_SYS_ERROR("Failed to delete " - "file '" << localFilename << - "'"); + BOX_LOG_SYS_ERROR("Failed to delete file " + "'" << localFilename << "'"); if (Params.ContinueAfterErrors) { @@ -856,7 +855,7 @@ int BackupClientRestore(BackupProtocolCallable &rConnection, params.mRestoreResumeInfoFilename += ".boxbackupresume"; // Target exists? - int targetExistance = ObjectExists(LocalDirectoryName); + object_exists_t targetExistance = ObjectExists(LocalDirectoryName); // Does any resumption information exist? bool doingResume = false; @@ -912,7 +911,7 @@ int BackupClientRestore(BackupProtocolCallable &rConnection, } // Delete the resume information file - ::unlink(params.mRestoreResumeInfoFilename.c_str()); + EMU_UNLINK(params.mRestoreResumeInfoFilename.c_str()); return params.ContinuedAfterError ? Restore_CompleteWithErrors : Restore_Complete; diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 865ee4132..891361827 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -48,11 +48,23 @@ static const ConfigurationVerifyKey verifyserverkeys[] = static const ConfigurationVerifyKey verifys3keys[] = { // These values are only required for Amazon S3-compatible stores + // HostName is the network address that we will connect to (e.g. localhost). ConfigurationVerifyKey("HostName", ConfigTest_Exists), ConfigurationVerifyKey("Port", ConfigTest_Exists | ConfigTest_IsInt, 80), ConfigurationVerifyKey("BasePath", ConfigTest_Exists), ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), - ConfigurationVerifyKey("SecretKey", ConfigTest_Exists | ConfigTest_LastEntry) + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), + // S3VirtualHostName is the Host header that we will send, e.g. + // "quotes.s3.amazonaws.com". If missing or empty, HostName will be used as a + // default. + ConfigurationVerifyKey("S3VirtualHostName", 0, ""), + ConfigurationVerifyKey("SimpleDBHostName", 0, "sdb.amazonaws.com"), + ConfigurationVerifyKey("SimpleDBPort", ConfigTest_IsInt, 80), + ConfigurationVerifyKey("SimpleDBEndpoint", 0, ""), + ConfigurationVerifyKey("SimpleDBDomain", 0, "boxbackup_locks"), + ConfigurationVerifyKey("SimpleDBLockName", 0), + ConfigurationVerifyKey("SimpleDBLockValue", 0), + ConfigurationVerifyKey("CacheDirectory", ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify verifyserver[] = diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp index 331ef8411..d9f5afe1f 100644 --- a/lib/backupstore/BackupAccountControl.cpp +++ b/lib/backupstore/BackupAccountControl.cpp @@ -10,16 +10,24 @@ #include "Box.h" #include +#include #include -#include "autogen_CommonException.h" #include "autogen_BackupStoreException.h" +#include "autogen_CommonException.h" +#include "autogen_RaidFileException.h" #include "BackupAccountControl.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreCheck.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreInfo.h" #include "Configuration.h" #include "HTTPResponse.h" +#include "HousekeepStoreAccount.h" +#include "RaidFileController.h" +#include "RaidFileWrite.h" +#include "UnixUser.h" #include "Utils.h" #include "MemLeakFindOn.h" @@ -40,6 +48,7 @@ void BackupAccountControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLi } } + int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSize) { // Get number @@ -50,7 +59,7 @@ int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSi BOX_FATAL("'" << string << "' is not a valid number."); exit(1); } - + // Check units switch(*endptr) { @@ -59,37 +68,42 @@ int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSi // Units: Mb return (number * 1024*1024) / blockSize; break; - + case 'G': case 'g': // Units: Gb return (number * 1024*1024*1024) / blockSize; break; - + case 'B': case 'b': // Units: Blocks // Easy! Just return the number specified. return number; break; - + default: BOX_FATAL(string << " has an invalid units specifier " "(use B for blocks, M for MB, G for GB, eg 2GB)"); exit(1); - break; + break; } } + std::string BackupAccountControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize) { return FormatUsageBar(Blocks, Blocks * BlockSize, MaxBlocks * BlockSize, mMachineReadableOutput); } -int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, - int BlockSize) + +int BackupAccountControl::PrintAccountInfo() { + OpenAccount(false); // !readWrite + BackupStoreInfo& info(mapFileSystem->GetBackupStoreInfo(true)); // ReadOnly + int BlockSize = GetBlockSize(); + // Then print out lots of info std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << BOX_FORMAT_ACCOUNT(info.GetAccountID()) << std::endl; @@ -135,133 +149,478 @@ int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, return 0; } -S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, - bool machineReadableOutput) -: BackupAccountControl(config, machineReadableOutput) + +int BackupStoreAccountControl::BlockSizeOfDiscSet(int discSetNum) { - if(!mConfig.SubConfigurationExists("S3Store")) + // Get controller, check disc set number + RaidFileController &controller(RaidFileController::GetController()); + if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) { - THROW_EXCEPTION_MESSAGE(CommonException, - InvalidConfiguration, - "The S3Store configuration subsection is required " - "when S3Store mode is enabled"); + BOX_FATAL("Disc set " << discSetNum << " does not exist."); + exit(1); } - const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); - mBasePath = s3config.GetKeyValue("BasePath"); - if(mBasePath.size() == 0) + // Return block size + return controller.GetDiscSet(discSetNum).GetBlockSize(); +} + + +int BackupAccountControl::SetLimit(const char *SoftLimitStr, + const char *HardLimitStr) +{ + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, GetBlockSize()); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, GetBlockSize()); + return BackupAccountControl::SetLimit(softlimit, hardlimit); +} + +int BackupAccountControl::SetLimit(int64_t softlimit, int64_t hardlimit) +{ + CheckSoftHardLimits(softlimit, hardlimit); + + // Change the limits + OpenAccount(true); // readWrite + BackupStoreInfo &info(mapFileSystem->GetBackupStoreInfo(false)); // !ReadOnly + info.ChangeLimits(softlimit, hardlimit); + mapFileSystem->PutBackupStoreInfo(info); + + BOX_NOTICE("Limits on account " << mapFileSystem->GetAccountIdentifier() << " " + "changed to " << softlimit << " blocks soft, " << hardlimit << " hard."); + + return 0; +} + + +int BackupAccountControl::SetAccountName(const std::string& rNewAccountName) +{ + OpenAccount(true); // readWrite + BackupStoreInfo &info(mapFileSystem->GetBackupStoreInfo(false)); // !ReadOnly + info.SetAccountName(rNewAccountName); + mapFileSystem->PutBackupStoreInfo(info); + + BOX_NOTICE("Name of account " << mapFileSystem->GetAccountIdentifier() << " " + "changed to " << rNewAccountName); + + return 0; +} + + +int BackupAccountControl::SetAccountEnabled(bool enabled) +{ + OpenAccount(true); // readWrite + BackupStoreInfo &info(mapFileSystem->GetBackupStoreInfo(false)); // !ReadOnly + info.SetAccountEnabled(enabled); + mapFileSystem->PutBackupStoreInfo(info); + + BOX_NOTICE("Account " << mapFileSystem->GetAccountIdentifier() << " is now " << + (enabled ? "enabled" : "disabled")); + + return 0; +} + + +int BackupAccountControl::CreateAccount(int32_t AccountID, int32_t SoftLimit, + int32_t HardLimit, const std::string& AccountName) +{ + // We definitely need a lock to do something this destructive! + mapFileSystem->GetLock(); + + BackupStoreInfo info(AccountID, SoftLimit, HardLimit); + info.SetAccountName(AccountName); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, + BACKUPSTORE_ROOT_DIRECTORY_ID); + mapFileSystem->PutDirectory(rootDir); + int64_t rootDirSize = rootDir.GetUserInfo1_SizeInBlocks(); + + // Update the store info to reflect the size of the root directory + info.ChangeBlocksUsed(rootDirSize); + info.ChangeBlocksInDirectories(rootDirSize); + info.AdjustNumDirectories(1); + int64_t id = info.AllocateObjectID(); + ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + + mapFileSystem->PutBackupStoreInfo(info); + + // Now get the info file again, and report any differences, to check that it + // really worked. + std::auto_ptr copy = mapFileSystem->GetBackupStoreInfoUncached(); + ASSERT(info.ReportChangesTo(*copy) == 0); + + // We also need to create and upload a fresh refcount DB. + BackupStoreRefCountDatabase& refcount( + mapFileSystem->GetPotentialRefCountDatabase()); + refcount.Commit(); + + return 0; +} + + +int BackupStoreAccountControl::DeleteAccount(bool AskForConfirmation) +{ + // Obtain a write lock, as the daemon user + // We definitely need a lock to do something this destructive! + OpenAccount(true); // readWrite + + // Check user really wants to do this + if(AskForConfirmation) { - mBasePath = "/"; + BOX_WARNING("Really delete account " << + BOX_FORMAT_ACCOUNT(mAccountID) << "? (type 'yes' to confirm)"); + char response[256]; + if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) + { + BOX_NOTICE("Deletion cancelled."); + return 0; + } } - else + + // Back to original user, but write lock is maintained + mapChangeUser.reset(); + + std::auto_ptr db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Delete from account database + db->DeleteEntry(mAccountID); + + // Write back to disc + db->Write(); + + // Remove the store files... + + // First, become the user specified in the config file + std::string username; { - if(mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) { - THROW_EXCEPTION_MESSAGE(CommonException, - InvalidConfiguration, - "If S3Store.BasePath is not empty then it must start and " - "end with a slash, e.g. '/subdir/', but it currently does not."); + username = rserverConfig.GetKeyValue("User"); } } - mapS3Client.reset(new S3Client( - s3config.GetKeyValue("HostName"), - s3config.GetKeyValueInt("Port"), - s3config.GetKeyValue("AccessKey"), - s3config.GetKeyValue("SecretKey"))); + // Become the right user + if(!username.empty()) + { + // Username specified, change... + mapChangeUser.reset(new UnixUser(username)); + mapChangeUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when user goes out of scope + } + + // Secondly, work out which directories need wiping + std::vector toDelete; + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet discSet(rcontroller.GetDiscSet(mDiscSetNum)); + for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) + { + if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) + { + toDelete.push_back((*i) + DIRECTORY_SEPARATOR + mRootDir); + } + } + + // NamedLock will throw an exception if it can't delete the lockfile, + // which it can't if it doesn't exist. Now that we've deleted the account, + // nobody can open it anyway, so it's safe to unlock. + mapFileSystem->ReleaseLock(); + + int retcode = 0; - mapFileSystem.reset(new S3BackupFileSystem(mConfig, mBasePath, *mapS3Client)); + // Thirdly, delete the directories... + for(std::vector::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + BOX_NOTICE("Deleting store directory " << (*d) << "..."); + // Just use the rm command to delete the files +#ifdef WIN32 + std::string cmd("rmdir /s/q "); + std::string dir = *d; + + // rmdir doesn't understand forward slashes, so replace them all. + for(std::string::iterator i = dir.begin(); i != dir.end(); i++) + { + if(*i == '/') + { + *i = '\\'; + } + } + cmd += dir; +#else + std::string cmd("rm -rf "); + cmd += *d; +#endif + // Run command + if(::system(cmd.c_str()) != 0) + { + BOX_ERROR("Failed to delete files in " << (*d) << + ", delete them manually."); + retcode = 1; + } + } + + // Success! + return retcode; } -std::string S3BackupAccountControl::GetFullURL(const std::string ObjectPath) const + +void BackupStoreAccountControl::OpenAccount(bool readWrite) { - const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); - return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + - s3config.GetKeyValue("Port") + GetFullPath(ObjectPath); + if(mapFileSystem.get()) + { + // Can use existing BackupFileSystem. + + if(readWrite && !mapFileSystem->HaveLock()) + { + // Need to acquire a lock first. + mapFileSystem->GetLock(); + } + + return; + } + + // Load in the account database + std::auto_ptr db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Exists? + if(!db->EntryExists(mAccountID)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, AccountDoesNotExist, + "Failed to open account " << BOX_FORMAT_ACCOUNT(mAccountID) << + ": does not exist"); + } + + // Get info from the database + BackupStoreAccounts acc(*db); + acc.GetAccountRoot(mAccountID, mRootDir, mDiscSetNum); + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + mapChangeUser.reset(new UnixUser(username)); + mapChangeUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when this BackupStoreAccountControl is destroyed + } + + mapFileSystem.reset(new RaidBackupFileSystem(mAccountID, mRootDir, mDiscSetNum)); + + if(readWrite) + { + mapFileSystem->GetLock(); + } } -int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + +int BackupStoreAccountControl::CheckAccount(bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound) +{ + // Don't need a write lock if not making changes. + OpenAccount(FixErrors); // readWrite + + // Check it + BackupStoreCheck check(*mapFileSystem, FixErrors, Quiet); + check.Check(); + + if(ReturnNumErrorsFound) + { + return check.GetNumErrorsFound(); + } + else + { + return check.ErrorsFound() ? 1 : 0; + } +} + + +int BackupStoreAccountControl::CreateAccount(int32_t DiscNumber, int32_t SoftLimit, int32_t HardLimit) { - // Try getting the info file. If we get a 200 response then it already - // exists, and we should bail out. If we get a 404 then it's safe to - // continue. Otherwise something else is wrong and we should bail out. - std::string info_url = GetFullURL(S3_INFO_FILE_NAME); + CheckSoftHardLimits(SoftLimit, HardLimit); + + // Load in the account database + std::auto_ptr db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Already exists? + if(db->EntryExists(mAccountID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(mAccountID) << + " already exists."); + return 1; + } - HTTPResponse response = GetObject(S3_INFO_FILE_NAME); - if(response.GetResponseCode() == HTTPResponse::Code_OK) + db->AddEntry(mAccountID, DiscNumber); + + // As the original user, write the modified account database back + db->Write(); + + // Get the user under which the daemon runs + std::string username; { - THROW_EXCEPTION_MESSAGE(BackupStoreException, AccountAlreadyExists, - "The BackupStoreInfo file already exists at this URL: " << - info_url); + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } } - if(response.GetResponseCode() != HTTPResponse::Code_NotFound) + // Become the user specified in the config file? + if(!username.empty()) { - mapS3Client->CheckResponse(response, std::string("Failed to check for an " - "existing BackupStoreInfo file at this URL: ") + info_url); + // Username specified, change... + mapChangeUser.reset(new UnixUser(username)); + mapChangeUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when this BackupStoreAccountControl is destroyed } - BackupStoreInfo info(0, // fake AccountID for S3 stores - info_url, // FileName, - SoftLimit, HardLimit); - info.SetAccountName(name); + // Get directory name: + BackupStoreAccounts acc(*db); + acc.GetAccountRoot(mAccountID, mRootDir, mDiscSetNum); - // And an empty directory - BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); - int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir); + // Create the account root directory on disc: + RaidFileWrite::CreateDirectory(mDiscSetNum, mRootDir, true /* recursive */); - // Update the store info to reflect the size of the root directory - info.ChangeBlocksUsed(rootDirSize); - info.ChangeBlocksInDirectories(rootDirSize); - info.AdjustNumDirectories(1); - int64_t id = info.AllocateObjectID(); - ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + // Create the BackupStoreInfo and BackupStoreRefCountDatabase files: + mapFileSystem.reset(new RaidBackupFileSystem(mAccountID, mRootDir, mDiscSetNum)); - CollectInBufferStream out; - info.Save(out); - out.SetForReading(); + { + // Taking the lock will try to GetAccountIdentifier() (for the log message) which + // will throw a RaidFileException(RaidFileDoesntExist) because the BackupStoreInfo + // file does not exist yet. We expect that! So take the lock early (before calling + // BackupAccountControl::CreateAccount) and silence the exception warning while we + // do so. + HideSpecificExceptionGuard guard(RaidFileException::ExceptionType, + RaidFileException::RaidFileDoesntExist); + mapFileSystem->GetLock(); + } - response = PutObject(S3_INFO_FILE_NAME, out); - mapS3Client->CheckResponse(response, std::string("Failed to upload the new BackupStoreInfo " - "file to this URL: ") + info_url); + BackupAccountControl::CreateAccount(mAccountID, SoftLimit, HardLimit, ""); - // Now get the file again, to check that it really worked. - response = GetObject(S3_INFO_FILE_NAME); - mapS3Client->CheckResponse(response, std::string("Failed to download the new BackupStoreInfo " - "file that we just created: ") + info_url); + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(mAccountID) << " created."); return 0; } -std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) + +int BackupStoreAccountControl::HousekeepAccountNow() { - std::ostringstream out; - out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; - return out.str(); + // Housekeeping locks the account itself, so we can't. + OpenAccount(false); // readWrite + + HousekeepStoreAccount housekeeping(*mapFileSystem, NULL); + bool success = housekeeping.DoHousekeeping(); + + if(!success) + { + BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(mAccountID) + << " for housekeeping: perhaps a client is " + "still connected?"); + return 1; + } + else + { + BOX_TRACE("Finished housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID)); + return 0; + } } -std::auto_ptr S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir) + +S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput) +: BackupAccountControl(config, machineReadableOutput) { - std::string uri = GetDirectoryURI(rDir.GetObjectID()); - HTTPResponse response = mrClient.GetObject(uri); - mrClient.CheckResponse(response, - std::string("Failed to download directory: ") + uri); - return std::auto_ptr(new HTTPResponse(response)); + if(!mConfig.SubConfigurationExists("S3Store")) + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "The S3Store configuration subsection is required " + "when S3Store mode is enabled"); + } + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + + std::string base_path = s3config.GetKeyValue("BasePath"); + if(base_path.size() == 0) + { + base_path = "/"; + } + else + { + if(base_path[0] != '/' || base_path[base_path.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "If S3Store.BasePath is not empty then it must start and " + "end with a slash, e.g. '/subdir/', but it currently does not."); + } + } + + std::string cache_dir = s3config.GetKeyValue("CacheDirectory"); + mapS3Client.reset(new S3Client(s3config)); + mapFileSystem.reset(new S3BackupFileSystem(mConfig, base_path, cache_dir, + *mapS3Client)); } -int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) + +int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + int32_t HardLimit) { - CollectInBufferStream out; - rDir.WriteToStream(out); - out.SetForReading(); + // Try getting the info file. If we get a 200 response then it already + // exists, and we should bail out. If we get a 404 then it's safe to + // continue. Otherwise something else is wrong and we should bail out. + S3BackupFileSystem& s3fs(*(S3BackupFileSystem *)(mapFileSystem.get())); + try + { + // We expect this to throw a FileNotFound HTTPException. + HideSpecificExceptionGuard guard(HTTPException::ExceptionType, + HTTPException::FileNotFound); + s3fs.GetBackupStoreInfoUncached(); - std::string uri = GetDirectoryURI(rDir.GetObjectID()); - HTTPResponse response = mrClient.PutObject(uri, out); - mrClient.CheckResponse(response, - std::string("Failed to upload directory: ") + uri); + // If it doesn't, then the file already exists, which is bad. + BOX_FATAL("The BackupStoreInfo file already exists at this URL: " << + s3fs.GetObjectURL(s3fs.GetMetadataURI(S3_INFO_FILE_NAME))); + return 1; + } + catch(HTTPException &e) + { + if(EXCEPTION_IS_TYPE(e, HTTPException, FileNotFound)) + { + // This is what we want to see, so don't do anything rash. + } + else + { + // This is not what we wanted to see, so reraise the exception. + throw; + } + } - int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; - return blocks; -} + { + // Taking the lock will try to GetAccountIdentifier() (for the log message) which + // will throw a RaidFileException(RaidFileDoesntExist) because the BackupStoreInfo + // file does not exist yet. We expect that! So take the lock early (before calling + // BackupAccountControl::CreateAccount) and silence the exception warning while we + // do so. + HideSpecificExceptionGuard guard(HTTPException::ExceptionType, + HTTPException::FileNotFound); + mapFileSystem->GetLock(); + } + // Create the BackupStoreInfo and BackupStoreRefCountDatabase files: + BackupAccountControl::CreateAccount(S3_FAKE_ACCOUNT_ID, SoftLimit, HardLimit, name); + + return 0; +} diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h index 00118ec26..3000132b5 100644 --- a/lib/backupstore/BackupAccountControl.h +++ b/lib/backupstore/BackupAccountControl.h @@ -13,18 +13,29 @@ #include #include "BackupStoreAccountDatabase.h" -#include "HTTPResponse.h" +#include "BackupFileSystem.h" #include "S3Client.h" +#include "UnixUser.h" class BackupStoreDirectory; class BackupStoreInfo; class Configuration; +class NamedLock; +class UnixUser; class BackupAccountControl { protected: const Configuration& mConfig; bool mMachineReadableOutput; + std::auto_ptr mapFileSystem; + + virtual void OpenAccount(bool readWrite) { } + virtual int GetBlockSize() + { + return mapFileSystem->GetBlockSize(); + } + virtual int SetLimit(int64_t softlimit, int64_t hardlimit); public: BackupAccountControl(const Configuration& config, @@ -32,60 +43,89 @@ class BackupAccountControl : mConfig(config), mMachineReadableOutput(machineReadableOutput) { } + virtual ~BackupAccountControl() { } void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); int64_t SizeStringToBlocks(const char *string, int BlockSize); std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize); - int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); + virtual int SetLimit(const char *SoftLimitStr, const char *HardLimitStr); + virtual int SetAccountName(const std::string& rNewAccountName); + virtual int PrintAccountInfo(); + virtual int SetAccountEnabled(bool enabled); + virtual BackupFileSystem& GetFileSystem() = 0; + virtual BackupFileSystem* GetCurrentFileSystem() { return mapFileSystem.get(); } + int CreateAccount(int32_t AccountID, int32_t SoftLimit, int32_t HardLimit, + const std::string& AccountName); }; -class S3BackupFileSystem + +class BackupStoreAccountControl : public BackupAccountControl { private: - std::string mBasePath; - S3Client& mrClient; + int32_t mAccountID; + std::string mRootDir; + int mDiscSetNum; + std::auto_ptr mapChangeUser; // used to reset uid when we return + public: - S3BackupFileSystem(const Configuration& config, const std::string& BasePath, - S3Client& rClient) - : mBasePath(BasePath), - mrClient(rClient) + BackupStoreAccountControl(const Configuration& config, int32_t AccountID, + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput), + mAccountID(AccountID), + mDiscSetNum(0) { } - std::string GetDirectoryURI(int64_t ObjectID); - std::auto_ptr GetDirectory(BackupStoreDirectory& rDir); - int PutDirectory(BackupStoreDirectory& rDir); + virtual int GetBlockSize() + { + return BlockSizeOfDiscSet(mDiscSetNum); + } + int BlockSizeOfDiscSet(int discSetNum); + int DeleteAccount(bool AskForConfirmation); + int CheckAccount(bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound = false); + int CreateAccount(int32_t DiscNumber, int32_t SoftLimit, int32_t HardLimit); + int HousekeepAccountNow(); + virtual BackupFileSystem& GetFileSystem() + { + if(mapFileSystem.get()) + { + return *mapFileSystem; + } + + // We don't know whether the caller wants a write-locked filesystem or + // not, but they can lock it themselves if they want to. + OpenAccount(false); // !ReadWrite + return *mapFileSystem; + } +protected: + virtual void OpenAccount(bool readWrite); }; class S3BackupAccountControl : public BackupAccountControl { private: - std::string mBasePath; std::auto_ptr mapS3Client; - std::auto_ptr mapFileSystem; + // mapFileSystem is inherited from BackupAccountControl + public: S3BackupAccountControl(const Configuration& config, bool machineReadableOutput = false); - std::string GetFullPath(const std::string ObjectPath) const + virtual ~S3BackupAccountControl() { - return mBasePath + ObjectPath; + // Destroy mapFileSystem before mapS3Client, because it may need it + // for cleanup. + mapFileSystem.reset(); } - std::string GetFullURL(const std::string ObjectPath) const; int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); int GetBlockSize() { return 4096; } - HTTPResponse GetObject(const std::string& name) - { - return mapS3Client->GetObject(GetFullPath(name)); - } - HTTPResponse PutObject(const std::string& name, IOStream& rStreamToSend, - const char* pContentType = NULL) + + virtual BackupFileSystem& GetFileSystem() { - return mapS3Client->PutObject(GetFullPath(name), rStreamToSend, - pContentType); + ASSERT(mapFileSystem.get() != NULL); + return *mapFileSystem; } }; // max size of soft limit as percent of hard limit #define MAX_SOFT_LIMIT_SIZE 97 -#define S3_INFO_FILE_NAME "boxbackup.info" -#define S3_NOTIONAL_BLOCK_SIZE 1048576 #endif // BACKUPACCOUNTCONTROL__H diff --git a/lib/backupstore/BackupClientFileAttributes.cpp b/lib/backupstore/BackupClientFileAttributes.cpp index 371403012..431e55877 100644 --- a/lib/backupstore/BackupClientFileAttributes.cpp +++ b/lib/backupstore/BackupClientFileAttributes.cpp @@ -814,7 +814,7 @@ void BackupClientFileAttributes::WriteAttributes(const std::string& Filename, Filename << "'"); #else // Make a symlink, first deleting anything in the way - ::unlink(Filename.c_str()); + EMU_UNLINK(Filename.c_str()); if(::symlink((char*)(pattr + 1), Filename.c_str()) != 0) { BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp index 22ef02153..0c14b33a4 100644 --- a/lib/backupstore/BackupCommands.cpp +++ b/lib/backupstore/BackupCommands.cpp @@ -21,11 +21,7 @@ #include "BackupStoreException.h" #include "BackupStoreFile.h" #include "BackupStoreInfo.h" -#include "BufferedStream.h" #include "CollectInBufferStream.h" -#include "FileStream.h" -#include "InvisibleTempFileStream.h" -#include "RaidFileController.h" #include "StreamableMemBlock.h" #include "MemLeakFindOn.h" @@ -71,7 +67,9 @@ std::auto_ptr BackupProtocolReplyable::HandleException(Bo } else if (e.GetType() == BackupStoreException::ExceptionType) { - if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + // Slightly broken or really broken, both thrown by VerifyStream: + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify || + e.GetSubType() == BackupStoreException::BadBackupStoreFile) { return PROTOCOL_ERROR(Err_FileDoesNotVerify); } @@ -226,7 +224,7 @@ std::auto_ptr BackupProtocolFinished::DoCommand(BackupPro "(name=" << rContext.GetAccountName() << ")"); // Let the context know about it - rContext.ReceivedFinishCommand(); + rContext.CleanUp(); return std::auto_ptr(new BackupProtocolFinished); } @@ -348,132 +346,22 @@ std::auto_ptr BackupProtocolGetObject::DoCommand(BackupPr std::auto_ptr BackupProtocolGetFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { CHECK_PHASE(Phase_Commands) - - // Check the objects exist - if(!rContext.ObjectExists(mObjectID) - || !rContext.ObjectExists(mInDirectory)) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - - // Get the directory it's in - const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory)); - - // Find the object within the directory - BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID); - if(pfileEntry == 0) - { - return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); - } - - // The result std::auto_ptr stream; - // Does this depend on anything? - if(pfileEntry->GetDependsNewer() != 0) + try { - // File exists, but is a patch from a new version. Generate the older version. - std::vector patchChain; - int64_t id = mObjectID; - BackupStoreDirectory::Entry *en = 0; - do - { - patchChain.push_back(id); - en = rdir.FindEntryByID(id); - if(en == 0) - { - BOX_ERROR("Object " << - BOX_FORMAT_OBJECTID(mObjectID) << - " in dir " << - BOX_FORMAT_OBJECTID(mInDirectory) << - " for account " << - BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << - " references object " << - BOX_FORMAT_OBJECTID(id) << - " which does not exist in dir"); - return PROTOCOL_ERROR(Err_PatchConsistencyError); - } - id = en->GetDependsNewer(); - } - while(en != 0 && id != 0); - - // OK! The last entry in the chain is the full file, the others are patches back from it. - // Open the last one, which is the current from file - std::auto_ptr from(rContext.OpenObject(patchChain[patchChain.size() - 1])); - - // Then, for each patch in the chain, do a combine - for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) - { - // ID of patch - int64_t patchID = patchChain[p]; - - // Open it a couple of times - std::auto_ptr diff(rContext.OpenObject(patchID)); - std::auto_ptr diff2(rContext.OpenObject(patchID)); - - // Choose a temporary filename for the result of the combination - std::ostringstream fs; - fs << rContext.GetAccountRoot() << ".recombinetemp." << p; - std::string tempFn = - RaidFileController::DiscSetPathToFileSystemPath( - rContext.GetStoreDiscSet(), fs.str(), - p + 16); - - // Open the temporary file - std::auto_ptr combined( - new InvisibleTempFileStream( - tempFn, O_RDWR | O_CREAT | O_EXCL | - O_BINARY | O_TRUNC)); - - // Do the combining - BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); - - // Move to the beginning of the combined file - combined->Seek(0, IOStream::SeekType_Absolute); - - // Then shuffle round for the next go - if (from.get()) from->Close(); - from = combined; - } - - // Now, from contains a nice file to send to the client. Reorder it - { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); - stream = t; - } - - // Release from file to avoid double deletion - from.release(); + stream = rContext.GetFile(mObjectID, mInDirectory); } - else + catch(BackupStoreException &e) { - // Simple case: file already exists on disc ready to go - - // Open the object - std::auto_ptr object(rContext.OpenObject(mObjectID)); - BufferedStream buf(*object); - - // Verify it - if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) + if(EXCEPTION_IS_TYPE(e, BackupStoreException, ObjectDoesNotExist)) { - return PROTOCOL_ERROR(Err_FileDoesNotVerify); + return PROTOCOL_ERROR(Err_DoesNotExist); } - - // Reset stream -- seek to beginning - object->Seek(0, IOStream::SeekType_Absolute); - - // Reorder the stream/file into stream order + else { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */)); - stream = t; + throw; } - - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid - // premature deletion - object.release(); } // Stream the reordered stream to the peer @@ -957,10 +845,6 @@ std::auto_ptr BackupProtocolGetAccountUsage::DoCommand(Ba // Get store info from context const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo()); - // Find block size - RaidFileController &rcontroller(RaidFileController::GetController()); - RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber())); - // Return info return std::auto_ptr(new BackupProtocolAccountUsage( rinfo.GetBlocksUsed(), @@ -969,7 +853,7 @@ std::auto_ptr BackupProtocolGetAccountUsage::DoCommand(Ba rinfo.GetBlocksInDirectories(), rinfo.GetBlocksSoftLimit(), rinfo.GetBlocksHardLimit(), - rdiscSet.GetBlockSize() + rContext.GetBlockSize() )); } @@ -1007,10 +891,6 @@ std::auto_ptr BackupProtocolGetAccountUsage2::DoCommand( // Get store info from context const BackupStoreInfo &info(rContext.GetBackupStoreInfo()); - // Find block size - RaidFileController &rcontroller(RaidFileController::GetController()); - RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(info.GetDiscSetNumber())); - // Return info BackupProtocolAccountUsage2* usage = new BackupProtocolAccountUsage2(); std::auto_ptr reply(usage); @@ -1018,7 +898,7 @@ std::auto_ptr BackupProtocolGetAccountUsage2::DoCommand( COPY(AccountName); usage->SetAccountEnabled(info.IsAccountEnabled()); COPY(ClientStoreMarker); - usage->SetBlockSize(rdiscSet.GetBlockSize()); + usage->SetBlockSize(rContext.GetBlockSize()); COPY(LastObjectIDUsed); COPY(BlocksUsed); COPY(BlocksInCurrentFiles); diff --git a/lib/backupstore/BackupConstants.h b/lib/backupstore/BackupConstants.h index 195dc6212..24acfde43 100644 --- a/lib/backupstore/BackupConstants.h +++ b/lib/backupstore/BackupConstants.h @@ -11,14 +11,15 @@ #define BACKUPCONSTANTS__H // 15 minutes to timeout (milliseconds) -#define BACKUP_STORE_TIMEOUT (15*60*1000) +#define BACKUP_STORE_TIMEOUT (15*60*1000) // Time to wait for retry after a backup error -#define BACKUP_ERROR_RETRY_SECONDS 100 +#define BACKUP_ERROR_RETRY_SECONDS 100 // Should the store daemon convert files to Raid immediately? -#define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true +#define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true -#endif // BACKUPCONSTANTS__H +#define WRITE_LOCK_FILENAME "write.lock" +#endif // BACKUPCONSTANTS__H diff --git a/lib/backupstore/BackupFileSystem.cpp b/lib/backupstore/BackupFileSystem.cpp new file mode 100644 index 000000000..40544871e --- /dev/null +++ b/lib/backupstore/BackupFileSystem.cpp @@ -0,0 +1,2379 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupFileSystem.cpp +// Purpose: Generic interface for reading and writing files and +// directories, abstracting over RaidFile, S3, FTP etc. +// Created: 2015/08/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef HAVE_PWD_H +# include +#endif + +#ifdef HAVE_LMCONS_H +# include +#endif + +#ifdef HAVE_PROCESS_H +# include // for getpid() on Windows +#endif + +#include + +#include + +#include "autogen_BackupStoreException.h" +#include "autogen_RaidFileException.h" +#include "BackupConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupFileSystem.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreFile.h" +#include "BackupStoreInfo.h" +#include "BackupStoreRefCountDatabase.h" +#include "BufferedStream.h" +#include "BufferedWriteStream.h" +#include "ByteCountingStream.h" +#include "CollectInBufferStream.h" +#include "Configuration.h" +#include "BackupStoreObjectMagic.h" +#include "HTTPResponse.h" +#include "IOStream.h" +#include "InvisibleTempFileStream.h" +#include "RaidFileController.h" +#include "RaidFileRead.h" +#include "RaidFileUtil.h" +#include "RaidFileWrite.h" +#include "S3Client.h" +#include "StoreStructure.h" +#include "Utils.h" // for FileExists, StartsWith, EndsWith + +#include "MemLeakFindOn.h" + +const FileSystemCategory BackupFileSystem::LOCKING("Locking"); + +void BackupFileSystem::GetLock(int max_tries) +{ + if(HaveLock()) + { + // Account already locked, nothing to do + return; + } + + int i = 0; + for(; i < max_tries || max_tries == KEEP_TRYING_FOREVER; i++) + { + try + { + TryGetLock(); + + // If that didn't throw an exception, then we should have successfully + // got a lock! + ASSERT(HaveLock()); + break; + } + catch(BackupStoreException &e) + { + if(EXCEPTION_IS_TYPE(e, BackupStoreException, CouldNotLockStoreAccount) && + ((i < max_tries - 1) || max_tries == KEEP_TRYING_FOREVER)) + { + // Sleep a bit, and try again, as long as we have retries left. + + if(i == 0) + { + BOX_LOG_CATEGORY(Log::INFO, LOCKING, "Failed to lock " + "account " << GetAccountIdentifier() << ", " + "still trying..."); + } + else if(i == 30) + { + BOX_LOG_CATEGORY(Log::WARNING, LOCKING, "Failed to lock " + "account " << GetAccountIdentifier() << " for " + "30 seconds, still trying..."); + } + + ShortSleep(MilliSecondsToBoxTime(1000), true); + // Will try again + } + else if(EXCEPTION_IS_TYPE(e, BackupStoreException, + CouldNotLockStoreAccount)) + { + BOX_LOG_CATEGORY(Log::INFO, LOCKING, "Failed to lock account " << + GetAccountIdentifier() << " after " << (i + 1) << " " + "attempts, giving up"); + throw; + } + else + { + BOX_LOG_CATEGORY(Log::INFO, LOCKING, "Failed to lock account " << + GetAccountIdentifier() << " " "with unexpected error: " << + e.what()); + throw; + } + } + } + + // If that didn't throw an exception, then we should have successfully got a lock! + ASSERT(HaveLock()); + BOX_LOG_CATEGORY(Log::TRACE, LOCKING, "Successfully locked account " << + GetAccountIdentifier() << " after " << (i + 1) << " attempts"); +} + +void BackupFileSystem::ReleaseLock() +{ + if(HaveLock()) + { + BOX_LOG_CATEGORY(Log::TRACE, LOCKING, "Releasing lock on account " << + GetAccountIdentifier()); + } + + mapBackupStoreInfo.reset(); + + if(mapPotentialRefCountDatabase.get()) + { + mapPotentialRefCountDatabase->Discard(); + mapPotentialRefCountDatabase.reset(); + } + + mapPermanentRefCountDatabase.reset(); +} + +// Refresh forces the any current BackupStoreInfo to be discarded and reloaded from the +// store. This would be dangerous if anyone was holding onto a reference to it! +BackupStoreInfo& BackupFileSystem::GetBackupStoreInfo(bool ReadOnly, bool Refresh) +{ + if(mapBackupStoreInfo.get()) + { + if(!Refresh && (ReadOnly || !mapBackupStoreInfo->IsReadOnly())) + { + // Return the current BackupStoreInfo + return *mapBackupStoreInfo; + } + else + { + // Need to reopen to change from read-only to read-write. + mapBackupStoreInfo.reset(); + } + } + + mapBackupStoreInfo = GetBackupStoreInfoInternal(ReadOnly); + return *mapBackupStoreInfo; +} + + +void BackupFileSystem::RefCountDatabaseBeforeCommit(BackupStoreRefCountDatabase& refcount_db) +{ + ASSERT(&refcount_db == mapPotentialRefCountDatabase.get()); + // This is the potential database, so it is about to be committed and become the permanent + // database, so we need to close the current permanent database (if any) first. + mapPermanentRefCountDatabase.reset(); +} + + +void BackupFileSystem::RefCountDatabaseAfterCommit(BackupStoreRefCountDatabase& refcount_db) +{ + // Can only commit a temporary database: + ASSERT(&refcount_db == mapPotentialRefCountDatabase.get()); + + // This was the temporary database, but it is now permanent, and has replaced the + // permanent file, so we need to change the databases returned by the temporary + // and permanent getter functions: + mapPermanentRefCountDatabase = mapPotentialRefCountDatabase; + + // And save it to permanent storage (TODO: avoid double Save() by AfterCommit + // handler and refcount DB destructor): + SaveRefCountDatabase(refcount_db); + + // Reopen the database that was closed by Commit(). + mapPermanentRefCountDatabase->Reopen(); +} + + +void BackupFileSystem::RefCountDatabaseAfterDiscard(BackupStoreRefCountDatabase& refcount_db) +{ + ASSERT(&refcount_db == mapPotentialRefCountDatabase.get()); + mapPotentialRefCountDatabase.reset(); +} + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabaseWrapper +// Purpose: Wrapper around BackupStoreRefCountDatabase that +// automatically notifies the BackupFileSystem when +// Commit() or Discard() is called. +// Created: 2016/04/16 +// +// -------------------------------------------------------------------------- + +typedef BackupStoreRefCountDatabase::refcount_t refcount_t; + +class BackupStoreRefCountDatabaseWrapper : public BackupStoreRefCountDatabase +{ +private: + // No copying allowed + BackupStoreRefCountDatabaseWrapper(const BackupStoreRefCountDatabase &); + std::auto_ptr mapUnderlying; + BackupFileSystem& mrFileSystem; + +public: + BackupStoreRefCountDatabaseWrapper( + std::auto_ptr apUnderlying, + BackupFileSystem& filesystem) + : mapUnderlying(apUnderlying), + mrFileSystem(filesystem) + { } + virtual ~BackupStoreRefCountDatabaseWrapper() + { + // If this is the permanent database, and not read-only, it to permanent + // storage on destruction. (TODO: avoid double Save() by AfterCommit + // handler and this destructor) + if(this == mrFileSystem.mapPermanentRefCountDatabase.get() && + // ReadOnly: don't make the database read-write if it isn't already + !IsReadOnly()) + { + mrFileSystem.SaveRefCountDatabase(*this); + } + } + + virtual void Commit() + { + mrFileSystem.RefCountDatabaseBeforeCommit(*this); + mapUnderlying->Commit(); + // Upload the changed file to the server. + mrFileSystem.RefCountDatabaseAfterCommit(*this); + } + virtual void Discard() + { + mapUnderlying->Discard(); + mrFileSystem.RefCountDatabaseAfterDiscard(*this); + } + virtual void Close() { mapUnderlying->Close(); } + virtual void Reopen() { mapUnderlying->Reopen(); } + virtual bool IsReadOnly() { return mapUnderlying->IsReadOnly(); } + + // Data access functions + virtual refcount_t GetRefCount(int64_t ObjectID) const + { + return mapUnderlying->GetRefCount(ObjectID); + } + virtual int64_t GetLastObjectIDUsed() const + { + return mapUnderlying->GetLastObjectIDUsed(); + } + + // Data modification functions + virtual void AddReference(int64_t ObjectID) + { + mapUnderlying->AddReference(ObjectID); + } + // RemoveReference returns false if refcount drops to zero + virtual bool RemoveReference(int64_t ObjectID) + { + return mapUnderlying->RemoveReference(ObjectID); + } + virtual int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) + { + return mapUnderlying->ReportChangesTo(rOldRefs); + } +}; + + +void RaidBackupFileSystem::TryGetLock() +{ + if(mWriteLock.GotLock()) + { + return; + } + + // Make the filename of the write lock file + std::string writeLockFile; + StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, + writeLockFile); + + // Request the lock + if(!mWriteLock.TryAndGetLock(writeLockFile.c_str(), + 0600 /* restrictive file permissions */)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Failed to get exclusive " + "lock on account"); + } +} + + +std::string RaidBackupFileSystem::GetAccountIdentifier() +{ + std::string account_name; + try + { + account_name = GetBackupStoreInfo(true).GetAccountName(); + } + catch(RaidFileException &e) + { + account_name = "unknown"; + } + + std::ostringstream oss; + oss << BOX_FORMAT_ACCOUNT(mAccountID) << " (" << account_name << ")"; + return oss.str(); +} + + +std::string RaidBackupFileSystem::GetObjectFileName(int64_t ObjectID, + bool EnsureDirectoryExists) +{ + std::string filename; + StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, + filename, EnsureDirectoryExists); + return filename; +} + + +int RaidBackupFileSystem::GetBlockSize() +{ + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(mStoreDiscSet)); + return rdiscSet.GetBlockSize(); +} + + +std::auto_ptr RaidBackupFileSystem::GetBackupStoreInfoInternal(bool ReadOnly) +{ + // Generate the filename + std::string fn(mAccountRootDir + INFO_FILENAME); + + // Open the file for reading (passing on optional request for revision ID) + std::auto_ptr rf(RaidFileRead::Open(mStoreDiscSet, fn)); + std::auto_ptr info = BackupStoreInfo::Load(*rf, fn, ReadOnly); + + // Check it + if(info->GetAccountID() != mAccountID) + { + THROW_FILE_ERROR("Found wrong account ID in store info", + fn, BackupStoreException, BadStoreInfoOnLoad); + } + + return info; +} + + +void RaidBackupFileSystem::PutBackupStoreInfo(BackupStoreInfo& rInfo) +{ + if(rInfo.IsReadOnly()) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, StoreInfoIsReadOnly, + "Tried to save BackupStoreInfo when configured as read-only"); + } + + std::string filename(mAccountRootDir + INFO_FILENAME); + RaidFileWrite rf(mStoreDiscSet, filename); + rf.Open(true); // AllowOverwrite + rInfo.Save(rf); + + // Commit it to disc, converting it to RAID now + rf.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} + + +BackupStoreRefCountDatabase& RaidBackupFileSystem::GetPermanentRefCountDatabase( + bool ReadOnly) +{ + if(mapPermanentRefCountDatabase.get()) + { + return *mapPermanentRefCountDatabase; + } + + // It's dangerous to have two read-write databases open at the same time (it would + // be too easy to update the refcounts in the wrong one by mistake), and temporary + // databases are always read-write, so if a temporary database is already open + // then we should only allow a read-only permanent database to be opened. + ASSERT(!mapPotentialRefCountDatabase.get() || ReadOnly); + + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + mapPermanentRefCountDatabase = BackupStoreRefCountDatabase::Load(account, + ReadOnly); + return *mapPermanentRefCountDatabase; +} + + +BackupStoreRefCountDatabase& RaidBackupFileSystem::GetPotentialRefCountDatabase() +{ + // Creating the "official" potential refcount DB is actually a change + // to the store, even if you don't commit it, because it's in the same + // directory and would conflict with another process trying to do the + // same thing, so it requires that you hold the write lock. + ASSERT(mWriteLock.GotLock()); + + if(mapPotentialRefCountDatabase.get()) + { + return *mapPotentialRefCountDatabase; + } + + // It's dangerous to have two read-write databases open at the same + // time (it would be too easy to update the refcounts in the wrong one + // by mistake), and temporary databases are always read-write, so if a + // permanent database is already open then it must be a read-only one. + ASSERT(!mapPermanentRefCountDatabase.get() || + mapPermanentRefCountDatabase->IsReadOnly()); + + // We deliberately do not give the caller control of the + // reuse_existing_file parameter to Create(), because that would make + // it easy to bypass the restriction of only one (committable) + // temporary database at a time, and to accidentally overwrite the main + // refcount DB. + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + std::auto_ptr ap_new_db = + BackupStoreRefCountDatabase::Create(account); + mapPotentialRefCountDatabase.reset( + new BackupStoreRefCountDatabaseWrapper(ap_new_db, *this)); + + return *mapPotentialRefCountDatabase; +} + + +void RaidBackupFileSystem::SaveRefCountDatabase(BackupStoreRefCountDatabase& refcount_db) +{ + // You can only save the permanent database. + ASSERT(&refcount_db == mapPermanentRefCountDatabase.get()); + + // The database is already saved in the store, so we don't need to do anything. +} + + +//! Returns whether an object (a file or directory) exists with this object ID, and its +//! revision ID, which for a RaidFile is based on its timestamp and file size. +bool RaidBackupFileSystem::ObjectExists(int64_t ObjectID, int64_t *pRevisionID) +{ + // Don't bother creating the containing directory if it doesn't exist. + std::string filename = GetObjectFileName(ObjectID, false); + return RaidFileRead::FileExists(mStoreDiscSet, filename, pRevisionID); +} + + +//! Reads a directory with the specified ID into the supplied BackupStoreDirectory +//! object, also initialising its revision ID and SizeInBlocks. +void RaidBackupFileSystem::GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut) +{ + int64_t revID = 0; + // Don't bother creating the containing directory if it doesn't exist. + std::string filename = GetObjectFileName(ObjectID, false); + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, + filename, &revID)); + + // Read it from the stream, then set it's revision ID + BufferedStream buf(*objectFile); + rDirOut.ReadFromStream(buf, IOStream::TimeOutInfinite); + rDirOut.SetRevisionID(revID); + + // Make sure the size of the directory is available for writing the dir back + int64_t dirSize = objectFile->GetDiscUsageInBlocks(); + ASSERT(dirSize > 0); + rDirOut.SetUserInfo1_SizeInBlocks(dirSize); +} + + +void RaidBackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + // Create the containing directory if it doesn't exist. + std::string filename = GetObjectFileName(rDir.GetObjectID(), true); + RaidFileWrite writeDir(mStoreDiscSet, filename); + writeDir.Open(true); // allow overwriting + + BufferedWriteStream buffer(writeDir); + rDir.WriteToStream(buffer); + buffer.Flush(); + + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + int64_t revid = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revid)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + rDir.SetUserInfo1_SizeInBlocks(dirSize); + rDir.SetRevisionID(revid); +} + + +class RaidPutFileCompleteTransaction : public BackupFileSystem::Transaction +{ +private: + RaidFileWrite mStoreFile; + std::string mFileName; +#ifndef BOX_RELEASE_BUILD + int mDiscSet; +#endif + bool mCommitted; + +public: + RaidPutFileCompleteTransaction(int StoreDiscSet, const std::string& filename, + BackupStoreRefCountDatabase::refcount_t refcount) + : mStoreFile(StoreDiscSet, filename, refcount), + mFileName(filename), +#ifndef BOX_RELEASE_BUILD + mDiscSet(StoreDiscSet), +#endif + mCommitted(false), + mNumBlocks(-1) + { } + ~RaidPutFileCompleteTransaction(); + virtual void Commit(); + virtual int64_t GetNumBlocks() + { + ASSERT(mNumBlocks != -1); + return mNumBlocks; + } + RaidFileWrite& GetRaidFile() { return mStoreFile; } + + // It doesn't matter what we return here, because this should never be called + // for a PutFileCompleteTransaction (the API is intended for + // PutFilePatchTransaction instead): + virtual bool IsNewFileIndependent() { return false; } + + int64_t mNumBlocks; +}; + + +void RaidPutFileCompleteTransaction::Commit() +{ + ASSERT(!mCommitted); + mStoreFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + +#ifndef BOX_RELEASE_BUILD + // Verify the file -- only necessary for non-diffed versions. + // + // We cannot use VerifyEncodedFileFormat() until the file is committed. We already + // verified it as we were saving it, so this is a double check that should not be + // necessary, and thus is only done in debug builds. + std::auto_ptr checkFile(RaidFileRead::Open(mDiscSet, mFileName)); + if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) + { + mStoreFile.Delete(); + THROW_EXCEPTION_MESSAGE(BackupStoreException, AddedFileDoesNotVerify, + "Newly saved file does not verify after write: this should not " + "happen: " << mFileName); + } +#endif // !BOX_RELEASE_BUILD + + mCommitted = true; +} + + +RaidPutFileCompleteTransaction::~RaidPutFileCompleteTransaction() +{ + if(!mCommitted) + { + mStoreFile.Discard(); + } +} + + +std::auto_ptr +RaidBackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData, + BackupStoreRefCountDatabase::refcount_t refcount) +{ + // Create the containing directory if it doesn't exist. + std::string filename = GetObjectFileName(ObjectID, true); + + // We can only do this when the file (ObjectID) doesn't already exist. + ASSERT(refcount == 0); + + RaidPutFileCompleteTransaction* pTrans = new RaidPutFileCompleteTransaction( + mStoreDiscSet, filename, refcount); + std::auto_ptr apTrans(pTrans); + + RaidFileWrite& rStoreFile(pTrans->GetRaidFile()); + rStoreFile.Open(false); // no overwriting + + BackupStoreFile::VerifyStream validator(rFileData); + + // A full file, just store to disc + if(!validator.CopyStreamTo(rStoreFile, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut); + } + + // Close() is necessary to perform final validation on the block index. + validator.Close(false); // Don't CloseCopyStream, RaidFile doesn't like it. + + // Need to do this before committing the RaidFile, can't do it after. + pTrans->mNumBlocks = rStoreFile.GetDiscUsageInBlocks(); + + return apTrans; +} + + +class RaidPutFilePatchTransaction : public BackupFileSystem::Transaction +{ +private: + RaidFileWrite mNewCompleteFile; + RaidFileWrite mReversedPatchFile; + bool mReversedDiffIsCompletelyDifferent; + int64_t mBlocksUsedByNewFile; + int64_t mChangeInBlocksUsedByOldFile; + +public: + RaidPutFilePatchTransaction(int StoreDiscSet, + const std::string& newCompleteFilename, + const std::string& reversedPatchFilename, + BackupStoreRefCountDatabase::refcount_t refcount) + // It's not quite true that mNewCompleteFile has 1 reference: it doesn't exist + // yet, so it has 0 right now. However when the transaction is committed it will + // have 1, and RaidFileWrite gets upset if we try to modify a file with != 1 + // references, so we need to pretend now that we already have the reference. + : mNewCompleteFile(StoreDiscSet, newCompleteFilename, 1), + mReversedPatchFile(StoreDiscSet, reversedPatchFilename, refcount), + mReversedDiffIsCompletelyDifferent(false), + mBlocksUsedByNewFile(0), + mChangeInBlocksUsedByOldFile(0) + { } + virtual void Commit(); + RaidFileWrite& GetNewCompleteFile() { return mNewCompleteFile; } + RaidFileWrite& GetReversedPatchFile() { return mReversedPatchFile; } + void SetReversedDiffIsCompletelyDifferent(bool IsCompletelyDifferent) + { + mReversedDiffIsCompletelyDifferent = IsCompletelyDifferent; + } + virtual bool IsNewFileIndependent() + { + return mReversedDiffIsCompletelyDifferent; + } + void SetBlocksUsedByNewFile(int64_t BlocksUsedByNewFile) + { + mBlocksUsedByNewFile = BlocksUsedByNewFile; + } + virtual int64_t GetNumBlocks() + { + return mBlocksUsedByNewFile; + } + void SetChangeInBlocksUsedByOldFile(int64_t ChangeInBlocksUsedByOldFile) + { + mChangeInBlocksUsedByOldFile = ChangeInBlocksUsedByOldFile; + } + virtual int64_t GetChangeInBlocksUsedByOldFile() + { + return mChangeInBlocksUsedByOldFile; + } +}; + + +void RaidPutFilePatchTransaction::Commit() +{ + mNewCompleteFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + mReversedPatchFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} + + +std::auto_ptr +RaidBackupFileSystem::PutFilePatch(int64_t ObjectID, int64_t DiffFromFileID, + IOStream& rPatchData, BackupStoreRefCountDatabase::refcount_t refcount) +{ + // Create the containing directory if it doesn't exist. + std::string newVersionFilename = GetObjectFileName(ObjectID, true); + + // Filename of the old version + std::string oldVersionFilename = GetObjectFileName(DiffFromFileID, + false); // no need to make sure the directory it's in exists + + RaidPutFilePatchTransaction* pTrans = new RaidPutFilePatchTransaction( + mStoreDiscSet, newVersionFilename, oldVersionFilename, refcount); + std::auto_ptr apTrans(pTrans); + + RaidFileWrite& rNewCompleteFile(pTrans->GetNewCompleteFile()); + RaidFileWrite& rReversedPatchFile(pTrans->GetReversedPatchFile()); + + rNewCompleteFile.Open(false); // no overwriting + + // Diff file, needs to be recreated. + // Choose a temporary filename. + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, + newVersionFilename + ".difftemp", + 1)); // NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily + + try + { + // Open it twice +#ifdef WIN32 + InvisibleTempFileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_BINARY); + InvisibleTempFileStream diff2(tempFn.c_str(), O_RDWR | O_BINARY); +#else + FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); + FileStream diff2(tempFn.c_str(), O_RDONLY); + + // Unlink it immediately, so it definitely goes away + if(EMU_UNLINK(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif + + // Stream the incoming diff to this temporary file + if(!rPatchData.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut); + } + + // Verify the diff + diff.Seek(0, IOStream::SeekType_Absolute); + if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify); + } + + // Seek to beginning of diff file + diff.Seek(0, IOStream::SeekType_Absolute); + + // Reassemble that diff -- open previous file, and combine the patch and file + std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + BackupStoreFile::CombineFile(diff, diff2, *from, rNewCompleteFile); + + // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) + std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + rReversedPatchFile.Open(true); // allow overwriting + from->Seek(0, IOStream::SeekType_Absolute); + diff.Seek(0, IOStream::SeekType_Absolute); + + bool isCompletelyDifferent; + BackupStoreFile::ReverseDiffFile(diff, *from, *from2, rReversedPatchFile, + DiffFromFileID, &isCompletelyDifferent); + pTrans->SetReversedDiffIsCompletelyDifferent(isCompletelyDifferent); + + // Store disc space used + int64_t oldVersionNewBlocksUsed = + rReversedPatchFile.GetDiscUsageInBlocks(); + + // And make a space adjustment for the size calculation + int64_t spaceSavedByConversionToPatch = from->GetDiscUsageInBlocks() - + oldVersionNewBlocksUsed; + pTrans->SetChangeInBlocksUsedByOldFile(-spaceSavedByConversionToPatch); + pTrans->SetBlocksUsedByNewFile(rNewCompleteFile.GetDiscUsageInBlocks()); + return apTrans; + + // Everything cleans up here... + } + catch(...) + { + // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway + EMU_UNLINK(tempFn.c_str()); + throw; + } +} + + +std::auto_ptr RaidBackupFileSystem::GetObject(int64_t ObjectID, bool required) +{ + std::string filename = GetObjectFileName(ObjectID, + false); // no need to make sure the directory it's in exists. + + if(!required && !RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + return std::auto_ptr(); + } + + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, + filename)); + return static_cast >(objectFile); +} + + +std::auto_ptr RaidBackupFileSystem::GetFilePatch(int64_t ObjectID, + std::vector& rPatchChain) +{ + // File exists, but is a patch from a new version. Generate the older version. + // The last entry in the chain is the full file, the others are patches back from it. + // Open the last one, which is the current full file. + std::auto_ptr from(GetFile(rPatchChain[rPatchChain.size() - 1])); + + // Then, for each patch in the chain, do a combine + for(int p = ((int)rPatchChain.size()) - 2; p >= 0; --p) + { + // ID of patch + int64_t patchID = rPatchChain[p]; + + // Open it a couple of times. TODO FIXME: this could be very inefficient. + std::auto_ptr diff(GetFile(patchID)); + std::auto_ptr diff2(GetFile(patchID)); + + // Choose a temporary filename for the result of the combination + std::ostringstream fs; + fs << mAccountRootDir << ".recombinetemp." << p; + std::string tempFn = + RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, + fs.str(), p + 16); + + // Open the temporary file + std::auto_ptr combined( + new InvisibleTempFileStream( + tempFn, O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); + + // Do the combining + BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); + + // Move to the beginning of the combined file + combined->Seek(0, IOStream::SeekType_Absolute); + + // Then shuffle round for the next go + if(from.get()) + { + from->Close(); + } + + from = combined; + } + + std::auto_ptr stream( + BackupStoreFile::ReorderFileToStreamOrder(from.get(), + true)); // take ownership + + // Release from file to avoid double deletion + from.release(); + + return stream; +} + + +// Delete an object whose type is unknown. For RaidFile, we don't need to know what type +// it is to use RaidFileWrite::Delete() on it. +void RaidBackupFileSystem::DeleteObjectUnknown(int64_t ObjectID) +{ + std::string filename = GetObjectFileName(ObjectID, false); + RaidFileWrite deleteFile(mStoreDiscSet, filename); + deleteFile.Delete(); +} + + +std::auto_ptr +RaidBackupFileSystem::CombineFileOrDiff(int64_t OlderPatchID, int64_t NewerObjectID, bool NewerIsPatch) +{ + // This normally only happens during housekeeping, which is using a temporary + // refcount database, so insist on that for now. + BackupStoreRefCountDatabase* pRefCount = mapPotentialRefCountDatabase.get(); + ASSERT(pRefCount != NULL); + ASSERT(mapPermanentRefCountDatabase.get() == NULL || + mapPermanentRefCountDatabase->IsReadOnly()); + + // Open the older object twice (the patch) + std::auto_ptr pdiff = GetFile(OlderPatchID); + std::auto_ptr pdiff2 = GetFile(OlderPatchID); + + // Open the newer object (the file to be deleted) + std::auto_ptr pobjectBeingDeleted = GetFile(NewerObjectID); + + // And open a write file to overwrite the older object (the patch) + std::string older_filename = GetObjectFileName(OlderPatchID, + false); // no need to make sure the directory it's in exists. + + std::auto_ptr + ap_overwrite_older(new RaidPutFileCompleteTransaction( + mStoreDiscSet, older_filename, + pRefCount->GetRefCount(OlderPatchID))); + RaidFileWrite& overwrite_older(ap_overwrite_older->GetRaidFile()); + overwrite_older.Open(true /* allow overwriting */); + + if(NewerIsPatch) + { + // Combine two adjacent patches (reverse diffs) into a single one object. + BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, overwrite_older); + } + else + { + // Combine an older patch (reverse diff) with the subsequent complete file. + BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, overwrite_older); + } + + // Need to do this before committing the RaidFile, can't do it after. + ap_overwrite_older->mNumBlocks = overwrite_older.GetDiscUsageInBlocks(); + + // The file will be committed later when the directory is safely commited. + return static_cast >(ap_overwrite_older); +} + + +std::auto_ptr +RaidBackupFileSystem::CombineFile(int64_t OlderPatchID, int64_t NewerFileID) +{ + return CombineFileOrDiff(OlderPatchID, NewerFileID, false); // !NewerIsPatch +} + + +std::auto_ptr +RaidBackupFileSystem::CombineDiffs(int64_t OlderPatchID, int64_t NewerPatchID) +{ + return CombineFileOrDiff(OlderPatchID, NewerPatchID, true); // NewerIsPatch +} + + +int64_t RaidBackupFileSystem::GetFileSizeInBlocks(int64_t ObjectID) +{ + std::string filename = GetObjectFileName(ObjectID, false); + std::auto_ptr apRead = RaidFileRead::Open(mStoreDiscSet, filename); + return apRead->GetDiscUsageInBlocks(); +} + + +BackupFileSystem::CheckObjectsResult +RaidBackupFileSystem::CheckObjects(bool fix_errors) +{ + // Find the maximum start ID of directories -- worked out by looking at disc + // contents, not trusting anything. + CheckObjectsResult result; + CheckObjectsScanDir(0, 1, mAccountRootDir, result, fix_errors); + + // Then go through and scan all the objects within those directories + for(int64_t start_id = 0; start_id <= result.maxObjectIDFound; + start_id += (1<= '0' && String[0] <= '9') + { + n = (String[0] - '0') << 4; + } + else if(String[0] >= 'a' && String[0] <= 'f') + { + n = ((String[0] - 'a') + 0xa) << 4; + } + else + { + return false; + } + // Char 1 + if(String[1] >= '0' && String[1] <= '9') + { + n |= String[1] - '0'; + } + else if(String[1] >= 'a' && String[1] <= 'f') + { + n |= (String[1] - 'a') + 0xa; + } + else + { + return false; + } + + // Return a valid number + rNumberOut = n; + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: RaidBackupFileSystem::CheckObjectsScanDir( +// int64_t StartID, int Level, +// const std::string &rDirName, +// BackupFileSystem::CheckObjectsResult& Result, +// bool fix_errors) +// Purpose: Read in the contents of the directory, recurse to +// lower levels, return the maximum starting ID of any +// directory found. Internal method. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void RaidBackupFileSystem::CheckObjectsScanDir(int64_t StartID, int Level, + const std::string &rDirName, BackupFileSystem::CheckObjectsResult& Result, + bool fix_errors) +{ + //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID); + + if(Result.maxObjectIDFound < StartID) + { + Result.maxObjectIDFound = StartID; + } + + // Read in all the directories, and recurse downwards. + // If any of the directories is missing, create it. + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mStoreDiscSet)); + + if(!rdiscSet.IsNonRaidSet()) + { + unsigned int numDiscs = rdiscSet.size(); + + for(unsigned int l = 0; l < numDiscs; ++l) + { + // build name + std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); + EMU_STRUCT_STAT st; + + if(EMU_STAT(dn.c_str(), &st) != 0 && + errno == ENOENT) + { + if(mkdir(dn.c_str(), 0755) != 0) + { + THROW_SYS_FILE_ERROR("Failed to create missing " + "RaidFile directory", dn, + RaidFileException, OSError); + } + } + } + } + + std::vector dirs; + RaidFileRead::ReadDirectoryContents(mStoreDiscSet, rDirName, + RaidFileRead::DirReadType_DirsOnly, dirs); + + for(std::vector::const_iterator i(dirs.begin()); + i != dirs.end(); ++i) + { + // Check to see if it's the right name + int n = 0; + if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) + && n < (1< 4 && + (dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR || + dirName[dirName.size() - 4] == '/')); + // Remove the filename from it + dirName.resize(dirName.size() - 4); // four chars for "/o00" + + // Check directory exists + if(!RaidFileRead::DirectoryExists(mStoreDiscSet, dirName)) + { + BOX_ERROR("RaidFile dir " << dirName << " does not exist"); + Result.numErrorsFound++; + return; + } + + // Read directory contents + std::vector files; + RaidFileRead::ReadDirectoryContents(mStoreDiscSet, dirName, + RaidFileRead::DirReadType_FilesOnly, files); + + // Parse each entry, building up a list of object IDs which are present in the dir. + // This is done so that whatever order is retured from the directory, objects are scanned + // in order. + // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted. + for(std::vector::const_iterator i(files.begin()); i != files.end(); ++i) + { + bool fileOK = true; + int n = 0; + if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) + && n < (1< read( + RaidFileRead::Open(mStoreDiscSet, + filename)); + RaidFileWrite write(mStoreDiscSet, filename); + write.Open(true /* overwrite */); + read->CopyStreamTo(write); + read.reset(); + write.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + } + } + } +} + + +S3BackupFileSystem::S3BackupFileSystem(const Configuration& config, + const std::string& BasePath, const std::string& CacheDirectory, S3Client& rClient) +: mrConfig(config), + mBasePath(BasePath), + mCacheDirectory(CacheDirectory), + mrClient(rClient), + mHaveLock(false) +{ + if(mBasePath.size() == 0 || mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "mBasePath is invalid: must start and end with a slash (/)"); + } + + const Configuration s3config = config.GetSubConfiguration("S3Store"); + const std::string& s3_hostname(s3config.GetKeyValue("HostName")); + const std::string& s3_base_path(s3config.GetKeyValue("BasePath")); + mSimpleDBDomain = s3config.GetKeyValue("SimpleDBDomain"); + + // The lock name should be the same for all hosts/files/daemons potentially + // writing to the same region of the S3 store. The default is the Amazon S3 bucket + // name and path, concatenated. + mLockName = s3config.GetKeyValueDefault("SimpleDBLockName", + s3_hostname + s3_base_path); + + // The lock value should be unique for each host potentially accessing the same + // region of the store, and should help you to identify which one is currently + // holding the lock. The default is username@hostname(pid). +#if HAVE_DECL_GETUSERNAMEA + char username_buffer[UNLEN + 1]; + DWORD buffer_size = sizeof(username_buffer); + if(!GetUserNameA(username_buffer, &buffer_size)) + { + THROW_WIN_ERROR("Failed to GetUserName()", CommonException, Internal); + } + mCurrentUserName = username_buffer; +#elif defined HAVE_GETPWUID + mCurrentUserName = getpwuid(getuid())->pw_name; +#else +# error "Don't know how to get current user name" +#endif + + char hostname_buf[1024]; + if(gethostname(hostname_buf, sizeof(hostname_buf)) != 0) + { + THROW_SOCKET_ERROR("Failed to get local hostname", CommonException, Internal); + } + mCurrentHostName = hostname_buf; + + std::ostringstream lock_value_buf; + lock_value_buf << mCurrentUserName << "@" << hostname_buf << "(" << getpid() << + ")"; + mLockValue = s3config.GetKeyValueDefault("SimpleDBLockValue", + lock_value_buf.str()); +} + + +int S3BackupFileSystem::GetBlockSize() +{ + return S3_NOTIONAL_BLOCK_SIZE; +} + + +std::string S3BackupFileSystem::GetAccountIdentifier() +{ + std::string name; + + try + { + name = GetBackupStoreInfo(true).GetAccountName(); + } + catch(HTTPException &e) + { + if(EXCEPTION_IS_TYPE(e, HTTPException, FileNotFound)) + { + std::string info_uri = GetMetadataURI(S3_INFO_FILE_NAME); + std::string info_url = GetObjectURL(info_uri); + return "unknown (BackupStoreInfo file not found) at " + info_url; + } + else + { + throw; + } + } + + std::ostringstream oss; + oss << "'" << name << "'"; + return oss.str(); +} + + +std::string S3BackupFileSystem::GetObjectURL(const std::string& ObjectPath) const +{ + const Configuration s3config = mrConfig.GetSubConfiguration("S3Store"); + return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + + s3config.GetKeyValue("Port") + ObjectPath; +} + + +int64_t S3BackupFileSystem::GetRevisionID(const std::string& uri, + HTTPResponse& response) const +{ + std::string etag; + + if(!response.GetHeader("etag", &etag)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, MissingEtagHeader, + "Failed to get the MD5 checksum of the file or directory " + "at this URL: " << GetObjectURL(uri)); + } + + if(etag[0] != '"') + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, InvalidEtagHeader, + "Failed to get the MD5 checksum of the file or directory " + "at this URL: " << GetObjectURL(uri)); + } + + const char * pEnd = NULL; + std::string checksum = etag.substr(1, 16); + int64_t revID = box_strtoui64(checksum.c_str(), &pEnd, 16); + if(*pEnd != '\0') + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, InvalidEtagHeader, + "Failed to get the MD5 checksum of the file or " + "directory at this URL: " << uri << ": invalid character '" << + *pEnd << "' in '" << etag << "'"); + } + + return revID; +} + + +// TODO FIXME ADD CACHING! +std::auto_ptr S3BackupFileSystem::GetBackupStoreInfoInternal(bool ReadOnly) +{ + std::string info_uri = GetMetadataURI(S3_INFO_FILE_NAME); + std::string info_url = GetObjectURL(info_uri); + HTTPResponse response = mrClient.GetObject(info_uri); + mrClient.CheckResponse(response, std::string("No BackupStoreInfo file exists " + "at this URL: ") + info_url); + + std::auto_ptr info = BackupStoreInfo::Load(response, info_url, + ReadOnly); + + // We don't actually use AccountID to distinguish accounts on S3 stores. + if(info->GetAccountID() != S3_FAKE_ACCOUNT_ID) + { + THROW_FILE_ERROR("Found wrong account ID in store info", + info_url, BackupStoreException, BadStoreInfoOnLoad); + } + + return info; +} + + +void S3BackupFileSystem::PutBackupStoreInfo(BackupStoreInfo& rInfo) +{ + if(rInfo.IsReadOnly()) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, StoreInfoIsReadOnly, + "Tried to save BackupStoreInfo when configured as read-only"); + } + + CollectInBufferStream out; + rInfo.Save(out); + out.SetForReading(); + + std::string info_uri = GetMetadataURI(S3_INFO_FILE_NAME); + HTTPResponse response = mrClient.PutObject(info_uri, out); + + std::string info_url = GetObjectURL(info_uri); + mrClient.CheckResponse(response, std::string("Failed to upload the new " + "BackupStoreInfo file to this URL: ") + info_url); +} + + +void S3BackupFileSystem::GetCacheLock() +{ + if(!mCacheLock.GotLock()) + { + std::string cache_lockfile_name = mCacheDirectory + DIRECTORY_SEPARATOR + + S3_CACHE_LOCK_NAME; + if(!mCacheLock.TryAndGetLock(cache_lockfile_name)) + { + THROW_FILE_ERROR("Cache directory is locked by another process", + mCacheDirectory, BackupStoreException, CacheDirectoryLocked); + } + } +} + + +BackupStoreRefCountDatabase& S3BackupFileSystem::GetPermanentRefCountDatabase( + bool ReadOnly) +{ + if(mapPermanentRefCountDatabase.get()) + { + return *mapPermanentRefCountDatabase; + } + + // It's dangerous to have two read-write databases open at the same time (it would + // be too easy to update the refcounts in the wrong one by mistake), and potential + // databases are always read-write, so if a potential database is already open + // then we should only allow a read-only permanent database to be opened. + ASSERT(!mapPotentialRefCountDatabase.get() || ReadOnly); + + GetCacheLock(); + + // If we have a cached copy, check that it's up to date. + std::string local_path = GetRefCountDatabaseCachePath(); + std::string digest; + + if(FileExists(local_path)) + { + FileStream fs(local_path); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + digester.Close(); + digest = digester.DigestAsString(); + } + + // Try to fetch it from the remote server. If we pass a digest, and if it matches, + // then the server won't send us the same file data again. + std::string uri = GetMetadataURI(S3_REFCOUNT_FILE_NAME); + HTTPResponse response = mrClient.GetObject(uri, digest); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + if(digest.size() > 0) + { + BOX_WARNING("We had a cached copy of the refcount DB, but it " + "didn't match the one in S3, so the server sent us a new " + "copy and the cache will be updated"); + } + + FileStream fs(local_path, O_CREAT | O_RDWR); + response.CopyStreamTo(fs); + + // Check that we got the full and correct file. TODO: calculate the MD5 + // digest while writing the file, instead of afterwards. + fs.Seek(0, IOStream::SeekType_Absolute); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + digester.Close(); + digest = digester.DigestAsString(); + + if(response.GetHeaders().GetHeaderValue("etag") != + "\"" + digest + "\"") + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + FileDownloadedIncorrectly, "Downloaded invalid file from " + "S3: expected MD5 digest to be " << + response.GetHeaders().GetHeaderValue("etag") << " but " + "it was actually " << digest); + } + } + else if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + // Do not create a new refcount DB here, it is not safe! Users may wonder + // why they have lost all their files, and/or unwittingly overwrite their + // backup data. + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CorruptReferenceCountDatabase, "Reference count database is " + "missing, cannot safely open account. Please run bbstoreaccounts " + "check on the account to recreate it. 404 Not Found: " << uri); + } + else if(response.GetResponseCode() == HTTPResponse::Code_NotModified) + { + // The copy on the server is the same as the one in our cache, so we don't + // need to download it again, nothing to do! + } + else + { + mrClient.CheckResponse(response, "Failed to download reference " + "count database"); + } + + mapPermanentRefCountDatabase = BackupStoreRefCountDatabase::Load(local_path, + S3_FAKE_ACCOUNT_ID, ReadOnly); + return *mapPermanentRefCountDatabase; +} + + +BackupStoreRefCountDatabase& S3BackupFileSystem::GetPotentialRefCountDatabase() +{ + // Creating the "official" temporary refcount DB is actually a change + // to the cache, even if you don't commit it, because it's in the same + // directory and would conflict with another process trying to do the + // same thing, so it requires that you hold the write and cache locks. + ASSERT(mHaveLock); + + if(mapPotentialRefCountDatabase.get()) + { + return *mapPotentialRefCountDatabase; + } + + // It's dangerous to have two read-write databases open at the same time (it would + // be too easy to update the refcounts in the wrong one by mistake), and temporary + // databases are always read-write, so if a permanent database is already open + // then it must be a read-only one. + ASSERT(!mapPermanentRefCountDatabase.get() || + mapPermanentRefCountDatabase->IsReadOnly()); + + GetCacheLock(); + + // The temporary database cannot be on the remote server, so there is no need to + // download it into the cache. Just create one and return it. + std::string local_path = GetRefCountDatabaseCachePath(); + + // We deliberately do not give the caller control of the + // reuse_existing_file parameter to Create(), because that would make + // it easy to bypass the restriction of only one (committable) + // temporary database at a time, and to accidentally overwrite the main + // refcount DB. + std::auto_ptr ap_new_db = + BackupStoreRefCountDatabase::Create(local_path, S3_FAKE_ACCOUNT_ID); + mapPotentialRefCountDatabase.reset( + new BackupStoreRefCountDatabaseWrapper(ap_new_db, *this)); + + return *mapPotentialRefCountDatabase; +} + + +void S3BackupFileSystem::SaveRefCountDatabase(BackupStoreRefCountDatabase& refcount_db) +{ + // You can only save the permanent database. + ASSERT(&refcount_db == mapPermanentRefCountDatabase.get()); + + std::string local_path = GetRefCountDatabaseCachePath(); + FileStream fs(local_path, O_RDONLY); + + // Try to upload it to the remote server. + HTTPResponse response = mrClient.PutObject(GetMetadataURI(S3_REFCOUNT_FILE_NAME), + fs); + mrClient.CheckResponse(response, "Failed to upload refcount db to S3"); +} + + +//! Returns whether an object (a file or directory) exists with this object ID, and its +//! revision ID, which for a RaidFile is based on its timestamp and file size. +//! +//! TODO FIXME: we should probably return a hint of whether the object is a file or a +//! directory (because we know), as BackupStoreCheck could use this to avoid repeated +//! wasted requests for the object as a file or a directory, when it's already known that +//! it has a different type. We should also use a cache for directory listings! +bool S3BackupFileSystem::ObjectExists(int64_t ObjectID, int64_t *pRevisionID) +{ + std::string uri = GetDirectoryURI(ObjectID); + HTTPResponse response = mrClient.HeadObject(uri); + + if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + // A file might exist, check that too. + uri = GetFileURI(ObjectID); + response = mrClient.HeadObject(uri); + } + + if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + return false; + } + + if(response.GetResponseCode() != HTTPResponse::Code_OK) + { + // Throw an appropriate exception. + mrClient.CheckResponse(response, std::string("Failed to check whether " + "a file or directory exists at this URL: ") + + GetObjectURL(uri)); + } + + if(pRevisionID) + { + *pRevisionID = GetRevisionID(uri, response); + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3BackupFileSystem::GetObjectURI(int64_t ObjectID, +// int Type) +// Purpose: Builds the object filename for a given object, +// including mBasePath. Very similar to +// StoreStructure::MakeObjectFilename(), but files and +// directories have different extensions, and the +// filename is the full object ID, not just the lower +// STORE_ID_SEGMENT_LENGTH bits. +// Created: 2016/03/21 +// +// -------------------------------------------------------------------------- +std::string S3BackupFileSystem::GetObjectURI(int64_t ObjectID, int Type) const +{ + const static char *hex = "0123456789abcdef"; + ASSERT(mBasePath.size() > 0 && mBasePath[0] == '/' && + mBasePath[mBasePath.size() - 1] == '/'); + std::ostringstream out; + out << mBasePath; + + // Get the id value from the stored object ID into an unsigned int64_t, so that + // we can do bitwise operations on it safely. + uint64_t id = (uint64_t)ObjectID; + + // Shift off the bits which make up the leafname + id >>= STORE_ID_SEGMENT_LENGTH; + + // build pathname + while(id != 0) + { + // assumes that the segments are no bigger than 8 bits + int v = id & STORE_ID_SEGMENT_MASK; + out << hex[(v & 0xf0) >> 4]; + out << hex[v & 0xf]; + out << "/"; + + // shift the bits we used off the pathname + id >>= STORE_ID_SEGMENT_LENGTH; + } + + // append the filename + out << BOX_FORMAT_OBJECTID(ObjectID); + if(Type == ObjectExists_File) + { + out << ".file"; + } + else if(Type == ObjectExists_Dir) + { + out << ".dir"; + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Unknown file type for object " << BOX_FORMAT_OBJECTID(ObjectID) << + ": " << Type); + } + + return out.str(); +} + + +//! Reads a directory with the specified ID into the supplied BackupStoreDirectory +//! object, also initialising its revision ID and SizeInBlocks. +void S3BackupFileSystem::GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut) +{ + std::string uri = GetDirectoryURI(ObjectID); + HTTPResponse response = mrClient.GetObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to download directory: ") + uri); + rDirOut.ReadFromStream(response, mrClient.GetNetworkTimeout()); + + rDirOut.SetRevisionID(GetRevisionID(uri, response)); + rDirOut.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(response.GetContentLength())); +} + + +//! Writes the supplied BackupStoreDirectory object to the store, and updates its revision +//! ID and SizeInBlocks. +void S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + CollectInBufferStream out; + rDir.WriteToStream(out); + out.SetForReading(); + + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.PutObject(uri, out); + mrClient.CheckResponse(response, + std::string("Failed to upload directory: ") + uri); + + rDir.SetRevisionID(GetRevisionID(uri, response)); + rDir.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(out.GetSize())); +} + + +void S3BackupFileSystem::DeleteDirectory(int64_t ObjectID) +{ + std::string uri = GetDirectoryURI(ObjectID); + HTTPResponse response = mrClient.DeleteObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to delete directory: ") + uri, + true); // ExpectNoContent +} + + +void S3BackupFileSystem::DeleteFile(int64_t ObjectID) +{ + std::string uri = GetFileURI(ObjectID); + HTTPResponse response = mrClient.DeleteObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to delete file: ") + uri, + true); // ExpectNoContent +} + +void S3BackupFileSystem::DeleteObjectUnknown(int64_t ObjectID) +{ + std::string uri = GetFileURI(ObjectID); + HTTPResponse response = mrClient.DeleteObject(uri); + // It might be a directory instead, try that before returning an error. + if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + std::string uri = GetDirectoryURI(ObjectID); + response = mrClient.DeleteObject(uri); + } + mrClient.CheckResponse(response, + std::string("Failed to delete file or directory: ") + uri, + true); // ExpectNoContent +} + + + +class S3PutFileCompleteTransaction : public BackupFileSystem::Transaction +{ +private: + S3Client& mrClient; + std::string mFileURI; + bool mCommitted; + int64_t mNumBlocks; + +public: + S3PutFileCompleteTransaction(S3BackupFileSystem& fs, S3Client& client, + const std::string& file_uri, IOStream& file_data); + ~S3PutFileCompleteTransaction(); + virtual void Commit(); + virtual int64_t GetNumBlocks() { return mNumBlocks; } + + // It doesn't matter what we return here, because this should never be called + // for a PutFileCompleteTransaction (the API is intended for + // PutFilePatchTransaction instead): + virtual bool IsNewFileIndependent() { return false; } +}; + + +S3PutFileCompleteTransaction::S3PutFileCompleteTransaction(S3BackupFileSystem& fs, + S3Client& client, const std::string& file_uri, IOStream& file_data) +: mrClient(client), + mFileURI(file_uri), + mCommitted(false), + mNumBlocks(0) +{ + ByteCountingStream counter(file_data); + HTTPResponse response = mrClient.PutObject(file_uri, counter); + if(response.GetResponseCode() != HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, FileUploadFailed, + "Failed to upload file to Amazon S3: " << + response.ResponseCodeString()); + } + mNumBlocks = fs.GetSizeInBlocks(counter.GetNumBytesRead()); +} + + +void S3PutFileCompleteTransaction::Commit() +{ + mCommitted = true; +} + + +S3PutFileCompleteTransaction::~S3PutFileCompleteTransaction() +{ + if(!mCommitted) + { + HTTPResponse response = mrClient.DeleteObject(mFileURI); + mrClient.CheckResponse(response, "Failed to delete uploaded file from Amazon S3", + true); // ExpectNoContent + } +} + + +std::auto_ptr +S3BackupFileSystem::PutFileComplete(int64_t ObjectID, IOStream& rFileData, + BackupStoreRefCountDatabase::refcount_t refcount) +{ + ASSERT(refcount == 0 || refcount == 1); + BackupStoreFile::VerifyStream validator(rFileData); + S3PutFileCompleteTransaction* pTrans = new S3PutFileCompleteTransaction(*this, + mrClient, GetFileURI(ObjectID), validator); + return std::auto_ptr(pTrans); +} + + +//! GetObject() can be for either a file or a directory, so we need to try both. +// TODO FIXME use a cached directory listing to determine which it is +std::auto_ptr S3BackupFileSystem::GetObject(int64_t ObjectID, bool required) +{ + std::string uri = GetFileURI(ObjectID); + std::auto_ptr ap_response( + new HTTPResponse(mrClient.GetObject(uri)) + ); + + if(ap_response->GetResponseCode() == HTTPResponse::Code_NotFound) + { + // It's not a file, try the directory URI instead. + uri = GetDirectoryURI(ObjectID); + ap_response.reset(new HTTPResponse(mrClient.GetObject(uri))); + } + + if(ap_response->GetResponseCode() == HTTPResponse::Code_NotFound) + { + if(required) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, ObjectDoesNotExist, + "Requested object " << BOX_FORMAT_OBJECTID(ObjectID) << " " + "does not exist, or is not a file or a directory"); + } + else + { + return std::auto_ptr(); + } + } + + mrClient.CheckResponse(*ap_response, + std::string("Failed to download requested file: " + uri)); + return static_cast >(ap_response); +} + + +std::auto_ptr S3BackupFileSystem::GetFile(int64_t ObjectID) +{ + std::string uri = GetFileURI(ObjectID); + std::auto_ptr ap_response( + new HTTPResponse(mrClient.GetObject(uri)) + ); + + if(ap_response->GetResponseCode() == HTTPResponse::Code_NotFound) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, ObjectDoesNotExist, + "Requested object " << BOX_FORMAT_OBJECTID(ObjectID) << " " + "does not exist, or is not a file."); + } + + mrClient.CheckResponse(*ap_response, + std::string("Failed to download requested file: " + uri)); + return static_cast >(ap_response); +} + + +void S3BackupFileSystem::ReportLockMismatches(str_map_diff_t mismatches) +{ + if(!mismatches.empty()) + { + std::ostringstream error_buf; + bool first_item = true; + for(str_map_diff_t::iterator i = mismatches.begin(); + i != mismatches.end(); i++) + { + if(!first_item) + { + error_buf << ", "; + } + first_item = false; + const std::string& name(i->first); + const std::string& expected(i->second.first); + const std::string& actual(i->second.second); + + error_buf << name << " was not '" << expected << "' but '" << + actual << "'"; + } + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Lock on '" << mLockName << + "' was concurrently modified: " << error_buf.str()); + } +} + + +void S3BackupFileSystem::TryGetLock() +{ + if(mHaveLock) + { + return; + } + + const Configuration s3config = mrConfig.GetSubConfiguration("S3Store"); + + if(!mapSimpleDBClient.get()) + { + mapSimpleDBClient.reset(new SimpleDBClient(s3config)); + // timeout left at the default 300 seconds. + } + + // Create the domain, to ensure that it exists. This is idempotent. + mapSimpleDBClient->CreateDomain(mSimpleDBDomain); + SimpleDBClient::str_map_t conditional; + + // Check to see whether someone already holds the lock + try + { + SimpleDBClient::str_map_t attributes; + { + HideSpecificExceptionGuard hex(HTTPException::ExceptionType, + HTTPException::SimpleDBItemNotFound); + attributes = mapSimpleDBClient->GetAttributes(mSimpleDBDomain, + mLockName); + } + + // This succeeded, which means that someone once held the lock. If the + // locked attribute is empty, then they released it cleanly, and we can + // access the account safely. + box_time_t since_time = box_strtoui64(attributes["since"].c_str(), NULL, 10); + + if(attributes["locked"] == "") + { + // The account was locked, but no longer. Make sure it stays that + // way, to avoid a race condition. + conditional = attributes; + } + // Otherwise, someone holds the lock right now. If the lock is held by + // this computer (same hostname) and the PID is no longer running, then + // it's reasonable to assume that we can override it because the original + // process is dead. + else if(attributes["hostname"] == mCurrentHostName) + { + char* end_ptr; + int locking_pid = strtol(attributes["pid"].c_str(), &end_ptr, 10); + if(*end_ptr != 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Failed to parse PID " + "from existing lock: " << attributes["pid"]); + } + + if(process_is_running(locking_pid)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Lock on '" << + mLockName << "' is held by '" << + attributes["locker"] << "' (process " << + locking_pid << " on this host, " << + mCurrentHostName << ", which is still running), " + "since " << FormatTime(since_time, + true)); // includeDate + } + else + { + BOX_WARNING( + "Lock on '" << mLockName << "' was held by '" << + attributes["locker"] << "' (process " << + locking_pid << " on this host, " << + mCurrentHostName << ", which appears to have ended) " + "since " << FormatTime(since_time, + true) // includeDate + << ", overriding it"); + conditional = attributes; + } + } + else + { + // If the account is locked by a process on a different host, then + // we have no way to check whether it is still running, so we can + // only give up. + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotLockStoreAccount, "Lock on '" << mLockName << + "' is held by '" << attributes["locker"] << " since " << + FormatTime(since_time, true)); // includeDate + } + } + catch(HTTPException &e) + { + if(EXCEPTION_IS_TYPE(e, HTTPException, SimpleDBItemNotFound)) + { + // The lock doesn't exist, so it's safe to create it. We can't + // make this request conditional, so there is a race condition + // here! We deal with that by reading back the attributes with + // a ConsistentRead after writing them. + } + else + { + // Something else went wrong. + throw; + } + } + + mLockAttributes["locked"] = "true"; + mLockAttributes["locker"] = mLockValue; + mLockAttributes["hostname"] = mCurrentHostName; + { + std::ostringstream pid_buf; + pid_buf << getpid(); + mLockAttributes["pid"] = pid_buf.str(); + } + { + std::ostringstream since_buf; + since_buf << GetCurrentBoxTime(); + mLockAttributes["since"] = since_buf.str(); + } + + // This will throw an exception if the conditional PUT fails: + mapSimpleDBClient->PutAttributes(mSimpleDBDomain, mLockName, mLockAttributes, + conditional); + + // To avoid the race condition, read back the attribute values with a consistent + // read, to check that nobody else sneaked in at the same time: + SimpleDBClient::str_map_t attributes_read = mapSimpleDBClient->GetAttributes( + mSimpleDBDomain, mLockName, true); // consistent_read + + str_map_diff_t mismatches = compare_str_maps(mLockAttributes, attributes_read); + + // This should throw an exception if there are any mismatches: + ReportLockMismatches(mismatches); + ASSERT(mismatches.empty()); + + // Now we have the lock! + mHaveLock = true; +} + + +void S3BackupFileSystem::ReleaseLock() +{ + BackupFileSystem::ReleaseLock(); + + // It's possible that neither the temporary nor the permanent refcount DB was + // requested while we had the write lock, so the cache lock may not have been + // acquired. + if(mCacheLock.GotLock()) + { + mCacheLock.ReleaseLock(); + } + + // Releasing is so much easier! + if(!mHaveLock) + { + return; + } + + // If we have a lock, we should also have the SimpleDBClient that we used to + // acquire it! + ASSERT(mapSimpleDBClient.get()); + + // Read the current values, and check that they match what we expected, i.e. that + // nobody stole the lock from under us + SimpleDBClient::str_map_t attributes_read = mapSimpleDBClient->GetAttributes( + mSimpleDBDomain, mLockName, true); // consistent_read + str_map_diff_t mismatches = compare_str_maps(mLockAttributes, attributes_read); + + // This should throw an exception if there are any mismatches: + ReportLockMismatches(mismatches); + ASSERT(mismatches.empty()); + + // Now write the same values back, except with "locked" = "" + mLockAttributes["locked"] = ""; + + // Conditional PUT, using the values that we just read, to ensure that nobody + // changes it under our feet right now. This will throw an exception if the + // conditional PUT fails: + mapSimpleDBClient->PutAttributes(mSimpleDBDomain, mLockName, mLockAttributes, + attributes_read); + + // Read back, to check that we unlocked successfully: + attributes_read = mapSimpleDBClient->GetAttributes(mSimpleDBDomain, mLockName, + true); // consistent_read + mismatches = compare_str_maps(mLockAttributes, attributes_read); + + // This should throw an exception if there are any mismatches: + ReportLockMismatches(mismatches); + ASSERT(mismatches.empty()); + + // Now we no longer have the lock! + mHaveLock = false; +} + + +S3BackupFileSystem::~S3BackupFileSystem() +{ + // Close any open refcount DBs before partially destroying the + // BackupFileSystem that they need to close down. Need to do this in + // the subclass to avoid calling SaveRefCountDatabase() when the + // subclass has already been partially destroyed. + // http://stackoverflow.com/questions/10707286/how-to-resolve-pure-virtual-method-called + mapPotentialRefCountDatabase.reset(); + mapPermanentRefCountDatabase.reset(); + + // This needs to be in the source file, not inline, as long as we don't include + // the whole of SimpleDBClient.h in BackupFileSystem.h. + if(mHaveLock) + { + try + { + ReleaseLock(); + } + catch(BoxException& e) + { + // Destructors aren't supposed to throw exceptions, so it's too late + // for us to do much except log a warning + BOX_WARNING("Failed to release a lock while destroying " + "S3BackupFileSystem: " << e.what()); + } + } +} + + +// TODO FIXME check if file is in cache and return size from there +int64_t S3BackupFileSystem::GetFileSizeInBlocks(int64_t ObjectID) +{ + std::string uri = GetFileURI(ObjectID); + HTTPResponse response = mrClient.HeadObject(uri); + return GetSizeInBlocks(response.GetContentLength()); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3BackupFileSystem::CheckObjects(bool fix_errors) +// Purpose: Scan directories on store recursively, counting +// files and identifying ones that should not exist, +// identify the highest object ID that exists, and +// check every object up to that ID. +// Created: 2016/03/21 +// +// -------------------------------------------------------------------------- + + +BackupFileSystem::CheckObjectsResult +S3BackupFileSystem::CheckObjects(bool fix_errors) +{ + // Find the maximum start ID of directories -- worked out by looking at list of + // objects on store, not trusting anything. + CheckObjectsResult result; + start_id_to_files_t start_id_to_files; + CheckObjectsScanDir(0, 1, "", result, fix_errors, start_id_to_files); + + // Then go through and scan all the objects within those directories + for(int64_t StartID = 0; StartID <= result.maxObjectIDFound; + StartID += (1< files; + std::vector dirs; + bool is_truncated; + int max_keys = 1000; + start_id_to_files[start_id] = std::vector(); + + // Remove the initial / from BasePath. It must also end with /, and dir_name + // must not start with / but must end with / (or be empty), so that when + // concatenated they make a valid URI with a trailing slash ("prefix"). + if(!StartsWith("/", mBasePath)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "BasePath must start and end with '/', but was: " << mBasePath); + } + std::string prefix = RemovePrefix("/", mBasePath); + + if(!EndsWith("/", mBasePath)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "BasePath must start and end with '/', but was: " << mBasePath); + } + + if(StartsWith("/", dir_name)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "dir_name must not start with '/': '" << dir_name << "'"); + } + + if(!dir_name.empty() && !EndsWith("/", dir_name)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "dir_name must be empty or end with '/': '" << dir_name << "'"); + } + prefix += dir_name; + ASSERT(EndsWith("/", prefix)); + + mrClient.ListBucket(&files, &dirs, prefix, "/", // delimiter + &is_truncated, max_keys); + if(is_truncated) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, TooManyFilesInDirectory, + "Failed to check directory: " << prefix << ": too many entries"); + } + + // Cache the list of files in this directory for later use. + for(std::vector::const_iterator i = files.begin(); + i != files.end(); i++) + { + // This will throw an exception if the filename does not start with + // the prefix: + std::string unprefixed_name = RemovePrefix(prefix, i->name()); + start_id_to_files[start_id].push_back(unprefixed_name); + } + + for(std::vector::const_iterator i(dirs.begin()); + i != dirs.end(); ++i) + { + std::string subdir_name = RemovePrefix(prefix, *i); + subdir_name = RemoveSuffix("/", subdir_name); + + // Check to see if it's the right name + int n = 0; + if(subdir_name.size() == 2 && TwoDigitHexToInt(subdir_name.c_str(), n) + && n < (1< files = start_id_to_files[start_id]; + std::vector files = start_id_to_files.find(start_id)->second; + + // Parse each entry, checking whether it has a valid name for an object + // file: must begin with '0x', followed by some hex digits and end with + // .file or .dir. The object ID must belong in this position in the + // directory hierarchy. Any other file present is an error, and if + // fix_errors is true, it will be deleted. + for(std::vector::const_iterator i(files.begin()); i != files.end(); ++i) + { + bool fileOK = false; + if(StartsWith("0x", (*i))) + { + std::string object_id_str; + if(EndsWith(".file", *i)) + { + object_id_str = RemoveSuffix(".file", *i); + } + else if(EndsWith(".dir", *i)) + { + object_id_str = RemoveSuffix(".dir", *i); + } + else + { + ASSERT(!fileOK); + } + + if(!object_id_str.empty()) + { + const char * p_end; + int64_t object_id = box_strtoui64(object_id_str.c_str() + 2, + &p_end, 16); + if(*p_end != 0) + { + ASSERT(!fileOK); + } + else if(object_id < start_id || + object_id >= start_id + (1< + +#include "autogen_BackupStoreException.h" +#include "BackupStoreRefCountDatabase.h" +#include "HTTPResponse.h" +#include "NamedLock.h" +#include "S3Client.h" +#include "SimpleDBClient.h" +#include "Utils.h" // for ObjectExists_* + +class BackupStoreDirectory; +class BackupStoreInfo; +class Configuration; +class IOStream; + +class FileSystemCategory : public Log::Category +{ + public: + FileSystemCategory(const std::string& name) + : Log::Category(std::string("FileSystem/") + name) + { } +}; + + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupFileSystem +// Purpose: Generic interface for reading and writing files and +// directories, abstracting over RaidFile, S3, FTP etc. +// Created: 2015/08/03 +// +// -------------------------------------------------------------------------- +class BackupFileSystem +{ +public: + class Transaction + { + public: + virtual ~Transaction() { } + virtual void Commit() = 0; + virtual int64_t GetNumBlocks() = 0; + virtual bool IsNewFileIndependent() = 0; + virtual int64_t GetChangeInBlocksUsedByOldFile() { return 0; } + }; + + BackupFileSystem() { } + virtual ~BackupFileSystem() { } + static const int KEEP_TRYING_FOREVER = -1; + virtual void GetLock(int max_tries = 8); + virtual void ReleaseLock(); + virtual bool HaveLock() = 0; + virtual int GetBlockSize() = 0; + virtual BackupStoreInfo& GetBackupStoreInfo(bool ReadOnly, bool Refresh = false); + virtual void PutBackupStoreInfo(BackupStoreInfo& rInfo) = 0; + + // DiscardBackupStoreInfo() discards the active BackupStoreInfo, invalidating any + // references to it! It is needed to allow a BackupStoreContext to be Finished, + // changes made to the BackupStoreInfo by BackupStoreCheck and HousekeepStoreAccount, + // and the Context to be reopened. + virtual void DiscardBackupStoreInfo(BackupStoreInfo& rInfo) + { + ASSERT(mapBackupStoreInfo.get() == &rInfo); + mapBackupStoreInfo.reset(); + } + virtual std::auto_ptr GetBackupStoreInfoUncached() + { + // Return a BackupStoreInfo freshly retrieved from storage, read-only to + // prevent accidentally making changes to this copy, which can't be saved + // back to the BackupFileSystem. + return GetBackupStoreInfoInternal(true); // ReadOnly + } + + // GetPotentialRefCountDatabase() returns the current potential database if it + // has been already obtained and not closed, otherwise creates a new one. + // This same database will never be returned by both this function and + // GetPermanentRefCountDatabase() at the same time; it must be committed to + // convert it to a permanent DB before GetPermanentRefCountDatabase() will + // return it, and GetPotentialRefCountDatabase() no longer will after that. + virtual BackupStoreRefCountDatabase& GetPotentialRefCountDatabase() = 0; + // GetPermanentRefCountDatabase returns the current permanent database, if already + // open, otherwise refreshes the cached copy, opens it, and returns it. + virtual BackupStoreRefCountDatabase& + GetPermanentRefCountDatabase(bool ReadOnly) = 0; + // SaveRefCountDatabase() uploads a modified database to permanent storage + // (if necessary). It must be called with the permanent database. Calling Commit() + // on the temporary database calls this function automatically. + virtual void SaveRefCountDatabase(BackupStoreRefCountDatabase& refcount_db) = 0; + + virtual bool ObjectExists(int64_t ObjectID, int64_t *pRevisionID = 0) = 0; + virtual void GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut) = 0; + virtual void PutDirectory(BackupStoreDirectory& rDir) = 0; + virtual std::auto_ptr PutFileComplete(int64_t ObjectID, + IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount) = 0; + virtual std::auto_ptr PutFilePatch(int64_t ObjectID, + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount) = 0; + // GetObject() will retrieve either a file or directory, whichever exists. + // GetFile() and GetDirectory() are only guaranteed to work on objects of the + // correct type, but may be faster (depending on the implementation). + virtual std::auto_ptr GetObject(int64_t ObjectID, bool required = true) = 0; + virtual std::auto_ptr GetFile(int64_t ObjectID) = 0; + virtual std::auto_ptr GetFilePatch(int64_t ObjectID, + std::vector& rPatchChain) = 0; + virtual void DeleteFile(int64_t ObjectID) = 0; + virtual void DeleteDirectory(int64_t ObjectID) = 0; + virtual void DeleteObjectUnknown(int64_t ObjectID) = 0; + virtual bool CanMergePatches() = 0; + virtual std::auto_ptr + CombineFile(int64_t OlderPatchID, int64_t NewerFileID) = 0; + virtual std::auto_ptr + CombineDiffs(int64_t OlderPatchID, int64_t NewerPatchID) = 0; + virtual std::string GetAccountIdentifier() = 0; + // Use of GetAccountID() is not recommended. It returns S3_FAKE_ACCOUNT_ID on + // S3BackupFileSystem. + virtual int GetAccountID() = 0; + virtual int64_t GetFileSizeInBlocks(int64_t ObjectID) = 0; + + class CheckObjectsResult + { + public: + CheckObjectsResult() + : maxObjectIDFound(0), + numErrorsFound(0) + { } + + int64_t maxObjectIDFound; + int numErrorsFound; + }; + + virtual CheckObjectsResult CheckObjects(bool fix_errors) = 0; + virtual void EnsureObjectIsPermanent(int64_t ObjectID, bool fix_errors) = 0; + + // CloseRefCountDatabase() closes the active database, saving changes to permanent + // storage if necessary. It invalidates any references to the current database! + // It is needed to allow a BackupStoreContext to be Finished, changes made to the + // BackupFileSystem's BackupStoreInfo by BackupStoreCheck and HousekeepStoreAccount, + // and the Context to be reopened. + void CloseRefCountDatabase(BackupStoreRefCountDatabase* p_refcount_db) + { + ASSERT(p_refcount_db == mapPermanentRefCountDatabase.get()); + SaveRefCountDatabase(*p_refcount_db); + mapPermanentRefCountDatabase.reset(); + } + BackupStoreRefCountDatabase* GetCurrentRefCountDatabase() + { + return mapPermanentRefCountDatabase.get(); + } + + static const FileSystemCategory LOCKING; + +protected: + virtual void TryGetLock() = 0; + virtual std::auto_ptr GetBackupStoreInfoInternal(bool ReadOnly) = 0; + std::auto_ptr mapBackupStoreInfo; + // You can have one temporary and one permanent refcound DB open at any time, + // obtained with GetPotentialRefCountDatabase() and + // GetPermanentRefCountDatabase() respectively: + std::auto_ptr mapPotentialRefCountDatabase; + std::auto_ptr mapPermanentRefCountDatabase; + + friend class BackupStoreRefCountDatabaseWrapper; + // These should only be called by BackupStoreRefCountDatabaseWrapper::Commit(): + virtual void RefCountDatabaseBeforeCommit(BackupStoreRefCountDatabase& refcount_db); + virtual void RefCountDatabaseAfterCommit(BackupStoreRefCountDatabase& refcount_db); + // AfterDiscard() destroys the temporary database object and therefore invalidates + // any held references to it! + virtual void RefCountDatabaseAfterDiscard(BackupStoreRefCountDatabase& refcount_db); +}; + +class RaidBackupFileSystem : public BackupFileSystem +{ +private: + const int64_t mAccountID; + const std::string mAccountRootDir; + const int mStoreDiscSet; + std::string GetObjectFileName(int64_t ObjectID, bool EnsureDirectoryExists); + NamedLock mWriteLock; + +public: + RaidBackupFileSystem(int64_t AccountID, const std::string &AccountRootDir, int discSet) + : BackupFileSystem(), + mAccountID(AccountID), + mAccountRootDir(AccountRootDir), + mStoreDiscSet(discSet) + { + if(AccountRootDir.size() == 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadConfiguration, + "Account root directory is empty"); + } + + ASSERT(AccountRootDir[AccountRootDir.size() - 1] == '/' || + AccountRootDir[AccountRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + } + virtual ~RaidBackupFileSystem() + { + // Call ReleaseLock() to close any open refcount DBs before + // partially destroying the BackupFileSystem that they need to + // close down. Need to do this in the subclass to avoid calling + // SaveRefCountDatabase() (from + // ~BackupStoreRefCountDatabaseWrapper) when the subclass has + // already been partially destroyed. + // http://stackoverflow.com/questions/10707286/how-to-resolve-pure-virtual-method-called + ReleaseLock(); + } + + virtual void ReleaseLock() + { + BackupFileSystem::ReleaseLock(); + if(HaveLock()) + { + mWriteLock.ReleaseLock(); + } + } + virtual bool HaveLock() + { + return mWriteLock.GotLock(); + } + virtual int GetBlockSize(); + virtual void PutBackupStoreInfo(BackupStoreInfo& rInfo); + + virtual BackupStoreRefCountDatabase& GetPotentialRefCountDatabase(); + virtual BackupStoreRefCountDatabase& GetPermanentRefCountDatabase(bool ReadOnly); + virtual void SaveRefCountDatabase(BackupStoreRefCountDatabase& refcount_db); + + virtual bool ObjectExists(int64_t ObjectID, int64_t *pRevisionID = 0); + virtual std::auto_ptr GetObject(int64_t ObjectID, bool required = true); + virtual void GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut); + virtual void PutDirectory(BackupStoreDirectory& rDir); + virtual std::auto_ptr PutFileComplete(int64_t ObjectID, + IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount); + virtual std::auto_ptr PutFilePatch(int64_t ObjectID, + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount); + virtual std::auto_ptr GetFile(int64_t ObjectID) + { + // For RaidBackupFileSystem, GetObject() is equivalent to GetFile(). + return GetObject(ObjectID); + } + virtual std::auto_ptr GetFilePatch(int64_t ObjectID, + std::vector& rPatchChain); + virtual void DeleteFile(int64_t ObjectID) + { + // RaidFile doesn't care what type of object it is + DeleteObjectUnknown(ObjectID); + } + virtual void DeleteDirectory(int64_t ObjectID) + { + // RaidFile doesn't care what type of object it is + DeleteObjectUnknown(ObjectID); + } + virtual void DeleteObjectUnknown(int64_t ObjectID); + virtual bool CanMergePatches() { return true; } + std::auto_ptr + CombineFile(int64_t OlderPatchID, int64_t NewerFileID); + std::auto_ptr + CombineDiffs(int64_t OlderPatchID, int64_t NewerPatchID); + virtual std::string GetAccountIdentifier(); + virtual int GetAccountID() { return mAccountID; } + virtual int64_t GetFileSizeInBlocks(int64_t ObjectID); + virtual CheckObjectsResult CheckObjects(bool fix_errors); + virtual void EnsureObjectIsPermanent(int64_t ObjectID, bool fix_errors); + +protected: + virtual void TryGetLock(); + virtual std::auto_ptr GetBackupStoreInfoInternal(bool ReadOnly); + std::auto_ptr + CombineFileOrDiff(int64_t OlderPatchID, int64_t NewerObjectID, bool NewerIsPatch); + +private: + void CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName, + CheckObjectsResult& Result, bool fix_errors); + void CheckObjectsDir(int64_t StartID, + BackupFileSystem::CheckObjectsResult& Result, bool fix_errors); +}; + +#define S3_INFO_FILE_NAME "boxbackup.info" +#define S3_REFCOUNT_FILE_NAME "boxbackup.refcount.db" +#define S3_FAKE_ACCOUNT_ID 0x53336964 // 'S3id' +#define S3_CACHE_LOCK_NAME "boxbackup.cache.lock" + +#ifdef BOX_RELEASE_BUILD + // Use a larger block size for efficiency + #define S3_NOTIONAL_BLOCK_SIZE 1048576 +#else + // Use a smaller block size to make tests run faster + #define S3_NOTIONAL_BLOCK_SIZE 16384 +#endif + +class S3BackupFileSystem : public BackupFileSystem +{ +private: + const Configuration& mrConfig; + std::string mBasePath, mCacheDirectory; + NamedLock mCacheLock; + S3Client& mrClient; + std::auto_ptr mapSimpleDBClient; + bool mHaveLock; + std::string mSimpleDBDomain, mLockName, mLockValue, mCurrentUserName, + mCurrentHostName; + SimpleDBClient::str_map_t mLockAttributes; + void ReportLockMismatches(str_map_diff_t mismatches); + + S3BackupFileSystem(const S3BackupFileSystem& forbidden); // no copying + S3BackupFileSystem& operator=(const S3BackupFileSystem& forbidden); // no assignment + std::string GetRefCountDatabaseCachePath() + { + return mCacheDirectory + DIRECTORY_SEPARATOR + S3_REFCOUNT_FILE_NAME; + } + void GetCacheLock(); + +public: + S3BackupFileSystem(const Configuration& config, const std::string& BasePath, + const std::string& CacheDirectory, S3Client& rClient); + virtual ~S3BackupFileSystem(); + + virtual void ReleaseLock(); + virtual bool HaveLock() + { + return mHaveLock; + } + virtual int GetBlockSize(); + virtual void PutBackupStoreInfo(BackupStoreInfo& rInfo); + virtual BackupStoreRefCountDatabase& GetPotentialRefCountDatabase(); + virtual BackupStoreRefCountDatabase& GetPermanentRefCountDatabase(bool ReadOnly); + virtual bool ObjectExists(int64_t ObjectID, int64_t *pRevisionID = 0); + virtual std::auto_ptr GetObject(int64_t ObjectID, bool required = true); + virtual void GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut); + virtual void PutDirectory(BackupStoreDirectory& rDir); + virtual std::auto_ptr PutFileComplete(int64_t ObjectID, + IOStream& rFileData, BackupStoreRefCountDatabase::refcount_t refcount); + virtual std::auto_ptr PutFilePatch(int64_t ObjectID, + int64_t DiffFromFileID, IOStream& rPatchData, + BackupStoreRefCountDatabase::refcount_t refcount) + { + return PutFileComplete(ObjectID, rPatchData, refcount); + } + virtual std::auto_ptr GetFile(int64_t ObjectID); + virtual std::auto_ptr GetFilePatch(int64_t ObjectID, + std::vector& rPatchChain) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void DeleteFile(int64_t ObjectID); + virtual void DeleteDirectory(int64_t ObjectID); + virtual void DeleteObjectUnknown(int64_t ObjectID); + + // These should not really be APIs, but they are public to make them testable: + const std::string& GetSimpleDBDomain() const { return mSimpleDBDomain; } + const std::string& GetSimpleDBLockName() const { return mLockName; } + const std::string& GetSimpleDBLockValue() const { return mLockValue; } + const std::string& GetCurrentUserName() const { return mCurrentUserName; } + const std::string& GetCurrentHostName() const { return mCurrentHostName; } + const box_time_t GetSinceTime() const + { + // Unfortunately operator[] is not const, so use a const_iterator to + // get the value that we want. + const std::string& since(mLockAttributes.find("since")->second); + return box_strtoui64(since.c_str(), NULL, 10); + } + + // And these are public to help with writing tests ONLY: + friend class S3BackupAccountControl; + + // The returned URI should start with mBasePath. + std::string GetMetadataURI(const std::string& MetadataFilename) const + { + return mBasePath + MetadataFilename; + } + // The returned URI should start with mBasePath. + std::string GetDirectoryURI(int64_t ObjectID) + { + return GetObjectURI(ObjectID, ObjectExists_Dir); + } + // The returned URI should start with mBasePath. + std::string GetFileURI(int64_t ObjectID) + { + return GetObjectURI(ObjectID, ObjectExists_File); + } + int GetSizeInBlocks(int64_t bytes) + { + return (bytes + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; + } + virtual bool CanMergePatches() { return false; } + std::auto_ptr + CombineFile(int64_t OlderPatchID, int64_t NewerFileID) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + std::auto_ptr + CombineDiffs(int64_t OlderPatchID, int64_t NewerPatchID) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual std::string GetAccountIdentifier(); + virtual int GetAccountID() { return S3_FAKE_ACCOUNT_ID; } + virtual int64_t GetFileSizeInBlocks(int64_t ObjectID); + virtual CheckObjectsResult CheckObjects(bool fix_errors); + virtual void EnsureObjectIsPermanent(int64_t ObjectID, bool fix_errors) + { + // Filesystem is not transactional, so nothing to do here. + } + +protected: + virtual void TryGetLock(); + virtual std::auto_ptr GetBackupStoreInfoInternal(bool ReadOnly); + +private: + // GetObjectURL() returns the complete URL for an object at the given + // path, by adding the hostname, port and the object's URI (which can + // be retrieved from GetMetadataURI or GetObjectURI). + std::string GetObjectURL(const std::string& ObjectURI) const; + + // GetObjectURI() is a private interface which converts an object ID + // and type into a URI, which starts with mBasePath: + std::string GetObjectURI(int64_t ObjectID, int Type) const; + + int64_t GetRevisionID(const std::string& uri, HTTPResponse& response) const; + + typedef std::map > start_id_to_files_t; + void CheckObjectsScanDir(int64_t start_id, int level, const std::string &dir_name, + CheckObjectsResult& result, bool fix_errors, + start_id_to_files_t& start_id_to_files); + void CheckObjectsDir(int64_t start_id, + BackupFileSystem::CheckObjectsResult& result, bool fix_errors, + const start_id_to_files_t& start_id_to_files); + virtual void SaveRefCountDatabase(BackupStoreRefCountDatabase& refcount_db); +}; + +#endif // BACKUPFILESYSTEM__H diff --git a/lib/backupstore/BackupProtocol.h b/lib/backupstore/BackupProtocol.h index d9070c73d..4512021e7 100644 --- a/lib/backupstore/BackupProtocol.h +++ b/lib/backupstore/BackupProtocol.h @@ -25,40 +25,54 @@ class BackupProtocolLocal2 : public BackupProtocolLocal { private: - BackupStoreContext mContext; + std::auto_ptr mapLocalContext; int32_t mAccountNumber; bool mReadOnly; -protected: - BackupStoreContext& GetContext() { return mContext; } - public: BackupProtocolLocal2(int32_t AccountNumber, const std::string& ConnectionDetails, const std::string& AccountRootDir, int DiscSetNumber, bool ReadOnly) - // This is rather ugly: the BackupProtocolLocal constructor must not - // touch the Context, because it's not initialised yet! - : BackupProtocolLocal(mContext), - mContext(AccountNumber, (HousekeepingInterface *)NULL, - ConnectionDetails), + // This is rather ugly: we need to pass a reference to a context to + // BackupProtocolLocal(), and we want it to be one that we've created ourselves, + // so we create one with new(), dereference it to pass the reference to the + // superclass, and then get the reference out again, take its address and stick + // that into the auto_ptr, which will delete it when we are destroyed. + : BackupProtocolLocal( + *(new BackupStoreContext(AccountNumber, (HousekeepingInterface *)NULL, + ConnectionDetails)) + ), + mapLocalContext(&GetContext()), + mAccountNumber(AccountNumber), + mReadOnly(ReadOnly) + { + GetContext().SetClientHasAccount(AccountRootDir, DiscSetNumber); + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(AccountNumber, + ReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } + + BackupProtocolLocal2(BackupStoreContext& rContext, int32_t AccountNumber, + bool ReadOnly) + : BackupProtocolLocal(rContext), mAccountNumber(AccountNumber), mReadOnly(ReadOnly) { - mContext.SetClientHasAccount(AccountRootDir, DiscSetNumber); + GetContext().SetClientHasAccount(); QueryVersion(BACKUP_STORE_SERVER_VERSION); QueryLogin(AccountNumber, ReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); } - virtual ~BackupProtocolLocal2() { } std::auto_ptr Query(const BackupProtocolFinished &rQuery) { std::auto_ptr finished = BackupProtocolLocal::Query(rQuery); - mContext.ReleaseWriteLock(); + GetContext().CleanUp(); return finished; } + using BackupProtocolLocal::Query; void Reopen() { diff --git a/lib/backupstore/BackupProtocol.txt b/lib/backupstore/BackupProtocol.txt index 5921d0094..8f3d79c31 100644 --- a/lib/backupstore/BackupProtocol.txt +++ b/lib/backupstore/BackupProtocol.txt @@ -42,6 +42,7 @@ Error 0 IsError(Type,SubType) Reply CONSTANT Err_MultiplyReferencedObject 15 CONSTANT Err_DisabledAccount 16 + Version 1 Command(Version) Reply int32 Version diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 7955b3c46..8177d760d 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -17,8 +17,6 @@ #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" -#include "BackupStoreCheck.h" -#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" @@ -31,7 +29,6 @@ #include "RaidFileWrite.h" #include "StoreStructure.h" #include "UnixUser.h" -#include "Utils.h" #include "MemLeakFindOn.h" @@ -48,6 +45,7 @@ BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase) { } + // -------------------------------------------------------------------------- // // Function @@ -61,74 +59,6 @@ BackupStoreAccounts::~BackupStoreAccounts() } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) -// Purpose: Create a new account on the specified disc set. -// If rAsUsername is not empty, then the account information will be written under the -// username specified. -// Created: 2003/08/21 -// -// -------------------------------------------------------------------------- -void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) -{ - // Create the entry in the database - BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID, - DiscSet)); - - { - // Become the user specified in the config file? - std::auto_ptr user; - if(!rAsUsername.empty()) - { - // Username specified, change... - user.reset(new UnixUser(rAsUsername.c_str())); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone at the end of this function - } - - // Get directory name - std::string dirName(MakeAccountRootDir(ID, DiscSet)); - - // Create a directory on disc - RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */); - - // Create an info file - BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit); - - // And an empty directory - BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); - int64_t rootDirSize = 0; - // Write it, knowing the directory scheme - { - RaidFileWrite rf(DiscSet, dirName + "o01"); - rf.Open(); - rootDir.WriteToStream(rf); - rootDirSize = rf.GetDiscUsageInBlocks(); - rf.Commit(true); - } - - // Update the store info to reflect the size of the root directory - std::auto_ptr info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); - info->ChangeBlocksUsed(rootDirSize); - info->ChangeBlocksInDirectories(rootDirSize); - info->AdjustNumDirectories(1); - - // Save it back - info->Save(); - - // Create the refcount database - BackupStoreRefCountDatabase::Create(Entry)->Commit(); - } - - // As the original user... - // Write the database back - mrDatabase.Write(); -} - - // -------------------------------------------------------------------------- // // Function @@ -141,7 +71,7 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i { // Find the account const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); - + rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet()); rDiscSetOut = en.GetDiscSet(); } @@ -177,419 +107,3 @@ bool BackupStoreAccounts::AccountExists(int32_t ID) return mrDatabase.EntryExists(ID); } -void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) -{ - const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); - std::string rootDir = MakeAccountRootDir(ID, en.GetDiscSet()); - int discSet = en.GetDiscSet(); - - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(rootDir, discSet, writeLockFilename); - - bool gotLock = false; - int triesLeft = 8; - do - { - gotLock = rNamedLock.TryAndGetLock(writeLockFilename, - 0600 /* restrictive file permissions */); - - if(!gotLock) - { - --triesLeft; - ::sleep(1); - } - } - while (!gotLock && triesLeft > 0); - - if (!gotLock) - { - THROW_EXCEPTION_MESSAGE(BackupStoreException, - CouldNotLockStoreAccount, "Failed to get exclusive " - "lock on account " << BOX_FORMAT_ACCOUNT(ID)); - } -} - -int BackupStoreAccountsControl::BlockSizeOfDiscSet(int discSetNum) -{ - // Get controller, check disc set number - RaidFileController &controller(RaidFileController::GetController()); - if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) - { - BOX_FATAL("Disc set " << discSetNum << " does not exist."); - exit(1); - } - - // Return block size - return controller.GetDiscSet(discSetNum).GetBlockSize(); -} - -int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, - const char *HardLimitStr) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change limits."); - return 1; - } - - // Load the info - std::auto_ptr info(BackupStoreInfo::Load(ID, rootDir, - discSetNum, false /* Read/Write */)); - - // Change the limits - int blocksize = BlockSizeOfDiscSet(discSetNum); - int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); - int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); - CheckSoftHardLimits(softlimit, hardlimit); - info->ChangeLimits(softlimit, hardlimit); - - // Save - info->Save(); - - BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << - " changed to " << softlimit << " soft, " << - hardlimit << " hard."); - - return 0; -} - -int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change name."); - return 1; - } - - // Load the info - std::auto_ptr info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, false /* Read/Write */)); - - info->SetAccountName(rNewAccountName); - - // Save - info->Save(); - - BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << - " name changed to " << rNewAccountName); - - return 0; -} - -int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - - if(!OpenAccount(ID, rootDir, discSetNum, user, - NULL /* no write lock needed for this read-only operation */)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to display info."); - return 1; - } - - // Load it in - std::auto_ptr info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, true /* ReadOnly */)); - - return BackupAccountControl::PrintAccountInfo(*info, - BlockSizeOfDiscSet(discSetNum)); -} - -int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change enabled flag."); - return 1; - } - - // Load it in - std::auto_ptr info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, false /* ReadOnly */)); - info->SetAccountEnabled(enabled); - info->Save(); - return 0; -} - -int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - NamedLock writeLock; - - // Obtain a write lock, as the daemon user - if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for deletion."); - return 1; - } - - // Check user really wants to do this - if(AskForConfirmation) - { - BOX_WARNING("Really delete account " << - BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); - char response[256]; - if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) - { - BOX_NOTICE("Deletion cancelled."); - return 0; - } - } - - // Back to original user, but write lock is maintained - user.reset(); - - std::auto_ptr db( - BackupStoreAccountDatabase::Read( - mConfig.GetKeyValue("AccountDatabase"))); - - // Delete from account database - db->DeleteEntry(ID); - - // Write back to disc - db->Write(); - - // Remove the store files... - - // First, become the user specified in the config file - std::string username; - { - const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Become the right user - if(!username.empty()) - { - // Username specified, change... - user.reset(new UnixUser(username)); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone when user goes out of scope - } - - // Secondly, work out which directories need wiping - std::vector toDelete; - RaidFileController &rcontroller(RaidFileController::GetController()); - RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); - for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) - { - if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) - { - toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); - } - } - - // NamedLock will throw an exception if it can't delete the lockfile, - // which it can't if it doesn't exist. Now that we've deleted the account, - // nobody can open it anyway, so it's safe to unlock. - writeLock.ReleaseLock(); - - int retcode = 0; - - // Thirdly, delete the directories... - for(std::vector::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) - { - BOX_NOTICE("Deleting store directory " << (*d) << "..."); - // Just use the rm command to delete the files -#ifdef WIN32 - std::string cmd("rmdir /s/q "); - std::string dir = *d; - - // rmdir doesn't understand forward slashes, so replace them all. - for(std::string::iterator i = dir.begin(); i != dir.end(); i++) - { - if(*i == '/') - { - *i = '\\'; - } - } - cmd += dir; -#else - std::string cmd("rm -rf "); - cmd += *d; -#endif - // Run command - if(::system(cmd.c_str()) != 0) - { - BOX_ERROR("Failed to delete files in " << (*d) << - ", delete them manually."); - retcode = 1; - } - } - - // Success! - return retcode; -} - -bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr apUser, NamedLock* pLock) -{ - // Load in the account database - std::auto_ptr db( - BackupStoreAccountDatabase::Read( - mConfig.GetKeyValue("AccountDatabase"))); - - // Exists? - if(!db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " does not exist."); - return false; - } - - // Get info from the database - BackupStoreAccounts acc(*db); - acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); - - // Get the user under which the daemon runs - std::string username; - { - const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Become the right user - if(!username.empty()) - { - // Username specified, change... - apUser.reset(new UnixUser(username)); - apUser->ChangeProcessUser(true /* temporary */); - // Change will be undone when apUser goes out of scope - // in the caller. - } - - if(pLock) - { - acc.LockAccount(ID, *pLock); - } - - return true; -} - -int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet, - bool ReturnNumErrorsFound) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(ID, rootDir, discSetNum, user, - FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for checking."); - return 1; - } - - // Check it - BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); - check.Check(); - - if(ReturnNumErrorsFound) - { - return check.GetNumErrorsFound(); - } - else - { - return check.ErrorsFound() ? 1 : 0; - } -} - -int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber, - int32_t SoftLimit, int32_t HardLimit) -{ - // Load in the account database - std::auto_ptr db( - BackupStoreAccountDatabase::Read( - mConfig.GetKeyValue("AccountDatabase"))); - - // Already exists? - if(db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " already exists."); - return 1; - } - - // Get the user under which the daemon runs - std::string username; - { - const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Create it. - BackupStoreAccounts acc(*db); - acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); - - BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); - - return 0; -} - -int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr user; // used to reset uid when we return - - if(!OpenAccount(ID, rootDir, discSetNum, user, - NULL /* housekeeping locks the account itself */)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for housekeeping."); - return 1; - } - - HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); - bool success = housekeeping.DoHousekeeping(); - - if(!success) - { - BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) - << " for housekeeping: perhaps a client is " - "still connected?"); - return 1; - } - else - { - BOX_TRACE("Finished housekeeping on account " << - BOX_FORMAT_ACCOUNT(ID)); - return 0; - } -} - diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index bcc3cf1c5..d982835b4 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,14 +13,15 @@ #include #include "BackupStoreAccountDatabase.h" -#include "BackupAccountControl.h" -#include "NamedLock.h" // -------------------------------------------------------------------------- // // Class // Name: BackupStoreAccounts -// Purpose: Account management for backup store server +// Purpose: Account management for backup store server. This +// class now serves very little purpose, and it should +// probably be folded into other classes, such as +// BackupStoreAccountDatabase and RaidBackupFileSystem. // Created: 2003/08/21 // // -------------------------------------------------------------------------- @@ -33,9 +34,6 @@ class BackupStoreAccounts BackupStoreAccounts(const BackupStoreAccounts &rToCopy); public: - void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, - int64_t SizeHardLimit, const std::string &rAsUsername); - bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; static std::string GetAccountRoot(const @@ -43,41 +41,13 @@ class BackupStoreAccounts { return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); } - void LockAccount(int32_t ID, NamedLock& rNamedLock); private: static std::string MakeAccountRootDir(int32_t ID, int DiscSet); -private: BackupStoreAccountDatabase &mrDatabase; }; -class Configuration; -class UnixUser; - -class BackupStoreAccountsControl : public BackupAccountControl -{ -public: - BackupStoreAccountsControl(const Configuration& config, - bool machineReadableOutput = false) - : BackupAccountControl(config, machineReadableOutput) - { } - int BlockSizeOfDiscSet(int discSetNum); - bool OpenAccount(int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr apUser, NamedLock* pLock); - int SetLimit(int32_t ID, const char *SoftLimitStr, - const char *HardLimitStr); - int SetAccountName(int32_t ID, const std::string& rNewAccountName); - int PrintAccountInfo(int32_t ID); - int SetAccountEnabled(int32_t ID, bool enabled); - int DeleteAccount(int32_t ID, bool AskForConfirmation); - int CheckAccount(int32_t ID, bool FixErrors, bool Quiet, - bool ReturnNumErrorsFound = false); - int CreateAccount(int32_t ID, int32_t DiscNumber, int32_t SoftLimit, - int32_t HardLimit); - int HousekeepAccountNow(int32_t ID); -}; - // max size of soft limit as percent of hard limit #define MAX_SOFT_LIMIT_SIZE 97 diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index b53ebf6d0..7221ad933 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -17,6 +17,7 @@ #endif #include "autogen_BackupStoreException.h" +#include "BackupFileSystem.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" @@ -30,7 +31,7 @@ #include "RaidFileUtil.h" #include "RaidFileWrite.h" #include "StoreStructure.h" -#include "Utils.h" +#include "Utils.h" // for ObjectExists_* (object_exists_t) #include "MemLeakFindOn.h" @@ -38,32 +39,33 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool) +// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, +// int, int32_t, bool, bool) // Purpose: Constructor // Created: 21/4/04 // // -------------------------------------------------------------------------- -BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet) - : mStoreRoot(rStoreRoot), - mDiscSetNumber(DiscSetNumber), - mAccountID(AccountID), - mFixErrors(FixErrors), - mQuiet(Quiet), - mNumberErrorsFound(0), - mLastIDInInfo(0), - mpInfoLastBlock(0), - mInfoLastBlockEntries(0), - mLostDirNameSerial(0), - mLostAndFoundDirectoryID(0), - mBlocksUsed(0), - mBlocksInCurrentFiles(0), - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0), - mBlocksInDirectories(0), - mNumCurrentFiles(0), - mNumOldFiles(0), - mNumDeletedFiles(0), - mNumDirectories(0) +BackupStoreCheck::BackupStoreCheck(BackupFileSystem& FileSystem, bool FixErrors, bool Quiet) +: mAccountID(FileSystem.GetAccountID()), // will be 0 for S3BackupFileSystem + mFixErrors(FixErrors), + mQuiet(Quiet), + mNumberErrorsFound(0), + mLastIDInInfo(0), + mpInfoLastBlock(0), + mInfoLastBlockEntries(0), + mrFileSystem(FileSystem), + mLostDirNameSerial(0), + mLostAndFoundDirectoryID(0), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mTimeout(600) // default timeout is 10 minutes { } @@ -82,13 +84,13 @@ BackupStoreCheck::~BackupStoreCheck() FreeInfo(); // Avoid an exception if we forget to discard mapNewRefs - if (mapNewRefs.get()) + if(mpNewRefs) { // Discard() can throw exception, but destructors aren't supposed to do that, so // just catch and log them. try { - mapNewRefs->Discard(); + mpNewRefs->Discard(); } catch(BoxException &e) { @@ -96,7 +98,7 @@ BackupStoreCheck::~BackupStoreCheck() "the refcount database threw an exception: " << e.what()); } - mapNewRefs.reset(); + mpNewRefs = NULL; } } @@ -114,9 +116,9 @@ void BackupStoreCheck::Check() { if(mFixErrors) { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); - ASSERT(FileExists(writeLockFilename)); + // Will throw an exception if it doesn't manage to get a lock + // within about 8 seconds: + mrFileSystem.GetLock(); } if(!mQuiet && mFixErrors) @@ -124,14 +126,48 @@ void BackupStoreCheck::Check() BOX_INFO("Will fix errors encountered during checking."); } - BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber); - mapNewRefs = BackupStoreRefCountDatabase::Create(account); + // If we are read-only, then we should not call GetPotentialRefCountDatabase because + // that does actually change the store: the temporary file would conflict with any other + // process which wants to do the same thing at the same time (e.g. housekeeping), and if + // neither process locks the store, they will break each other. We can still create a + // refcount DB in a temporary directory, and Commit() will not really commit it in that + // case (it will rename it, but still in the temporary directory). + if(mFixErrors) + { + mpNewRefs = &mrFileSystem.GetPotentialRefCountDatabase(); + } + else + { + std::string temp_file = GetTempDirPath() + "boxbackup_refcount_db_XXXXXX"; + char temp_file_buf[PATH_MAX]; + strncpy(temp_file_buf, temp_file.c_str(), sizeof(temp_file_buf)); +#ifdef WIN32 + if(_mktemp_s(temp_file_buf, sizeof(temp_file_buf)) != 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, FailedToCreateTemporaryFile, + "Failed to get a temporary file name based on " << temp_file); + } +#else + int fd = mkstemp(temp_file_buf); + if(fd == -1) + { + THROW_SYS_FILE_ERROR("Failed to get a temporary file name based on", + temp_file, BackupStoreException, FailedToCreateTemporaryFile); + } + close(fd); +#endif + + BOX_TRACE("Creating temporary refcount DB in a temporary file: " << temp_file_buf); + mapOwnNewRefs = BackupStoreRefCountDatabase::Create(temp_file_buf, + mrFileSystem.GetAccountID(), true); // reuse_existing_file + mpNewRefs = mapOwnNewRefs.get(); + } // Phase 1, check objects if(!mQuiet) { - BOX_INFO("Checking store account ID " << - BOX_FORMAT_ACCOUNT(mAccountID) << "..."); + BOX_INFO("Checking account " << mrFileSystem.GetAccountIdentifier() << + "..."); BOX_INFO("Phase 1, check objects..."); } CheckObjects(); @@ -174,43 +210,50 @@ void BackupStoreCheck::Check() try { - std::auto_ptr apOldRefs = - BackupStoreRefCountDatabase::Load(account, false); - mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs); + // We should be able to load a reference to the old refcount database + // (read-only) at the same time that we have a reference to the new one + // (temporary) open but not yet committed. + BackupStoreRefCountDatabase& old_refs( + mrFileSystem.GetPermanentRefCountDatabase(true)); // ReadOnly + mNumberErrorsFound += mpNewRefs->ReportChangesTo(old_refs); } catch(BoxException &e) { - BOX_WARNING("Reference count database was missing or " - "corrupted, cannot check it for errors."); + BOX_WARNING("Old reference count database was missing or corrupted, " + "cannot check it for errors."); mNumberErrorsFound++; } // force file to be saved and closed before releasing the lock below if(mFixErrors) { - mapNewRefs->Commit(); + mpNewRefs->Commit(); } else { - mapNewRefs->Discard(); + mpNewRefs->Discard(); + mapOwnNewRefs.reset(); } - mapNewRefs.reset(); + mpNewRefs = NULL; if(mNumberErrorsFound > 0) { - BOX_WARNING("Finished checking store account ID " << - BOX_FORMAT_ACCOUNT(mAccountID) << ": " << + BOX_WARNING("Finished checking account " << + mrFileSystem.GetAccountIdentifier() << ": " << mNumberErrorsFound << " errors found"); + if(!mFixErrors) { BOX_WARNING("No changes to the store account " "have been made."); } + if(!mFixErrors && mNumberErrorsFound > 0) { BOX_WARNING("Run again with fix option to " "fix these errors"); } + if(mFixErrors && mNumberErrorsFound > 0) { BOX_WARNING("You should now use bbackupquery " @@ -229,57 +272,13 @@ void BackupStoreCheck::Check() } else { - BOX_NOTICE("Finished checking store account ID " << - BOX_FORMAT_ACCOUNT(mAccountID) << ": " + BOX_NOTICE("Finished checking account " << + mrFileSystem.GetAccountIdentifier() << ": " << "no errors found"); } } -// -------------------------------------------------------------------------- -// -// Function -// Name: static TwoDigitHexToInt(const char *, int &) -// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not -// Created: 21/4/04 -// -// -------------------------------------------------------------------------- -static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut) -{ - int n = 0; - // Char 0 - if(String[0] >= '0' && String[0] <= '9') - { - n = (String[0] - '0') << 4; - } - else if(String[0] >= 'a' && String[0] <= 'f') - { - n = ((String[0] - 'a') + 0xa) << 4; - } - else - { - return false; - } - // Char 1 - if(String[1] >= '0' && String[1] <= '9') - { - n |= String[1] - '0'; - } - else if(String[1] >= 'a' && String[1] <= 'f') - { - n |= (String[1] - 'a') + 0xa; - } - else - { - return false; - } - - // Return a valid number - rNumberOut = n; - return true; -} - - // -------------------------------------------------------------------------- // // Function @@ -291,217 +290,25 @@ static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut) // -------------------------------------------------------------------------- void BackupStoreCheck::CheckObjects() { - // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything - int64_t maxDir = 0; - - // Find the maximum directory starting ID - { - // Make sure the starting root dir doesn't end with '/'. - std::string start(mStoreRoot); - if(start.size() > 0 && ( - start[start.size() - 1] == '/' || - start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR)) - { - start.resize(start.size() - 1); - } - - maxDir = CheckObjectsScanDir(0, 1, start); - BOX_TRACE("Max dir starting ID is " << - BOX_FORMAT_OBJECTID(maxDir)); - } - - // Then go through and scan all the objects within those directories - for(int64_t d = 0; d <= maxDir; d += (1< dirs; - RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, - RaidFileRead::DirReadType_DirsOnly, dirs); - - for(std::vector::const_iterator i(dirs.begin()); i != dirs.end(); ++i) + if(CheckAndAddObject(ObjectID) == ObjectExists_Unknown) { - // Check to see if it's the right name - int n = 0; - if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) - && n < (1< maxID) - { - maxID = mi; - } - } - else - { - BOX_ERROR("Spurious or invalid directory " << - rDirName << DIRECTORY_SEPARATOR << - (*i) << " found, " << - (mFixErrors?"deleting":"delete manually")); - ++mNumberErrorsFound; - } - } - } - - return maxID; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreCheck::CheckObjectsDir(int64_t) -// Purpose: Check all the files within this directory which has -// the given starting ID. -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreCheck::CheckObjectsDir(int64_t StartID) -{ - // Make directory name -- first generate the filename of an entry in it - std::string dirName; - StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); - // Check expectations - ASSERT(dirName.size() > 4 && - dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); - // Remove the filename from it - dirName.resize(dirName.size() - 4); // four chars for "/o00" - - // Check directory exists - if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) - { - BOX_WARNING("RaidFile dir " << dirName << " does not exist"); - return; - } - - // Read directory contents - std::vector files; - RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, - RaidFileRead::DirReadType_FilesOnly, files); - - // Array of things present - bool idsPresent[(1<::const_iterator i(files.begin()); i != files.end(); ++i) - { - bool fileOK = true; - int n = 0; - if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) - && n < (1< file = mrFileSystem.GetObject(ObjectID, false); // !required + if(!file.get()) { - // Open file - std::auto_ptr file( - RaidFileRead::Open(mDiscSetNumber, rFilename)); - size = file->GetDiscUsageInBlocks(); + // This file just doesn't exist. Saves the caller from calling + // ObjectExists() before calling us, which is expensive on S3. + return ObjectExists_NoObject; + } + try + { // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile uint32_t signature; - if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) + int bytes_read; + if(!file->ReadFullBuffer(&signature, sizeof(signature), + 0, // don't care about number of bytes actually read + mTimeout)) { // Too short, can't read signature from it - return false; + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " is " + "too small to have a valid header"); + return ObjectExists_Unknown; } + // Seek back to beginning file->Seek(0, IOStream::SeekType_Absolute); @@ -552,34 +365,41 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: #endif - // File... check + // Check it as a file. containerID = CheckFile(ObjectID, *file); break; case OBJECTMAGIC_DIR_MAGIC_VALUE: + // Check it as a directory. isFile = false; + isDirectory = true; containerID = CheckDirInitial(ObjectID, *file); break; default: // Unknown signature. Bad file. Very bad file. - return false; + return ObjectExists_Unknown; break; } } - catch(...) + catch(std::exception &e) { // Error caught, not a good file then, let it be deleted - return false; + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " failed initial " + "validation: " << e.what()); + return ObjectExists_Unknown; } + ASSERT((isFile || isDirectory) && !(isFile && isDirectory)); + // Got a container ID? (ie check was successful) if(containerID == -1) { - return false; + return ObjectExists_Unknown; } // Add to list of IDs known about + int64_t size = mrFileSystem.GetFileSizeInBlocks(ObjectID); AddID(ObjectID, containerID, size, isFile); // Add to usage counts @@ -591,46 +411,10 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // If it looks like a good object, and it's non-RAID, and // this is a RAID set, then convert it to RAID. - - RaidFileController &rcontroller(RaidFileController::GetController()); - RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); - if(!rdiscSet.IsNonRaidSet()) - { - // See if the file exists - RaidFileUtil::ExistType existance = - RaidFileUtil::RaidFileExists(rdiscSet, rFilename); - if(existance == RaidFileUtil::NonRaid) - { - BOX_WARNING("Found non-RAID write file in RAID set" << - (mFixErrors?", transforming to RAID: ":"") << - (mFixErrors?rFilename:"")); - if(mFixErrors) - { - RaidFileWrite write(mDiscSetNumber, rFilename); - write.TransformToRaidStorage(); - } - } - else if(existance == RaidFileUtil::AsRaidWithMissingReadable) - { - BOX_WARNING("Found damaged but repairable RAID file" << - (mFixErrors?", repairing: ":"") << - (mFixErrors?rFilename:"")); - if(mFixErrors) - { - std::auto_ptr read( - RaidFileRead::Open(mDiscSetNumber, - rFilename)); - RaidFileWrite write(mDiscSetNumber, rFilename); - write.Open(true /* overwrite */); - read->CopyStreamTo(write); - read.reset(); - write.Commit(true /* transform to RAID */); - } - } - } + mrFileSystem.EnsureObjectIsPermanent(ObjectID, mFixErrors); // Report success - return true; + return isFile ? ObjectExists_File : ObjectExists_Dir; } @@ -660,7 +444,8 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) 0 /* don't want diffing from ID */, &originalContainerID)) { - // Didn't verify + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " does not " + "verify as a file"); return -1; } @@ -686,7 +471,9 @@ int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) // Check object ID if(dir.GetObjectID() != ObjectID) { - // Wrong object ID + BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(ObjectID) << " has a " + "different internal object ID than expected: " << + BOX_FORMAT_OBJECTID(dir.GetObjectID())); return -1; } @@ -729,14 +516,9 @@ void BackupStoreCheck::CheckDirectories() if(flags & Flags_IsDir) { // Found a directory. Read it in. - std::string filename; - StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); BackupStoreDirectory dir; - { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - + mrFileSystem.GetDirectory(pblock->mID[e], dir); + // Flag for modifications bool isModified = CheckDirectory(dir); @@ -759,12 +541,10 @@ void BackupStoreCheck::CheckDirectories() if(isModified && mFixErrors) { - BOX_WARNING("Writing modified directory to disk: " << + BOX_WARNING("Writing modified directory back to " + "storage: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); - RaidFileWrite fixed(mDiscSetNumber, filename); - fixed.Open(true /* allow overwriting */); - dir.WriteToStream(fixed); - fixed.Commit(true /* convert to raid representation now */); + mrFileSystem.PutDirectory(dir); } CountDirectoryEntries(dir); @@ -868,7 +648,6 @@ void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) { int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); - bool badEntry = false; bool wasAlreadyContained = false; ASSERT(piBlock != 0 || @@ -906,11 +685,8 @@ void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) { // Add to sizes? // If piBlock was zero, then wasAlreadyContained - // might be uninitialized; but we only process - // files here, and if a file's piBlock was zero - // then badEntry would be set above, so we - // wouldn't be here. - ASSERT(!badEntry) + // might be uninitialized. + ASSERT(piBlock != NULL) // It can be both old and deleted. // If neither, then it's current. @@ -933,7 +709,7 @@ void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) } } - mapNewRefs->AddReference(en->GetObjectID()); + mpNewRefs->AddReference(en->GetObjectID()); } } diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 5353c968a..ac87b5bb1 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -15,11 +15,11 @@ #include #include -#include "NamedLock.h" #include "BackupStoreDirectory.h" +#include "Utils.h" // for object_exists_t class IOStream; -class BackupStoreFilename; +class BackupFileSystem; class BackupStoreRefCountDatabase; /* @@ -74,17 +74,18 @@ typedef int64_t BackupStoreCheck_Size_t; class BackupStoreCheck { public: - BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet); + BackupStoreCheck(BackupFileSystem& rFileSystem, bool FixErrors, bool Quiet); ~BackupStoreCheck(); + private: // no copying BackupStoreCheck(const BackupStoreCheck &); BackupStoreCheck &operator=(const BackupStoreCheck &); -public: +public: // Do the exciting things void Check(); - + bool ErrorsFound() {return mNumberErrorsFound > 0;} inline int64_t GetNumErrorsFound() { @@ -114,7 +115,7 @@ class BackupStoreCheck BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE]; BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE]; } IDBlock; - + // Phases of the check void CheckObjects(); void CheckDirectories(); @@ -127,7 +128,7 @@ class BackupStoreCheck // Checking functions int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); void CheckObjectsDir(int64_t StartID); - bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); + object_exists_t CheckAndAddObject(int64_t ObjectID); bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); @@ -150,7 +151,7 @@ class BackupStoreCheck ASSERT(pBlock != 0); ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); ASSERT(Flags < (1 << Flags__NumFlags)); - + pBlock->mFlags[Index / Flags__NumItemsPerEntry] |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)); } @@ -161,7 +162,7 @@ class BackupStoreCheck return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; } - + #ifndef BOX_RELEASE_BUILD void DumpObjectInfo(); #define DUMP_OBJECT_INFO DumpObjectInfo(); @@ -170,41 +171,42 @@ class BackupStoreCheck #endif private: - std::string mStoreRoot; - int mDiscSetNumber; int32_t mAccountID; std::string mAccountName; bool mFixErrors; bool mQuiet; - + int64_t mNumberErrorsFound; - - // Lock for the store account - NamedLock mAccountLock; - + // Storage for ID data typedef std::map Info_t; Info_t mInfo; BackupStoreCheck_ID_t mLastIDInInfo; IDBlock *mpInfoLastBlock; int32_t mInfoLastBlockEntries; - + // List of stuff to fix std::vector mDirsWithWrongContainerID; // This is a map of lost dir ID -> existing dir ID std::map mDirsWhichContainLostDirs; - + // Set of extra directories added std::set mDirsAdded; // The refcount database, being reconstructed as the check/fix progresses - std::auto_ptr mapNewRefs; - + BackupStoreRefCountDatabase* mpNewRefs; + // And a holder for the auto_ptr to a new refcount DB in the temporary directory + // (not the one created by BackupFileSystem::GetPotentialRefCountDatabase()): + std::auto_ptr mapOwnNewRefs; + + // Abstracted interface to software-RAID filesystem + BackupFileSystem& mrFileSystem; + // Misc stuff int32_t mLostDirNameSerial; int64_t mLostAndFoundDirectoryID; - + // Usage int64_t mBlocksUsed; int64_t mBlocksInCurrentFiles; @@ -215,6 +217,7 @@ class BackupStoreCheck int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; + int mTimeout; }; #endif // BACKUPSTORECHECK__H diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 13831a099..6da94123e 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -13,6 +13,7 @@ #include #include "autogen_BackupStoreException.h" +#include "BackupFileSystem.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" @@ -22,9 +23,6 @@ #include "BackupStoreObjectMagic.h" #include "BackupStoreRefCountDatabase.h" #include "MemBlockStream.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "StoreStructure.h" #include "MemLeakFindOn.h" @@ -49,8 +47,7 @@ void BackupStoreCheck::CheckRoot() } else { - BOX_WARNING("Root directory doesn't exist"); - + BOX_ERROR("Root directory doesn't exist"); ++mNumberErrorsFound; if(mFixErrors) @@ -79,37 +76,25 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain } BackupStoreDirectory dir(DirectoryID, ContainingDirID); - - // Serialise to disc - std::string filename; - StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); - RaidFileWrite obj(mDiscSetNumber, filename); - obj.Open(false /* don't allow overwriting */); - dir.WriteToStream(obj); - int64_t size = obj.GetDiscUsageInBlocks(); - obj.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(dir); // Record the fact we've done this mDirsAdded.insert(DirectoryID); // Add to sizes - mBlocksUsed += size; - mBlocksInDirectories += size; + mBlocksUsed += dir.GetUserInfo1_SizeInBlocks(); + mBlocksInDirectories += dir.GetUserInfo1_SizeInBlocks(); } class BackupStoreDirectoryFixer { private: BackupStoreDirectory mDirectory; - std::string mFilename; - std::string mStoreRoot; - int mDiscSetNumber; + BackupFileSystem& mrFileSystem; public: - BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber, - int64_t ID); - void InsertObject(int64_t ObjectID, bool IsDirectory, - int32_t lostDirNameSerial); + BackupStoreDirectoryFixer(BackupFileSystem& rFileSystem, int64_t ID); + void InsertObject(int64_t ObjectID, bool IsDirectory, int32_t lostDirNameSerial); ~BackupStoreDirectoryFixer(); }; @@ -160,15 +145,11 @@ void BackupStoreCheck::CheckUnattachedObjects() // File. Only attempt to attach it somewhere if it isn't a patch { int64_t diffFromObjectID = 0; - std::string filename; - StoreStructure::MakeObjectFilename(ObjectID, - mStoreRoot, mDiscSetNumber, filename, - false /* don't attempt to make sure the dir exists */); // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + std::auto_ptr file = mrFileSystem.GetFile(ObjectID); BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID); } @@ -181,8 +162,7 @@ void BackupStoreCheck::CheckUnattachedObjects() // Delete this object instead if(mFixErrors) { - RaidFileWrite del(mDiscSetNumber, filename); - del.Delete(); + mrFileSystem.DeleteFile(ObjectID); } mBlocksUsed -= pblock->mObjectSizeInBlocks[e]; @@ -239,8 +219,7 @@ void BackupStoreCheck::CheckUnattachedObjects() { // no match, create a new one pFixer = new BackupStoreDirectoryFixer( - mStoreRoot, mDiscSetNumber, - putIntoDirectoryID); + mrFileSystem, putIntoDirectoryID); fixers.insert(fixer_pair_t( putIntoDirectoryID, pFixer)); } @@ -260,7 +239,7 @@ void BackupStoreCheck::CheckUnattachedObjects() pFixer->InsertObject(ObjectID, ((flags & Flags_IsDir) == Flags_IsDir), lostDirNameSerial); - mapNewRefs->AddReference(ObjectID); + mpNewRefs->AddReference(ObjectID); } } } @@ -309,16 +288,10 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) // Create a blank directory BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); + // Note that this directory already contains a directory entry pointing to // this dir, so it doesn't have to be added. - - // Serialise to disc - std::string filename; - StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(false /* don't allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(dir); // Record the fact we've done this mDirsAdded.insert(MissingDirectoryID); @@ -329,19 +302,11 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) return true; } -BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, - int discSetNumber, int64_t ID) -: mStoreRoot(storeRoot), - mDiscSetNumber(discSetNumber) +BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(BackupFileSystem& rFileSystem, + int64_t ID) +: mrFileSystem(rFileSystem) { - // Generate filename - StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, - mFilename, false /* don't make sure the dir exists */); - - // Read it in - std::auto_ptr file( - RaidFileRead::Open(mDiscSetNumber, mFilename)); - mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite); + mrFileSystem.GetDirectory(ID, mDirectory); } void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, @@ -361,19 +326,11 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, } else { - // Files require a little more work... - // Open file - std::string fileFilename; - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, - mDiscSetNumber, fileFilename, - false /* don't make sure the dir exists */); - std::auto_ptr file( - RaidFileRead::Open(mDiscSetNumber, fileFilename)); - - // Fill in size information - sizeInBlocks = file->GetDiscUsageInBlocks(); + // Files require a little more work... Fill in size information. + sizeInBlocks = mrFileSystem.GetFileSizeInBlocks(ObjectID); // Read in header + std::auto_ptr file = mrFileSystem.GetFile(ObjectID); file_StreamFormat hdr; if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 @@ -404,10 +361,7 @@ BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() mDirectory.CheckAndFix(); // Write it out - RaidFileWrite root(mDiscSetNumber, mFilename); - root.Open(true /* allow overwriting */); - mDirectory.WriteToStream(root); - root.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(mDirectory); } // -------------------------------------------------------------------------- @@ -434,12 +388,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // Load up the root directory BackupStoreDirectory dir; - std::string filename; - StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } + mrFileSystem.GetDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, dir); // Find a suitable name BackupStoreFilename lostAndFound; @@ -467,10 +416,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); // Write out root dir - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(dir); // Store mLostAndFoundDirectoryID = id; @@ -506,21 +452,13 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() // Load in BackupStoreDirectory dir; - std::string filename; - StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } + mrFileSystem.GetDirectory(*i, dir); // Adjust container ID dir.SetContainerID(pblock->mContainer[index]); // Write it out - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(dir); } } @@ -551,12 +489,7 @@ void BackupStoreCheck::FixDirsWithLostDirs() // Load in BackupStoreDirectory dir; - std::string filename; - StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } + mrFileSystem.GetDirectory(i->second, dir); // Delete the dodgy entry dir.DeleteEntry(i->first); @@ -565,10 +498,7 @@ void BackupStoreCheck::FixDirsWithLostDirs() dir.CheckAndFix(); // Write it out - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); + mrFileSystem.PutDirectory(dir); } } @@ -584,11 +514,11 @@ void BackupStoreCheck::FixDirsWithLostDirs() void BackupStoreCheck::WriteNewStoreInfo() { // Attempt to load the existing store info file - std::auto_ptr pOldInfo; + std::auto_ptr apOldInfo; try { - pOldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); - mAccountName = pOldInfo->GetAccountName(); + apOldInfo = mrFileSystem.GetBackupStoreInfoUncached(); + mAccountName = apOldInfo->GetAccountName(); } catch(...) { @@ -606,14 +536,14 @@ void BackupStoreCheck::WriteNewStoreInfo() int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; - int64_t softLimit = pOldInfo.get() ? pOldInfo->GetBlocksSoftLimit() : minSoft; - int64_t hardLimit = pOldInfo.get() ? pOldInfo->GetBlocksHardLimit() : minHard; + int64_t softLimit = apOldInfo.get() ? apOldInfo->GetBlocksSoftLimit() : minSoft; + int64_t hardLimit = apOldInfo.get() ? apOldInfo->GetBlocksHardLimit() : minHard; - if(mNumberErrorsFound && pOldInfo.get()) + if(mNumberErrorsFound && apOldInfo.get()) { - if(pOldInfo->GetBlocksSoftLimit() > minSoft) + if(apOldInfo->GetBlocksSoftLimit() > minSoft) { - softLimit = pOldInfo->GetBlocksSoftLimit(); + softLimit = apOldInfo->GetBlocksSoftLimit(); } else { @@ -621,9 +551,9 @@ void BackupStoreCheck::WriteNewStoreInfo() "housekeeping doesn't delete files on next run."); } - if(pOldInfo->GetBlocksHardLimit() > minHard) + if(apOldInfo->GetBlocksHardLimit() > minHard) { - hardLimit = pOldInfo->GetBlocksHardLimit(); + hardLimit = apOldInfo->GetBlocksHardLimit(); } else { @@ -641,19 +571,17 @@ void BackupStoreCheck::WriteNewStoreInfo() // Build a new store info std::auto_ptr extra_data; - if(pOldInfo.get()) + if(apOldInfo.get()) { - extra_data.reset(new MemBlockStream(pOldInfo->GetExtraData())); + extra_data.reset(new MemBlockStream(apOldInfo->GetExtraData())); } else { extra_data.reset(new MemBlockStream(/* empty */)); } - std::auto_ptr info(BackupStoreInfo::CreateForRegeneration( + std::auto_ptr apNewInfo(BackupStoreInfo::CreateForRegeneration( mAccountID, mAccountName, - mStoreRoot, - mDiscSetNumber, lastObjID, mBlocksUsed, mBlocksInCurrentFiles, @@ -662,35 +590,35 @@ void BackupStoreCheck::WriteNewStoreInfo() mBlocksInDirectories, softLimit, hardLimit, - (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), + (apOldInfo.get() ? apOldInfo->IsAccountEnabled() : true), *extra_data)); - info->AdjustNumCurrentFiles(mNumCurrentFiles); - info->AdjustNumOldFiles(mNumOldFiles); - info->AdjustNumDeletedFiles(mNumDeletedFiles); - info->AdjustNumDirectories(mNumDirectories); + apNewInfo->AdjustNumCurrentFiles(mNumCurrentFiles); + apNewInfo->AdjustNumOldFiles(mNumOldFiles); + apNewInfo->AdjustNumDeletedFiles(mNumDeletedFiles); + apNewInfo->AdjustNumDirectories(mNumDirectories); // If there are any errors (apart from wrong block counts), then we // should reset the ClientStoreMarker to zero, which // CreateForRegeneration does. But if there are no major errors, then // we should maintain the old ClientStoreMarker, to avoid invalidating // the client's directory cache. - if (pOldInfo.get() && !mNumberErrorsFound) + if(apOldInfo.get() && !mNumberErrorsFound) { BOX_INFO("No major errors found, preserving old " "ClientStoreMarker: " << - pOldInfo->GetClientStoreMarker()); - info->SetClientStoreMarker(pOldInfo->GetClientStoreMarker()); + apOldInfo->GetClientStoreMarker()); + apNewInfo->SetClientStoreMarker(apOldInfo->GetClientStoreMarker()); } - if(pOldInfo.get()) + if(apOldInfo.get()) { - mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); + mNumberErrorsFound += apNewInfo->ReportChangesTo(*apOldInfo); } // Save to disc? if(mFixErrors) { - info->Save(); + mrFileSystem.PutBackupStoreInfo(*apNewInfo); BOX_INFO("New store info file written successfully."); } } diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp index 1a782df44..fd7b8c7af 100644 --- a/lib/backupstore/BackupStoreContext.cpp +++ b/lib/backupstore/BackupStoreContext.cpp @@ -21,24 +21,13 @@ #include "BufferedStream.h" #include "BufferedWriteStream.h" #include "FileStream.h" -#include "InvisibleTempFileStream.h" -#include "RaidFileController.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "StoreStructure.h" #include "MemLeakFindOn.h" // Maximum number of directories to keep in the cache When the cache is bigger -// than this, everything gets deleted. In tests, we set the cache size to zero -// to ensure that it's always flushed, which is very inefficient but helps to -// catch programming errors (use of freed data). -#ifdef BOX_RELEASE_BUILD - #define MAX_CACHE_SIZE 32 -#else - #define MAX_CACHE_SIZE 0 -#endif +// than this, everything gets deleted. +#define MAX_CACHE_SIZE 32 // Allow the housekeeping process 4 seconds to release an account #define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 @@ -46,11 +35,17 @@ // Maximum amount of store info updates before it's actually saved to disc. #define STORE_INFO_SAVE_DELAY 96 +#define CHECK_FILESYSTEM_INITIALISED() \ + if(!mpFileSystem) \ + { \ + THROW_EXCEPTION(BackupStoreException, FileSystemNotInitialised); \ + } + // -------------------------------------------------------------------------- // // Function // Name: BackupStoreContext::BackupStoreContext() -// Purpose: Constructor +// Purpose: Traditional constructor (for RAID filesystems only) // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -61,9 +56,34 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, mpHousekeeping(pHousekeeping), mProtocolPhase(Phase_START), mClientHasAccount(false), - mStoreDiscSet(-1), mReadOnly(true), mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpFileSystem(NULL), + mpTestHook(NULL) +// If you change the initialisers, be sure to update +// BackupStoreContext::ReceivedFinishCommand as well! +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::BackupStoreContext() +// Purpose: New constructor (for any type of BackupFileSystem) +// Created: 2015/11/02 +// +// -------------------------------------------------------------------------- +BackupStoreContext::BackupStoreContext(BackupFileSystem& rFileSystem, int32_t ClientID, + HousekeepingInterface* pHousekeeping, const std::string& rConnectionDetails) +: mConnectionDetails(rConnectionDetails), + mClientID(ClientID), + mpHousekeeping(pHousekeeping), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpFileSystem(&rFileSystem), mpTestHook(NULL) // If you change the initialisers, be sure to update // BackupStoreContext::ReceivedFinishCommand as well! @@ -81,17 +101,24 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, // -------------------------------------------------------------------------- BackupStoreContext::~BackupStoreContext() { - ClearDirectoryCache(); + try + { + ClearDirectoryCache(); + ReleaseWriteLock(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to clean up BackupStoreContext: " << e.what()); + } } void BackupStoreContext::ClearDirectoryCache() { // Delete the objects in the cache - for(std::map::iterator i(mDirectoryCache.begin()); - i != mDirectoryCache.end(); ++i) + for(auto i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) { - delete (i->second); + delete i->second; } mDirectoryCache.clear(); } @@ -107,43 +134,46 @@ void BackupStoreContext::ClearDirectoryCache() // -------------------------------------------------------------------------- void BackupStoreContext::CleanUp() { - // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified - if(mapStoreInfo.get() && !(mapStoreInfo->IsReadOnly()) && - mapStoreInfo->IsModified()) + if(!mpFileSystem) { - mapStoreInfo->Save(); + return; } -} + CHECK_FILESYSTEM_INITIALISED(); -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreContext::ReceivedFinishCommand() -// Purpose: Called when the finish command is received by the protocol -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupStoreContext::ReceivedFinishCommand() -{ - if(!mReadOnly && mapStoreInfo.get()) + if(!mReadOnly) { - // Save the store info, not delayed - SaveStoreInfo(false); + // Make sure the store info is saved, if it has been loaded, isn't + // read only and has been modified. + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + if(!info.IsReadOnly() && info.IsModified()) + { + // Save the store info, not delayed + SaveStoreInfo(false); + } + // Ask the BackupFileSystem to clear its BackupStoreInfo, so that we don't use + // a stale one if the Context is later reused. + mpFileSystem->DiscardBackupStoreInfo(info); + + // Make sure the refcount database is saved too, and removed from the + // BackupFileSystem (in case it's modified before reopening). + if(mpRefCount) + { + mpFileSystem->CloseRefCountDatabase(mpRefCount); + mpRefCount = NULL; + } } + ReleaseWriteLock(); + // Just in case someone wants to reuse a local protocol object, // put the context back to its initial state. mProtocolPhase = BackupStoreContext::Phase_Version; - // Avoid the need to check version again, by not resetting - // mClientHasAccount, mAccountRootDir or mStoreDiscSet - + // Avoid the need to check version again, by not resetting mClientHasAccount. mReadOnly = true; mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; mpTestHook = NULL; - mapStoreInfo.reset(); - mapRefCount.reset(); ClearDirectoryCache(); } @@ -158,42 +188,73 @@ void BackupStoreContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupStoreContext::AttemptToGetWriteLock() { - // Make the filename of the write lock file - std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, writeLockFile); + CHECK_FILESYSTEM_INITIALISED(); // Request the lock - bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + bool gotLock = false; - if(!gotLock && mpHousekeeping) + for(int i = 0; i < (mpHousekeeping ? 2 : 1); i++) { - // The housekeeping process might have the thing open -- ask it to stop - char msg[256]; - int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); - // Send message - mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); - - // Then try again a few times - int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; - do + try { - ::sleep(1 /* second */); - --tries; - gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + // On the first pass, only try once. If it fails, and we have a + // housekeeping process, then we'll notify it and try again, + // with more retries. + mpFileSystem->GetLock((i == 0) ? 1 : + MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT); + + // If we got to here, then it worked! + gotLock = true; + } + catch(BackupStoreException &e) + { + if(!EXCEPTION_IS_TYPE(e, BackupStoreException, CouldNotLockStoreAccount)) + { + // We don't know what this error is. + throw; + } - } while(!gotLock && tries > 0); + if(mpHousekeeping) + { + // The housekeeping process might have the account locked. Ask it to stop. + char msg[256]; + int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); + + // Send message + mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); + + // Then loop again, with more retries this time + } + } } if(gotLock) { // Got the lock, mark as not read only mReadOnly = false; + + // GetDirectoryInternal assumes that if we have the write lock, everything in the + // cache was loaded with that lock, and cannot be stale. That would be violated if + // we had anything in the cache already when the lock was obtained, so clear it now. + ClearDirectoryCache(); } return gotLock; } +void BackupStoreContext::SetClientHasAccount(const std::string &rStoreRoot, + int StoreDiscSet) +{ + // Check that the BackupStoreContext hasn't already been initialised, or already + // created its own BackupFileSystem. + ASSERT(!mpFileSystem); + mClientHasAccount = true; + mapOwnFileSystem.reset( + new RaidBackupFileSystem(mClientID, rStoreRoot, StoreDiscSet)); + mpFileSystem = mapOwnFileSystem.get(); +} + // -------------------------------------------------------------------------- // // Function @@ -204,37 +265,27 @@ bool BackupStoreContext::AttemptToGetWriteLock() // -------------------------------------------------------------------------- void BackupStoreContext::LoadStoreInfo() { - if(mapStoreInfo.get() != 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); - // Load it up! - std::auto_ptr i(BackupStoreInfo::Load(mClientID, mAccountRootDir, mStoreDiscSet, mReadOnly)); + // Load it up! This checks the account ID on RaidBackupFileSystem backends, + // but not on S3BackupFileSystem which don't use account IDs. + GetBackupStoreInfo(); - // Check it - if(i->GetAccountID() != mClientID) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) - } - - // Keep the pointer to it - mapStoreInfo = i; - - BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet); - - // try to load the reference count database + // Try to load the reference count database try { - mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + mpRefCount = &(mpFileSystem->GetPermanentRefCountDatabase(mReadOnly)); } catch(BoxException &e) { + // Do not create a new refcount DB here, it is not safe! Users may wonder + // why they have lost all their files, and/or unwittingly overwrite their + // backup data. THROW_EXCEPTION_MESSAGE(BackupStoreException, - CorruptReferenceCountDatabase, "Reference count " - "database is missing or corrupted, cannot safely open " - "account. Housekeeping will fix this automatically " - "when it next runs."); + CorruptReferenceCountDatabase, "Account " << + BOX_FORMAT_ACCOUNT(mClientID) << " reference count database is " + "missing or corrupted, cannot safely open account. Housekeeping " + "will fix this automatically when it next runs."); } } @@ -249,7 +300,7 @@ void BackupStoreContext::LoadStoreInfo() // -------------------------------------------------------------------------- void BackupStoreContext::SaveStoreInfo(bool AllowDelay) { - if(mapStoreInfo.get() == 0) + if(!mpFileSystem) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } @@ -270,83 +321,93 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) } // Want to save now - mapStoreInfo->Save(); + CHECK_FILESYSTEM_INITIALISED(); + mpFileSystem->PutBackupStoreInfo(GetBackupStoreInfoInternal()); - // Set count for next delay + // Reset counter for next delayed save. mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool) -// Purpose: Create the filename of an object in the store, optionally creating the -// containing directory if it doesn't already exist. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) -{ - // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, rOutput, EnsureDirectoryExists); -} - - // -------------------------------------------------------------------------- // // Function // Name: BackupStoreContext::GetDirectoryInternal(int64_t, // bool) -// Purpose: Return a reference to a directory. Valid only until -// the next time a function which affects directories -// is called. Mainly this function, and creation of -// files. Private version of this, which returns -// non-const directories. Unless called with -// AllowFlushCache == false, the cache may be flushed, -// invalidating all directory references that you may -// be holding, so beware. +// Purpose: Return a reference to a directory, valid only until +// the next time a function which may flush the +// directory cache is called: mainly this function +// (with AllowFlushCache == true) or creation of files. +// This is a private function which returns non-const +// references to directories in the cache. It will +// invalidate all directory references that you may be +// holding, except for the one that it returns. // Created: 2003/09/02 // // -------------------------------------------------------------------------- BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, bool AllowFlushCache) { + CHECK_FILESYSTEM_INITIALISED(); + +#ifndef BOX_RELEASE_BUILD + // In debug builds, if AllowFlushCache is true, we invalidate the entire cache. That's + // because it could be flushed at any time, invalidating any pointers held to any entry + // except the one returned by this function. Invalidating makes all attempted accesses + // throw exceptions, so it should catch any such programming error. We will need to + // uninvalidate whatever entry we return, before returning it, otherwise it cannot be used. + if(AllowFlushCache) + { + for(auto i = mDirectoryCache.begin(); i != mDirectoryCache.end(); i++) + { + i->second->Invalidate(true); + } + } +#endif + // Get the filename - std::string filename; - MakeObjectFilename(ObjectID, filename); int64_t oldRevID = 0, newRevID = 0; + bool gotRevID = false; // Already in cache? - std::map::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) { -#ifndef BOX_RELEASE_BUILD // it might be in the cache, but invalidated - // in which case, delete it instead of returning it. - if(!item->second->IsInvalidated()) -#else - if(true) + auto item = mDirectoryCache.find(ObjectID); + if(item != mDirectoryCache.end()) + { +#ifndef BOX_RELEASE_BUILD + // Uninvalidate this one entry (we invalidated them all above): + item->second->Invalidate(false); #endif + oldRevID = item->second->GetRevisionID(); + + // Check the revision ID of the file -- does it need refreshing? + // We assume that if we have the write lock, everything in the cache was loaded + // with that lock held, and therefore cannot be stale. + if(!mReadOnly) { - oldRevID = item->second->GetRevisionID(); + // Looks good... return the cached object + BOX_TRACE("Returning directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache (locked), modtime = " << oldRevID) + return *(item->second); + } - // Check the revision ID of the file -- does it need refreshing? - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &newRevID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } + if(!mpFileSystem->ObjectExists(ObjectID, &newRevID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } - if(newRevID == oldRevID) - { - // Looks good... return the cached object - BOX_TRACE("Returning object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " from cache, modtime = " << newRevID) - return *(item->second); - } + gotRevID = true; + + if(newRevID == oldRevID) + { + // Looks good... return the cached object + BOX_TRACE("Returning directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache (validated), modtime = " << newRevID) + return *(item->second); } - // Delete this cached object + // The cached object is stale, so remove it from the cache. delete item->second; mDirectoryCache.erase(item); } @@ -356,64 +417,44 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, // First check to see if the cache is too big if(mDirectoryCache.size() > MAX_CACHE_SIZE && AllowFlushCache) { - // Very simple. Just delete everything! But in debug builds, - // leave the entries in the cache and invalidate them instead, - // so that any attempt to access them will cause an assertion - // failure that helps to track down the error. -#ifdef BOX_RELEASE_BUILD + // Trivial policy: just delete everything! ClearDirectoryCache(); -#else - for(std::map::iterator - i = mDirectoryCache.begin(); - i != mDirectoryCache.end(); i++) + } + + if(!gotRevID) + { + // We failed to find it in the cache, so it might not exist at all (if it was in + // the cache then it definitely does). Check for it now: + if(!mpFileSystem->ObjectExists(ObjectID, &newRevID)) { - i->second->Invalidate(); + THROW_EXCEPTION(BackupStoreException, ObjectDoesNotExist); } -#endif } - // Get a RaidFileRead to read it - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, - filename, &newRevID)); - + // Get an IOStream to read it in ASSERT(newRevID != 0); if (oldRevID == 0) { - BOX_TRACE("Loading object " << BOX_FORMAT_OBJECTID(ObjectID) << + BOX_TRACE("Loading directory " << BOX_FORMAT_OBJECTID(ObjectID) << " with modtime " << newRevID); } else { - BOX_TRACE("Refreshing object " << BOX_FORMAT_OBJECTID(ObjectID) << - " in cache, modtime changed from " << oldRevID << - " to " << newRevID); + BOX_TRACE("Refreshing directory " << BOX_FORMAT_OBJECTID(ObjectID) << + " in cache (modtime changed from " << oldRevID << + " to " << newRevID << ")"); } // Read it from the stream, then set it's revision ID - BufferedStream buf(*objectFile); - std::auto_ptr dir(new BackupStoreDirectory(buf)); - dir->SetRevisionID(newRevID); - - // Make sure the size of the directory is available for writing the dir back - int64_t dirSize = objectFile->GetDiscUsageInBlocks(); - ASSERT(dirSize > 0); - dir->SetUserInfo1_SizeInBlocks(dirSize); + std::auto_ptr dir(new BackupStoreDirectory); + mpFileSystem->GetDirectory(ObjectID, *dir); // Store in cache - BackupStoreDirectory *pdir = dir.release(); - try - { - mDirectoryCache[ObjectID] = pdir; - } - catch(...) - { - delete pdir; - throw; - } + mDirectoryCache[ObjectID] = dir.get(); - // Return it - return *pdir; + // Since it's freshly loaded, it won't be invalidated, and we can just return it: + return *(dir.release()); } @@ -427,10 +468,7 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, // -------------------------------------------------------------------------- int64_t BackupStoreContext::AllocateObjectID() { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY // times after it has been updated, this is a reasonable number of times @@ -441,13 +479,11 @@ int64_t BackupStoreContext::AllocateObjectID() while(retryLimit > 0) { // Attempt to allocate an ID from the store - int64_t id = mapStoreInfo->AllocateObjectID(); + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + int64_t id = info.AllocateObjectID(); - // Generate filename - std::string filename; - MakeObjectFilename(id, filename); // Check it doesn't exist - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + if(!mpFileSystem->ObjectExists(id)) { // Success! return id; @@ -483,10 +519,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); if(mReadOnly) { @@ -509,171 +542,52 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t id = AllocateObjectID(); // Stream the file to disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - int64_t newObjectBlocksUsed = 0; - RaidFileWrite *ppreviousVerStoreFile = 0; - bool reversedDiffIsCompletelyDifferent = false; - int64_t oldVersionNewBlocksUsed = 0; BackupStoreInfo::Adjustment adjustment = {}; + std::auto_ptr apTransaction; - try + // Diff or full file? + if(DiffFromFileID == 0) { - RaidFileWrite storeFile(mStoreDiscSet, fn); - storeFile.Open(false /* no overwriting */); - - int64_t spaceSavedByConversionToPatch = 0; - - // Diff or full file? - if(DiffFromFileID == 0) - { - // A full file, just store to disc - if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - } - else - { - // Check that the diffed from ID actually exists in the directory - if(dir.FindEntryByID(DiffFromFileID) == 0) - { - THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) - } - - // Diff file, needs to be recreated. - // Choose a temporary filename. - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", - 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - - try - { - // Open it twice -#ifdef WIN32 - InvisibleTempFileStream diff(tempFn.c_str(), - O_RDWR | O_CREAT | O_BINARY); - InvisibleTempFileStream diff2(tempFn.c_str(), - O_RDWR | O_BINARY); -#else - FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); - FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif - - // Stream the incoming diff to this temporary file - if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - - // Verify the diff - diff.Seek(0, IOStream::SeekType_Absolute); - if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) - { - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - - // Seek to beginning of diff file - diff.Seek(0, IOStream::SeekType_Absolute); - - // Filename of the old version - std::string oldVersionFilename; - MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - - // Reassemble that diff -- open previous file, and combine the patch and file - std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); - - // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) - std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); - ppreviousVerStoreFile->Open(true /* allow overwriting */); - from->Seek(0, IOStream::SeekType_Absolute); - diff.Seek(0, IOStream::SeekType_Absolute); - BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, - DiffFromFileID, &reversedDiffIsCompletelyDifferent); - - // Store disc space used - oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - - // And make a space adjustment for the size calculation - spaceSavedByConversionToPatch = - from->GetDiscUsageInBlocks() - - oldVersionNewBlocksUsed; - - adjustment.mBlocksUsed -= spaceSavedByConversionToPatch; - // The code below will change the patch from a - // Current file to an Old file, so we need to - // account for it as a Current file here. - adjustment.mBlocksInCurrentFiles -= - spaceSavedByConversionToPatch; - - // Don't adjust anything else here. We'll do it - // when we update the directory just below, - // which also accounts for non-diff replacements. - - // Everything cleans up here... - } - catch(...) - { - // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway - ::unlink(tempFn.c_str()); - throw; - } - } - - // Get the blocks used - newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); - adjustment.mBlocksUsed += newObjectBlocksUsed; - adjustment.mBlocksInCurrentFiles += newObjectBlocksUsed; - adjustment.mNumCurrentFiles++; - - // Exceeds the hard limit? - int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + - adjustment.mBlocksUsed; - if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) - { - THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The store file will be deleted automatically by the RaidFile object - } - - // Commit the file - storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + apTransaction = mpFileSystem->PutFileComplete(id, rFile, + 0); // refcount: BackupStoreFile requires us to pass 0 to assert that + // the file doesn't already exist, because it will refuse to overwrite an + // existing file. The refcount will increase to 1 when we commit the change + // to the directory, dir. } - catch(...) + else { - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) + // Check that the diffed from ID actually exists in the directory + if(dir.FindEntryByID(DiffFromFileID) == 0) { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; + THROW_EXCEPTION(BackupStoreException, + DiffFromIDNotFoundInDirectory) } - throw; + apTransaction = mpFileSystem->PutFilePatch(id, DiffFromFileID, + rFile, mpRefCount->GetRefCount(DiffFromFileID)); } - // Verify the file -- only necessary for non-diffed versions - // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because - // in the non-diffed code path it's never allocated. - if(DiffFromFileID == 0) - { - std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); - if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) - { - // Error! Delete the file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); + // Get the blocks used + int64_t changeInBlocksUsed = apTransaction->GetNumBlocks() + + apTransaction->GetChangeInBlocksUsedByOldFile(); + adjustment.mBlocksUsed += changeInBlocksUsed; + adjustment.mBlocksInCurrentFiles += changeInBlocksUsed; + adjustment.mNumCurrentFiles++; - // Exception - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } + // Exceeds the hard limit? + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + int64_t newTotalBlocksUsed = info.GetBlocksUsed() + changeInBlocksUsed; + if(newTotalBlocksUsed > info.GetBlocksHardLimit()) + { + // This will cancel the Transaction and delete the RaidFile(s). + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) } + // Can only get this before we commit the RaidFiles. + int64_t numBlocksInNewFile = apTransaction->GetNumBlocks(); + int64_t changeInBlocksUsedByOldFile = + apTransaction->GetChangeInBlocksUsedByOldFile(); + // Modify the directory -- first make all files with the same name // marked as an old version try @@ -690,7 +604,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Adjust size of old entry int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + poldEntry->SetSizeInBlocks(oldSize + changeInBlocksUsedByOldFile); } if(MarkFileWithSameNameAsOldVersions) @@ -722,67 +636,52 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Then the new entry BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, newObjectBlocksUsed, - BackupStoreDirectory::Entry::Flags_File, - AttributesHash); + ModificationTime, id, numBlocksInNewFile, + BackupStoreDirectory::Entry::Flags_File, AttributesHash); // Adjust dependency info of file? - if(DiffFromFileID && poldEntry && !reversedDiffIsCompletelyDifferent) + if(DiffFromFileID && poldEntry && !apTransaction->IsNewFileIndependent()) { poldEntry->SetDependsNewer(id); pnewEntry->SetDependsOlder(DiffFromFileID); } - // Write the directory back to disc + // Save the directory back SaveDirectory(dir); - // Commit the old version's new patched version, now that the directory safely reflects - // the state of the files on disc. - if(ppreviousVerStoreFile != 0) - { - ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } + // It is now safe to commit the old version's new patched version, now + // that the directory safely reflects the state of the files on disc. } catch(...) { - // Back out on adding that file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - // Don't worry about the incremented number in the store info + // Leaving this function without committing the Transaction will cancel + // it, deleting the new file and not modifying the old one. Don't worry + // about the incremented numbers in the store info, they won't cause + // any real problem and bbstoreaccounts check can fix them. throw; } - // Check logic - ASSERT(ppreviousVerStoreFile == 0); + // Commit the new file, and replace the old file (if any) with a patch. + apTransaction->Commit(); // Modify the store info - mapStoreInfo->AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); - mapStoreInfo->AdjustNumOldFiles(adjustment.mNumOldFiles); - mapStoreInfo->AdjustNumDeletedFiles(adjustment.mNumDeletedFiles); - mapStoreInfo->AdjustNumDirectories(adjustment.mNumDirectories); - mapStoreInfo->ChangeBlocksUsed(adjustment.mBlocksUsed); - mapStoreInfo->ChangeBlocksInCurrentFiles(adjustment.mBlocksInCurrentFiles); - mapStoreInfo->ChangeBlocksInOldFiles(adjustment.mBlocksInOldFiles); - mapStoreInfo->ChangeBlocksInDeletedFiles(adjustment.mBlocksInDeletedFiles); - mapStoreInfo->ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); + info.AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); + info.AdjustNumOldFiles(adjustment.mNumOldFiles); + info.AdjustNumDeletedFiles(adjustment.mNumDeletedFiles); + info.AdjustNumDirectories(adjustment.mNumDirectories); + info.ChangeBlocksUsed(adjustment.mBlocksUsed); + info.ChangeBlocksInCurrentFiles(adjustment.mBlocksInCurrentFiles); + info.ChangeBlocksInOldFiles(adjustment.mBlocksInOldFiles); + info.ChangeBlocksInDeletedFiles(adjustment.mBlocksInDeletedFiles); + info.ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); // Increment reference count on the new directory to one - mapRefCount->AddReference(id); + mpRefCount->AddReference(id); - // Save the store info -- can cope if this exceptions because infomation + // Save the store info -- can cope if this exceptions because information // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(false); @@ -805,12 +704,9 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // -------------------------------------------------------------------------- bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) { - // Essential checks! - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); + // Essential checks! if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -843,8 +739,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ madeChanges = true; int64_t blocks = e->GetSizeInBlocks(); - mapStoreInfo->AdjustNumDeletedFiles(1); - mapStoreInfo->ChangeBlocksInDeletedFiles(blocks); + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + info.AdjustNumDeletedFiles(1); + info.ChangeBlocksInDeletedFiles(blocks); // We're marking all old versions as deleted. // This is how a file can be old and deleted @@ -854,8 +751,8 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ // we do need to adjust the current counts. if(!e->IsOld()) { - mapStoreInfo->AdjustNumCurrentFiles(-1); - mapStoreInfo->ChangeBlocksInCurrentFiles(-blocks); + info.AdjustNumCurrentFiles(-1); + info.ChangeBlocksInCurrentFiles(-blocks); } // Is this the last version? @@ -873,6 +770,8 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ { // Save the directory back SaveDirectory(dir); + + // Maybe postponed save of store info SaveStoreInfo(false); } } @@ -897,12 +796,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ // -------------------------------------------------------------------------- bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) { - // Essential checks! - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); + // Essential checks! if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -953,7 +849,8 @@ bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) SaveDirectory(dir); // Modify the store info, and write - mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + info.ChangeBlocksInDeletedFiles(blocksDel); // Maybe postponed save of store info SaveStoreInfo(); @@ -979,12 +876,12 @@ bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) // -------------------------------------------------------------------------- void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) { - std::map::iterator item(mDirectoryCache.find(ObjectID)); + auto item = mDirectoryCache.find(ObjectID); if(item != mDirectoryCache.end()) { // Delete this cached object delete item->second; - // Erase the entry form the map + // Erase the entry from the map mDirectoryCache.erase(item); } } @@ -994,79 +891,56 @@ void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) // // Function // Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &) -// Purpose: Save directory back to disc, update time in cache +// Purpose: Save directory back to disc, update time in cache. +// Since this updates the parent directory, it needs to +// fetch it, which invalidates rDir along with the rest +// of the cache. But since it's usually the last thing +// we do to rDir, that should be fine. // Created: 2003/09/04 // // -------------------------------------------------------------------------- void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); int64_t ObjectID = rDir.GetObjectID(); + auto i = mDirectoryCache.find(ObjectID); + ASSERT(i != mDirectoryCache.end()); + + int64_t new_dir_size; try { // Write to disc, adjust size in store info - std::string dirfn; - MakeObjectFilename(ObjectID, dirfn); int64_t old_dir_size = rDir.GetUserInfo1_SizeInBlocks(); - { - RaidFileWrite writeDir(mStoreDiscSet, dirfn); - writeDir.Open(true /* allow overwriting */); - - BufferedWriteStream buffer(writeDir); - rDir.WriteToStream(buffer); - buffer.Flush(); - - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is available for writing the dir back - ASSERT(dirSize > 0); - int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); - mapStoreInfo->ChangeBlocksUsed(sizeAdjustment); - mapStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); - // Update size stored in directory - rDir.SetUserInfo1_SizeInBlocks(dirSize); - } + mpFileSystem->PutDirectory(rDir); + new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); - // Refresh revision ID in cache { - int64_t revid = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - BOX_TRACE("Saved directory " << - BOX_FORMAT_OBJECTID(ObjectID) << - ", modtime = " << revid); - - rDir.SetRevisionID(revid); + int64_t sizeAdjustment = new_dir_size - old_dir_size; + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + info.ChangeBlocksUsed(sizeAdjustment); + info.ChangeBlocksInDirectories(sizeAdjustment); } + // Update the directory entry in the grandparent, to ensure // that it reflects the current size of the parent directory. - int64_t new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); if(new_dir_size != old_dir_size && ObjectID != BACKUPSTORE_ROOT_DIRECTORY_ID) { int64_t ContainerID = rDir.GetContainerID(); BackupStoreDirectory& parent( GetDirectoryInternal(ContainerID)); - // rDir is now invalid + // i and rDir are now invalid BackupStoreDirectory::Entry* en = parent.FindEntryByID(ObjectID); if(!en) { - BOX_ERROR("Missing entry for directory " << + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CouldNotFindEntryInDirectory, + "Missing entry for directory " << BOX_FORMAT_OBJECTID(ObjectID) << " in directory " << BOX_FORMAT_OBJECTID(ContainerID) << @@ -1106,10 +980,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, int64_t ModificationTime, bool &rAlreadyExists) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); if(mReadOnly) { @@ -1138,43 +1009,38 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, } } + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + // Allocate the next ID int64_t id = AllocateObjectID(); // Create an empty directory with the given attributes on disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); int64_t dirSize; { BackupStoreDirectory emptyDir(id, InDirectory); - // add the atttribues + // Add the attributes: emptyDir.SetAttributes(Attributes, AttributesModTime); - // Write... - RaidFileWrite dirFile(mStoreDiscSet, fn); - dirFile.Open(false /* no overwriting */); - emptyDir.WriteToStream(dirFile); - // Get disc usage, before it's commited - dirSize = dirFile.GetDiscUsageInBlocks(); + // Write, but not using SaveDirectory() because that tries to update the entry + // in the parent directory with the new size, and that entry hasn't been added yet! + mpFileSystem->PutDirectory(emptyDir); + dirSize = emptyDir.GetUserInfo1_SizeInBlocks(); + } + { // Exceeds the hard limit? - int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + - dirSize; - if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + int64_t newTotalBlocksUsed = info.GetBlocksUsed() + dirSize; + if(newTotalBlocksUsed > info.GetBlocksHardLimit()) { + mpFileSystem->DeleteDirectory(id); THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The file will be deleted automatically by the RaidFile object } - // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // Make sure the size of the directory is added to the usage counts in the info ASSERT(dirSize > 0); - mapStoreInfo->ChangeBlocksUsed(dirSize); - mapStoreInfo->ChangeBlocksInDirectories(dirSize); - // Not added to cache, so don't set the size in the directory + info.ChangeBlocksUsed(dirSize); + info.ChangeBlocksInDirectories(dirSize); } // Then add it into the parent directory @@ -1185,25 +1051,36 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, 0 /* attributes hash */); SaveDirectory(dir); - // Increment reference count on the new directory to one - mapRefCount->AddReference(id); + // Increment reference count on the new directory to one: + mpRefCount->AddReference(id); } catch(...) { - // Back out on adding that directory - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); + // Back out on adding that directory: + mpFileSystem->DeleteDirectory(id); - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); + info.ChangeBlocksUsed(-dirSize); + info.ChangeBlocksInDirectories(-dirSize); + + // Remove the newly created directory from the cache: + RemoveDirectoryFromCache(id); + + // Try to remove the new entry from this directory. If + // SaveDirectory() above failed because there was something + // wrong with this directory, then it's likely to fail again, + // so we do this last. Also, SaveDirectory() above invalidated + // dir, so we need to fetch it again. + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + dir.DeleteEntry(id); + SaveDirectory(dir); - // Don't worry about the incremented number in the store info + // Don't worry about the incremented number in the store info. throw; } - // Save the store info (may not be postponed) - mapStoreInfo->AdjustNumDirectories(1); - SaveStoreInfo(false); + // Update and save the store info + info.AdjustNumDirectories(1); + SaveStoreInfo(true); // Allow defer: it's just an empty directory. // tell caller what the ID was return id; @@ -1220,12 +1097,9 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, // -------------------------------------------------------------------------- void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { - // Essential checks! - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); + // Essential checks! if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1298,6 +1172,8 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // -------------------------------------------------------------------------- void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) { + CHECK_FILESYSTEM_INITIALISED(); + try { // Does things carefully to avoid using a directory in the cache after recursive call @@ -1361,13 +1237,14 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) ASSERT(en->IsDeleted() == Undelete); // Don't adjust counters for old files, // because it can be both old and deleted. + BackupStoreInfo& info(GetBackupStoreInfoInternal()); if(!en->IsOld()) { - mapStoreInfo->ChangeBlocksInCurrentFiles(Undelete ? size : -size); - mapStoreInfo->AdjustNumCurrentFiles(Undelete ? 1 : -1); + info.ChangeBlocksInCurrentFiles(Undelete ? size : -size); + info.AdjustNumCurrentFiles(Undelete ? 1 : -1); } - mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete ? -size : size); - mapStoreInfo->AdjustNumDeletedFiles(Undelete ? -1 : 1); + info.ChangeBlocksInDeletedFiles(Undelete ? -size : size); + info.AdjustNumDeletedFiles(Undelete ? -1 : 1); } // Add/remove the deleted flags @@ -1384,7 +1261,6 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) changesMade = true; } - // Save the directory if(changesMade) { SaveDirectory(dir); @@ -1409,10 +1285,8 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) // -------------------------------------------------------------------------- void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1447,10 +1321,8 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable // -------------------------------------------------------------------------- bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1512,34 +1384,28 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena // -------------------------------------------------------------------------- bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } + CHECK_FILESYSTEM_INITIALISED(); // Note that we need to allow object IDs a little bit greater than the last one in the store info, // because the store info may not have got saved in an error condition. Max greater ID is // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. - if(ObjectID <= 0 || ObjectID > (mapStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) + if(ObjectID <= 0 || ObjectID > (GetBackupStoreInfo().GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) { // Obviously bad object ID return false; } - // Test to see if it exists on the disc - std::string filename; - MakeObjectFilename(ObjectID, filename); - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + if (!mpFileSystem->ObjectExists(ObjectID)) { - // RaidFile reports no file there return false; } // Do we need to be more specific? if(MustBe != ObjectExists_Anything) { - // Open the file - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); + // Open the file. TODO FIXME: don't download the entire file from S3 + // to read the first four bytes. + std::auto_ptr objectFile = mpFileSystem->GetObject(ObjectID); // Read the first integer uint32_t magic; @@ -1583,15 +1449,100 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // -------------------------------------------------------------------------- std::auto_ptr BackupStoreContext::OpenObject(int64_t ObjectID) { - if(mapStoreInfo.get() == 0) + CHECK_FILESYSTEM_INITIALISED(); + + return mpFileSystem->GetObject(ObjectID); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetFile() +// Purpose: Retrieve a file from the store +// Created: 2015/08/10 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreContext::GetFile(int64_t ObjectID, int64_t InDirectory) +{ + CHECK_FILESYSTEM_INITIALISED(); + + // Get the directory it's in + const BackupStoreDirectory &rdir(GetDirectoryInternal(InDirectory, + true)); // AllowFlushCache + + // Find the object within the directory + BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(ObjectID); + if(pfileEntry == 0) { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory); } - // Attempt to open the file - std::string fn; - MakeObjectFilename(ObjectID, fn); - return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); + // The result + std::auto_ptr stream; + + // Does this depend on anything? + if(pfileEntry->GetDependsNewer() != 0) + { + // File exists, but is a patch from a new version. Generate the older version. + std::vector patchChain; + int64_t id = ObjectID; + BackupStoreDirectory::Entry *en = 0; + do + { + patchChain.push_back(id); + en = rdir.FindEntryByID(id); + if(en == 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + PatchChainInfoBadInDirectory, + "Object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in dir " << + BOX_FORMAT_OBJECTID(InDirectory) << + " for account " << + BOX_FORMAT_ACCOUNT(mClientID) << + " references object " << + BOX_FORMAT_OBJECTID(id) << + " which does not exist in dir"); + } + id = en->GetDependsNewer(); + } + while(en != 0 && id != 0); + + stream = mpFileSystem->GetFilePatch(ObjectID, patchChain); + } + else + { + // Simple case: file already exists on disc ready to go + + // Open the object + std::auto_ptr object(OpenObject(ObjectID)); + BufferedStream buf(*object); + + // Verify it + if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify); + } + + // Reset stream -- seek to beginning + object->Seek(0, IOStream::SeekType_Absolute); + + // Reorder the stream/file into stream order + { + // Write nastily to allow this to work with gcc 2.x + std::auto_ptr t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */)); + stream = t; + } + + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion + object.release(); + } + + return stream; } @@ -1605,12 +1556,7 @@ std::auto_ptr BackupStoreContext::OpenObject(int64_t ObjectID) // -------------------------------------------------------------------------- int64_t BackupStoreContext::GetClientStoreMarker() { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mapStoreInfo->GetClientStoreMarker(); + return GetBackupStoreInfo().GetClientStoreMarker(); } @@ -1624,14 +1570,10 @@ int64_t BackupStoreContext::GetClientStoreMarker() // -------------------------------------------------------------------------- void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - rBlocksUsed = mapStoreInfo->GetBlocksUsed(); - rBlocksSoftLimit = mapStoreInfo->GetBlocksSoftLimit(); - rBlocksHardLimit = mapStoreInfo->GetBlocksHardLimit(); + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + rBlocksUsed = info.GetBlocksUsed(); + rBlocksSoftLimit = info.GetBlocksSoftLimit(); + rBlocksHardLimit = info.GetBlocksHardLimit(); } @@ -1645,12 +1587,8 @@ void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rB // -------------------------------------------------------------------------- bool BackupStoreContext::HardLimitExceeded() { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mapStoreInfo->GetBlocksUsed() > mapStoreInfo->GetBlocksHardLimit(); + BackupStoreInfo& info(GetBackupStoreInfoInternal()); + return info.GetBlocksUsed() > info.GetBlocksHardLimit(); } @@ -1664,16 +1602,12 @@ bool BackupStoreContext::HardLimitExceeded() // -------------------------------------------------------------------------- void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) { - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - mapStoreInfo->SetClientStoreMarker(ClientStoreMarker); + GetBackupStoreInfoInternal().SetClientStoreMarker(ClientStoreMarker); SaveStoreInfo(false /* don't delay saving this */); } @@ -1924,22 +1858,3 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreContext::GetBackupStoreInfo() -// Purpose: Return the backup store info object, exception if it isn't loaded -// Created: 19/4/04 -// -// -------------------------------------------------------------------------- -const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const -{ - if(mapStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return *(mapStoreInfo.get()); -} - diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h index 484483608..9221d60ef 100644 --- a/lib/backupstore/BackupStoreContext.h +++ b/lib/backupstore/BackupStoreContext.h @@ -15,9 +15,10 @@ #include #include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" +#include "BackupFileSystem.h" #include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" -#include "NamedLock.h" #include "Message.h" #include "Utils.h" @@ -48,22 +49,24 @@ class BackupStoreContext BackupStoreContext(int32_t ClientID, HousekeepingInterface* mpHousekeeping, const std::string& rConnectionDetails); - ~BackupStoreContext(); + BackupStoreContext(BackupFileSystem& rFileSystem, int32_t ClientID, + HousekeepingInterface* mpHousekeeping, + const std::string& rConnectionDetails); + virtual ~BackupStoreContext(); + private: - BackupStoreContext(const BackupStoreContext &rToCopy); -public: + BackupStoreContext(const BackupStoreContext &rToCopy); // no copying - void ReceivedFinishCommand(); +public: void CleanUp(); - int32_t GetClientID() {return mClientID;} enum { - Phase_START = 0, - Phase_Version = 0, - Phase_Login = 1, - Phase_Commands = 2 + Phase_START = 0, + Phase_Version = 0, + Phase_Login = 1, + Phase_Commands = 2 }; int GetPhase() const {return mProtocolPhase;} @@ -89,28 +92,41 @@ class BackupStoreContext // Not really an API, but useful for BackupProtocolLocal2. void ReleaseWriteLock() { - if(mWriteLock.GotLock()) - { - mWriteLock.ReleaseLock(); - } + // Even a read-only filesystem may hold some locks, for example + // S3BackupFileSystem's cache lock, so we always notify the filesystem + // to release any locks that it holds, even if we are read-only. + mpFileSystem->ReleaseLock(); } - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mAccountRootDir = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + // TODO: stop using this version, which has the side-effect of creating a + // BackupStoreFileSystem: + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet); + void SetClientHasAccount() + { + mClientHasAccount = true; + } bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetAccountRoot() const {return mAccountRootDir;} - int GetStoreDiscSet() const {return mStoreDiscSet;} // Store info void LoadStoreInfo(); void SaveStoreInfo(bool AllowDelay = true); - const BackupStoreInfo &GetBackupStoreInfo() const; + + // Const version for external use: + const BackupStoreInfo& GetBackupStoreInfo() const + { + return GetBackupStoreInfoInternal(); + } + const std::string GetAccountName() { - if(!mapStoreInfo.get()) + if(!mpFileSystem) { - return "Unknown"; + // This can happen if the account doesn't exist on the server, e.g. not + // created yet, because BackupStoreDaemon doesn't call + // SetClientHasAccount(), which creates the BackupStoreFileSystem. + return "no such account"; } - return mapStoreInfo->GetAccountName(); + return mpFileSystem->GetBackupStoreInfo(true).GetAccountName(); // ReadOnly } // Client marker @@ -153,12 +169,18 @@ class BackupStoreContext int64_t AttributesModTime, int64_t ModificationTime, bool &rAlreadyExists); - void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); - bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); - bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); + void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, + int64_t AttributesModTime); + bool ChangeFileAttributes(const BackupStoreFilename &rFilename, + int64_t InDirectory, const StreamableMemBlock &Attributes, + int64_t AttributesHash, int64_t &rObjectIDOut); + bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, + int64_t &rObjectIDOut); bool UndeleteFile(int64_t ObjectID, int64_t InDirectory); void DeleteDirectory(int64_t ObjectID, bool Undelete = false); - void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); + void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, + int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, + bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); // Manipulating objects enum @@ -169,18 +191,22 @@ class BackupStoreContext }; bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); std::auto_ptr OpenObject(int64_t ObjectID); - + std::auto_ptr GetFile(int64_t ObjectID, int64_t InDirectory); + // Info int32_t GetClientID() const {return mClientID;} const std::string& GetConnectionDetails() { return mConnectionDetails; } + virtual int GetBlockSize() { return mpFileSystem->GetBlockSize(); } + + // This is not an API, but it's useful in tests with multiple contexts, to allow + // synchronisation between them: + void ClearDirectoryCache(); private: - void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID, bool AllowFlushCache = true); void SaveDirectory(BackupStoreDirectory &rDir); void RemoveDirectoryFromCache(int64_t ObjectID); - void ClearDirectoryCache(); void DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete); int64_t AllocateObjectID(); @@ -189,18 +215,32 @@ class BackupStoreContext HousekeepingInterface *mpHousekeeping; int mProtocolPhase; bool mClientHasAccount; - std::string mAccountRootDir; // has final directory separator - int mStoreDiscSet; - bool mReadOnly; - NamedLock mWriteLock; int mSaveStoreInfoDelay; // how many times to delay saving the store info - // Store info - std::auto_ptr mapStoreInfo; + // mapOwnFileSystem is initialised when we created our own BackupFileSystem, + // using the old constructor. It ensures that the BackupFileSystem is deleted + // when this BackupStoreContext is destroyed. TODO: stop using that old + // constructor, and remove this member. + std::auto_ptr mapOwnFileSystem; + + // mpFileSystem is always initialised when SetClientHasAccount() has been called, + // whether or not we created it ourselves, and all internal functions use this + // member instead of mapOwnFileSystem. + BackupFileSystem* mpFileSystem; + + // Non-const version for internal use: + BackupStoreInfo& GetBackupStoreInfoInternal() const + { + if(!mpFileSystem) + { + THROW_EXCEPTION(BackupStoreException, FileSystemNotInitialised); + } + return mpFileSystem->GetBackupStoreInfo(mReadOnly); + } // Refcount database - std::auto_ptr mapRefCount; + BackupStoreRefCountDatabase* mpRefCount; // Directory cache std::map mDirectoryCache; diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h index 788a3ad03..25cf2d754 100644 --- a/lib/backupstore/BackupStoreDirectory.h +++ b/lib/backupstore/BackupStoreDirectory.h @@ -36,13 +36,13 @@ class BackupStoreDirectory public: #ifndef BOX_RELEASE_BUILD - void Invalidate() + void Invalidate(bool invalid = true) { - mInvalidated = true; + mInvalidated = invalid; for (std::vector::iterator i = mEntries.begin(); i != mEntries.end(); i++) { - (*i)->Invalidate(); + (*i)->Invalidate(invalid); } } #endif @@ -86,7 +86,7 @@ class BackupStoreDirectory public: #ifndef BOX_RELEASE_BUILD - void Invalidate() { mInvalidated = true; } + void Invalidate(bool invalid) { mInvalidated = invalid; } #endif friend class BackupStoreDirectory; diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index efdbcf683..811bf0f04 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -64,13 +64,23 @@ InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing 60 CouldNotLockStoreAccount 61 Another process is accessing this account -- is a client connected to the server? AttributeHashSecretNotSet 62 AEScipherNotSupportedByInstalledOpenSSL 63 The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES -SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection. +SignalReceived 64 A signal was received by the process, restart or terminate needed. Exception thrown to abort connection IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory -PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. -UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. +PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it +UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first -CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping. -CancelledByBackgroundTask 71 The current task was cancelled on request by the background task. -ObjectDoesNotExist 72 The specified object ID does not exist in the store. -AccountAlreadyExists 73 Tried to create an account that already exists. +CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping +CancelledByBackgroundTask 71 The current task was cancelled on request by the background task +MissingEtagHeader 72 The S3 HTTP response did not contain an ETag header as expected +InvalidEtagHeader 73 The S3 HTTP response contain a malformed or unexpected ETag header +FileSystemNotInitialised 74 No BackupFileSystem has been configured for this account +ObjectDoesNotExist 75 The specified object ID does not exist in the store, or is not of this type +AccountAlreadyExists 76 Tried to create an account that already exists +AccountDoesNotExist 77 Tried to open an account that does not exist +FileDownloadedIncorrectly 78 The downloaded file had different data than expected, so was downloaded badly +FileUploadFailed 79 Failed to upload the file to the storage server +CacheDirectoryLocked 80 The cache directory is already locked by another process +BadConfiguration 82 The configuration file contains an error or invalid value +TooManyFilesInDirectory 83 The S3 directory contains too many files for it to be a Boxbackup store, please check and/or empty it +FailedToCreateTemporaryFile 84 Failed to create a temporary file diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index 99562685f..49b29263d 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -279,24 +279,21 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreFile::VerifyStream::Write() -// Purpose: Handles writes to the verifying stream. If the write -// completes the current unit, then verify it, copy it -// to mpCopyToStream if not NULL, and move on to the -// next unit, otherwise throw an exception. +// Name: BackupStoreFile::VerifyStream::Read() +// Purpose: Handles reads from the verifying stream. If the read +// completes the current unit, then verify it and move +// on to the next unit. // Created: 2015/08/07 // // -------------------------------------------------------------------------- -void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int Timeout) +int BackupStoreFile::VerifyStream::Read(void *pBuffer, int NBytes, int Timeout) { - // Check that we haven't already written too much to the current unit - size_t BytesToAdd; + // Check that we haven't already read too much for the current unit if(mState == State_Blocks) { // We don't know how many bytes to expect ASSERT(mCurrentUnitSize == 0); - BytesToAdd = NBytes; } else { @@ -304,31 +301,30 @@ void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int T size_t BytesLeftInCurrentUnit = mCurrentUnitSize - mCurrentUnitData.GetSize(); // Add however many bytes are needed/available to the current unit's buffer. - BytesToAdd = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); + NBytes = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); } + if(NBytes == 0) + { + return 0; + } // We must make progress here, or we could have infinite recursion. - ASSERT(BytesToAdd > 0); + ASSERT(NBytes >= 0); + NBytes = mReadFromStream.Read(pBuffer, NBytes, Timeout); CollectInBufferStream* pCurrentBuffer = (mCurrentBufferIsAlternate ? &mAlternateData : &mCurrentUnitData); - pCurrentBuffer->Write(pBuffer, BytesToAdd, Timeout); - if(mpCopyToStream) - { - mpCopyToStream->Write(pBuffer, BytesToAdd, Timeout); - } - - pBuffer = (uint8_t *)pBuffer + BytesToAdd; - NBytes -= BytesToAdd; - mCurrentPosition += BytesToAdd; + pCurrentBuffer->Write(pBuffer, NBytes); + mCurrentPosition += NBytes; if(mState == State_Blocks) { // The block index follows the blocks themselves, but without seeing the // index we don't know how big the blocks are. So we just have to keep - // reading, holding the last mBlockIndexSize bytes in two buffers, until - // we reach the end of the file (when Close() is called) when we can look - // back over those buffers and extract the block index from them. + // reading, holding (at least) the last mBlockIndexSize bytes in two + // buffers, until we reach the end of the file (when Close() is called) + // when we can look back over those buffers and extract the block index + // from them. if(pCurrentBuffer->GetSize() >= mBlockIndexSize) { // Time to swap buffers, and clear the one we're about to @@ -340,8 +336,8 @@ void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int T } // We don't want to move data into the finished buffer while we're in this - // state, and we don't need to call ourselves recursively, so just return. - return; + // state, so just return. + return NBytes; } ASSERT(mState != State_Blocks); @@ -349,7 +345,7 @@ void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int T // If the current unit is not complete, just return now. if(mCurrentUnitData.GetSize() < mCurrentUnitSize) { - return; + return NBytes; } ASSERT(mCurrentUnitData.GetSize() == mCurrentUnitSize); @@ -460,11 +456,9 @@ void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int T } } - if(NBytes > 0) - { - // Still some data to process, so call recursively to deal with it. - Write(pBuffer, NBytes, Timeout); - } + // It's OK to read less data than requested, so we don't need to call ourselves + // recursively to read more data, we can just return what we have. + return NBytes; } // -------------------------------------------------------------------------- @@ -479,14 +473,8 @@ void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int T // // -------------------------------------------------------------------------- - -void BackupStoreFile::VerifyStream::Close(bool CloseCopyStream) +void BackupStoreFile::VerifyStream::Close() { - if(mpCopyToStream && CloseCopyStream) - { - mpCopyToStream->Close(); - } - if(mState != State_Blocks) { THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, @@ -677,7 +665,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile } catch(...) { - ::unlink(DecodedFilename); + EMU_UNLINK(DecodedFilename); throw; } } @@ -1006,7 +994,8 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) // // Function // Name: BackupStoreFile::DecodedStream::Read(void *, int, int) -// Purpose: As interface. Reads decrpyted data. +// Purpose: An interface to read decrypted data from an +// encrypted file stream. // Created: 9/12/03 // // -------------------------------------------------------------------------- diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h index fe69caeb8..e357bfc2e 100644 --- a/lib/backupstore/BackupStoreFile.h +++ b/lib/backupstore/BackupStoreFile.h @@ -91,6 +91,8 @@ class BackupStoreFile virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -138,7 +140,7 @@ class BackupStoreFile }; int mState; - IOStream* mpCopyToStream; + IOStream& mReadFromStream; CollectInBufferStream mCurrentUnitData; size_t mCurrentUnitSize; int64_t mNumBlocks; @@ -150,11 +152,12 @@ class BackupStoreFile bool mBlockFromOtherFileReferenced; int64_t mContainerID; int64_t mDiffFromObjectID; + bool mClosed; public: - VerifyStream(IOStream* pCopyToStream = NULL) + VerifyStream(IOStream& ReadFromStream) : mState(State_Header), - mpCopyToStream(pCopyToStream), + mReadFromStream(ReadFromStream), mCurrentUnitSize(sizeof(file_StreamFormat)), mNumBlocks(0), mBlockIndexSize(0), @@ -163,23 +166,36 @@ class BackupStoreFile mCurrentBufferIsAlternate(false), mBlockFromOtherFileReferenced(false), mContainerID(0), - mDiffFromObjectID(0) + mDiffFromObjectID(0), + mClosed(false) { } virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) { THROW_EXCEPTION(CommonException, NotSupported); } - virtual void Write(const void *pBuffer, int NBytes, - int Timeout = IOStream::TimeOutInfinite); - virtual void Close(bool CloseCopyStream = true); + using IOStream::Write; + + // Declare twice (with different parameters) to avoid warnings that + // Close(bool) hides overloaded virtual function. + virtual void Close(bool CloseReadFromStream) + { + if(CloseReadFromStream) + { + mReadFromStream.Close(); + } + Close(); + } + virtual void Close(); virtual bool StreamDataLeft() { - THROW_EXCEPTION(CommonException, NotSupported); + return mReadFromStream.StreamDataLeft(); } virtual bool StreamClosed() { - THROW_EXCEPTION(CommonException, NotSupported); + return mClosed; } }; diff --git a/lib/backupstore/BackupStoreFileCmbIdx.cpp b/lib/backupstore/BackupStoreFileCmbIdx.cpp index 0eec3872e..1250f1177 100644 --- a/lib/backupstore/BackupStoreFileCmbIdx.cpp +++ b/lib/backupstore/BackupStoreFileCmbIdx.cpp @@ -34,6 +34,8 @@ class BSFCombinedIndexStream : public IOStream virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual void Initialise(IOStream &rFrom); diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp index 83333a5b2..accfaec73 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.cpp +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -713,7 +713,6 @@ BackupStoreFileEncodeStream::Recipe::Recipe( BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex, int64_t OtherFileID) : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), mOtherFileID(OtherFileID) { ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index 5b9b4a612..6aac78bd2 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -71,7 +71,6 @@ class BackupStoreFileEncodeStream : public IOStream private: BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex; - int64_t mNumBlocksInIndex; int64_t mOtherFileID; }; @@ -85,6 +84,8 @@ class BackupStoreFileEncodeStream : public IOStream virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); int64_t GetBytesToUpload() { return mBytesToUpload; } diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index efe3f7bb9..12bd7611e 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -14,8 +14,6 @@ #include "Archive.h" #include "BackupStoreInfo.h" #include "BackupStoreException.h" -#include "RaidFileWrite.h" -#include "RaidFileRead.h" #include "MemLeakFindOn.h" @@ -35,7 +33,6 @@ // -------------------------------------------------------------------------- BackupStoreInfo::BackupStoreInfo() : mAccountID(-1), - mDiscSet(-1), mReadOnly(true), mIsModified(false), mClientStoreMarker(0), @@ -67,38 +64,9 @@ BackupStoreInfo::~BackupStoreInfo() { } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int) -// Purpose: Create a new info file on disc -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) -{ - BackupStoreInfo info; - info.mAccountID = AccountID; - info.mDiscSet = DiscSet; - info.mReadOnly = false; - info.mLastObjectIDUsed = 1; - info.mBlocksSoftLimit = BlockSoftLimit; - info.mBlocksHardLimit = BlockHardLimit; - - // Generate the filename - ASSERT(rRootDir[rRootDir.size() - 1] == '/' || - rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); - info.mFilename = rRootDir + INFO_FILENAME; - info.mExtraData.SetForReading(); // extra data is empty in this case - - info.Save(false); -} - -BackupStoreInfo::BackupStoreInfo(int32_t AccountID, const std::string &FileName, - int64_t BlockSoftLimit, int64_t BlockHardLimit) +BackupStoreInfo::BackupStoreInfo(int32_t AccountID, int64_t BlockSoftLimit, + int64_t BlockHardLimit) : mAccountID(AccountID), - mDiscSet(-1), - mFilename(FileName), mReadOnly(false), mIsModified(false), mClientStoreMarker(0), @@ -119,40 +87,8 @@ BackupStoreInfo::BackupStoreInfo(int32_t AccountID, const std::string &FileName, mExtraData.SetForReading(); // extra data is empty in this case } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreInfo::Load(int32_t, const std::string &, -// int, bool) -// Purpose: Loads the info from disc, given the root -// information. Can be marked as read only. -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -std::auto_ptr BackupStoreInfo::Load(int32_t AccountID, - const std::string &rRootDir, int DiscSet, bool ReadOnly, - int64_t *pRevisionID) -{ - // Generate the filename - std::string fn(rRootDir + INFO_FILENAME); - - // Open the file for reading (passing on optional request for revision ID) - std::auto_ptr rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); - std::auto_ptr info = Load(*rf, fn, ReadOnly); - - // Check it - if(info->GetAccountID() != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store info", - fn, BackupStoreException, BadStoreInfoOnLoad); - } - - info->mDiscSet = DiscSet; - return info; -} - std::auto_ptr BackupStoreInfo::Load(IOStream& rStream, - const std::string FileName, bool ReadOnly) + const std::string& FileName, bool ReadOnly) { // Read in format and version int32_t magic; @@ -184,7 +120,6 @@ std::auto_ptr BackupStoreInfo::Load(IOStream& rStream, std::auto_ptr info(new BackupStoreInfo); // Put in basic location info - info->mFilename = FileName; info->mReadOnly = ReadOnly; int64_t numDelObj = 0; @@ -306,24 +241,18 @@ std::auto_ptr BackupStoreInfo::Load(IOStream& rStream, // -------------------------------------------------------------------------- std::auto_ptr BackupStoreInfo::CreateForRegeneration( int32_t AccountID, const std::string& rAccountName, - const std::string &rRootDir, int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInCurrentFiles, int64_t BlocksInOldFiles, int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit, bool AccountEnabled, IOStream& ExtraData) { - // Generate the filename - std::string fn(rRootDir + INFO_FILENAME); - // Make new object std::auto_ptr info(new BackupStoreInfo); // Put in basic info info->mAccountID = AccountID; info->mAccountName = rAccountName; - info->mDiscSet = DiscSet; - info->mFilename = fn; info->mReadOnly = false; // Insert info starting info @@ -346,37 +275,6 @@ std::auto_ptr BackupStoreInfo::CreateForRegeneration( } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreInfo::Save(bool allowOverwrite) -// Purpose: Save modified info back to disc -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -void BackupStoreInfo::Save(bool allowOverwrite) -{ - // Make sure we're initialised (although should never come to this) - if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - // Can we do this? - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) - } - - // Then... open a write file - RaidFileWrite rf(mDiscSet, mFilename); - rf.Open(allowOverwrite); - Save(rf); - - // Commit it to disc, converting it to RAID now - rf.Commit(true); -} - void BackupStoreInfo::Save(IOStream& rOutStream) { // Make header diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index ec6cd2cb1..909da1e7c 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -70,35 +70,26 @@ END_STRUCTURE_PACKING_FOR_WIRE class BackupStoreInfo { friend class BackupStoreCheck; -public: - ~BackupStoreInfo(); -private: - // Creation through static functions only - BackupStoreInfo(); + // No copying allowed BackupStoreInfo(const BackupStoreInfo &); -public: - // Create a New account, saving a blank info object to the disc - static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, - int64_t BlockSoftLimit, int64_t BlockHardLimit); - BackupStoreInfo(int32_t AccountID, const std::string &FileName, - int64_t BlockSoftLimit, int64_t BlockHardLimit); +protected: + BackupStoreInfo(); - // Load it from the store - static std::auto_ptr Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); +public: + BackupStoreInfo(int32_t AccountID, int64_t BlockSoftLimit, + int64_t BlockHardLimit); + ~BackupStoreInfo(); // Load it from a stream (file or RaidFile) static std::auto_ptr Load(IOStream& rStream, - const std::string FileName, bool ReadOnly); + const std::string& FileName, bool ReadOnly); + void Save(IOStream& rOutStream); // Has info been modified? bool IsModified() const {return mIsModified;} - // Save modified infomation back to store - void Save(bool allowOverwrite = true); - void Save(IOStream& rOutStream); - // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} @@ -116,7 +107,6 @@ class BackupStoreInfo int64_t GetNumDirectories() const {return mNumDirectories;} bool IsAccountEnabled() const {return mAccountEnabled;} bool IsReadOnly() const {return mReadOnly;} - int GetDiscSetNumber() const {return mDiscSet;} int ReportChangesTo(BackupStoreInfo& rOldInfo); @@ -153,7 +143,6 @@ class BackupStoreInfo */ static std::auto_ptr CreateForRegeneration( int32_t AccountID, const std::string &rAccountName, - const std::string &rRootDir, int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInCurrentFiles, int64_t BlocksInOldFiles, int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, @@ -180,8 +169,6 @@ class BackupStoreInfo // they now define the sizes of fields on disk (via Archive). int32_t mAccountID; std::string mAccountName; - int mDiscSet; - std::string mFilename; bool mReadOnly; bool mIsModified; diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupstore/BackupStoreObjectDump.cpp similarity index 100% rename from lib/backupclient/BackupStoreObjectDump.cpp rename to lib/backupstore/BackupStoreObjectDump.cpp diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index b2ea1abdf..abdf10ec9 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -27,30 +27,138 @@ #define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC #define REFCOUNT_FILENAME "refcount" +typedef BackupStoreRefCountDatabase::refcount_t refcount_t; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabaseImpl +// Purpose: Implementation of the BackupStoreRefCountDatabase +// interface. +// Created: 2016/04/17 +// +// -------------------------------------------------------------------------- +class BackupStoreRefCountDatabaseImpl : public BackupStoreRefCountDatabase +{ +public: + ~BackupStoreRefCountDatabaseImpl(); + + BackupStoreRefCountDatabaseImpl(const std::string& Filename, int64_t AccountID, + bool ReadOnly, bool PotentialDB, bool TemporaryDB, + std::auto_ptr apDatabaseFile); + +private: + // No copying allowed + BackupStoreRefCountDatabaseImpl(const BackupStoreRefCountDatabase &); + +public: + // Create a blank database, using a temporary file that you must + // Discard() or Commit() to make permanent. + static std::auto_ptr Create + (const BackupStoreAccountDatabase::Entry& rAccount); + static std::auto_ptr Create + (const std::string& Filename, int64_t AccountID); + void Commit(); + void Discard(); + void Close() + { + // If this was a potential database, it should have been + // Commit()ed or Discard()ed first. + ASSERT(!mIsPotentialDB); + // If this is a temporary database, we should Discard() it to + // delete the file: + if(mIsTemporaryDB) + { + Discard(); + } + mapDatabaseFile.reset(); + } + void Reopen(); + bool IsReadOnly() { return mReadOnly; } + + // Load it from the store + static std::auto_ptr Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + // Load it from a stream (file or RaidFile) + static std::auto_ptr Load( + const std::string& FileName, int64_t AccountID, bool ReadOnly); + + // Data access functions + refcount_t GetRefCount(int64_t ObjectID) const; + int64_t GetLastObjectIDUsed() const; + + // SetRefCount is not private, but this whole implementation is effectively + // private, and SetRefCount is not in the interface, so it's not callable from + // anywhere else. + void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); + + // Data modification functions + void AddReference(int64_t ObjectID); + // RemoveReference returns false if refcount drops to zero + bool RemoveReference(int64_t ObjectID); + int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs); + +private: + IOStream::pos_type GetSize() const + { + return mapDatabaseFile->GetPosition() + + mapDatabaseFile->BytesLeftToRead(); + } + IOStream::pos_type GetEntrySize() const + { + return sizeof(refcount_t); + } + IOStream::pos_type GetOffset(int64_t ObjectID) const + { + return ((ObjectID - 1) * GetEntrySize()) + + sizeof(refcount_StreamFormat); + } + + // Location information + int64_t mAccountID; + std::string mFilename, mFinalFilename; + bool mReadOnly; + bool mIsModified; + bool mIsPotentialDB; + bool mIsTemporaryDB; + std::auto_ptr mapDatabaseFile; + + bool NeedsCommitOrDiscard() + { + return mapDatabaseFile.get() && mIsModified && mIsPotentialDB; + } +}; + + // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase() +// Name: BackupStoreRefCountDatabaseImpl::BackupStoreRefCountDatabase() // Purpose: Default constructor // Created: 2003/08/28 // // -------------------------------------------------------------------------- -BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const - BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly, - bool Temporary, std::auto_ptr apDatabaseFile) -: mAccount(rAccount), - mFilename(GetFilename(rAccount, Temporary)), +BackupStoreRefCountDatabaseImpl::BackupStoreRefCountDatabaseImpl( + const std::string& Filename, int64_t AccountID, bool ReadOnly, bool PotentialDB, + bool TemporaryDB, std::auto_ptr apDatabaseFile) +: mAccountID(AccountID), + mFilename(Filename + (PotentialDB ? "X" : "")), + mFinalFilename(TemporaryDB ? "" : Filename), mReadOnly(ReadOnly), mIsModified(false), - mIsTemporaryFile(Temporary), + mIsPotentialDB(PotentialDB), + mIsTemporaryDB(TemporaryDB), mapDatabaseFile(apDatabaseFile) { - ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense + ASSERT(!(PotentialDB && TemporaryDB)); // being both doesn't make sense + ASSERT(!(ReadOnly && PotentialDB)); // being both doesn't make sense } -void BackupStoreRefCountDatabase::Commit() +void BackupStoreRefCountDatabaseImpl::Commit() { - if (!mIsTemporaryFile) + ASSERT(!mIsTemporaryDB); + + if (!mIsPotentialDB) { THROW_EXCEPTION_MESSAGE(CommonException, Internal, "Cannot commit a permanent reference count database"); @@ -65,32 +173,30 @@ void BackupStoreRefCountDatabase::Commit() mapDatabaseFile->Close(); mapDatabaseFile.reset(); - std::string Final_Filename = GetFilename(mAccount, false); - #ifdef WIN32 - if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0) + if(FileExists(mFinalFilename) && EMU_UNLINK(mFinalFilename.c_str()) != 0) { THROW_EMU_FILE_ERROR("Failed to delete old permanent refcount " - "database file", mFilename, CommonException, + "database file", mFinalFilename, CommonException, OSFileError); } #endif - if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0) + if(rename(mFilename.c_str(), mFinalFilename.c_str()) != 0) { THROW_EMU_ERROR("Failed to rename temporary refcount database " "file from " << mFilename << " to " << - Final_Filename, CommonException, OSFileError); + mFinalFilename, CommonException, OSFileError); } - mFilename = Final_Filename; + mFilename = mFinalFilename; mIsModified = false; - mIsTemporaryFile = false; + mIsPotentialDB = false; } -void BackupStoreRefCountDatabase::Discard() +void BackupStoreRefCountDatabaseImpl::Discard() { - if (!mIsTemporaryFile) + if (!mIsTemporaryDB && !mIsPotentialDB) { THROW_EXCEPTION_MESSAGE(CommonException, Internal, "Cannot discard a permanent reference count database"); @@ -106,32 +212,37 @@ void BackupStoreRefCountDatabase::Discard() mapDatabaseFile.reset(); } - if(unlink(mFilename.c_str()) != 0) + if(EMU_UNLINK(mFilename.c_str()) != 0) { - THROW_EMU_FILE_ERROR("Failed to delete temporary refcount " + THROW_EMU_FILE_ERROR("Failed to delete temporary/potential refcount " "database file", mFilename, CommonException, OSFileError); } mIsModified = false; - mIsTemporaryFile = false; + mIsTemporaryDB = false; + mIsPotentialDB = false; } // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase +// Name: BackupStoreRefCountDatabaseImpl::~BackupStoreRefCountDatabase // Purpose: Destructor // Created: 2003/08/28 // // -------------------------------------------------------------------------- -BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() +BackupStoreRefCountDatabaseImpl::~BackupStoreRefCountDatabaseImpl() { - if (mIsTemporaryFile) + if (mIsTemporaryDB || mIsPotentialDB) { // Don't throw exceptions in a destructor. - BOX_ERROR("BackupStoreRefCountDatabase destroyed without " - "explicit commit or discard"); + if(mIsPotentialDB) + { + BOX_ERROR("Potential new BackupStoreRefCountDatabase destroyed " + "without explicit commit or discard"); + } + try { Discard(); @@ -145,17 +256,13 @@ BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() } std::string BackupStoreRefCountDatabase::GetFilename(const - BackupStoreAccountDatabase::Entry& rAccount, bool Temporary) + BackupStoreAccountDatabase::Entry& rAccount) { std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); std::string fn(RootDir + REFCOUNT_FILENAME ".rdb"); - if(Temporary) - { - fn += "X"; - } RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -164,52 +271,89 @@ std::string BackupStoreRefCountDatabase::GetFilename(const // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreRefCountDatabase::Create(int32_t, -// const std::string &, int, bool) +// Name: BackupStoreRefCountDatabase::Create( +// const BackupStoreAccountDatabase::Entry& rAccount) // Purpose: Create a blank database, using a temporary file that // you must Discard() or Commit() to make permanent. // Created: 2003/08/28 // // -------------------------------------------------------------------------- std::auto_ptr - BackupStoreRefCountDatabase::Create - (const BackupStoreAccountDatabase::Entry& rAccount) +BackupStoreRefCountDatabase::Create(const BackupStoreAccountDatabase::Entry& rAccount) { - // Initial header - refcount_StreamFormat hdr; - hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); - hdr.mAccountID = htonl(rAccount.GetID()); - - std::string Filename = GetFilename(rAccount, true); // temporary + std::string Filename = GetFilename(rAccount); + return Create(Filename, rAccount.GetID()); +} +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Create( +// const std::string& Filename, int64_t AccountID, +// bool reuse_existing_file) +// Purpose: Create a blank database, using a temporary file that +// you must Discard() or Commit() to make permanent. +// Be careful with reuse_existing_file, because it +// makes it easy to bypass the restriction of only one +// (committable) temporary database at a time, and to +// accidentally overwrite the main DB. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr +BackupStoreRefCountDatabase::Create(const std::string& Filename, int64_t AccountID, + bool reuse_existing_file) +{ // Open the file for writing - if (FileExists(Filename)) + std::string temp_filename = Filename + (reuse_existing_file ? "" : "X"); + int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL; + + if(FileExists(temp_filename)) { - BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing " - "temporary reference count database")); - if (unlink(Filename.c_str()) != 0) + if(reuse_existing_file) { - THROW_SYS_FILE_ERROR("Failed to delete old temporary " - "reference count database file", Filename, - CommonException, OSFileError); + // Don't warn, and don't fail because the file already exists. This allows + // creating a temporary file securely with mkstemp() and then opening it as + // a refcount database, avoiding a race condition. + flags &= ~O_EXCL; + } + else + { + BOX_WARNING(BOX_FILE_MESSAGE(temp_filename, "Overwriting existing " + "temporary reference count database")); + if (EMU_UNLINK(temp_filename.c_str()) != 0) + { + THROW_SYS_FILE_ERROR("Failed to delete old temporary " + "reference count database file", temp_filename, + CommonException, OSFileError); + } } } - int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL; - std::auto_ptr DatabaseFile(new FileStream(Filename, flags)); - +#ifdef BOX_OPEN_LOCK + flags |= BOX_OPEN_LOCK; +#endif + std::auto_ptr database_file(new FileStream(temp_filename, flags)); + // Write header - DatabaseFile->Write(&hdr, sizeof(hdr)); + refcount_StreamFormat hdr; + hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); + hdr.mAccountID = htonl(AccountID); + database_file->Write(&hdr, sizeof(hdr)); // Make new object - std::auto_ptr refcount( - new BackupStoreRefCountDatabase(rAccount, false, true, - DatabaseFile)); - + BackupStoreRefCountDatabaseImpl* p_impl = new BackupStoreRefCountDatabaseImpl( + Filename, AccountID, + false, // ReadOnly + !reuse_existing_file, // PotentialDB + reuse_existing_file, // TemporaryDB + database_file); + std::auto_ptr refcount(p_impl); + // The root directory must always have one reference for a database // to be valid, so set that now on the new database. This will leave // mIsModified set to true. - refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + p_impl->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); // return it to caller return refcount; @@ -229,55 +373,93 @@ std::auto_ptr std::auto_ptr BackupStoreRefCountDatabase::Load( const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) { - // Generate the filename. Cannot open a temporary database, so it must - // be a permanent one. - std::string Filename = GetFilename(rAccount, false); + return BackupStoreRefCountDatabase::Load(GetFilename(rAccount), + rAccount.GetID(), ReadOnly); +} + +std::auto_ptr OpenDatabaseFile(const std::string& Filename, int64_t AccountID, + bool ReadOnly) +{ int flags = ReadOnly ? O_RDONLY : O_RDWR; + std::auto_ptr database_file(new FileStream(Filename, flags | O_BINARY)); - // Open the file for read/write - std::auto_ptr dbfile(new FileStream(Filename, - flags | O_BINARY)); - // Read in a header refcount_StreamFormat hdr; - if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + if(!database_file->ReadFullBuffer(&hdr, sizeof(hdr), + 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read refcount database: " "short read", Filename, BackupStoreException, CouldNotLoadStoreInfo); } - + // Check it if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || - (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) + (int32_t)ntohl(hdr.mAccountID) != AccountID) { THROW_FILE_ERROR("Failed to read refcount database: " "bad magic number", Filename, BackupStoreException, BadStoreInfoOnLoad); } - - // Make new object + + return database_file; +} + + +std::auto_ptr +BackupStoreRefCountDatabase::Load(const std::string& Filename, int64_t AccountID, + bool ReadOnly) +{ + // You cannot reopen a temporary database, so it must be the permanent filename, + // so no need to append an X to it. + ASSERT(Filename.size() > 0 && Filename[Filename.size() - 1] != 'X'); + + std::auto_ptr database_file = OpenDatabaseFile(Filename, AccountID, ReadOnly); std::auto_ptr refcount( - new BackupStoreRefCountDatabase(rAccount, ReadOnly, false, - dbfile)); - + new BackupStoreRefCountDatabaseImpl(Filename, AccountID, ReadOnly, + false, // PotentialDB + false, // TemporaryDB + database_file)); + // return it to caller return refcount; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabaseImpl::Reopen() +// Purpose: Reopen a previously-opened and then closed refcount +// database. +// Created: 2016/04/25 +// +// -------------------------------------------------------------------------- +void BackupStoreRefCountDatabaseImpl::Reopen() +{ + ASSERT(!mapDatabaseFile.get()); + + // You cannot reopen a temporary database, so it must be the permanent filename, + // so no need to append an X to it. + ASSERT(mFilename.size() > 0 && mFilename[mFilename.size() - 1] != 'X'); + + mapDatabaseFile = OpenDatabaseFile(mFilename, mAccountID, mReadOnly); +} + + // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t +// Name: BackupStoreRefCountDatabaseImpl::GetRefCount(int64_t // ObjectID) // Purpose: Get the number of references to the specified object // out of the database // Created: 2009/06/01 // // -------------------------------------------------------------------------- -BackupStoreRefCountDatabase::refcount_t -BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +refcount_t BackupStoreRefCountDatabaseImpl::GetRefCount(int64_t ObjectID) const { + ASSERT(mapDatabaseFile.get()); IOStream::pos_type offset = GetOffset(ObjectID); if (GetSize() < offset + GetEntrySize()) @@ -302,13 +484,13 @@ BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const return ntohl(refcount); } -int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const +int64_t BackupStoreRefCountDatabaseImpl::GetLastObjectIDUsed() const { return (GetSize() - sizeof(refcount_StreamFormat)) / sizeof(refcount_t); } -void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) +void BackupStoreRefCountDatabaseImpl::AddReference(int64_t ObjectID) { refcount_t refcount; @@ -328,9 +510,10 @@ void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) SetRefCount(ObjectID, refcount); } -void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, +void BackupStoreRefCountDatabaseImpl::SetRefCount(int64_t ObjectID, refcount_t NewRefCount) { + ASSERT(mapDatabaseFile.get()); IOStream::pos_type offset = GetOffset(ObjectID); mapDatabaseFile->Seek(offset, SEEK_SET); refcount_t RefCountNetOrder = htonl(NewRefCount); @@ -338,7 +521,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, mIsModified = true; } -bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) +bool BackupStoreRefCountDatabaseImpl::RemoveReference(int64_t ObjectID) { refcount_t refcount = GetRefCount(ObjectID); // must exist in database ASSERT(refcount > 0); @@ -347,7 +530,7 @@ bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) return (refcount > 0); } -int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) +int BackupStoreRefCountDatabaseImpl::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) { int ErrorCount = 0; int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed(); @@ -357,7 +540,6 @@ int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rO ObjectID < std::max(MaxOldObjectId, MaxNewObjectId); ObjectID++) { - typedef BackupStoreRefCountDatabase::refcount_t refcount_t; refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ? rOldRefs.GetRefCount(ObjectID) : 0; refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ? diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h index 915653a4b..c0b71503d 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.h +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -45,7 +45,8 @@ END_STRUCTURE_PACKING_FOR_WIRE // // Class // Name: BackupStoreRefCountDatabase -// Purpose: Backup store reference count database storage +// Purpose: Abstract interface for an object reference count +// database. // Created: 2009/06/01 // // -------------------------------------------------------------------------- @@ -55,72 +56,60 @@ class BackupStoreRefCountDatabase friend class BackupStoreContext; friend class HousekeepStoreAccount; -public: - ~BackupStoreRefCountDatabase(); private: - // Creation through static functions only - BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& - rAccount, bool ReadOnly, bool Temporary, - std::auto_ptr apDatabaseFile); // No copying allowed BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); - + +protected: + // Protected constructor which does nothing, to allow concrete implementations + // to initialise themselves. + BackupStoreRefCountDatabase() { } + public: + virtual ~BackupStoreRefCountDatabase() { } + // Create a blank database, using a temporary file that you must // Discard() or Commit() to make permanent. + virtual void Commit() = 0; + virtual void Discard() = 0; + virtual void Close() = 0; + + // I'm not sure that Reopen() is a good idea, but it's part of BackupFileSystem's + // API that it manages the lifetime of two BackupStoreRefCountDatabases, and + // Commit() changes the temporary DB to permanent, and it should not be + // invalidated by this, so we need a way to reopen it to make the existing object + // usable again. + virtual void Reopen() = 0; + virtual bool IsReadOnly() = 0; + + // These static methods actually create instances of + // BackupStoreRefCountDatabaseImpl. + + // Create a new empty database: static std::auto_ptr Create (const BackupStoreAccountDatabase::Entry& rAccount); - void Commit(); - void Discard(); - + static std::auto_ptr Create + (const std::string& Filename, int64_t AccountID, bool reuse_existing_file = false); // Load it from the store - static std::auto_ptr Load(const - BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + static std::auto_ptr Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + // Load it from a stream (file or RaidFile) + static std::auto_ptr Load( + const std::string& FileName, int64_t AccountID, bool ReadOnly); + static std::string GetFilename(const BackupStoreAccountDatabase::Entry& + rAccount); typedef uint32_t refcount_t; // Data access functions - refcount_t GetRefCount(int64_t ObjectID) const; - int64_t GetLastObjectIDUsed() const; + virtual refcount_t GetRefCount(int64_t ObjectID) const = 0; + virtual int64_t GetLastObjectIDUsed() const = 0; // Data modification functions - void AddReference(int64_t ObjectID); + virtual void AddReference(int64_t ObjectID) = 0; // RemoveReference returns false if refcount drops to zero - bool RemoveReference(int64_t ObjectID); - int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs); - -private: - static std::string GetFilename(const BackupStoreAccountDatabase::Entry& - rAccount, bool Temporary); - - IOStream::pos_type GetSize() const - { - return mapDatabaseFile->GetPosition() + - mapDatabaseFile->BytesLeftToRead(); - } - IOStream::pos_type GetEntrySize() const - { - return sizeof(refcount_t); - } - IOStream::pos_type GetOffset(int64_t ObjectID) const - { - return ((ObjectID - 1) * GetEntrySize()) + - sizeof(refcount_StreamFormat); - } - void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); - - // Location information - BackupStoreAccountDatabase::Entry mAccount; - std::string mFilename; - bool mReadOnly; - bool mIsModified; - bool mIsTemporaryFile; - std::auto_ptr mapDatabaseFile; - - bool NeedsCommitOrDiscard() - { - return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile; - } + virtual bool RemoveReference(int64_t ObjectID) = 0; + virtual int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) = 0; }; #endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp index d5acf62c2..0a0ab410d 100644 --- a/lib/backupstore/HousekeepStoreAccount.cpp +++ b/lib/backupstore/HousekeepStoreAccount.cpp @@ -2,7 +2,10 @@ // // File // Name: HousekeepStoreAccount.cpp -// Purpose: +// Purpose: Run housekeeping on a server-side account. Removes +// files and directories which are marked as RemoveASAP, +// and Old and Deleted objects as necessary to bring the +// account back under its soft limit. // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -15,6 +18,7 @@ #include "autogen_BackupStoreException.h" #include "BackupConstants.h" +#include "BackupFileSystem.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" @@ -24,9 +28,6 @@ #include "BufferedStream.h" #include "HousekeepStoreAccount.h" #include "NamedLock.h" -#include "RaidFileRead.h" -#include "RaidFileWrite.h" -#include "StoreStructure.h" #include "MemLeakFindOn.h" @@ -36,20 +37,18 @@ // -------------------------------------------------------------------------- // // Function -// Name: HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &) +// Name: HousekeepStoreAccount::HousekeepStoreAccount( +// BackupFileSystem&, HousekeepingCallback*) // Purpose: Constructor // Created: 11/12/03 // // -------------------------------------------------------------------------- -HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, - const std::string &rStoreRoot, int StoreDiscSet, +HousekeepStoreAccount::HousekeepStoreAccount(BackupFileSystem& FileSystem, HousekeepingCallback* pHousekeepingCallback) - : mAccountID(AccountID), - mStoreRoot(rStoreRoot), - mStoreDiscSet(StoreDiscSet), + : mrFileSystem(FileSystem), mpHousekeepingCallback(pHousekeepingCallback), mDeletionSizeTarget(0), - mPotentialDeletionsTotalSize(0), + mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), mErrorCount(0), mBlocksUsed(0), @@ -62,10 +61,11 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), + mpNewRefs(NULL), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; - tag << "hk=" << BOX_FORMAT_ACCOUNT(mAccountID); + tag << "hk=" << FileSystem.GetAccountIdentifier(); mTagWithClientID.Change(tag.str()); } @@ -79,19 +79,21 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, // -------------------------------------------------------------------------- HousekeepStoreAccount::~HousekeepStoreAccount() { - if(mapNewRefs.get()) + if(mpNewRefs) { // Discard() can throw exception, but destructors aren't supposed to do that, so // just catch and log them. try { - mapNewRefs->Discard(); + mpNewRefs->Discard(); } catch(BoxException &e) { BOX_ERROR("Failed to destroy housekeeper: discarding the refcount " "database threw an exception: " << e.what()); } + + mpNewRefs = NULL; } } @@ -106,94 +108,67 @@ HousekeepStoreAccount::~HousekeepStoreAccount() bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { BOX_TRACE("Starting housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID)); - - // Attempt to lock the account - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, - writeLockFilename); - NamedLock writeLock; - if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), - 0600 /* restrictive file permissions */)) + mrFileSystem.GetAccountIdentifier()); + + // Attempt to lock the account. If KeepTryingForever is false, then only + // try once, and return false if that fails. + try { - if(KeepTryingForever) + mrFileSystem.GetLock(KeepTryingForever ? BackupFileSystem::KEEP_TRYING_FOREVER : 1); + } + catch(BackupStoreException &e) + { + if(EXCEPTION_IS_TYPE(e, BackupStoreException, CouldNotLockStoreAccount)) { - BOX_INFO("Failed to lock account for housekeeping, " - "still trying..."); - while(!writeLock.TryAndGetLock(writeLockFilename, - 0600 /* restrictive file permissions */)) - { - sleep(1); - } + // Couldn't lock the account -- just stop now + return false; } else { - // Couldn't lock the account -- just stop now - return false; + // something unexpected went wrong + throw; } } // Load the store info to find necessary info for the housekeeping - std::auto_ptr info(BackupStoreInfo::Load(mAccountID, - mStoreRoot, mStoreDiscSet, false /* Read/Write */)); - std::auto_ptr pOldInfo( - BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, - true /* Read Only */)); - - // If the account has a name, change the logging tag to include it - if(!(info->GetAccountName().empty())) - { - std::ostringstream tag; - tag << "hk=" << BOX_FORMAT_ACCOUNT(mAccountID) << "/" << - info->GetAccountName(); - mTagWithClientID.Change(tag.str()); - } + BackupStoreInfo* pInfo = &(mrFileSystem.GetBackupStoreInfo(false)); // !ReadOnly + std::auto_ptr apOldInfo = mrFileSystem.GetBackupStoreInfoUncached(); // Calculate how much should be deleted - mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit(); + mDeletionSizeTarget = pInfo->GetBlocksUsed() - pInfo->GetBlocksSoftLimit(); if(mDeletionSizeTarget < 0) { mDeletionSizeTarget = 0; } - BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); - mapNewRefs = BackupStoreRefCountDatabase::Create(account); + mpNewRefs = &mrFileSystem.GetPotentialRefCountDatabase(); // Scan the directory for potential things to delete - // This will also remove eligible items marked with RemoveASAP - bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, - *info); + // This will also find and enqueue eligible items marked with RemoveASAP + bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, *pInfo); if(!continueHousekeeping) { // The scan was incomplete, so the new block counts are - // incorrect, we can't rely on them. It's better to discard - // the new info and adjust the old one instead. - info = pOldInfo; + // incorrect, we can't rely on them, so discard them. + mrFileSystem.DiscardBackupStoreInfo(*pInfo); + pInfo = &(mrFileSystem.GetBackupStoreInfo(false)); // !ReadOnly // We're about to reset counters and exit, so report what // happened now. BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " removed " << - (0 - mBlocksUsedDelta) << " blocks (" << - mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs) and the directory " - "scan was interrupted"); + mrFileSystem.GetAccountIdentifier() << " removed " << + (0 - mBlocksUsedDelta) << " blocks (" << mFilesDeleted << + " files, " << mEmptyDirectoriesDeleted << " dirs) and the " + "directory scan was interrupted"); } - // If housekeeping made any changes, such as deleting RemoveASAP files, - // the differences in block counts will be recorded in the deltas. - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - - // Reset the delta counts for files, as they will include - // RemoveASAP flagged files deleted during the initial scan. - // keep removeASAPBlocksUsedDelta for reporting - int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; - mBlocksUsedDelta = 0; - mBlocksInOldFilesDelta = 0; - mBlocksInDeletedFilesDelta = 0; + if(!continueHousekeeping) + { + // Report any UNexpected changes, and consider them to be errors. + // Do this before applying the expected changes below. + mErrorCount += pInfo->ReportChangesTo(*apOldInfo); + } // If scan directory stopped for some reason, probably parent // instructed to terminate, stop now. @@ -205,26 +180,22 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) if(!continueHousekeeping) { - mapNewRefs->Discard(); - info->Save(); + mpNewRefs->Discard(); + mpNewRefs = NULL; + mrFileSystem.PutBackupStoreInfo(*pInfo); return false; } - // Report any UNexpected changes, and consider them to be errors. - // Do this before applying the expected changes below. - mErrorCount += info->ReportChangesTo(*pOldInfo); - info->Save(); - // Try to load the old reference count database and check whether - // any counts have changed. We want to compare the mapNewRefs to + // any counts have changed. We want to compare the mpNewRefs to // apOldRefs before we delete any files, because that will also change // the reference count in a way that's not an error. try { - std::auto_ptr apOldRefs = - BackupStoreRefCountDatabase::Load(account, false); - mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs); + BackupStoreRefCountDatabase& old_refs( + mrFileSystem.GetPermanentRefCountDatabase(true)); // ReadOnly + mErrorCount += mpNewRefs->ReportChangesTo(old_refs); } catch(BoxException &e) { @@ -235,86 +206,67 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) } // Go and delete items from the accounts - bool deleteInterrupted = DeleteFiles(*info); + bool deleteInterrupted = DeleteFiles(*pInfo); // If that wasn't interrupted, remove any empty directories which // are also marked as deleted in their containing directory if(!deleteInterrupted) { - deleteInterrupted = DeleteEmptyDirectories(*info); + deleteInterrupted = DeleteEmptyDirectories(*pInfo); } // Log deletion if anything was deleted if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " - "removed " << - (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << - " blocks (" << mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs)" << + BOX_INFO("Housekeeping on account " << mrFileSystem.GetAccountIdentifier() << " " + "removed " << -mBlocksUsedDelta << " blocks (" << mFilesDeleted << " " + "files, " << mEmptyDirectoriesDeleted << " dirs)" << (deleteInterrupted?" and was interrupted":"")); } // Make sure the delta's won't cause problems if the counts are // really wrong, and it wasn't fixed because the store was // updated during the scan. - if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) + if(mBlocksUsedDelta < (0 - pInfo->GetBlocksUsed())) { - mBlocksUsedDelta = (0 - info->GetBlocksUsed()); + mBlocksUsedDelta = (0 - pInfo->GetBlocksUsed()); } - if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) + if(mBlocksInOldFilesDelta < (0 - pInfo->GetBlocksInOldFiles())) { - mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + mBlocksInOldFilesDelta = (0 - pInfo->GetBlocksInOldFiles()); } - if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) + if(mBlocksInDeletedFilesDelta < (0 - pInfo->GetBlocksInDeletedFiles())) { - mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + mBlocksInDeletedFilesDelta = (0 - pInfo->GetBlocksInDeletedFiles()); } - if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) + if(mBlocksInDirectoriesDelta < (0 - pInfo->GetBlocksInDirectories())) { - mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); + mBlocksInDirectoriesDelta = (0 - pInfo->GetBlocksInDirectories()); } // Update the usage counts in the store - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); + pInfo->ChangeBlocksUsed(mBlocksUsedDelta); + pInfo->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); + pInfo->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); + pInfo->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); // Save the store info back - info->Save(); + mrFileSystem.PutBackupStoreInfo(*pInfo); // force file to be saved and closed before releasing the lock below - mapNewRefs->Commit(); - mapNewRefs.reset(); + mpNewRefs->Commit(); + mpNewRefs = NULL; - // Explicity release the lock (would happen automatically on - // going out of scope, included for code clarity) - writeLock.ReleaseLock(); + // Explicitly release the lock (would happen automatically on going out of scope, + // included for code clarity) + mrFileSystem.ReleaseLock(); BOX_TRACE("Finished housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID)); + mrFileSystem.GetAccountIdentifier()); return true; } - -// -------------------------------------------------------------------------- -// -// Function -// Name: HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &) -// Purpose: Generate and place the filename for a given object ID -// Created: 11/12/03 -// -// -------------------------------------------------------------------------- -void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut) -{ - // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */); -} - - // -------------------------------------------------------------------------- // // Function @@ -336,7 +288,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, // Check for having to stop // Include account ID here as the specified account is locked - if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) + int account_id = mrFileSystem.GetAccountID(); + if(mpHousekeepingCallback && + mpHousekeepingCallback->CheckForInterProcessMsg(account_id)) { // Need to abort now return false; @@ -344,26 +298,16 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, } #endif - // Get the filename - std::string objectFilename; - MakeObjectFilename(ObjectID, objectFilename); - - // Open it. - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, - objectFilename)); + // Read the directory in + BackupStoreDirectory dir; + mrFileSystem.GetDirectory(ObjectID, dir); // Add the size of the directory on disc to the size being calculated - int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); + int64_t originalDirSizeInBlocks = dir.GetUserInfo1_SizeInBlocks(); + ASSERT(originalDirSizeInBlocks > 0); mBlocksInDirectories += originalDirSizeInBlocks; mBlocksUsed += originalDirSizeInBlocks; - // Read the directory in - BackupStoreDirectory dir; - BufferedStream buf(*dirStream); - dir.ReadFromStream(buf, IOStream::TimeOutInfinite); - dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks); - dirStream->Close(); - // Is it empty? if(dir.GetNumberOfEntries() == 0) { @@ -381,40 +325,55 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, while((en = i.Next()) != 0) { // This directory references this object - mapNewRefs->AddReference(en->GetObjectID()); + mpNewRefs->AddReference(en->GetObjectID()); } } // BLOCK { - // Remove any files which are marked for removal as soon - // as they become old or deleted. - bool deletedSomething = false; - do + // Add to mDefiniteDeletions any files which are marked for removal as soon as + // they become old or deleted. + + // Iterate through the directory + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) { - // Iterate through the directory - deletedSomething = false; - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + int16_t enFlags = en->GetFlags(); + if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 + && (en->IsDeleted() || en->IsOld())) { - int16_t enFlags = en->GetFlags(); - if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 - && (en->IsDeleted() || en->IsOld())) + if(!mrFileSystem.CanMergePatches() && + en->GetDependsNewer() != 0) { - // Delete this immediately. - DeleteFile(ObjectID, en->GetObjectID(), dir, - objectFilename, rBackupStoreInfo); + BOX_ERROR("Cannot delete file " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " flagged as RemoveASAP because " + "another file depends on it (" << + BOX_FORMAT_OBJECTID(en->GetDependsNewer()) << + " and the filesystem does not " + "support merging patches"); + continue; + } - // flag as having done something - deletedSomething = true; + mDefiniteDeletions.push_back( + std::pair(en->GetObjectID(), + ObjectID)); // of the directory - // Must start the loop from the beginning again, as iterator is now - // probably invalid. - break; + // Because we are definitely deleting this file, we don't need + // housekeeping to delete potential files to free up the space + // that it occupies, so reduce the deletion target by this file's + // size. + if(mDeletionSizeTarget > 0) + { + mDeletionSizeTarget -= en->GetSizeInBlocks(); + if(mDeletionSizeTarget < 0) + { + mDeletionSizeTarget = 0; + } } } - } while(deletedSomething); + } } // BLOCK @@ -433,7 +392,6 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) { // Update recalculated usage sizes - int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); mBlocksUsed += enSizeInBlocks; if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks; @@ -456,9 +414,22 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, } // enVersionAge is now the age of this version. - // Potentially add it to the list if it's deleted, if it's an old version or deleted + // Add it to the list of potential files to remove, if it's an old version + // or deleted: if(en->IsOld() || en->IsDeleted()) { + if(!mrFileSystem.CanMergePatches() && + en->GetDependsNewer() != 0) + { + BOX_TRACE("Cannot remove old/deleted file " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " now, because another file depends on it (" << + BOX_FORMAT_OBJECTID(en->GetDependsNewer()) << + " and the filesystem does not support merging " + "patches"); + continue; + } + // Is deleted / old version. DelEn d; d.mObjectID = en->GetObjectID(); @@ -605,7 +576,19 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount // -------------------------------------------------------------------------- bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { - // Only delete files if the deletion target is greater than zero + // Delete all the definite deletions first, because we promised that we would, and because + // the deletion target might only be zero because we are definitely deleting enough files + // to free up all required space. So if we didn't delete them, the store would remain over + // its target size. + for(std::vector >::iterator i = mDefiniteDeletions.begin(); + i != mDefiniteDeletions.end(); i++) + { + int64_t FileID = i->first; + int64_t DirID = i->second; + RemoveReferenceAndMaybeDeleteFile(FileID, DirID, "RemoveASAP", rBackupStoreInfo); + } + + // Only delete potentially deletable files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) if(mDeletionSizeTarget <= 0) { @@ -621,45 +604,20 @@ bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + int account_id = mrFileSystem.GetAccountID(); // Check for having to stop - if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked + if(mpHousekeepingCallback && + // include account ID here as the specified account is now locked + mpHousekeepingCallback->CheckForInterProcessMsg(account_id)) { - // Need to abort now + // Need to abort now. Return true to signal that we were interrupted. return true; } } #endif - // Load up the directory it's in - // Get the filename - std::string dirFilename; - BackupStoreDirectory dir; - { - MakeObjectFilename(i->mInDirectory, dirFilename); - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); - dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); - } - - // Delete the file - BackupStoreRefCountDatabase::refcount_t refs = - DeleteFile(i->mInDirectory, i->mObjectID, dir, - dirFilename, rBackupStoreInfo); - if(refs == 0) - { - BOX_INFO("Housekeeping removed " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); - } - else - { - BOX_TRACE("Housekeeping preserved " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << - " with " << refs << " references"); - } + RemoveReferenceAndMaybeDeleteFile(i->mObjectID, i->mInDirectory, + (i->mIsFlagDeleted ? "deleted" : "old"), rBackupStoreInfo); // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the @@ -674,6 +632,30 @@ bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) return false; } +void HousekeepStoreAccount::RemoveReferenceAndMaybeDeleteFile(int64_t FileID, int64_t DirID, + const std::string& reason, BackupStoreInfo& rBackupStoreInfo) +{ + // Load up the directory it's in + // Get the filename + BackupStoreDirectory dir; + mrFileSystem.GetDirectory(DirID, dir); + + // Delete the file + BackupStoreRefCountDatabase::refcount_t refs = + DeleteFile(DirID, FileID, dir, rBackupStoreInfo); + if(refs == 0) + { + BOX_INFO("Housekeeping removed " << reason << " file " << + BOX_FORMAT_OBJECTID(FileID) << " from dir " << BOX_FORMAT_OBJECTID(DirID)); + } + else + { + BOX_TRACE("Housekeeping preserved " << reason << " file " << + BOX_FORMAT_OBJECTID(FileID) << " in dir " << BOX_FORMAT_OBJECTID(DirID) << + " with " << refs << " references"); + } +} + // -------------------------------------------------------------------------- // @@ -691,25 +673,26 @@ bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, - const std::string &rDirectoryFilename, BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; bool wasOldVersion = false; int64_t deletedFileSizeInBlocks = 0; + // A pointer to an object which requires committing if the directory save goes OK - std::auto_ptr padjustedEntry; + std::auto_ptr apAdjustedEntry; + // BLOCK { BackupStoreRefCountDatabase::refcount_t refs = - mapNewRefs->GetRefCount(ObjectID); + mpNewRefs->GetRefCount(ObjectID); BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { BOX_ERROR("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " + mrFileSystem.GetAccountIdentifier() << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " "not found in dir " << @@ -749,12 +732,13 @@ BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( rBackupStoreInfo.AdjustNumOldFiles(-1); } - mapNewRefs->RemoveReference(ObjectID); + mpNewRefs->RemoveReference(ObjectID); return refs - 1; } - // If the entry is involved in a chain of patches, it needs to be handled - // a bit more carefully. + // If the entry is involved in a chain of patches, it needs to be handled a bit + // more carefully. + if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) { // This entry is a patch from a newer entry. Just need to update the info on that entry. @@ -768,66 +752,62 @@ BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( } else if(pentry->GetDependsOlder() != 0) { + // We should have checked whether the BackupFileSystem can merge patches + // before this point: + ASSERT(mrFileSystem.CanMergePatches()); + BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder()); if(pentry->GetDependsNewer() == 0) { - // There exists an older version which depends on this one. Need to combine the two over that one. + // There exists an older version which depends on this + // one. Need to combine the two over that one. // Adjust the other entry in the directory if(polder == 0 || polder->GetDependsNewer() != ObjectID) { - THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); + THROW_EXCEPTION(BackupStoreException, + PatchChainInfoBadInDirectory); } - // Change the info in the older entry so that this no longer points to this entry + // Change the info in the older entry so that this no + // longer points to this entry. polder->SetDependsNewer(0); + + // Actually combine the patch and file, but don't commit + // the resulting file yet. + apAdjustedEntry = mrFileSystem.CombineFile( + pentry->GetDependsOlder(), ObjectID); } else { - // This entry is in the middle of a chain, and two patches need combining. + // This entry is in the middle of a chain, and two + // patches need combining. // First, adjust the directory entries - BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); - if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID - || polder == 0 || polder->GetDependsNewer() != ObjectID) + BackupStoreDirectory::Entry *pnewer = + rDirectory.FindEntryByID(pentry->GetDependsNewer()); + if(pnewer == 0 || + pnewer->GetDependsOlder() != ObjectID || + polder == 0 || + polder->GetDependsNewer() != ObjectID) { - THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory); + THROW_EXCEPTION(BackupStoreException, + PatchChainInfoBadInDirectory); } // Remove the middle entry from the linked list by simply using the values from this entry pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); - } - // COMMON CODE to both cases - - // Generate the filename of the older version - std::string objFilenameOlder; - MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder); - // Open it twice (it's the diff) - std::auto_ptr pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); - std::auto_ptr pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder)); - // Open this file - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - std::auto_ptr pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); - // And open a write file to overwrite the other directory entry - padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, - objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); - padjustedEntry->Open(true /* allow overwriting */); - - if(pentry->GetDependsNewer() == 0) - { - // There exists an older version which depends on this one. Need to combine the two over that one. - BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry); + // Actually combine the patch and file, but don't commit + // the resulting file yet. + apAdjustedEntry = mrFileSystem.CombineDiffs( + pentry->GetDependsOlder(), ObjectID); } - else - { - // This entry is in the middle of a chain, and two patches need combining. - BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); - } - // The file will be committed later when the directory is safely commited. + + // COMMON CODE to both cases. The file will be committed later, + // when the directory is safely commited. // Work out the adjusted size - int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); + int64_t newSize = apAdjustedEntry->GetNumBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; if(polder->IsDeleted()) @@ -850,44 +830,32 @@ BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( // Save directory back to disc // BLOCK { - RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, - mapNewRefs->GetRefCount(InDirectory)); - writeDir.Open(true /* allow overwriting */); - rDirectory.WriteToStream(writeDir); - - // Get the disc usage (must do this before commiting it) - int64_t new_size = writeDir.GetDiscUsageInBlocks(); - - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + mrFileSystem.PutDirectory(rDirectory); // Adjust block counts if the directory itself changed in size - int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + int64_t new_size = rDirectory.GetUserInfo1_SizeInBlocks(); int64_t adjust = new_size - original_size; mBlocksUsedDelta += adjust; mBlocksInDirectoriesDelta += adjust; - UpdateDirectorySize(rDirectory, new_size); + UpdateDirectorySize(rDirectory, original_size, new_size); } // Commit any new adjusted entry - if(padjustedEntry.get() != 0) + if(apAdjustedEntry.get() != 0) { - padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - padjustedEntry.reset(); // delete it now + apAdjustedEntry->Commit(); + apAdjustedEntry.reset(); // delete it now } // Drop reference count by one. Must now be zero, to delete the file. - bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + bool remaining_refs = mpNewRefs->RemoveReference(ObjectID); ASSERT(!remaining_refs); // Delete from disc - BOX_TRACE("Removing unreferenced object " << - BOX_FORMAT_OBJECTID(ObjectID)); - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); - del.Delete(); + BOX_TRACE("Removing unreferenced object " << BOX_FORMAT_OBJECTID(ObjectID)); + mrFileSystem.DeleteFile(ObjectID); // Adjust counts for the file ++mFilesDeleted; @@ -921,6 +889,7 @@ BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( // Function // Name: HousekeepStoreAccount::UpdateDirectorySize( // BackupStoreDirectory& rDirectory, +// IOStream::pos_type old_size_in_blocks, // IOStream::pos_type new_size_in_blocks) // Purpose: Update the directory size, modifying the parent // directory's entry for this directory if necessary. @@ -930,45 +899,33 @@ BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( void HousekeepStoreAccount::UpdateDirectorySize( BackupStoreDirectory& rDirectory, + IOStream::pos_type old_size_in_blocks, IOStream::pos_type new_size_in_blocks) { -#ifndef BOX_RELEASE_BUILD - { - std::string dirFilename; - MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); - std::auto_ptr dirStream( - RaidFileRead::Open(mStoreDiscSet, dirFilename)); - ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); - } -#endif - - IOStream::pos_type old_size_in_blocks = - rDirectory.GetUserInfo1_SizeInBlocks(); + // The directory itself should already have been updated by the FileSystem. + ASSERT(rDirectory.GetUserInfo1_SizeInBlocks() == new_size_in_blocks); if(new_size_in_blocks == old_size_in_blocks) { + // No need to update the entry for this directory in its parent directory. return; } - rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); - - if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) + if(rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) { + // The root directory has no parent, so no entry for it that might need + // updating. return; } - std::string parentFilename; - MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); - std::auto_ptr parentStream( - RaidFileRead::Open(mStoreDiscSet, parentFilename)); - BackupStoreDirectory parent(*parentStream); - parentStream.reset(); + BackupStoreDirectory parent; + mrFileSystem.GetDirectory(rDirectory.GetContainerID(), parent); BackupStoreDirectory::Entry* en = parent.FindEntryByID(rDirectory.GetObjectID()); ASSERT(en); - if (en->GetSizeInBlocks() != old_size_in_blocks) + if(en->GetSizeInBlocks() != old_size_in_blocks) { BOX_WARNING("Directory " << BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << @@ -980,12 +937,7 @@ void HousekeepStoreAccount::UpdateDirectorySize( } en->SetSizeInBlocks(new_size_in_blocks); - - RaidFileWrite writeDir(mStoreDiscSet, parentFilename, - mapNewRefs->GetRefCount(rDirectory.GetContainerID())); - writeDir.Open(true /* allow overwriting */); - parent.WriteToStream(writeDir); - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + mrFileSystem.PutDirectory(parent); } // -------------------------------------------------------------------------- @@ -1010,8 +962,11 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories(BackupStoreInfo& rBackupStore if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + int account_id = mrFileSystem.GetAccountID(); // Check for having to stop - if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is now locked + if(mpHousekeepingCallback && + // include account ID here as the specified account is now locked + mpHousekeepingCallback->CheckForInterProcessMsg(account_id)) { // Need to abort now return true; @@ -1044,23 +999,18 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Load up the directory to potentially delete std::string dirFilename; BackupStoreDirectory dir; - int64_t dirSizeInBlocks = 0; // BLOCK { - MakeObjectFilename(dirId, dirFilename); - // Check it actually exists (just in case it gets - // added twice to the list) - if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) + // Check it actually exists (just in case it gets added twice to the list) + ASSERT(mrFileSystem.ObjectExists(dirId)); + if(!mrFileSystem.ObjectExists(dirId)) { // doesn't exist, next! return; } // load - std::auto_ptr dirStream( - RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + mrFileSystem.GetDirectory(dirId, dir); } // Make sure this directory is actually empty @@ -1071,25 +1021,13 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, } // Candidate for deletion... open containing directory - std::string containingDirFilename; BackupStoreDirectory containingDir; - int64_t containingDirSizeInBlocksOrig = 0; - { - MakeObjectFilename(dir.GetContainerID(), containingDirFilename); - std::auto_ptr containingDirStream( - RaidFileRead::Open(mStoreDiscSet, - containingDirFilename)); - containingDirSizeInBlocksOrig = - containingDirStream->GetDiscUsageInBlocks(); - containingDir.ReadFromStream(*containingDirStream, - IOStream::TimeOutInfinite); - containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); - } + mrFileSystem.GetDirectory(dir.GetContainerID(), containingDir); // Find the entry BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - // TODO FIXME invert test and reduce indentation + if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted @@ -1102,45 +1040,40 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, } // Write revised parent directory - RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, - mapNewRefs->GetRefCount(containingDir.GetObjectID())); - writeDir.Open(true /* allow overwriting */); - containingDir.WriteToStream(writeDir); - - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + int64_t old_size = containingDir.GetUserInfo1_SizeInBlocks(); + mrFileSystem.PutDirectory(containingDir); + int64_t new_size = containingDir.GetUserInfo1_SizeInBlocks(); - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - UpdateDirectorySize(containingDir, dirSize); + // Removing an entry from the directory may have changed its size, so we + // might need to update its parent as well. + UpdateDirectorySize(containingDir, old_size, new_size); // adjust usage counts for this directory - if(dirSize > 0) + if(new_size > 0) { - int64_t adjust = dirSize - containingDirSizeInBlocksOrig; + int64_t adjust = new_size - old_size; mBlocksUsedDelta += adjust; mBlocksInDirectoriesDelta += adjust; } - if (mapNewRefs->RemoveReference(dir.GetObjectID())) + if (mpNewRefs->RemoveReference(dir.GetObjectID())) { // Still referenced BOX_TRACE("Housekeeping spared empty deleted dir " << BOX_FORMAT_OBJECTID(dirId) << " due to " << - mapNewRefs->GetRefCount(dir.GetObjectID()) << - "remaining references"); + mpNewRefs->GetRefCount(dir.GetObjectID()) << + " remaining references"); return; } // Delete the directory itself BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); - RaidFileWrite del(mStoreDiscSet, dirFilename, - mapNewRefs->GetRefCount(dir.GetObjectID())); - del.Delete(); + mrFileSystem.DeleteDirectory(dirId); // And adjust usage counts for the directory that's // just been deleted + int64_t dirSizeInBlocks = dir.GetUserInfo1_SizeInBlocks(); mBlocksUsedDelta -= dirSizeInBlocks; mBlocksInDirectoriesDelta -= dirSizeInBlocks; diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h index ff9e9ffea..588db1038 100644 --- a/lib/backupstore/HousekeepStoreAccount.h +++ b/lib/backupstore/HousekeepStoreAccount.h @@ -17,6 +17,7 @@ #include "BackupStoreRefCountDatabase.h" class BackupStoreDirectory; +class BackupFileSystem; class HousekeepingCallback { @@ -36,28 +37,28 @@ class HousekeepingCallback class HousekeepStoreAccount { public: - HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, - int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback); + HousekeepStoreAccount(BackupFileSystem& FileSystem, + HousekeepingCallback* pHousekeepingCallback); ~HousekeepStoreAccount(); - + bool DoHousekeeping(bool KeepTryingForever = false); int GetErrorCount() { return mErrorCount; } - + private: // utility functions - void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut); - bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo); bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo); + void RemoveReferenceAndMaybeDeleteFile(int64_t FileID, int64_t DirID, + const std::string& reason, BackupStoreInfo& rBackupStoreInfo); bool DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo); void DeleteEmptyDirectory(int64_t dirId, std::vector& rToExamine, BackupStoreInfo& rBackupStoreInfo); BackupStoreRefCountDatabase::refcount_t DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, - const std::string &rDirectoryFilename, BackupStoreInfo& rBackupStoreInfo); void UpdateDirectorySize(BackupStoreDirectory &rDirectory, + IOStream::pos_type old_size_in_blocks, IOStream::pos_type new_size_in_blocks); typedef struct @@ -69,29 +70,28 @@ class HousekeepStoreAccount int32_t mVersionAgeWithinMark; // 0 == current, 1 latest old version, etc bool mIsFlagDeleted; // false for files flagged "Old" } DelEn; - + struct DelEnCompare { bool operator()(const DelEn &x, const DelEn &y); }; - - int mAccountID; - std::string mStoreRoot; - int mStoreDiscSet; + + BackupFileSystem& mrFileSystem; HousekeepingCallback* mpHousekeepingCallback; - + int64_t mDeletionSizeTarget; - + + std::vector > mDefiniteDeletions; std::set mPotentialDeletions; int64_t mPotentialDeletionsTotalSize; int64_t mMaxSizeInPotentialDeletions; - + // List of directories which are empty, and might be good for deleting std::vector mEmptyDirectories; // Count of errors found and fixed int64_t mErrorCount; - + // The re-calculated blocks used stats int64_t mBlocksUsed; int64_t mBlocksInOldFiles; @@ -103,14 +103,14 @@ class HousekeepStoreAccount int64_t mBlocksInOldFilesDelta; int64_t mBlocksInDeletedFilesDelta; int64_t mBlocksInDirectoriesDelta; - + // Deletion count int64_t mFilesDeleted; int64_t mEmptyDirectoriesDeleted; // New reference count list - std::auto_ptr mapNewRefs; - + BackupStoreRefCountDatabase* mpNewRefs; + // Poll frequency int mCountUntilNextInterprocessMsgCheck; diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp index 45a1ce910..32191f0bf 100644 --- a/lib/backupstore/StoreStructure.cpp +++ b/lib/backupstore/StoreStructure.cpp @@ -9,6 +9,7 @@ #include "Box.h" +#include "BackupConstants.h" #include "StoreStructure.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" @@ -86,7 +87,8 @@ void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int Di RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet)); // Make the filename - std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock"); + std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + + WRITE_LOCK_FILENAME); // Return it to the caller rFilenameOut = writeLockFile; diff --git a/lib/backupstore/StoreTestUtils.cpp b/lib/backupstore/StoreTestUtils.cpp index 2b773cb12..f1f5b199b 100644 --- a/lib/backupstore/StoreTestUtils.cpp +++ b/lib/backupstore/StoreTestUtils.cpp @@ -13,12 +13,14 @@ #include #include "autogen_BackupProtocol.h" -#include "BoxPortsAndFiles.h" +#include "BackupAccountControl.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreCheck.h" #include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreInfo.h" +#include "BoxPortsAndFiles.h" #include "HousekeepStoreAccount.h" #include "Logging.h" #include "ServerControl.h" @@ -33,10 +35,10 @@ bool create_account(int soft, int hard) std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); - BackupStoreAccountsControl control(*config); + BackupStoreAccountControl control(*config, 0x01234567); Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); - int result = control.CreateAccount(0x01234567, 0, soft, hard); + int result = control.CreateAccount(0, soft, hard); TEST_EQUAL(0, result); return (result == 0); } @@ -47,9 +49,9 @@ bool delete_account() std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); - BackupStoreAccountsControl control(*config); + BackupStoreAccountControl control(*config, 0x01234567); Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); - TEST_THAT_THROWONFAIL(control.DeleteAccount(0x01234567, false) == 0); + TEST_THAT_THROWONFAIL(control.DeleteAccount(false) == 0); return true; } @@ -62,17 +64,26 @@ void set_refcount(int64_t ObjectID, uint32_t RefCount) { ExpectedRefCounts.resize(ObjectID + 1, 0); } + ExpectedRefCounts[ObjectID] = RefCount; + + // BackupStoreCheck and housekeeping will both regenerate the refcount + // DB without any missing items at the end, so we need to prune + // ourselves of all items with no references to match. for (size_t i = ExpectedRefCounts.size() - 1; i >= 1; i--) { if (ExpectedRefCounts[i] == 0) { - // BackupStoreCheck and housekeeping will both - // regenerate the refcount DB without any missing - // items at the end, so we need to prune ourselves - // of all items with no references to match. ExpectedRefCounts.resize(i); } + else + { + // Don't keep going back up the list, as if we found a + // zero-referenced file higher up, we'd end up deleting + // the refcounts of referenced files further down the + // list (higher IDs). + break; + } } } @@ -121,24 +132,23 @@ std::auto_ptr connect_and_login(TLSContext& rContext, return protocol; } -bool check_num_files(int files, int old, int deleted, int dirs) +bool check_num_files(BackupFileSystem& fs, int files, int old, int deleted, int dirs) { - std::auto_ptr apInfo = - BackupStoreInfo::Load(0x1234567, - "backup/01234567/", 0, true); - TEST_EQUAL_LINE(files, apInfo->GetNumCurrentFiles(), - "current files"); - TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), - "old files"); - TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), - "deleted files"); - TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), - "directories"); - - return (files == apInfo->GetNumCurrentFiles() && - old == apInfo->GetNumOldFiles() && - deleted == apInfo->GetNumDeletedFiles() && - dirs == apInfo->GetNumDirectories()); + std::auto_ptr apInfo = fs.GetBackupStoreInfoUncached(); + return check_num_files(*apInfo, files, old, deleted, dirs); +} + +bool check_num_files(BackupStoreInfo& info, int files, int old, int deleted, int dirs) +{ + TEST_EQUAL_LINE(files, info.GetNumCurrentFiles(), "current files"); + TEST_EQUAL_LINE(old, info.GetNumOldFiles(), "old files"); + TEST_EQUAL_LINE(deleted, info.GetNumDeletedFiles(), "deleted files"); + TEST_EQUAL_LINE(dirs, info.GetNumDirectories(), "directories"); + + return (files == info.GetNumCurrentFiles() && + old == info.GetNumOldFiles() && + deleted == info.GetNumDeletedFiles() && + dirs == info.GetNumDirectories()); } bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, @@ -168,45 +178,67 @@ bool change_account_limits(const char* soft, const char* hard) std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); - BackupStoreAccountsControl control(*config); - int result = control.SetLimit(0x01234567, soft, hard); + BackupStoreAccountControl control(*config, 0x01234567); + return change_account_limits(control, soft, hard); +} + +bool change_account_limits(BackupAccountControl& control, const char* soft, + const char* hard) +{ + int result = control.SetLimit(soft, hard); TEST_EQUAL(0, result); return (result == 0); } int check_account_for_errors(Log::Level log_level) +{ + // TODO FIXME remove this backward-compatibility function + RaidBackupFileSystem fs(0x1234567, "backup/01234567/", 0); + return check_account_for_errors(fs, log_level); +} + +int check_account_for_errors(BackupFileSystem& filesystem, Log::Level log_level) { Logger::LevelGuard guard(Logging::GetConsole(), log_level); Logging::Tagger tag("check fix", true); Logging::ShowTagOnConsole show; - std::string errs; - std::auto_ptr config( - Configuration::LoadAndVerify("testfiles/bbstored.conf", - &BackupConfigFileVerify, errs)); - BackupStoreAccountsControl control(*config); - int errors_fixed = control.CheckAccount(0x01234567, - true, // FixErrors - false, // Quiet - true); // ReturnNumErrorsFound - return errors_fixed; -} -bool check_account(Log::Level log_level) -{ - int errors_fixed = check_account_for_errors(log_level); - TEST_EQUAL(0, errors_fixed); - return (errors_fixed == 0); + // The caller may already have opened a permanent read-write database, but + // BackupStoreCheck needs to open a temporary one, so we need to close the + // permanent one in that case. + if(filesystem.GetCurrentRefCountDatabase() && + !filesystem.GetCurrentRefCountDatabase()->IsReadOnly()) + { + filesystem.CloseRefCountDatabase(filesystem.GetCurrentRefCountDatabase()); + } + + BackupStoreCheck check(filesystem, + true, // FixErrors + false); // Quiet + check.Check(); + return check.GetNumErrorsFound(); } int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) { + // TODO FIXME remove this backward-compatibility function std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); int discSet = rAccount.GetDiscSet(); + RaidBackupFileSystem fs(rAccount.GetID(), rootDir, discSet); + + // Do housekeeping on this account + return run_housekeeping(fs); +} +int64_t run_housekeeping(BackupFileSystem& filesystem) +{ // Do housekeeping on this account - HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, - discSet, NULL); - TEST_THAT(housekeeping.DoHousekeeping(true /* keep trying forever */)); + HousekeepStoreAccount housekeeping(filesystem, NULL); + // Take a lock before calling DoHousekeeping, because although it does try to get a lock + // itself, it doesn't give us much control over how long it retries for. We want to retry + // for ~30 seconds, but not forever, because we don't want tests to hang. + // filesystem.GetLock(30); + TEST_THAT(housekeeping.DoHousekeeping(true)); // keep trying forever return housekeeping.GetErrorCount(); } @@ -217,24 +249,35 @@ int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) bool run_housekeeping_and_check_account() { - int error_count; + // TODO FIXME remove this backward-compatibility function + RaidBackupFileSystem fs(0x1234567, "backup/01234567/", 0); + return run_housekeeping_and_check_account(fs); +} + +bool run_housekeeping_and_check_account(BackupFileSystem& filesystem) +{ + if(filesystem.GetCurrentRefCountDatabase() != NULL) + { + filesystem.CloseRefCountDatabase(filesystem.GetCurrentRefCountDatabase()); + } + + int num_housekeeping_errors; { Logging::Tagger tag("", true); Logging::ShowTagOnConsole show; - std::auto_ptr apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - BackupStoreAccountDatabase::Entry account = - apAccounts->GetEntry(0x1234567); - error_count = run_housekeeping(account); + num_housekeeping_errors = run_housekeeping(filesystem); } + TEST_EQUAL_LINE(0, num_housekeeping_errors, "run_housekeeping"); + + filesystem.CloseRefCountDatabase(filesystem.GetCurrentRefCountDatabase()); - TEST_EQUAL_LINE(0, error_count, "housekeeping errors"); + int num_check_errors = check_account_for_errors(filesystem); + TEST_EQUAL_LINE(0, num_check_errors, "check_account_for_errors"); - bool check_account_is_ok = check_account(); - TEST_THAT(check_account_is_ok); + filesystem.CloseRefCountDatabase(filesystem.GetCurrentRefCountDatabase()); - return error_count == 0 && check_account_is_ok; + return num_housekeeping_errors == 0 && num_check_errors == 0; } bool check_reference_counts() @@ -244,20 +287,29 @@ bool check_reference_counts() BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); - std::auto_ptr apReferences( - BackupStoreRefCountDatabase::Load(account, true)); + std::auto_ptr apReferences = + BackupStoreRefCountDatabase::Load(account, true); TEST_EQUAL(ExpectedRefCounts.size(), apReferences->GetLastObjectIDUsed() + 1); + return check_reference_counts(*apReferences); +} + +bool check_reference_counts(BackupStoreRefCountDatabase& references) +{ bool counts_ok = true; + TEST_EQUAL_OR(ExpectedRefCounts.size(), + references.GetLastObjectIDUsed() + 1, + counts_ok = false); + for (unsigned int i = BackupProtocolListDirectory::RootDirectory; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], - apReferences->GetRefCount(i), + references.GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); - if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) + if (ExpectedRefCounts[i] != references.GetRefCount(i)) { counts_ok = false; } @@ -266,11 +318,11 @@ bool check_reference_counts() return counts_ok; } -bool StartServer() +bool StartServer(const std::string& daemon_args) { - bbstored_pid = StartDaemon(bbstored_pid, - BBSTORED " " + bbstored_args + " testfiles/bbstored.conf", - "testfiles/bbstored.pid"); + const std::string& daemon_args_final(daemon_args.size() ? daemon_args : bbstored_args); + bbstored_pid = StartDaemon(bbstored_pid, BBSTORED " " + daemon_args_final + + " testfiles/bbstored.conf", "testfiles/bbstored.pid"); return bbstored_pid != 0; } @@ -282,11 +334,11 @@ bool StopServer(bool wait_for_process) return result; } -bool StartClient(const std::string& bbackupd_conf_file) +bool StartClient(const std::string& bbackupd_conf_file, const std::string& daemon_args) { - bbackupd_pid = StartDaemon(bbackupd_pid, - BBACKUPD " " + bbackupd_args + " " + bbackupd_conf_file, - "testfiles/bbackupd.pid"); + const std::string& daemon_args_final(daemon_args.size() ? daemon_args : bbackupd_args); + bbackupd_pid = StartDaemon(bbackupd_pid, BBACKUPD " " + daemon_args_final + + " -c " + bbackupd_conf_file, "testfiles/bbackupd.pid"); return bbackupd_pid != 0; } diff --git a/lib/backupstore/StoreTestUtils.h b/lib/backupstore/StoreTestUtils.h index b3faebb5f..cde13efae 100644 --- a/lib/backupstore/StoreTestUtils.h +++ b/lib/backupstore/StoreTestUtils.h @@ -12,8 +12,10 @@ #include "Test.h" +class BackupAccountControl; class BackupProtocolCallable; class BackupProtocolClient; +class BackupFileSystem; class SocketStreamTLS; class TLSContext; @@ -41,38 +43,63 @@ std::auto_ptr connect_and_login(TLSContext& rContext, int flags = 0); //! Checks the number of files of each type in the store against expectations. -bool check_num_files(int files, int old, int deleted, int dirs); +bool check_num_files(BackupFileSystem& fs, int files, int old, int deleted, int dirs); +bool check_num_files(BackupStoreInfo& info, int files, int old, int deleted, int dirs); //! Checks the number of blocks in files of each type against expectations. bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, int Deleted, int Dirs, int Total); //! Change the soft and hard limits on the test account. +//! Old interface, only works with RaidBackupFileSystem, deprecated. bool change_account_limits(const char* soft, const char* hard); +//! Change the soft and hard limits on the test account. +//! New interface, takes a BackupAccountControl. +bool change_account_limits(BackupAccountControl& control, const char* soft, + const char* hard); + //! Checks an account for errors, returning the number of errors found and fixed. +//! Old interface, only works with RaidBackupFileSystem, deprecated. int check_account_for_errors(Log::Level log_level = Log::WARNING); -//! Checks an account for errors, returning true if it's OK, for use in assertions. -bool check_account(Log::Level log_level = Log::WARNING); +//! Checks an account for errors, returning number of errors found. +//! New interface, takes a BackupFileSystem. +int check_account_for_errors(BackupFileSystem& filesystem, + Log::Level log_level = Log::WARNING); //! Runs housekeeping on an account, to remove old and deleted files if necessary. +//! Old interface, only works with RaidBackupFileSystem, deprecated. int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount); +//! Runs housekeeping on an account, to remove old and deleted files if necessary. +//! New interface, takes a BackupFileSystem. +int64_t run_housekeeping(BackupFileSystem& filesystem); + //! Runs housekeeping and checks the account, returning true if it's OK. +//! Old interface, only works with RaidBackupFileSystem, deprecated. bool run_housekeeping_and_check_account(); +//! Runs housekeeping and checks the account, returning true if it's OK. +//! New interface, takes a BackupFileSystem. +bool run_housekeeping_and_check_account(BackupFileSystem& filesystem); + //! Tests that all object reference counts have the expected values. +//! Old interface, only works with RaidBackupFileSystem, deprecated. bool check_reference_counts(); +//! Tests that all object reference counts have the expected values. +bool check_reference_counts(BackupStoreRefCountDatabase& references); + //! Starts the bbstored test server running, which must not already be running. -bool StartServer(); +bool StartServer(const std::string& daemon_args = ""); //! Stops the currently running bbstored test server. bool StopServer(bool wait_for_process = false); //! Starts the bbackupd client running, which must not already be running. -bool StartClient(const std::string& bbackupd_conf_file = "testfiles/bbackupd.conf"); +bool StartClient(const std::string& bbackupd_conf_file = "testfiles/bbackupd.conf", + const std::string& daemon_args = ""); //! Stops the currently running bbackupd client. bool StopClient(bool wait_for_process = false); @@ -83,6 +110,7 @@ bool create_account(int soft, int hard); //! Deletes the standard test account, for testing behaviour with no account. bool delete_account(); +// You might want to use TEST_COMMAND_RETURNS_ERROR instead of TEST_PROTOCOL_ERROR_OR for new code: #define TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) \ { \ int type, subtype; \ @@ -91,7 +119,7 @@ bool delete_account(); { \ TEST_EQUAL_LINE(BackupProtocolError::error, subtype, \ "command returned error: " << \ - BackupProtocolError::GetMessage(subtype)); \ + (protocol).GetLastErrorMessage()); \ if (subtype != BackupProtocolError::error) \ { \ or_statements; \ @@ -99,9 +127,9 @@ bool delete_account(); } \ else \ { \ - TEST_FAIL_WITH_MESSAGE("command did not return an error, but a " \ - "response of type " << type << ", subtype " << subtype << \ - " instead"); \ + TEST_FAIL_WITH_MESSAGE("command did not return an error, " \ + "but " << (protocol).GetLastErrorMessage() << " " \ + "instead (" << type << "/" << subtype << ")"); \ or_statements; \ } \ } diff --git a/lib/bbackupd/BackupClientContext.cpp b/lib/bbackupd/BackupClientContext.cpp index 4c0b01ce5..a43c4368c 100644 --- a/lib/bbackupd/BackupClientContext.cpp +++ b/lib/bbackupd/BackupClientContext.cpp @@ -73,8 +73,10 @@ BackupClientContext::BackupClientContext mKeepAliveTimer(0, "KeepAliveTime"), mbIsManaged(false), mrProgressNotifier(rProgressNotifier), - mTcpNiceMode(TcpNiceMode), - mpNice(NULL) + mTcpNiceMode(TcpNiceMode) +#ifdef ENABLE_TCP_NICE + , mpNice(NULL) +#endif { } @@ -135,6 +137,7 @@ BackupProtocolCallable &BackupClientContext::GetConnection() if(mTcpNiceMode) { +#ifdef ENABLE_TCP_NICE // Pass control of apSocket to NiceSocketStream, // which will take care of destroying it for us. // But we need to hang onto a pointer to the nice @@ -142,6 +145,9 @@ BackupProtocolCallable &BackupClientContext::GetConnection() // This is scary, it could be deallocated under us. mpNice = new NiceSocketStream(apSocket); apSocket.reset(mpNice); +#else + BOX_WARNING("TcpNice option is enabled but not supported on this system"); +#endif } // We need to call some methods that aren't defined in @@ -310,7 +316,7 @@ int BackupClientContext::GetTimeout() const { return pConnection->GetTimeout(); } - + return (15*60*1000); } @@ -555,7 +561,7 @@ void BackupClientContext::DoKeepAlive() { return; } - + if (mKeepAliveTime == 0) { return; @@ -565,14 +571,14 @@ void BackupClientContext::DoKeepAlive() { return; } - + BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); pConnection->QueryGetIsAlive(); - + mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } -int BackupClientContext::GetMaximumDiffingTime() +int BackupClientContext::GetMaximumDiffingTime() { return mMaximumDiffingTime; } diff --git a/lib/bbackupd/BackupClientContext.h b/lib/bbackupd/BackupClientContext.h index df43a2321..854938b50 100644 --- a/lib/bbackupd/BackupClientContext.h +++ b/lib/bbackupd/BackupClientContext.h @@ -216,7 +216,9 @@ class BackupClientContext : public DiffTimer { if(mTcpNiceMode) { +#ifdef ENABLE_TCP_NICE mpNice->SetEnabled(enabled); +#endif } } @@ -246,7 +248,9 @@ class BackupClientContext : public DiffTimer int mMaximumDiffingTime; ProgressNotifier &mrProgressNotifier; bool mTcpNiceMode; +#ifdef ENABLE_TCP_NICE NiceSocketStream *mpNice; +#endif }; #endif // BACKUPCLIENTCONTEXT__H diff --git a/lib/bbackupd/BackupClientDirectoryRecord.cpp b/lib/bbackupd/BackupClientDirectoryRecord.cpp index 94cb7965b..f39c2b707 100644 --- a/lib/bbackupd/BackupClientDirectoryRecord.cpp +++ b/lib/bbackupd/BackupClientDirectoryRecord.cpp @@ -435,13 +435,9 @@ bool BackupClientDirectoryRecord::SyncDirectoryEntry( #ifdef WIN32 // Don't stat the file just yet, to ensure that users can exclude // unreadable files to suppress warnings that they are not accessible. - // - // Our emulated readdir() abuses en->d_type, which would normally - // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to - // have the full file attributes. int type; - if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + if (en->d_type == DT_DIR) { type = S_IFDIR; } @@ -514,24 +510,24 @@ bool BackupClientDirectoryRecord::SyncDirectoryEntry( return false; } - #ifdef WIN32 +#ifdef WIN32 // exclude reparse points, as Application Data points to the // parent directory under Vista and later, and causes an // infinite loop: // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ - if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + if (en->win_attrs & FILE_ATTRIBUTE_REPARSE_POINT) { rNotifier.NotifyMountPointSkipped(this, realFileName); return false; } - #endif +#endif // WIN32 } else // not a file or directory, what is it? { if (type == S_IFSOCK #ifndef WIN32 || type == S_IFIFO -#endif +#endif // !WIN32 ) { // removed notification for these types diff --git a/lib/bbackupd/BackupClientInodeToIDMap.cpp b/lib/bbackupd/BackupClientInodeToIDMap.cpp index 6eaf73947..9e3895505 100644 --- a/lib/bbackupd/BackupClientInodeToIDMap.cpp +++ b/lib/bbackupd/BackupClientInodeToIDMap.cpp @@ -19,6 +19,7 @@ #include "Archive.h" #include "BackupStoreException.h" #include "CollectInBufferStream.h" +#include "Database.h" #include "MemBlockStream.h" #include "autogen_CommonException.h" @@ -27,22 +28,6 @@ #define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" #define BOX_DBM_INODE_DB_VERSION_CURRENT 2 -#define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) - -#define BOX_LOG_DBM_ERROR(stuff) \ - BOX_ERROR(BOX_DBM_MESSAGE(stuff)) - -#define THROW_DBM_ERROR(message, filename, exception, subtype) \ - BOX_LOG_DBM_ERROR(message << ": " << filename); \ - THROW_EXCEPTION_MESSAGE(exception, subtype, \ - BOX_DBM_MESSAGE(message << ": " << filename)); - -#define ASSERT_DBM_OK(operation, message, filename, exception, subtype) \ - if(!(operation)) \ - { \ - THROW_DBM_ERROR(message, filename, exception, subtype); \ - } - #define ASSERT_DBM_OPEN() \ if(mpDepot == 0) \ { \ diff --git a/lib/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp index 996c1919c..5556ca333 100644 --- a/lib/bbackupd/BackupDaemon.cpp +++ b/lib/bbackupd/BackupDaemon.cpp @@ -291,7 +291,7 @@ const char *BackupDaemon::DaemonName() const // -------------------------------------------------------------------------- std::string BackupDaemon::DaemonBanner() const { - return BANNER_TEXT("Backup Client"); + return BANNER_TEXT("client daemon (bbackupd)"); } void BackupDaemon::Usage() @@ -531,7 +531,7 @@ void BackupDaemon::Run() mapCommandSocketInfo->mListeningSocket.Listen( socketName); #else - ::unlink(socketName); + EMU_UNLINK(socketName); mapCommandSocketInfo->mListeningSocket.Listen( Socket::TypeUNIX, socketName); #endif @@ -1848,36 +1848,51 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() // Run it? pid_t pid = 0; - try + while(true) { - std::auto_ptr pscript(LocalProcessStream(script, - pid)); - - // Read in the result - IOStreamGetLine getLine(*pscript); - std::string line; - if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough + try { + std::auto_ptr pscript(LocalProcessStream(script, pid)); + + // Read in and parse the script output: + IOStreamGetLine getLine(*pscript); + std::string line = getLine.GetLine(true, 30000); // 30 seconds should be enough waitInSeconds = BackupDaemon::ParseSyncAllowScriptOutput(script, line); } - else + catch(BoxException &e) { - BOX_ERROR("SyncAllowScript output nothing within " - "30 seconds, waiting 5 minutes to try again" - " (" << script << ")"); + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + BOX_ERROR("SyncAllowScript exited without any output " + "(" << script << ")"); + } + else if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + BOX_ERROR("SyncAllowScript output nothing within " + "30 seconds, waiting 5 minutes to try again " + "(" << script << ")"); + waitInSeconds = 5 * 60; + } + else + { + // Ignore any other exceptions, and log that something bad happened. + BOX_ERROR("Unexpected error while trying to execute the " + "SyncAllowScript: " << e.what() << " (" << script << ")"); + } } - } - catch(std::exception &e) - { - BOX_ERROR("Internal error running SyncAllowScript: " - << e.what() << " (" << script << ")"); - } - catch(...) - { - // Ignore any exceptions - // Log that something bad happened - BOX_ERROR("Unknown error running SyncAllowScript (" << - script << ")"); + catch(...) + { + // Ignore any other exceptions, and log that something bad happened. + BOX_ERROR("Unexpected error while trying to execute the SyncAllowScript " + "(" << script << ")"); + } + + break; } // Wait and then cleanup child process, if any @@ -2139,17 +2154,19 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla ASSERT(mapCommandSocketInfo->mapGetLine.get() != 0); // Ping the remote side, to provide errors which will mean the socket gets closed - mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, - timeout); + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, timeout); // Wait for a command or something on the socket std::string command; - while(mapCommandSocketInfo->mapGetLine.get() != 0 - && !mapCommandSocketInfo->mapGetLine->IsEOF() - && mapCommandSocketInfo->mapGetLine->GetLine(command, false /* no preprocessing */, timeout)) + while(mapCommandSocketInfo->mapGetLine.get() != 0) { - BOX_TRACE("Receiving command '" << command - << "' over command socket"); + { + HideExceptionMessageGuard hide_exceptions; + command = mapCommandSocketInfo->mapGetLine->GetLine( + false /* no preprocessing */, timeout); + } + + BOX_TRACE("Received command '" << command << "' over command socket"); bool sendOK = false; bool sendResponse = true; @@ -2199,13 +2216,6 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Set timeout to something very small, so this just checks for data which is waiting timeout = 1; } - - // Close on EOF? - if(mapCommandSocketInfo->mapGetLine.get() != 0 && - mapCommandSocketInfo->mapGetLine->IsEOF()) - { - CloseCommandConnection(); - } } catch(ConnectionException &ce) { @@ -2225,10 +2235,30 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla CloseCommandConnection(); } } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived) || + EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + // No special handling, wait for us to be called again + return; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + // Need to close the connection, then we can just return, and next time + // we're called we'll listen for a new one. + CloseCommandConnection(); + return; + } + else + { + BOX_ERROR("Failed to read from command socket: " << e.what()); + return; + } + } catch(std::exception &e) { - BOX_ERROR("Failed to write to command socket: " << - e.what()); + BOX_ERROR("Failed to write to command socket: " << e.what()); // If an error occurs, and there is a connection active, // just close that connection and continue. Otherwise, @@ -2311,6 +2341,7 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) mapCommandSocketInfo->mpConnectedSocket.get() != 0) { std::string message = SendStart ? "start-sync" : "finish-sync"; + BOX_TRACE("Writing to command socket: " << message); try { message += "\n"; @@ -2828,7 +2859,7 @@ void BackupDaemon::FillIDMapVector(std::vector &rVec BOX_NOTICE("Found an incomplete ID map " "database, deleting it to start " "afresh: " << filename); - if(unlink(filename.c_str()) != 0) + if(EMU_UNLINK(filename.c_str()) != 0) { BOX_LOG_NATIVE_ERROR(BOX_FILE_MESSAGE( filename, "Failed to delete " @@ -2877,14 +2908,14 @@ void BackupDaemon::DeleteCorruptBerkelyDbFiles() // Delete the file BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); + EMU_UNLINK(filename.c_str()); // Add a suffix for the new map filename += ".n"; // Delete that too BOX_TRACE("Deleting " << filename); - ::unlink(filename.c_str()); + EMU_UNLINK(filename.c_str()); } } @@ -3635,7 +3666,7 @@ bool BackupDaemon::DeleteStoreObjectInfo() const } // Actually delete it - if(::unlink(storeObjectInfoFile.c_str()) != 0) + if(EMU_UNLINK(storeObjectInfoFile.c_str()) != 0) { BOX_LOG_SYS_ERROR("Failed to delete the old " "StoreObjectInfoFile: " << storeObjectInfoFile); diff --git a/lib/bbackupquery/BackupQueries.cpp b/lib/bbackupquery/BackupQueries.cpp index bcb1827e7..f7a511c93 100644 --- a/lib/bbackupquery/BackupQueries.cpp +++ b/lib/bbackupquery/BackupQueries.cpp @@ -1003,17 +1003,17 @@ void BackupQueries::CommandGetObject(const std::vector &args, const { BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << " does not exist on store."); - ::unlink(args[1].c_str()); + EMU_UNLINK(args[1].c_str()); } else { BOX_ERROR("Error occured fetching object."); - ::unlink(args[1].c_str()); + EMU_UNLINK(args[1].c_str()); } } catch(...) { - ::unlink(args[1].c_str()); + EMU_UNLINK(args[1].c_str()); BOX_ERROR("Error occured fetching object."); } } @@ -1242,18 +1242,18 @@ void BackupQueries::CommandGet(std::vector args, const bool *opts) { BOX_ERROR("Failed to fetch file: " << e.what()); - ::unlink(localName.c_str()); + EMU_UNLINK(localName.c_str()); } catch(std::exception &e) { BOX_ERROR("Failed to fetch file: " << e.what()); - ::unlink(localName.c_str()); + EMU_UNLINK(localName.c_str()); } catch(...) { BOX_ERROR("Failed to fetch file: unknown error"); - ::unlink(localName.c_str()); + EMU_UNLINK(localName.c_str()); } } diff --git a/lib/bbstored/BBStoreDHousekeeping.cpp b/lib/bbstored/BBStoreDHousekeeping.cpp index 86d6409c3..130a5b8a0 100644 --- a/lib/bbstored/BBStoreDHousekeeping.cpp +++ b/lib/bbstored/BBStoreDHousekeeping.cpp @@ -61,7 +61,7 @@ void BackupStoreDaemon::HousekeepingProcess() if(secondsToGo < 1) secondsToGo = 1; if(secondsToGo > 60) secondsToGo = 60; int32_t millisecondsToGo = ((int)secondsToGo) * 1000; - + // Check to see if there's any message pending CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); } @@ -103,9 +103,9 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() { mpAccountDatabase->GetAllAccountIDs(accounts); } - + SetProcessTitle("housekeeping, active"); - + // Check them all for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) { @@ -128,10 +128,11 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Happens automatically when tagWithClientID // goes out of scope. } - + + RaidBackupFileSystem fs(*i, rootDir, discSet); + // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, - discSet, this); + HousekeepStoreAccount housekeeping(fs, this); housekeeping.DoHousekeeping(); } catch(BoxException &e) @@ -156,7 +157,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() "aborting run for this account: " "unknown exception"); } - + int64_t timeNow = GetCurrentBoxTime(); time_t secondsToGo = BoxTimeToSeconds( (mLastHousekeepingRun + housekeepingInterval) - @@ -174,7 +175,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() break; } } - + BOX_INFO("Finished housekeeping"); // Placed here for accuracy, if StopRun() is true, for example. @@ -183,12 +184,15 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() void BackupStoreDaemon::OnIdle() { - if (!IsSingleProcess()) + if(IsForkPerClient()) { + // Housekeeping will be done by a specialised child process. return; } - if (!mHousekeepingInited) + // Housekeeping will be done in the main process, in idle time. + + if(!mHousekeepingInited) { HousekeepingInit(); mHousekeepingInited = true; @@ -216,45 +220,77 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. if(mInterProcessComms.IsEOF()) { + BOX_INFO("Housekeeping process was hungup by main daemon, terminating"); SetTerminateWanted(); return true; } // Get a line, and process the message std::string line; - if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) + + try { - BOX_TRACE("Housekeeping received command '" << line << - "' over interprocess comms"); - - int account = 0; - - if(line == "h") + HideExceptionMessageGuard hide_exceptions; + line = mInterProcessComms.GetLine(false /* no pre-processing */, + MaximumWaitTime); + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived) || + EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) { - // HUP signal received by main process - SetReloadConfigWanted(); - return true; + // No special handling, just return empty-handed. We will be called again. + return false; } - else if(line == "t") + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) { - // Terminate signal received by main process + // This means something has gone wrong with the main server process, and the + // housekeeping process should also terminate. + BOX_INFO("Housekeeping process was hungup by main daemon, terminating"); SetTerminateWanted(); return true; } - else if(sscanf(line.c_str(), "r%x", &account) == 1) + else { - // Main process is trying to lock an account -- are we processing it? - if(account == AccountNum) - { - // Yes! -- need to stop now so when it retries to get the lock, it will succeed - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(AccountNum) << - "giving way to client connection"); - return true; - } + throw; } } - + + BOX_TRACE("Housekeeping received command '" << line << + "' over interprocess comms"); + + int account = 0; + + if(line == "h") + { + // HUP signal received by main process + SetReloadConfigWanted(); + return true; + } + else if(line == "t") + { + // Terminate signal received by main process + SetTerminateWanted(); + return true; + } + else if(sscanf(line.c_str(), "r%x", &account) == 1) + { + // Main process is trying to lock an account -- are we processing it? + if(account == AccountNum) + { + // Yes! -- need to stop now so when it retries to get the lock, it will succeed + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(AccountNum) << + "giving way to client connection"); + return true; + } + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, "Housekeeping received " + "unexpected command from main daemon: " << line); + } + return false; } diff --git a/lib/bbstored/BackupStoreDaemon.cpp b/lib/bbstored/BackupStoreDaemon.cpp index 8fddf1251..daa9f6235 100644 --- a/lib/bbstored/BackupStoreDaemon.cpp +++ b/lib/bbstored/BackupStoreDaemon.cpp @@ -96,7 +96,7 @@ const char *BackupStoreDaemon::DaemonName() const // -------------------------------------------------------------------------- std::string BackupStoreDaemon::DaemonBanner() const { - return BANNER_TEXT("Backup Store Server"); + return BANNER_TEXT("store server (bbstored)"); } @@ -171,14 +171,17 @@ void BackupStoreDaemon::Run() mExtendedLogging = false; const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); + int housekeeping_pid; // Fork off housekeeping daemon -- must only do this the first // time Run() is called. Housekeeping runs synchronously on Win32 // because IsSingleProcess() is always true #ifndef WIN32 - if(!IsSingleProcess() && !mHaveForkedHousekeeping) + if(IsForkPerClient() && !mHaveForkedHousekeeping) { + // Housekeeping will be done by a specialised child process. + // Open a socket pair for communication int sv[2] = {-1,-1}; if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0) @@ -188,7 +191,8 @@ void BackupStoreDaemon::Run() int whichSocket = 0; // Fork - switch(::fork()) + housekeeping_pid = ::fork(); + switch(housekeeping_pid) { case -1: { @@ -259,6 +263,39 @@ void BackupStoreDaemon::Run() if(IsTerminateWanted()) { mInterProcessCommsSocket.Write("t\n", 2); + +#ifndef WIN32 // waitpid() only works with fork(), and thus not on Windows + // Wait for housekeeping to finish, and report its status, otherwise it + // can remain running and write to the console after the main process has + // died, which can confuse the tests. + int child_state; + if(waitpid(housekeeping_pid, &child_state, 0) == -1) + { + BOX_LOG_SYS_ERROR("Failed to wait for housekeeping child process " + "to finish"); + } + else if(WIFEXITED(child_state)) + { + if(WEXITSTATUS(child_state) == 0) + { + BOX_TRACE("Housekeeping child process exited normally"); + } + else + { + BOX_ERROR("Housekeeping child process exited with " + "status " << WEXITSTATUS(child_state)); + } + } + else if(WIFSIGNALED(child_state)) + { + BOX_ERROR("Housekeeping child process terminated with signal " << + WTERMSIG(child_state)); + } + else + { + BOX_ERROR("Housekeeping child process terminated in unexpected manner"); + } +#endif // !WIN32 } } } @@ -331,7 +368,7 @@ void BackupStoreDaemon::Connection2(std::auto_ptr apStream) // Create a context, using this ID BackupStoreContext context(id, this, GetConnectionDetails()); - if (mpTestHook) + if(mpTestHook) { context.SetTestHook(*mpTestHook); } diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index 9ca0c11cf..f9ffadaad 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -15,8 +15,8 @@ #endif #define BANNER_TEXT(UtilityName) \ - "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ - "contributors 2003-2014" + "Box Backup " UtilityName " v" BOX_VERSION "\n" \ + "(c) Ben Summers and contributors 2003-2017" #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index 8ce2a6255..ebe4894d0 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -25,13 +25,13 @@ // Show backtraces on exceptions in release builds until further notice // (they are only logged at TRACE level anyway) -#ifdef HAVE_EXECINFO_H +#if defined WIN32 || defined HAVE_EXECINFO_H #define SHOW_BACKTRACE_ON_EXCEPTION #endif #ifdef SHOW_BACKTRACE_ON_EXCEPTION #include "Utils.h" - #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace(); + #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace(__FILE__); #else #define OPTIONAL_DO_BACKTRACE #endif @@ -40,8 +40,6 @@ #include "Logging.h" #ifndef BOX_RELEASE_BUILD - extern bool AssertFailuresToSyslog; - #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} void BoxDebugAssertFailed(const char *cond, const char *file, int line); #define ASSERT(cond) \ { \ @@ -53,17 +51,6 @@ } \ } - // Note that syslog tracing is independent of BoxDebugTraceOn, - // but stdout tracing is not - extern bool BoxDebugTraceToSyslog; - #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;} - extern bool BoxDebugTraceToStdout; - #define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;} - - extern bool BoxDebugTraceOn; - int BoxDebug_printf(const char *format, ...); - int BoxDebugTrace(const char *format, ...); - #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING #define BOX_MEMORY_LEAK_TESTING #endif @@ -71,12 +58,8 @@ // Exception names #define EXCEPTION_CODENAMES_EXTENDED #else - #define ASSERT_FAILS_TO_SYSLOG_ON #define ASSERT(cond) - #define TRACE_TO_SYSLOG(x) {} - #define TRACE_TO_STDOUT(x) {} - // Box Backup builds release get extra information for exception logging #define EXCEPTION_CODENAMES_EXTENDED #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h index 361f04e89..6b284b0cc 100644 --- a/lib/common/BoxException.h +++ b/lib/common/BoxException.h @@ -29,11 +29,17 @@ class BoxException : public std::exception virtual unsigned int GetType() const throw() = 0; virtual unsigned int GetSubType() const throw() = 0; + bool IsType(unsigned int Type, unsigned int SubType) + { + return GetType() == Type && GetSubType() == SubType; + } virtual const std::string& GetMessage() const = 0; private: }; +#define EXCEPTION_IS_TYPE(exception_obj, type, subtype) \ + exception_obj.IsType(type::ExceptionType, type::subtype) #endif // BOXEXCEPTION__H diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index f7c74bfc1..86e4e75e9 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -57,17 +57,11 @@ #endif // Slight hack; disable interception in raidfile test on Darwin and Windows -#if defined __APPLE__ || defined WIN32 +#if defined WIN32 // TODO: Replace with autoconf test #define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif -// Disable memory testing under Darwin, it just doesn't like it very much. -#ifdef __APPLE__ - // TODO: We really should get some decent leak detection code. - #define PLATFORM_DISABLE_MEM_LEAK_TESTING -#endif - // Darwin also has a weird idea of permissions and dates on symlinks: // perms are fixed at creation time by your umask, and dates can't be // changed. This breaks unit tests if we try to compare these things. @@ -87,15 +81,7 @@ #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET #endif -#ifdef HAVE_DEFINE_PRAGMA - // set packing to one bytes (can't use push/pop on gcc) - #define BEGIN_STRUCTURE_PACKING_FOR_WIRE #pragma pack(1) - - // Use default packing - #define END_STRUCTURE_PACKING_FOR_WIRE #pragma pack() -#else - #define STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS -#endif +#define STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS // Handle differing xattr APIs #if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h index 3984aceb3..e0b99904d 100644 --- a/lib/common/BufferedStream.h +++ b/lib/common/BufferedStream.h @@ -27,6 +27,8 @@ class BufferedStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); diff --git a/lib/common/BufferedWriteStream.h b/lib/common/BufferedWriteStream.h index 5f6d5f19d..f2b019114 100644 --- a/lib/common/BufferedWriteStream.h +++ b/lib/common/BufferedWriteStream.h @@ -27,6 +27,8 @@ class BufferedWriteStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Flush(int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/common/ByteCountingStream.h b/lib/common/ByteCountingStream.h new file mode 100644 index 000000000..b62c2e362 --- /dev/null +++ b/lib/common/ByteCountingStream.h @@ -0,0 +1,81 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ByteCountingStream.h +// Purpose: A stream wrapper that counts the number of bytes +// transferred through it. +// Created: 2016/02/05 +// +// -------------------------------------------------------------------------- + +#ifndef BYTECOUNTINGSTREAM__H +#define BYTECOUNTINGSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: ByteCountingStream +// Purpose: A stream wrapper that counts the number of bytes +// transferred through it. +// Created: 2016/02/05 +// +// -------------------------------------------------------------------------- +class ByteCountingStream : public IOStream +{ +public: + ByteCountingStream(IOStream &underlying) + : mrUnderlying(underlying), + mNumBytesRead(0), + mNumBytesWritten(0) + { } + + ByteCountingStream(const ByteCountingStream &rToCopy) + : mrUnderlying(rToCopy.mrUnderlying), + mNumBytesRead(0), + mNumBytesWritten(0) + { } + +private: + // no copying from IOStream allowed + ByteCountingStream(const IOStream& rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + int bytes_read = mrUnderlying.Read(pBuffer, NBytes, Timeout); + mNumBytesRead += bytes_read; + return bytes_read; + } + virtual pos_type BytesLeftToRead() + { + return mrUnderlying.BytesLeftToRead(); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + mrUnderlying.Write(pBuffer, NBytes, Timeout); + mNumBytesWritten += NBytes; + } + using IOStream::Write; + + virtual bool StreamDataLeft() + { + return mrUnderlying.StreamDataLeft(); + } + virtual bool StreamClosed() + { + return mrUnderlying.StreamClosed(); + } + int64_t GetNumBytesRead() { return mNumBytesRead; } + int64_t GetNumBytesWritten() { return mNumBytesWritten; } + +private: + IOStream &mrUnderlying; + int64_t mNumBytesRead, mNumBytesWritten; +}; + +#endif // BYTECOUNTINGSTREAM__H + diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp index 47b271f06..2b802ff4f 100644 --- a/lib/common/CollectInBufferStream.cpp +++ b/lib/common/CollectInBufferStream.cpp @@ -58,7 +58,10 @@ CollectInBufferStream::~CollectInBufferStream() // -------------------------------------------------------------------------- int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout) { - if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + if(mInWritePhase != false) + { + THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase); + } // Adjust to number of bytes left if(NBytes > (mBytesInBuffer - mReadPosition)) @@ -100,6 +103,7 @@ IOStream::pos_type CollectInBufferStream::BytesLeftToRead() // -------------------------------------------------------------------------- void CollectInBufferStream::Write(const void *pBuffer, int NBytes, int Timeout) { + ASSERT(NBytes >= 0); if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } // Enough space in the buffer @@ -225,7 +229,10 @@ bool CollectInBufferStream::StreamClosed() // -------------------------------------------------------------------------- void CollectInBufferStream::SetForReading() { - if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } + if(mInWritePhase != true) + { + THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase); + } // Move to read phase mInWritePhase = false; diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h index 297d28513..cc4cfb84d 100644 --- a/lib/common/CollectInBufferStream.h +++ b/lib/common/CollectInBufferStream.h @@ -42,6 +42,8 @@ class CollectInBufferStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index 610ba1a88..4413e8ef1 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -42,9 +42,9 @@ KEventErrorWait 34 KEventErrorRemove 35 KQueueNotSupportedOnThisPlatform 36 IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData() -TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables. -ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes. -AccessDenied 40 Access to the file or directory was denied. Please check the permissions. +TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables +ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes +AccessDenied 40 Access to the file or directory was denied. Please check the permissions DatabaseOpenFailed 41 Failed to open the database file DatabaseReadFailed 42 Failed to read a record from the database file DatabaseWriteFailed 43 Failed to write a record from the database file @@ -56,4 +56,8 @@ DatabaseRecordBadSize 48 The database contains a record with an invalid size DatabaseIterateFailed 49 Failed to iterate over the database keys ReferenceNotFound 50 The database does not contain an expected reference TimersNotInitialised 51 The timer framework should have been ready at this point -InvalidConfiguration 52 Some required values are missing or incorrect in the configuration file. +InvalidConfiguration 52 Some required values are missing or incorrect in the configuration file +IOStreamTimedOut 53 A network operation timed out +SignalReceived 54 The function call returned because the process received a signal +NamedLockFailed 55 Failed to acquire a named lock +FileLockingConflict 56 The file is already locked by another process diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index 8ce8d389b..1d348dfef 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -195,17 +195,17 @@ std::auto_ptr Configuration::LoadAndVerify( { // Just to make sure rErrorMsg.erase(); - + // Open the file FileHandleGuard file(rFilename); - + // GetLine object FdGetLine getline(file); - + // Object to create std::auto_ptr apConfig( new Configuration(std::string(""))); - + try { // Load @@ -238,7 +238,7 @@ std::auto_ptr Configuration::LoadAndVerify( // Clean up throw; } - + // Success. Return result. return apConfig; } @@ -263,13 +263,13 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s while(!rGetLine.IsEOF()) { std::string line(rGetLine.GetLine(true)); /* preprocess out whitespace and comments */ - + if(line.empty()) { // Ignore blank lines continue; } - + // Line an open block string? if(line == "{") { @@ -277,7 +277,7 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s { // New config object Configuration subConfig(blockName); - + // Continue processing into this block if(!LoadInto(subConfig, rGetLine, rErrorMsg, false)) { @@ -364,7 +364,7 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s startBlockExpected = true; } } - } + } } // End of file? @@ -373,7 +373,7 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s // Error if EOF and this isn't the root level rErrorMsg += "File ended without terminating all subblocks\n"; } - + return true; } @@ -391,7 +391,7 @@ void Configuration::AddKeyValue(const std::string& rKey, { // Store mKeys[rKey] = rValue; - } + } } void Configuration::AddSubConfig(const std::string& rName, @@ -427,17 +427,14 @@ bool Configuration::KeyExists(const std::string& rKeyName) const const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const { std::map::const_iterator i(mKeys.find(rKeyName)); - + if(i == mKeys.end()) { - BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + THROW_EXCEPTION_MESSAGE(CommonException, ConfigNoKey, "Missing configuration key: " << rKeyName); - THROW_EXCEPTION(CommonException, ConfigNoKey) - } - else - { - return i->second; } + + return i->second; } @@ -451,22 +448,17 @@ const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const // -------------------------------------------------------------------------- int Configuration::GetKeyValueInt(const std::string& rKeyName) const { - std::map::const_iterator i(mKeys.find(rKeyName)); - - if(i == mKeys.end()) - { - THROW_EXCEPTION(CommonException, ConfigNoKey) - } - else + std::string value_str = GetKeyValue(rKeyName); + long value = ::strtol(value_str.c_str(), NULL, 0 /* C style handling */); + + if(value == LONG_MAX || value == LONG_MIN) { - long value = ::strtol((i->second).c_str(), NULL, - 0 /* C style handling */); - if(value == LONG_MAX || value == LONG_MIN) - { - THROW_EXCEPTION(CommonException, ConfigBadIntValue) - } - return (int)value; + THROW_EXCEPTION_MESSAGE(CommonException, ConfigBadIntValue, + "Invalid integer value for configuration key: " << + rKeyName << ": '" << value_str << "'"); } + + return (int)value; } @@ -480,23 +472,18 @@ int Configuration::GetKeyValueInt(const std::string& rKeyName) const // -------------------------------------------------------------------------- uint32_t Configuration::GetKeyValueUint32(const std::string& rKeyName) const { - std::map::const_iterator i(mKeys.find(rKeyName)); - - if(i == mKeys.end()) - { - THROW_EXCEPTION(CommonException, ConfigNoKey) - } - else + std::string value_str = GetKeyValue(rKeyName); + errno = 0; + long value = ::strtoul(value_str.c_str(), NULL, 0 /* C style handling */); + + if(errno != 0) { - errno = 0; - long value = ::strtoul((i->second).c_str(), NULL, - 0 /* C style handling */); - if(errno != 0) - { - THROW_EXCEPTION(CommonException, ConfigBadIntValue) - } - return (int)value; + THROW_EXCEPTION_MESSAGE(CommonException, ConfigBadIntValue, + "Invalid integer value for configuration key: " << + rKeyName << ": '" << value_str << "'"); } + + return (int)value; } @@ -510,33 +497,24 @@ uint32_t Configuration::GetKeyValueUint32(const std::string& rKeyName) const // -------------------------------------------------------------------------- bool Configuration::GetKeyValueBool(const std::string& rKeyName) const { - std::map::const_iterator i(mKeys.find(rKeyName)); - - if(i == mKeys.end()) - { - THROW_EXCEPTION(CommonException, ConfigNoKey) - } - else + std::string value_str = GetKeyValue(rKeyName); + bool value = false; + + // Anything this is called for should have been verified as having a correct + // string in the verification section. However, this does default to false + // if it isn't in the string table. + + for(int l = 0; sValueBooleanStrings[l] != 0; ++l) { - bool value = false; - - // Anything this is called for should have been verified as having a correct - // string in the verification section. However, this does default to false - // if it isn't in the string table. - - for(int l = 0; sValueBooleanStrings[l] != 0; ++l) + if(::strcasecmp(value_str.c_str(), sValueBooleanStrings[l]) == 0) { - if(::strcasecmp((i->second).c_str(), sValueBooleanStrings[l]) == 0) - { - // Found. - value = sValueBooleanValue[l]; - break; - } + // Found. + value = sValueBooleanValue[l]; + break; } - - return value; } + return value; } @@ -552,14 +530,13 @@ bool Configuration::GetKeyValueBool(const std::string& rKeyName) const std::vector Configuration::GetKeyNames() const { std::map::const_iterator i(mKeys.begin()); - std::vector r; - + for(; i != mKeys.end(); ++i) { r.push_back(i->first); } - + return r; } @@ -577,7 +554,7 @@ bool Configuration::SubConfigurationExists(const std::string& rSubName) const { // Attempt to find it... std::list >::const_iterator i(mSubConfigurations.begin()); - + for(; i != mSubConfigurations.end(); ++i) { // This the one? @@ -607,7 +584,7 @@ const Configuration &Configuration::GetSubConfiguration(const std::string& { // Attempt to find it... std::list >::const_iterator i(mSubConfigurations.begin()); - + for(; i != mSubConfigurations.end(); ++i) { // This the one? @@ -618,7 +595,8 @@ const Configuration &Configuration::GetSubConfiguration(const std::string& } } - THROW_EXCEPTION(CommonException, ConfigNoSubConfig) + THROW_EXCEPTION_MESSAGE(CommonException, ConfigNoSubConfig, + "Missing sub-configuration section: " << rSubName); } @@ -635,7 +613,7 @@ Configuration &Configuration::GetSubConfigurationEditable(const std::string& rSubName) { // Attempt to find it... - + for(SubConfigListType::iterator i = mSubConfigurations.begin(); i != mSubConfigurations.end(); ++i) @@ -648,7 +626,8 @@ Configuration &Configuration::GetSubConfigurationEditable(const std::string& } } - THROW_EXCEPTION(CommonException, ConfigNoSubConfig) + THROW_EXCEPTION_MESSAGE(CommonException, ConfigNoSubConfig, + "Missing sub-configuration section: " << rSubName); } @@ -662,15 +641,15 @@ Configuration &Configuration::GetSubConfigurationEditable(const std::string& // -------------------------------------------------------------------------- std::vector Configuration::GetSubConfigurationNames() const { - std::list >::const_iterator i(mSubConfigurations.begin()); - + std::list >::const_iterator + i(mSubConfigurations.begin()); std::vector r; - + for(; i != mSubConfigurations.end(); ++i) { r.push_back(i->first); } - + return r; } @@ -678,7 +657,8 @@ std::vector Configuration::GetSubConfigurationNames() const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::Verify(const ConfigurationVerify &, const std::string &, std::string &) +// Name: Configuration::Verify(const ConfigurationVerify &, +// const std::string &, std::string &) // Purpose: Checks that the configuration is valid according to the // supplied verifier // Created: 2003/07/24 @@ -693,7 +673,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, if(rVerify.mpKeys != 0) { const ConfigurationVerifyKey *pvkey = rVerify.mpKeys; - + bool todo = true; do { @@ -706,7 +686,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, // Check it's a number? if((pvkey->Flags() & ConfigTest_IsInt) == ConfigTest_IsInt) - { + { // Test it... char *end; long r = ::strtol(val, &end, 0); @@ -720,7 +700,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, // Check it's a number? if(pvkey->Flags() & ConfigTest_IsUint32) - { + { // Test it... char *end; errno = 0; @@ -732,10 +712,10 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid unsigned 32-bit integer.\n"; } } - + // Check it's a bool? if((pvkey->Flags() & ConfigTest_IsBool) == ConfigTest_IsBool) - { + { // See if it's one of the allowed strings. bool found = false; for(int l = 0; sValueBooleanStrings[l] != 0; ++l) @@ -747,7 +727,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, break; } } - + // Error if it's not one of them. if(!found) { @@ -755,7 +735,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid boolean value.\n"; } } - + // Check for multi valued statments where they're not allowed if((pvkey->Flags() & ConfigTest_MultiValueAllowed) == 0) { @@ -765,7 +745,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, ok = false; rErrorMsg += rLevel + mName +"." + pvkey->Name() + " (key) multi value not allowed (duplicated key?).\n"; } - } + } } else { @@ -782,16 +762,16 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, pvkey->DefaultValue(); } } - + if((pvkey->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { // No more! todo = false; } - + // next pvkey++; - + } while(todo); // Check for additional keys @@ -808,7 +788,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, found = true; break; } - + // Next? if((scan->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { @@ -816,7 +796,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, } scan++; } - + if(!found) { // Shouldn't exist, but does. @@ -825,13 +805,13 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, } } } - + // Then the sub configurations if(rVerify.mpSubConfigurations) { // Find the wildcard entry, if it exists, and check that required subconfigs are there const ConfigurationVerify *wildcardverify = 0; - + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; while(scan) { @@ -839,7 +819,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, { wildcardverify = scan; } - + // Required? if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists) { @@ -873,7 +853,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, } scan++; } - + // Go through the sub configurations, one by one for(SubConfigListType::iterator i = mSubConfigurations.begin(); @@ -881,7 +861,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, { // Can this be found? const ConfigurationVerify *subverify = 0; - + const ConfigurationVerify *scan = rVerify.mpSubConfigurations; const char *name = i->first.c_str(); ASSERT(name); @@ -892,7 +872,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, // found it! subverify = scan; } - + // Next? if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) { @@ -900,13 +880,13 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, } scan++; } - + // Use wildcard? if(subverify == 0) { subverify = wildcardverify; } - + // Verify if(subverify) { @@ -919,7 +899,7 @@ bool Configuration::Verify(const ConfigurationVerify &rVerify, } } } - + return ok; } diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index e6498e801..779e60ee9 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -81,7 +81,7 @@ class ConfigurationVerify std::string mName; // "*" for all other sub config names const ConfigurationVerify *mpSubConfigurations; const ConfigurationVerifyKey *mpKeys; - int Tests; + int Tests; void *TestFunction; // set to zero for now, will implement later static const ConfigurationCategory VERIFY_ERROR; }; @@ -102,13 +102,13 @@ class Configuration Configuration(const std::string &rName); Configuration(const Configuration &rToCopy); ~Configuration(); - + enum { // The character to separate multi-values MultiValueSeparator = '\x01' }; - + static std::auto_ptr LoadAndVerify( const std::string& rFilename, const ConfigurationVerify *pVerify, @@ -118,35 +118,44 @@ class Configuration const std::string& rFilename, std::string &rErrorMsg) { return LoadAndVerify(rFilename, 0, rErrorMsg); } - + bool KeyExists(const std::string& rKeyName) const; const std::string &GetKeyValue(const std::string& rKeyName) const; + const std::string &GetKeyValueDefault(const std::string& rKeyName, + const std::string& rDefaultValue) const + { + // Don't call this for an item that has a default value defined, + // because rDefaultValue will never be used. + std::map::const_iterator i = + mKeys.find(rKeyName); + return (i != mKeys.end()) ? i->second : rDefaultValue; + } int GetKeyValueInt(const std::string& rKeyName) const; uint32_t GetKeyValueUint32(const std::string& rKeyName) const; bool GetKeyValueBool(const std::string& rKeyName) const; std::vector GetKeyNames() const; - + bool SubConfigurationExists(const std::string& rSubName) const; const Configuration &GetSubConfiguration(const std::string& rSubName) const; Configuration &GetSubConfigurationEditable(const std::string& rSubName); std::vector GetSubConfigurationNames() const; - + void AddKeyValue(const std::string& rKey, const std::string& rValue); void AddSubConfig(const std::string& rName, const Configuration& rSubConfig); - + bool Verify(const ConfigurationVerify &rVerify, std::string &rErrorMsg) { return Verify(rVerify, std::string(), rErrorMsg); } -private: +private: std::string mName; // Order of keys not preserved std::map mKeys; // Order of sub blocks preserved typedef std::list > SubConfigListType; SubConfigListType mSubConfigurations; - + static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel); bool Verify(const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg); diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 58a82c0e7..c3fc4b593 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -20,6 +20,10 @@ #include #include +#ifdef HAVE_EXECINFO_H +# include +#endif + #ifdef HAVE_PROCESS_H # include #endif @@ -33,15 +37,24 @@ #include #include "MemLeakFinder.h" +#include "Utils.h" static bool memleakfinder_initialised = false; bool memleakfinder_global_enable = false; +#if !defined BOX_RELEASE_BUILD && defined HAVE_EXECINFO_H +# define BOX_MEMORY_LEAK_BACKTRACE_ENABLED +#endif + typedef struct { size_t size; const char *file; int line; +#ifdef BOX_MEMORY_LEAK_BACKTRACE_ENABLED + void *stack_frames[20]; + size_t stack_size; +#endif } MallocBlockInfo; typedef struct @@ -50,6 +63,10 @@ typedef struct const char *file; int line; bool array; +#ifdef BOX_MEMORY_LEAK_BACKTRACE_ENABLED + void *stack_frames[20]; + size_t stack_size; +#endif } ObjectInfo; namespace @@ -529,8 +546,8 @@ void memleakfinder_reportleaks_file(FILE *file) { if(is_leak(i->first)) { - ::fprintf(file, "Block %p size %d allocated at " - "%s:%d\n", i->first, i->second.size, + ::fprintf(file, "Block %p size %lu allocated at " + "%s:%d\n", i->first, (unsigned long)i->second.size, i->second.file, i->second.line); } } @@ -540,10 +557,17 @@ void memleakfinder_reportleaks_file(FILE *file) { if(is_leak(i->first)) { - ::fprintf(file, "Object%s %p size %d allocated at " + ::fprintf(file, "Object%s %p size %lu allocated at " "%s:%d\n", i->second.array?" []":"", - i->first, i->second.size, i->second.file, + i->first, (unsigned long)i->second.size, i->second.file, i->second.line); +#ifdef BOX_MEMORY_LEAK_BACKTRACE_ENABLED + if(file == stdout) + { + DumpStackBacktrace(__FILE__, i->second.stack_size, + i->second.stack_frames); + } +#endif } } } @@ -628,6 +652,9 @@ void add_object_block(void *block, size_t size, const char *file, int line, bool i.file = file; i.line = line; i.array = array; +#ifdef BOX_MEMORY_LEAK_BACKTRACE_ENABLED + i.stack_size = backtrace(i.stack_frames, 20); +#endif sObjectBlocks[block] = i; if(sTrackObjectsInSection) @@ -727,4 +754,30 @@ void operator delete(void *ptr) throw () internal_delete(ptr); } +/* + We need to implement a placement operator delete too: + + "If the object is being created as part of a new expression, and an exception + is thrown, the object’s memory is deallocated by calling the appropriate + deallocation function. If the object is being created with a placement new + operator, the corresponding placement delete operator is called—that is, the + delete function that takes the same additional parameters as the placement new + operator. If no matching placement delete is found, no deallocation takes + place." + + So to avoid memory leaks, we need to implement placement delete operators that + correspond to our placement new, which we use for leak detection (ironically) + in debug builds. +*/ + +void operator delete(void *ptr, const char *file, int line) +{ + internal_delete(ptr); +} + +void operator delete[](void *ptr, const char *file, int line) +{ + internal_delete(ptr); +} + #endif // BOX_MEMORY_LEAK_TESTING diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp deleted file mode 100644 index 1335d473d..000000000 --- a/lib/common/DebugPrintf.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: DebugPrintf.cpp -// Purpose: Implementation of a printf function, to avoid a stdio.h include in Box.h -// Created: 2003/09/06 -// -// -------------------------------------------------------------------------- - -#ifndef BOX_RELEASE_BUILD - -#include "Box.h" - -#include -#include - -#ifdef WIN32 - #include "emu.h" -#else - #include -#endif - -#include "MemLeakFindOn.h" - -// Use this apparently superflous printf function to avoid having to -// include stdio.h in every file in the project. - -int BoxDebug_printf(const char *format, ...) -{ - va_list ap; - va_start(ap, format); - int r = vprintf(format, ap); - va_end(ap); - return r; -} - - -bool BoxDebugTraceOn = true; -bool BoxDebugTraceToStdout = true; -bool BoxDebugTraceToSyslog = false; - -int BoxDebugTrace(const char *format, ...) -{ - char text[512]; - int r = 0; - if(BoxDebugTraceOn || BoxDebugTraceToSyslog) - { - va_list ap; - va_start(ap, format); - r = vsnprintf(text, sizeof(text), format, ap); - va_end(ap); - } - - // Send to stdout if trace is on and std out is enabled - if(BoxDebugTraceOn && BoxDebugTraceToStdout) - { - printf("%s", text); - } - - // But tracing to syslog is independent of tracing being on or not - if(BoxDebugTraceToSyslog) - { -#ifdef WIN32 - // Remove trailing '\n', if it's there - if(r > 0 && text[r-1] == '\n') - { - text[r-1] = '\0'; -#else - if(r > 0 && text[r] == '\n') - { - text[r] = '\0'; -#endif - --r; - } - // Log it - ::syslog(LOG_INFO, "TRACE: %s", text); - } - - return r; -} - - -#endif // BOX_RELEASE_BUILD diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp index 30409d92f..cde9f8877 100644 --- a/lib/common/FdGetLine.cpp +++ b/lib/common/FdGetLine.cpp @@ -49,33 +49,6 @@ FdGetLine::~FdGetLine() } -// -------------------------------------------------------------------------- -// -// Function -// Name: FdGetLine::GetLine(bool) -// Purpose: Returns a file from the file. If Preprocess is true, leading -// and trailing whitespace is removed, and comments (after #) -// are deleted. -// Created: 2003/07/24 -// -// -------------------------------------------------------------------------- -std::string FdGetLine::GetLine(bool Preprocess) -{ - if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)} - - std::string r; - bool result = GetLineInternal(r, Preprocess); - - if(!result) - { - // should never fail for FdGetLine - THROW_EXCEPTION(CommonException, Internal); - } - - return r; -} - - // -------------------------------------------------------------------------- // // Function @@ -88,6 +61,11 @@ std::string FdGetLine::GetLine(bool Preprocess) // -------------------------------------------------------------------------- int FdGetLine::ReadMore(int Timeout) { + if(mFileHandle == -1) + { + THROW_EXCEPTION(CommonException, GetLineNoHandle); + } + int bytes; #ifdef WIN32 diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h index 2b9c268ff..2bd301213 100644 --- a/lib/common/FdGetLine.h +++ b/lib/common/FdGetLine.h @@ -12,7 +12,7 @@ #include -#include "GetLine.h" +#include "LineBuffer.h" // -------------------------------------------------------------------------- // @@ -22,16 +22,15 @@ // Created: 2003/07/24 // // -------------------------------------------------------------------------- -class FdGetLine : public GetLine +class FdGetLine : public LineBuffer { public: FdGetLine(int fd); virtual ~FdGetLine(); private: - FdGetLine(const FdGetLine &rToCopy); + FdGetLine(const FdGetLine &forbidden); public: - virtual std::string GetLine(bool Preprocess = false); // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. void DetachFile(); diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index 51752f858..d3e1eee70 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -8,12 +8,21 @@ // -------------------------------------------------------------------------- #include "Box.h" + +#include + +#ifdef HAVE_FCNTL_G + #include +#endif + +#ifdef HAVE_SYS_FILE_H + #include +#endif + #include "FileStream.h" #include "CommonException.h" #include "Logging.h" -#include - #include "MemLeakFindOn.h" // -------------------------------------------------------------------------- @@ -24,16 +33,13 @@ // Created: 2003/07/31 // // -------------------------------------------------------------------------- -FileStream::FileStream(const std::string& rFilename, int flags, int mode) -#ifdef WIN32 - : mOSFileHandle(::openfile(rFilename.c_str(), flags, mode)), -#else - : mOSFileHandle(::open(rFilename.c_str(), flags, mode)), -#endif - mIsEOF(false), - mFileName(rFilename) +FileStream::FileStream(const std::string& mFileName, int flags, int mode, + lock_mode_t lock_mode) +: mOSFileHandle(INVALID_FILE), + mIsEOF(false), + mFileName(mFileName) { - AfterOpen(); + OpenFile(flags, mode, lock_mode); } // -------------------------------------------------------------------------- @@ -45,51 +51,159 @@ FileStream::FileStream(const std::string& rFilename, int flags, int mode) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -FileStream::FileStream(const char *pFilename, int flags, int mode) -#ifdef WIN32 - : mOSFileHandle(::openfile(pFilename, flags, mode)), -#else - : mOSFileHandle(::open(pFilename, flags, mode)), -#endif - mIsEOF(false), - mFileName(pFilename) +FileStream::FileStream(const char *pFilename, int flags, int mode, + lock_mode_t lock_mode) +: mOSFileHandle(INVALID_FILE), + mIsEOF(false), + mFileName(pFilename) { - AfterOpen(); + OpenFile(flags, mode, lock_mode); } -void FileStream::AfterOpen() +void FileStream::OpenFile(int flags, int mode, lock_mode_t lock_mode) { -#ifdef WIN32 - if(mOSFileHandle == INVALID_HANDLE_VALUE) + std::string lock_method_name, lock_message; + + if(lock_mode == EXCLUSIVE) + { +#ifdef BOX_LOCK_TYPE_O_EXLOCK + flags |= O_NONBLOCK | O_EXLOCK; + lock_method_name = "O_EXLOCK"; +#elif defined BOX_LOCK_TYPE_WIN32 + flags |= BOX_OPEN_LOCK; + lock_method_name = "dwShareMode 0"; +#elif defined BOX_LOCK_TYPE_F_OFD_SETLK + lock_method_name = "F_OFD_SETLK, F_WRLCK"; +#elif defined BOX_LOCK_TYPE_F_SETLK + lock_method_name = "F_SETLK, F_WRLCK"; +#elif defined BOX_LOCK_TYPE_FLOCK + lock_method_name = "flock(LOCK_EX)"; +#elif defined BOX_LOCK_TYPE_DUMB + // We have no other way to get a lock, so this is equivalent to O_EXCL. + flags |= O_EXCL; + lock_method_name = "O_EXCL"; #else - if(mOSFileHandle < 0) +# error "Unknown locking type" #endif + lock_message = std::string("exclusively using ") + lock_method_name; + } + else { - MEMLEAKFINDER_NOT_A_LEAK(this); +#ifdef BOX_LOCK_TYPE_O_EXLOCK + flags |= O_NONBLOCK | O_SHLOCK; + lock_method_name = "O_SHLOCK"; +#elif defined BOX_LOCK_TYPE_WIN32 + // no extra flags needed for FILE_SHARE_READ | FILE_SHARE_WRITE + lock_method_name = "dwShareMode FILE_SHARE_READ | FILE_SHARE_WRITE"; +#elif defined BOX_LOCK_TYPE_F_OFD_SETLK + lock_method_name = "F_OFD_SETLK, F_RDLCK"; +#elif defined BOX_LOCK_TYPE_F_SETLK + lock_method_name = "F_SETLK, F_RDLCK"; +#elif defined BOX_LOCK_TYPE_FLOCK + lock_method_name = "flock(LOCK_SH)"; +#elif defined BOX_LOCK_TYPE_DUMB + lock_method_name = "no locking at all!"; +#else +# error "Unknown locking type" +#endif + lock_message = std::string("shared using ") + lock_method_name; + } + + BOX_TRACE("Trying to " << (mode & O_CREAT ? "create" : "open") << " " << mFileName << " " << + lock_message); #ifdef WIN32 - if(errno == EACCES) + mOSFileHandle = ::openfile(mFileName.c_str(), flags, mode); +#else + mOSFileHandle = ::open(mFileName.c_str(), flags, mode); +#endif + + if(mOSFileHandle == INVALID_FILE) + { + // Failed to open the file. What's the reason? The errno which indicates a lock + // conflict depends on the locking method. + +#ifdef BOX_LOCK_TYPE_O_EXLOCK + if(errno == EWOULDBLOCK) +#elif defined BOX_LOCK_TYPE_WIN32 + if(errno == EBUSY) +#elif defined BOX_LOCK_TYPE_DUMB + if(errno == EEXIST) +#else // F_OFD_SETLK, F_SETLK or FLOCK + if(false) +#endif { - THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, - winerrno, CommonException, AccessDenied); + // We failed to lock the file, which means that it's locked by someone else. + // Need to throw a specific exception that's expected by NamedLock, which + // should just return false in this case (and only this case). + THROW_EXCEPTION_MESSAGE(CommonException, FileLockingConflict, + BOX_FILE_MESSAGE(mFileName, "File already locked by another process")); + } + else if(errno == EACCES) + { + THROW_EMU_ERROR(BOX_FILE_MESSAGE(mFileName, "Failed to open file"), + CommonException, AccessDenied); } else { - THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, - winerrno, CommonException, OSFileOpenError); + THROW_EMU_ERROR(BOX_FILE_MESSAGE(mFileName, "Failed to open file"), + CommonException, OSFileOpenError); } -#else - if(errno == EACCES) + } + + bool lock_failed = false; + +#ifdef BOX_LOCK_TYPE_FLOCK + BOX_TRACE("Trying to lock " << mFileName << " " << lock_message); + if(::flock(mOSFileHandle, (lock_mode == SHARED ? LOCK_SH : LOCK_EX) | LOCK_NB) != 0) + { + Close(); + + if(errno == EWOULDBLOCK) { - THROW_SYS_FILE_ERROR("Failed to open file", mFileName, - CommonException, AccessDenied); + lock_failed = true; } else { - THROW_SYS_FILE_ERROR("Failed to open file", mFileName, - CommonException, OSFileOpenError); + THROW_SYS_FILE_ERROR("Failed to lock lockfile " << lock_method_name, + mFileName, CommonException, OSFileError); + } + } +#elif defined BOX_LOCK_TYPE_F_SETLK || defined BOX_LOCK_TYPE_F_OFD_SETLK + struct flock desc; + desc.l_type = (lock_mode == SHARED ? F_RDLCK : F_WRLCK); + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + desc.l_pid = 0; + BOX_TRACE("Trying to lock " << mFileName << " " << lock_message); +# if defined BOX_LOCK_TYPE_F_OFD_SETLK + if(::fcntl(mOSFileHandle, F_OFD_SETLK, &desc) != 0) +# else // BOX_LOCK_TYPE_F_SETLK + if(::fcntl(mOSFileHandle, F_SETLK, &desc) != 0) +# endif + { + Close(); + + if(errno == EAGAIN) + { + lock_failed = true; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile " << lock_method_name, + mFileName, CommonException, OSFileError); } + } #endif + + if(lock_failed) + { + // We failed to lock the file, which means that it's locked by someone else. + // Need to throw a specific exception that's expected by NamedLock, which + // should just return false in this case (and only this case). + THROW_EXCEPTION_MESSAGE(CommonException, FileLockingConflict, + BOX_FILE_MESSAGE(mFileName, "File already locked by another process")); } } @@ -107,13 +221,8 @@ FileStream::FileStream(tOSFileHandle FileDescriptor) mIsEOF(false), mFileName("HANDLE") { -#ifdef WIN32 - if(mOSFileHandle == INVALID_HANDLE_VALUE) -#else - if(mOSFileHandle < 0) -#endif + if(mOSFileHandle == INVALID_FILE) { - MEMLEAKFINDER_NOT_A_LEAK(this); BOX_ERROR("FileStream: called with invalid file handle"); THROW_EXCEPTION(CommonException, OSFileOpenError) } @@ -138,7 +247,6 @@ FileStream::FileStream(const FileStream &rToCopy) if(mOSFileHandle < 0) #endif { - MEMLEAKFINDER_NOT_A_LEAK(this); BOX_ERROR("FileStream: copying unopened file"); THROW_EXCEPTION(CommonException, OSFileOpenError) } @@ -171,7 +279,7 @@ FileStream::~FileStream() // -------------------------------------------------------------------------- int FileStream::Read(void *pBuffer, int NBytes, int Timeout) { - if(mOSFileHandle == INVALID_FILE) + if(mOSFileHandle == INVALID_FILE) { THROW_EXCEPTION(CommonException, FileClosed) } @@ -293,7 +401,7 @@ void FileStream::Write(const void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- IOStream::pos_type FileStream::GetPosition() const { - if(mOSFileHandle == INVALID_FILE) + if(mOSFileHandle == INVALID_FILE) { THROW_EXCEPTION(CommonException, FileClosed) } @@ -333,7 +441,7 @@ IOStream::pos_type FileStream::GetPosition() const // -------------------------------------------------------------------------- void FileStream::Seek(IOStream::pos_type Offset, int SeekType) { - if(mOSFileHandle == INVALID_FILE) + if(mOSFileHandle == INVALID_FILE) { THROW_EXCEPTION(CommonException, FileClosed) } diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 1426d8a2a..e3629dc5e 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -20,19 +20,46 @@ #include #endif +#if HAVE_DECL_O_EXLOCK +# define BOX_LOCK_TYPE_O_EXLOCK +#elif defined BOX_OPEN_LOCK +# define BOX_LOCK_TYPE_WIN32 +#elif defined HAVE_FLOCK +// This is preferable to F_OFD_SETLK because no byte ranges are involved +# define BOX_LOCK_TYPE_FLOCK +#elif HAVE_DECL_F_OFD_SETLK +// This is preferable to F_SETLK because it's non-reentrant +# define BOX_LOCK_TYPE_F_OFD_SETLK +#elif HAVE_DECL_F_SETLK +// This is not ideal because it's reentrant, but better than a dumb lock +// (reentrancy only matters in tests; in real use it's as good as F_OFD_SETLK). +# define BOX_LOCK_TYPE_F_SETLK +#else +// We have no other way to get a lock, so all we can do is fail if the +// file already exists, and take the risk of stale locks. +# define BOX_LOCK_TYPE_DUMB +#endif + class FileStream : public IOStream { public: + enum lock_mode_t { + SHARED, + EXCLUSIVE, + }; + FileStream(const std::string& rFilename, int flags = (O_RDONLY | O_BINARY), - int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH), + lock_mode_t lock_mode = SHARED); // Ensure that const char * name doesn't end up as a handle // on Windows! FileStream(const char *pFilename, int flags = (O_RDONLY | O_BINARY), - int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH), + lock_mode_t lock_mode = SHARED); FileStream(tOSFileHandle FileDescriptor); @@ -42,6 +69,7 @@ class FileStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -60,7 +88,7 @@ class FileStream : public IOStream tOSFileHandle mOSFileHandle; bool mIsEOF; FileStream(const FileStream &rToCopy) { /* do not call */ } - void AfterOpen(); + void OpenFile(int flags, int mode, lock_mode_t lock_mode); // for debugging.. std::string mFileName; diff --git a/lib/common/GetLine.cpp b/lib/common/GetLine.cpp deleted file mode 100644 index e6b26c8af..000000000 --- a/lib/common/GetLine.cpp +++ /dev/null @@ -1,176 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: GetLine.cpp -// Purpose: Common base class for line based file descriptor reading -// Created: 2011/04/22 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include - -#ifdef HAVE_UNISTD_H - #include -#endif - -#include "GetLine.h" -#include "CommonException.h" - -#include "MemLeakFindOn.h" - -// utility whitespace function -inline bool iw(int c) -{ - return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: GetLine::GetLine(int) -// Purpose: Constructor, taking file descriptor -// Created: 2011/04/22 -// -// -------------------------------------------------------------------------- -GetLine::GetLine() -: mLineNumber(0), - mBufferBegin(0), - mBytesInBuffer(0), - mPendingEOF(false), - mEOF(false) -{ } - -// -------------------------------------------------------------------------- -// -// Function -// Name: GetLine::GetLineInternal(std::string &, bool, int) -// Purpose: Gets a line from the file, returning it in rOutput. -// If Preprocess is true, leading and trailing -// whitespace is removed, and comments (after #) are -// deleted. Returns true if a line is available now, -// false if retrying may get a line (eg timeout, -// signal), and exceptions if it's EOF. -// Created: 2011/04/22 -// -// -------------------------------------------------------------------------- -bool GetLine::GetLineInternal(std::string &rOutput, bool Preprocess, - int Timeout) -{ - // EOF? - if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)} - - // Initialise string to stored into - rOutput = mPendingString; - mPendingString.erase(); - - bool foundLineEnd = false; - - while(!foundLineEnd && !mEOF) - { - // Use any bytes left in the buffer - while(mBufferBegin < mBytesInBuffer) - { - int c = mBuffer[mBufferBegin++]; - if(c == '\r') - { - // Ignore nasty Windows line ending extra chars - } - else if(c == '\n') - { - // Line end! - foundLineEnd = true; - break; - } - else - { - // Add to string - rOutput += c; - } - - // Implicit line ending at EOF - if(mBufferBegin >= mBytesInBuffer && mPendingEOF) - { - foundLineEnd = true; - } - } - - // Check size - if(rOutput.size() > GETLINE_MAX_LINE_SIZE) - { - THROW_EXCEPTION(CommonException, GetLineTooLarge) - } - - // Read more in? - if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF) - { - int bytes = ReadMore(Timeout); - - // Error? - if(bytes == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } - - // Adjust buffer info - mBytesInBuffer = bytes; - mBufferBegin = 0; - - // No data returned? - if(bytes == 0 && IsStreamDataLeft()) - { - // store string away - mPendingString = rOutput; - // Return false; - return false; - } - } - - // EOF? - if(mPendingEOF && mBufferBegin >= mBytesInBuffer) - { - // File is EOF, and now we've depleted the buffer completely, so tell caller as well. - mEOF = true; - } - } - - if(Preprocess) - { - // Check for comment char, but char before must be whitespace - // end points to a gap between characters, may equal start if - // the string to be extracted has zero length, and indexes the - // first character not in the string (== length, or a # mark - // or whitespace) - int end = 0; - int size = rOutput.size(); - while(end < size) - { - if(rOutput[end] == '#' && (end == 0 || (iw(rOutput[end-1])))) - { - break; - } - end++; - } - - // Remove whitespace - int begin = 0; - while(begin < size && iw(rOutput[begin])) - { - begin++; - } - - while(end > begin && end <= size && iw(rOutput[end-1])) - { - end--; - } - - // Return a sub string - rOutput = rOutput.substr(begin, end - begin); - } - - return true; -} - - diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp index 3e126d3f7..bf945f0c4 100644 --- a/lib/common/IOStream.cpp +++ b/lib/common/IOStream.cpp @@ -143,8 +143,15 @@ bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int T int bytesRead = Read(buffer, bytesToGo, Timeout); if(bytesRead == 0) { - // Timeout or something - return false; + if(errno == EINTR) + { + THROW_EXCEPTION(CommonException, SignalReceived); + } + else + { + // Timeout or something + return false; + } } // Increment things bytesToGo -= bytesRead; @@ -188,23 +195,25 @@ IOStream::pos_type IOStream::BytesLeftToRead() // // Function // Name: IOStream::CopyStreamTo(IOStream &, int Timeout) -// Purpose: Copies the entire stream to another stream (reading from this, -// writing to rCopyTo). Returns whether the copy completed (ie -// StreamDataLeft() returns false) +// Purpose: Copies the entire stream to another stream (reading +// from this, writing to rCopyTo). Returns the number +// of bytes copied. Throws an exception if a network +// timeout occurs. // Created: 2003/08/26 // // -------------------------------------------------------------------------- -bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) +IOStream::pos_type IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) { // Make sure there's something to do before allocating that buffer if(!StreamDataLeft()) { - return true; // complete, even though nothing happened + return 0; } // Buffer MemoryBlockGuard buffer(BufferSize); - + IOStream::pos_type bytes_copied = 0; + // Get copying! while(StreamDataLeft()) { @@ -212,17 +221,20 @@ bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) int bytes = Read(buffer, BufferSize, Timeout); if(bytes == 0 && StreamDataLeft()) { - return false; // incomplete, timed out + THROW_EXCEPTION_MESSAGE(CommonException, IOStreamTimedOut, + "Timed out copying stream"); } // Write some data if(bytes != 0) { - rCopyTo.Write(buffer, bytes); + rCopyTo.Write(buffer, bytes, Timeout); } + + bytes_copied += bytes; } - - return true; // completed + + return bytes_copied; // completed } // -------------------------------------------------------------------------- diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h index df7216c32..3e336028b 100644 --- a/lib/common/IOStream.h +++ b/lib/common/IOStream.h @@ -63,7 +63,8 @@ class IOStream // Utility functions bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); - bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); + IOStream::pos_type CopyStreamTo(IOStream &rCopyTo, + int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); void Flush(int Timeout = IOStream::TimeOutInfinite); static int ConvertSeekTypeToOSWhence(int SeekType); diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp index ef8930b8e..5837bba72 100644 --- a/lib/common/IOStreamGetLine.cpp +++ b/lib/common/IOStreamGetLine.cpp @@ -40,24 +40,6 @@ IOStreamGetLine::~IOStreamGetLine() } -// -------------------------------------------------------------------------- -// -// Function -// Name: IOStreamGetLine::GetLine(std::string &, bool, int) -// Purpose: Gets a line from the file, returning it in rOutput. If Preprocess is true, leading -// and trailing whitespace is removed, and comments (after #) -// are deleted. -// Returns true if a line is available now, false if retrying may get a line (eg timeout, signal), -// and exceptions if it's EOF. -// Created: 2003/07/24 -// -// -------------------------------------------------------------------------- -bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout) -{ - return GetLineInternal(rOutput, Preprocess, Timeout); -} - - // -------------------------------------------------------------------------- // // Function @@ -106,10 +88,10 @@ void IOStreamGetLine::DetachFile() // // Function // Name: IOStreamGetLine::IgnoreBufferedData(int) -// Purpose: Ignore buffered bytes (effectively removing them from the -// beginning of the buffered data.) -// Cannot remove more bytes than are currently in the buffer. -// Be careful when this is used! +// Purpose: Ignore buffered bytes (effectively removing them +// from the beginning of the buffered data.) Cannot +// remove more bytes than are currently in the buffer. +// Be careful when this is used! // Created: 22/12/04 // // -------------------------------------------------------------------------- diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index 1b5370316..a39805fc9 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -12,7 +12,7 @@ #include -#include "GetLine.h" +#include "LineBuffer.h" #include "IOStream.h" // -------------------------------------------------------------------------- @@ -23,7 +23,7 @@ // Created: 2003/07/24 // // -------------------------------------------------------------------------- -class IOStreamGetLine : public GetLine +class IOStreamGetLine : public LineBuffer { public: IOStreamGetLine(IOStream &Stream); @@ -32,14 +32,6 @@ class IOStreamGetLine : public GetLine IOStreamGetLine(const IOStreamGetLine &rToCopy); public: - bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); - std::string GetLine() - { - std::string output; - GetLine(output); - return output; - } - // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. void DetachFile(); diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp index 1a9d6d5a4..d6d044897 100644 --- a/lib/common/InvisibleTempFileStream.cpp +++ b/lib/common/InvisibleTempFileStream.cpp @@ -31,7 +31,7 @@ InvisibleTempFileStream::InvisibleTempFileStream(const std::string& Filename, #endif { #ifndef WIN32 - if(unlink(Filename.c_str()) != 0) + if(EMU_UNLINK(Filename.c_str()) != 0) { MEMLEAKFINDER_NOT_A_LEAK(this); THROW_EXCEPTION(CommonException, OSFileOpenError) diff --git a/lib/common/LineBuffer.cpp b/lib/common/LineBuffer.cpp new file mode 100644 index 000000000..3b0452ec0 --- /dev/null +++ b/lib/common/LineBuffer.cpp @@ -0,0 +1,216 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: GetLine.cpp +// Purpose: Common base class for line based file descriptor reading +// Created: 2011/04/22 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#ifdef HAVE_UNISTD_H + #include +#endif + +#include "autogen_CommonException.h" +#include "BoxTime.h" +#include "LineBuffer.h" + +#include "MemLeakFindOn.h" + +// utility whitespace function +inline bool iw(int c) +{ + return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: LineBuffer::LineBuffer() +// Purpose: Constructor +// Created: 2011/04/22 +// +// -------------------------------------------------------------------------- +LineBuffer::LineBuffer() +: mLineNumber(0), + mBufferBegin(0), + mBytesInBuffer(0), + mPendingEOF(false), + mEOF(false) +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: LineBuffer::GetLine(bool, int) +// Purpose: Gets a line from the input using ReadMore, and +// returns it. If Preprocess is true, leading and +// trailing whitespace is removed, and comments (after +// #) are deleted. Throws exceptions on timeout, signal +// received and EOF. +// Created: 2011/04/22 +// +// -------------------------------------------------------------------------- +std::string LineBuffer::GetLine(bool preprocess, int timeout) +{ + // EOF? + if(mEOF) + { + THROW_EXCEPTION(CommonException, GetLineEOF); + } + + // Initialise string to stored into + std::string output = mPendingString; + mPendingString.erase(); + + box_time_t start_time = GetCurrentBoxTime(); + box_time_t remaining_time = MilliSecondsToBoxTime(timeout); + box_time_t end_time = start_time + remaining_time; + bool found_line_end = false; + + while(!found_line_end && !mEOF) + { + // Use any bytes left in the buffer + while(mBufferBegin < mBytesInBuffer) + { + int c = mBuffer[mBufferBegin++]; + if(c == '\r') + { + // Ignore nasty Windows line ending extra chars + } + else if(c == '\n') + { + // Line end! + found_line_end = true; + break; + } + else + { + // Add to string + output += c; + } + + // Implicit line ending at EOF + if(mBufferBegin >= mBytesInBuffer && mPendingEOF) + { + found_line_end = true; + } + } + + // Check size + if(output.size() > GETLINE_MAX_LINE_SIZE) + { + THROW_EXCEPTION(CommonException, GetLineTooLarge); + } + + if(timeout != IOStream::TimeOutInfinite) + { + // Update remaining time, and if we have run out and not yet found EOL, then + // stash what we've read so far, and return false. (If the timeout is infinite, + // the only way out is EOL or EOF or a signal). + remaining_time = end_time - GetCurrentBoxTime(); + if(!found_line_end && remaining_time < 0) + { + mPendingString = output; + THROW_EXCEPTION(CommonException, IOStreamTimedOut); + } + } + + // Read more in? + if(!found_line_end && mBufferBegin >= mBytesInBuffer && !mPendingEOF) + { + int64_t read_timeout_ms; + if(timeout == IOStream::TimeOutInfinite) + { + read_timeout_ms = IOStream::TimeOutInfinite; + } + else + { + // We should have exited above, if remaining_time < 0. + ASSERT(remaining_time >= 0); + read_timeout_ms = BoxTimeToMilliSeconds(remaining_time); + } + + int bytes = ReadMore(read_timeout_ms); + + // Error? + if(bytes == -1) + { + THROW_EXCEPTION(CommonException, OSFileError); + } + + // Adjust buffer info + mBytesInBuffer = bytes; + mBufferBegin = 0; + + // No data returned? This could mean that (1) we reached EOF: + if(bytes == 0) + { + // This could mean that (1) we reached EOF: + if(!IsStreamDataLeft()) + { + mPendingEOF = true; + // Exception will be thrown at next start of loop. + } + // Or (2) we ran out of time waiting to read enough data + // (signals throw an exception in SocketStream instead). + else + { + // Store string away, ready for when the caller asks us again. + mPendingString = output; + THROW_EXCEPTION(CommonException, IOStreamTimedOut); + } + } + } + + // EOF? + if(mPendingEOF && mBufferBegin >= mBytesInBuffer) + { + // File is EOF, and now we've depleted the buffer completely, so tell caller as well. + mEOF = true; + } + } + + if(preprocess) + { + // Check for comment char, but char before must be whitespace + // end points to a gap between characters, may equal start if + // the string to be extracted has zero length, and indexes the + // first character not in the string (== length, or a # mark + // or whitespace) + int end = 0; + int size = output.size(); + while(end < size) + { + if(output[end] == '#' && (end == 0 || (iw(output[end-1])))) + { + break; + } + end++; + } + + // Remove whitespace + int begin = 0; + while(begin < size && iw(output[begin])) + { + begin++; + } + + while(end > begin && end <= size && iw(output[end-1])) + { + end--; + } + + // Return a sub string + output = output.substr(begin, end - begin); + } + + return output; +} + + diff --git a/lib/common/GetLine.h b/lib/common/LineBuffer.h similarity index 89% rename from lib/common/GetLine.h rename to lib/common/LineBuffer.h index 0eeb3c714..1083ad206 100644 --- a/lib/common/GetLine.h +++ b/lib/common/LineBuffer.h @@ -29,28 +29,27 @@ // -------------------------------------------------------------------------- // // Class -// Name: GetLine +// Name: LineBuffer // Purpose: Common base class for line based file descriptor reading // Created: 2011/04/22 // // -------------------------------------------------------------------------- -class GetLine +class LineBuffer { protected: - GetLine(); + LineBuffer(); private: - GetLine(const GetLine &rToCopy); + LineBuffer(const LineBuffer &forbidden); public: virtual bool IsEOF() {return mEOF;} int GetLineNumber() {return mLineNumber;} - virtual ~GetLine() { } + virtual ~LineBuffer() { } + std::string GetLine(bool Preprocess, + int Timeout = IOStream::TimeOutInfinite); protected: - bool GetLineInternal(std::string &rOutput, - bool Preprocess = false, - int Timeout = IOStream::TimeOutInfinite); virtual int ReadMore(int Timeout = IOStream::TimeOutInfinite) = 0; virtual bool IsStreamDataLeft() = 0; diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 0928a4d45..3bda6731d 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -45,7 +45,7 @@ Syslog* Logging::spSyslog = NULL; Logging Logging::sGlobalLogging; // automatic initialisation std::string Logging::sProgramName; const Log::Category Logging::UNCATEGORISED("Uncategorised"); -std::auto_ptr Logging::sapHideFileGuard; +std::vector Logging::sLogLevelOverrideByFileGuards; HideSpecificExceptionGuard::SuppressedExceptions_t HideSpecificExceptionGuard::sSuppressedExceptions; @@ -280,12 +280,43 @@ void Console::SetShowPID(bool enabled) sShowPID = enabled; } +bool Logging::ShouldLog(Log::Level default_level, Log::Level message_level, + const std::string& file, int line, const std::string& function, const Log::Category& category, + const std::string& message) +{ + // Based on http://stackoverflow.com/a/40947954/648162 and + // http://stackoverflow.com/a/41367919/648162, but since we have to do this at runtime for + // each logging call, don't do the strrchr unless we have at least one override configured + // (otherwise it's never used, and a waste of CPU): + const char* current_file = NULL; + if(Logging::sLogLevelOverrideByFileGuards.begin() != + Logging::sLogLevelOverrideByFileGuards.end()) + { + current_file = strrchr(file.c_str(), DIRECTORY_SEPARATOR_ASCHAR) + 1; + } + + Log::Level level_filter = default_level; + for(std::vector::iterator + i = Logging::sLogLevelOverrideByFileGuards.begin(); + i != Logging::sLogLevelOverrideByFileGuards.end(); i++) + { + if(i->IsOverridden(message_level, current_file, line, function, category, message)) + { + level_filter = i->GetNewLevel(); + } + } + + // Should log if the message level is less than the filter level, otherwise not. + return (message_level <= level_filter); +} + bool Console::Log(Log::Level level, const std::string& file, int line, const std::string& function, const Log::Category& category, const std::string& message) { - if (level > GetLevel()) + if(!Logging::ShouldLog(GetLevel(), level, file, line, function, category, message)) { + // Skip the rest of this logger, but continue with the others return true; } @@ -365,8 +396,9 @@ bool Syslog::Log(Log::Level level, const std::string& file, int line, const std::string& function, const Log::Category& category, const std::string& message) { - if (level > GetLevel()) + if(!Logging::ShouldLog(GetLevel(), level, file, line, function, category, message)) { + // Skip the rest of this logger, but continue with the others return true; } @@ -587,15 +619,39 @@ int Logging::OptionParser::ProcessOption(signed int option) { case 'L': { - if(sapHideFileGuard.get()) + std::string arg_value(optarg); + std::string::size_type equals_pos = arg_value.find('='); + if(equals_pos == std::string::npos) { - sapHideFileGuard->Add(optarg); + BOX_FATAL("Option -L argument should be 'filename[/category]=level'"); + return 2; + } + + std::string key = arg_value.substr(0, equals_pos); + std::string filename, category; + std::string::size_type slash_pos = key.find('/'); + if(slash_pos == std::string::npos) + { + // If no category is supplied, assume that the entire key is the filename + filename = key; } else { - sapHideFileGuard.reset(new HideFileGuard( - optarg, true)); // HideAllButSelected + filename = key.substr(0, slash_pos); + category = key.substr(slash_pos + 1); + } + + std::string level_name = arg_value.substr(equals_pos + 1); + Log::Level level = Logging::GetNamedLevel(level_name); + if (level == Log::INVALID) + { + BOX_FATAL("Invalid logging level: " << level_name); + return 2; } + + sLogLevelOverrideByFileGuards.push_back( + LogLevelOverrideByFileGuard(filename, category, level, false) // !OverrideAllButSelected + ); } break; @@ -713,9 +769,16 @@ int Logging::OptionParser::ProcessOption(signed int option) // -------------------------------------------------------------------------- std::string Logging::OptionParser::GetUsageString() { - return - " -L Filter out log messages except from specified file, can repeat\n" - " (for example, -L " __FILE__ ")\n" + // Based on http://stackoverflow.com/a/40947954/648162 and + // http://stackoverflow.com/a/41367919/648162: + const char* current_file = strrchr(__FILE__, DIRECTORY_SEPARATOR_ASCHAR) + 1; + + std::ostringstream buf; + buf << + " -L [][/]= Override log level for specified file or\n" + " category (for example, -L '" << current_file << "/Configuration=trace').\n" + " can be one of: {Uncategorised, Backtrace,\n" + " Configuration, RaidFileRead, FileSystem/Locking}\n" " -N Truncate log file at startup and on backup start\n" " -P Show process ID (PID) in console output\n" " -q Run more quietly, reduce verbosity level by one, can repeat\n" @@ -726,6 +789,7 @@ std::string Logging::OptionParser::GetUsageString() " -v Run more verbosely, increase verbosity level by one, can repeat\n" " -V Run at maximum verbosity, log everything to console and system\n" " -W Set verbosity to error/warning/notice/info/trace/everything\n"; + return buf.str(); } bool HideCategoryGuard::Log(Log::Level level, const std::string& file, int line, @@ -740,27 +804,46 @@ bool HideCategoryGuard::Log(Log::Level level, const std::string& file, int line, return (i == mCategories.end()); } -bool HideFileGuard::Log(Log::Level level, const std::string& file, int line, +LogLevelOverrideByFileGuard::~LogLevelOverrideByFileGuard() +{ + if(mInstalled) + { + auto this_pos = std::find(Logging::sLogLevelOverrideByFileGuards.begin(), + Logging::sLogLevelOverrideByFileGuards.end(), *this); + ASSERT(this_pos != Logging::sLogLevelOverrideByFileGuards.end()); + Logging::sLogLevelOverrideByFileGuards.erase(this_pos); + } +} + +void LogLevelOverrideByFileGuard::Install() +{ + ASSERT(!mInstalled); + Logging::sLogLevelOverrideByFileGuards.push_back(*this); + mInstalled = true; +} + +bool LogLevelOverrideByFileGuard::IsOverridden(Log::Level level, const std::string& file, int line, const std::string& function, const Log::Category& category, const std::string& message) { std::list::iterator i = std::find(mFileNames.begin(), mFileNames.end(), file); - bool allow_log_message; - if(mHideAllButSelected) + + // Return true if filename is in our list, i.e. its level is overridden, or if there + // are no filenames in the list (which corresponds to a wildcard match on filename): + bool override_matches = (i != mFileNames.end()) || mFileNames.empty(); + + // If we have a category, then that must match as well: + if(override_matches && !mCategoryNamePrefix.empty()) { - // Return true if filename is in our list, to allow further - // logging (thus, return false if it's not in our list, i.e. we - // found nothing, to suppress it). - allow_log_message = (i != mFileNames.end()); + override_matches = StartsWith(mCategoryNamePrefix, category.ToString()); } - else + + if(mOverrideAllButSelected) { - // Return false if filename is in our list, to suppress further - // logging (thus, return true if it's not in our list, i.e. we - // found nothing, to allow it). - allow_log_message = (i == mFileNames.end()); + override_matches = !override_matches; } - return allow_log_message; + + return override_matches; } diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 3dc3e69ca..f6d3af8ff 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -112,12 +112,16 @@ #define THROW_WIN_ERROR_NUMBER(message, error_number, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ BOX_WIN_ERRNO_MESSAGE(error_number, message)) + #define THROW_WIN_ERROR(message, exception, subtype) \ + THROW_WIN_ERROR_NUMBER(message, GetLastError(), exception, subtype) #define THROW_WIN_FILE_ERRNO(message, filename, error_number, exception, subtype) \ THROW_WIN_ERROR_NUMBER(BOX_FILE_MESSAGE(filename, message), \ error_number, exception, subtype) #define THROW_WIN_FILE_ERROR(message, filename, exception, subtype) \ THROW_WIN_FILE_ERRNO(message, filename, GetLastError(), \ exception, subtype) + #define THROW_SOCKET_ERROR(message, exception, subtype) \ + THROW_WIN_ERROR_NUMBER(message, WSAGetLastError(), exception, subtype) #define EMU_ERRNO winerrno #define THROW_EMU_ERROR(message, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ @@ -127,6 +131,8 @@ BOX_SYS_ERRNO_MESSAGE(error_number, stuff) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) + #define THROW_SOCKET_ERROR(message, exception, subtype) \ + THROW_SYS_ERROR(message, exception, subtype) #define EMU_ERRNO errno #define THROW_EMU_ERROR(message, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ @@ -203,7 +209,7 @@ namespace Log Category(const std::string& name) : mName(name) { } - const std::string& ToString() { return mName; } + const std::string& ToString() const { return mName; } bool operator==(const Category& other) { return mName == other.mName; } }; } @@ -410,8 +416,48 @@ class Capture : public Logger } }; -// Forward declaration -class HideFileGuard; +class LogLevelOverrideByFileGuard +{ + private: + std::list mFileNames; + std::string mCategoryNamePrefix; + Log::Level mNewLevel; + bool mOverrideAllButSelected; + bool mInstalled; + + public: + LogLevelOverrideByFileGuard(const std::string& rFileName, + const std::string& rCategoryNamePrefix, Log::Level NewLevel, + bool OverrideAllButSelected = false) + : mCategoryNamePrefix(rCategoryNamePrefix), + mNewLevel(NewLevel), + mOverrideAllButSelected(OverrideAllButSelected), + mInstalled(false) + { + if(rFileName.size() > 0) + { + mFileNames.push_back(rFileName); + } + } + virtual ~LogLevelOverrideByFileGuard(); + void Add(const std::string& rFileName) + { + mFileNames.push_back(rFileName); + } + void Install(); + bool IsOverridden(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + Log::Level GetNewLevel() { return mNewLevel; } + bool operator==(const LogLevelOverrideByFileGuard& rOther) + { + return (mFileNames == rOther.mFileNames) && + (mCategoryNamePrefix == rOther.mCategoryNamePrefix) && + (mNewLevel == rOther.mNewLevel) && + (mOverrideAllButSelected == rOther.mOverrideAllButSelected); + } +}; + // -------------------------------------------------------------------------- // @@ -434,7 +480,8 @@ class Logging static Syslog* spSyslog; static Logging sGlobalLogging; static std::string sProgramName; - static std::auto_ptr sapHideFileGuard; + static std::vector sLogLevelOverrideByFileGuards; + friend class LogLevelOverrideByFileGuard; public: Logging (); @@ -445,6 +492,9 @@ class Logging static void FilterConsole (Log::Level level); static void Add (Logger* pNewLogger); static void Remove (Logger* pOldLogger); + static bool ShouldLog(Log::Level default_level, Log::Level message_level, + const std::string& file, int line, const std::string& function, + const Log::Category& category, const std::string& message); static void Log(Log::Level level, const std::string& file, int line, const std::string& function, const Log::Category& category, const std::string& message); @@ -663,36 +713,6 @@ class HideCategoryGuard : public Logger virtual void SetProgramName(const std::string& rProgramName) { } }; -class HideFileGuard : public Logger -{ - private: - std::list mFileNames; - HideFileGuard(const HideFileGuard& other); // no copying - HideFileGuard& operator=(const HideFileGuard& other); // no assignment - bool mHideAllButSelected; - - public: - HideFileGuard(const std::string& rFileName, bool HideAllButSelected = false) - : mHideAllButSelected(HideAllButSelected) - { - mFileNames.push_back(rFileName); - Logging::Add(this); - } - ~HideFileGuard() - { - Logging::Remove(this); - } - void Add(const std::string& rFileName) - { - mFileNames.push_back(rFileName); - } - virtual bool Log(Log::Level level, const std::string& file, int line, - const std::string& function, const Log::Category& category, - const std::string& message); - virtual const char* GetType() { return "HideFileGuard"; } - virtual void SetProgramName(const std::string& rProgramName) { } -}; - std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H diff --git a/lib/common/MainHelper.cpp b/lib/common/MainHelper.cpp new file mode 100644 index 000000000..a39c639f8 --- /dev/null +++ b/lib/common/MainHelper.cpp @@ -0,0 +1,45 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: MainHelper.cpp +// Purpose: Helper stuff for main() programs +// Created: 2017/02/19 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifdef WIN32 +# include +# include +#endif + +#include "autogen_CommonException.h" +#include "Logging.h" + +// Windows requires winsock to be initialised before use, unlike every other platform. +void mainhelper_init_win32() +{ +#ifdef WIN32 + WSADATA info; + + // Under Win32 we must initialise the Winsock library + // before using it. + + if (WSAStartup(0x0101, &info) == SOCKET_ERROR) + { + // throw error? perhaps give it its own id in the future + DWORD wserrno = WSAGetLastError(); + THROW_WIN_ERROR_NUMBER("Failed to initialise Windows Sockets library", wserrno, + CommonException, Internal) + } + + HANDLE hProcess = GetCurrentProcess(); + if(!SymInitialize(hProcess, NULL, TRUE)) + { + BOX_LOG_WIN_WARNING("Failed to initialize symbol lookup for exception " + "backtraces"); + } +#endif +} + diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h index 0303090e8..c7c31206c 100644 --- a/lib/common/MainHelper.h +++ b/lib/common/MainHelper.h @@ -19,17 +19,17 @@ #include "BoxException.h" #include "Logging.h" +void mainhelper_init_win32(); + #define MAINHELPER_START \ if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ { printf(BOX_VERSION "\n"); return 0; } \ MEMLEAKFINDER_INIT \ MEMLEAKFINDER_START \ + mainhelper_init_win32(); \ try { #define MAINHELPER_END \ - } catch(BoxException &e) { \ - BOX_FATAL(e.what() << ": " << e.GetMessage()); \ - return 1; \ } catch(std::exception &e) { \ BOX_FATAL(e.what()); \ return 1; \ @@ -45,6 +45,5 @@ #define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker) #endif // BOX_MEMORY_LEAK_TESTING - #endif // MAINHELPER__H diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h index 1ba4b0a68..a0ff049bb 100644 --- a/lib/common/MemBlockStream.h +++ b/lib/common/MemBlockStream.h @@ -40,6 +40,8 @@ class MemBlockStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h index 07b52e26e..8b4d34826 100644 --- a/lib/common/MemLeakFinder.h +++ b/lib/common/MemLeakFinder.h @@ -53,6 +53,14 @@ void memleakfinder_notaleak(void *ptr); void *operator new (size_t size, const char *file, int line); void *operator new[](size_t size, const char *file, int line); +// "If the object is being created as part of a new expression, and an exception is thrown, the +// object’s memory is deallocated by calling the appropriate deallocation function. If the object +// is being created with a placement new operator, the corresponding placement delete operator is +// called—that is, the delete function that takes the same additional parameters as the placement +// new operator. If no matching placement delete is found, no deallocation takes place." +// So to avoid memory leaks, we need to implement placement delete operators too. +void operator delete (void *ptr, const char *file, int line); +void operator delete[](void *ptr, const char *file, int line); // Define the malloc functions now, if required. These should match the definitions // in MemLeakFindOn.h. diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index 8e672ff56..af3e9ef30 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -10,40 +10,12 @@ #include "Box.h" -#include -#include - -#ifdef HAVE_UNISTD_H - #include -#endif - -#ifdef HAVE_FLOCK - #include -#endif - #include "CommonException.h" #include "NamedLock.h" #include "Utils.h" #include "MemLeakFindOn.h" -// -------------------------------------------------------------------------- -// -// Function -// Name: NamedLock::NamedLock() -// Purpose: Constructor -// Created: 2003/08/28 -// -// -------------------------------------------------------------------------- -NamedLock::NamedLock() -#ifdef WIN32 -: mFileDescriptor(INVALID_HANDLE_VALUE) -#else -: mFileDescriptor(-1) -#endif -{ -} - // -------------------------------------------------------------------------- // // Function @@ -54,11 +26,7 @@ NamedLock::NamedLock() // -------------------------------------------------------------------------- NamedLock::~NamedLock() { -#ifdef WIN32 - if(mFileDescriptor != INVALID_HANDLE_VALUE) -#else - if(mFileDescriptor != -1) -#endif + if(mapLockFile.get()) { ReleaseLock(); } @@ -77,150 +45,49 @@ NamedLock::~NamedLock() bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) { // Check -#ifdef WIN32 - if(mFileDescriptor != INVALID_HANDLE_VALUE) -#else - if(mFileDescriptor != -1) -#endif + if(mapLockFile.get()) { THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething) } - mFileName = rFilename; - - // See if the lock can be got - int flags = O_WRONLY | O_CREAT | O_TRUNC; - -#if HAVE_DECL_O_EXLOCK - flags |= O_NONBLOCK | O_EXLOCK; - BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); -#elif defined BOX_OPEN_LOCK - flags |= BOX_OPEN_LOCK; - BOX_TRACE("Trying to create lockfile " << rFilename << " using BOX_OPEN_LOCK"); -#elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK - // We have no other way to get a lock, so all we can do is fail if - // the file already exists, and take the risk of stale locks. - flags |= O_EXCL; - BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXCL"); -#else - BOX_TRACE("Trying to create lockfile " << rFilename << " without special flags"); -#endif - + try + { + int flags = O_WRONLY | O_CREAT | O_TRUNC; #ifdef WIN32 - HANDLE fd = openfile(rFilename.c_str(), flags, mode); - if(fd == INVALID_HANDLE_VALUE) -#else - int fd = ::open(rFilename.c_str(), flags, mode); - if(fd == -1) + // Automatically delete file on close + flags |= O_TEMPORARY; #endif -#if HAVE_DECL_O_EXLOCK - { // if() - if(errno == EWOULDBLOCK) - { - // Lockfile already exists, and we tried to open it - // exclusively, which means we failed to lock it. - BOX_NOTICE("Failed to lock lockfile with O_EXLOCK: " << rFilename - << ": already locked by another process?"); - return false; - } - else - { - THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXLOCK", - rFilename, CommonException, OSFileError); - } + + // This exception message doesn't add any useful information, so hide it: + HideSpecificExceptionGuard hide(CommonException::ExceptionType, + CommonException::FileLockingConflict); + + mapLockFile.reset(new FileStream(rFilename, flags, mode, FileStream::EXCLUSIVE)); } -#else // !HAVE_DECL_O_EXLOCK - { // if() -# if defined BOX_OPEN_LOCK - if(errno == EBUSY) -# else // !BOX_OPEN_LOCK - if(errno == EEXIST && (flags & O_EXCL)) -# endif + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, FileLockingConflict)) { - // Lockfile already exists, and we tried to open it - // exclusively, which means we failed to lock it. - BOX_NOTICE("Failed to lock lockfile with O_EXCL: " << rFilename - << ": already locked by another process?"); + BOX_NOTICE("Failed to lock lockfile: " << rFilename << ": already locked " + "by another process"); return false; } else { - THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXCL", - rFilename, CommonException, OSFileError); + throw; } } - try - { -# ifdef HAVE_FLOCK - BOX_TRACE("Trying to lock lockfile " << rFilename << " using flock()"); - if(::flock(fd, LOCK_EX | LOCK_NB) != 0) - { - if(errno == EWOULDBLOCK) - { - ::close(fd); - BOX_NOTICE("Failed to lock lockfile with flock(): " << rFilename - << ": already locked by another process"); - return false; - } - else - { - THROW_SYS_FILE_ERROR("Failed to lock lockfile with flock()", - rFilename, CommonException, OSFileError); - } - } -# elif HAVE_DECL_F_SETLK - struct flock desc; - desc.l_type = F_WRLCK; - desc.l_whence = SEEK_SET; - desc.l_start = 0; - desc.l_len = 0; - BOX_TRACE("Trying to lock lockfile " << rFilename << " using fcntl()"); - if(::fcntl(fd, F_SETLK, &desc) != 0) - { - if(errno == EAGAIN) - { - ::close(fd); - BOX_NOTICE("Failed to lock lockfile with fcntl(): " << rFilename - << ": already locked by another process"); - return false; - } - else - { - THROW_SYS_FILE_ERROR("Failed to lock lockfile with fcntl()", - rFilename, CommonException, OSFileError); - } - } -# endif - } - catch(BoxException &e) - { -# ifdef WIN32 - CloseHandle(fd); -# else - ::close(fd); -# endif - BOX_NOTICE("Failed to lock lockfile " << rFilename << ": " << e.what()); - throw; - } -#endif // HAVE_DECL_O_EXLOCK - if(!FileExists(rFilename)) { - BOX_ERROR("Locked lockfile " << rFilename << ", but lockfile no longer " - "exists, bailing out"); -# ifdef WIN32 - CloseHandle(fd); -# else - ::close(fd); -# endif + BOX_ERROR("Locked lockfile " << rFilename << ", but lockfile no longer exists, " + "bailing out"); + mapLockFile.reset(); return false; } // Success - mFileDescriptor = fd; BOX_TRACE("Successfully locked lockfile " << rFilename); - return true; } @@ -235,65 +102,46 @@ bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) void NamedLock::ReleaseLock() { // Got a lock? -#ifdef WIN32 - if(mFileDescriptor == INVALID_HANDLE_VALUE) -#else - if(mFileDescriptor == -1) -#endif + if(!mapLockFile.get()) { THROW_EXCEPTION(CommonException, NamedLockNotHeld) } + std::string filename = mapLockFile->GetFileName(); + bool success = true; + #ifndef WIN32 - // Delete the file. We need to do this before closing the filehandle, + // Delete the file. We need to do this before closing the filehandle, // if we used flock() or fcntl() to lock it, otherwise someone could // acquire the lock, release and delete it between us closing (and // hence releasing) and deleting it, and we'd fail when it came to // deleting the file. This happens in tests much more often than // you'd expect! // - // This doesn't apply on systems using plain lockfile locking, such as - // Windows, and there we need to close the file before deleting it, - // otherwise the system won't let us delete it. + // This doesn't apply on Windows (of course) because we can't delete the file while we still + // have it open. So we open it with the O_TEMPORARY flag so that it's deleted automatically + // when we close it, avoiding any race condition. - if(::unlink(mFileName.c_str()) != 0) + if(EMU_UNLINK(filename.c_str()) != 0) { - THROW_EMU_ERROR( - BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), - CommonException, OSFileError); + // Let the function continue so that we close the file and reset the handle + // before returning the error to the user. + success = false; } #endif // !WIN32 // Close the file -# ifdef WIN32 - if(!CloseHandle(mFileDescriptor)) -# else - if(::close(mFileDescriptor) != 0) -# endif - { - THROW_EMU_ERROR( - BOX_FILE_MESSAGE(mFileName, "Failed to close lockfile"), - CommonException, OSFileError); - } + mapLockFile->Close(); - // Mark as unlocked, so we don't try to close it again if the unlink() fails. -#ifdef WIN32 - mFileDescriptor = INVALID_HANDLE_VALUE; -#else - mFileDescriptor = -1; -#endif - -#ifdef WIN32 - // On Windows we need to close the file before deleting it, otherwise - // the system won't let us delete it. + // Don't try to release it again + mapLockFile.reset(); - if(::unlink(mFileName.c_str()) != 0) + if(!success) { THROW_EMU_ERROR( - BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), - CommonException, OSFileError); + BOX_FILE_MESSAGE(filename, "Failed to delete lockfile after unlocking it"), + CommonException, NamedLockFailed); } -#endif // WIN32 - BOX_TRACE("Released lock and deleted lockfile " << mFileName); + BOX_TRACE("Released lock and deleted lockfile " << filename); } diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h index a7d0d7782..8cc78f489 100644 --- a/lib/common/NamedLock.h +++ b/lib/common/NamedLock.h @@ -21,7 +21,7 @@ class NamedLock { public: - NamedLock(); + NamedLock() { } ~NamedLock(); private: // No copying allowed @@ -29,21 +29,11 @@ class NamedLock public: bool TryAndGetLock(const std::string& rFilename, int mode = 0755); -# ifdef WIN32 - bool GotLock() {return mFileDescriptor != INVALID_HANDLE_VALUE;} -# else - bool GotLock() {return mFileDescriptor != -1;} -# endif + bool GotLock() {return bool(mapLockFile.get());} void ReleaseLock(); - -private: -# ifdef WIN32 - HANDLE mFileDescriptor; -# else - int mFileDescriptor; -# endif - std::string mFileName; +private: + std::auto_ptr mapLockFile; }; #endif // NAMEDLOCK__H diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h index 61bdd7d1e..f52bfb77d 100644 --- a/lib/common/PartialReadStream.h +++ b/lib/common/PartialReadStream.h @@ -35,6 +35,8 @@ class PartialReadStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/common/RateLimitingStream.h b/lib/common/RateLimitingStream.h index 818c90af7..82480cb06 100644 --- a/lib/common/RateLimitingStream.h +++ b/lib/common/RateLimitingStream.h @@ -35,6 +35,7 @@ class RateLimitingStream : public IOStream { mrSink.Write(pBuffer, NBytes, Timeout); } + using IOStream::Write; virtual pos_type BytesLeftToRead() { return mrSink.BytesLeftToRead(); diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h index 9a44480be..842cdf8f2 100644 --- a/lib/common/ReadGatherStream.h +++ b/lib/common/ReadGatherStream.h @@ -39,6 +39,8 @@ class ReadGatherStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual pos_type GetPosition() const; diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp index df4933449..5e5baf0a1 100644 --- a/lib/common/ReadLoggingStream.cpp +++ b/lib/common/ReadLoggingStream.cpp @@ -54,7 +54,7 @@ int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) } if (mLength == 0) - { + { mrLogger.Log(numBytesRead, mOffset); } else if (mTotalRead == 0) @@ -62,14 +62,14 @@ int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) mrLogger.Log(numBytesRead, mOffset, mLength); } else - { + { box_time_t timeNow = GetCurrentBoxTime(); box_time_t elapsed = timeNow - mStartTime; - box_time_t finish = (elapsed * mLength) / mTotalRead; + box_time_t finish = ((float)elapsed * (float)mLength) / mTotalRead; // box_time_t remain = finish - elapsed; mrLogger.Log(numBytesRead, mOffset, mLength, elapsed, finish); } - + return numBytesRead; } diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h index bee7e1d62..4f955a6f2 100644 --- a/lib/common/ReadLoggingStream.h +++ b/lib/common/ReadLoggingStream.h @@ -41,6 +41,8 @@ class ReadLoggingStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h index b4efa2942..e95aaee1e 100644 --- a/lib/common/SelfFlushingStream.h +++ b/lib/common/SelfFlushingStream.h @@ -61,6 +61,8 @@ class SelfFlushingStream : public IOStream { mrSource.Write(pBuffer, NBytes, Timeout); } + using IOStream::Write; + virtual bool StreamDataLeft() { return mrSource.StreamDataLeft(); diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index 2c51cd61a..92f18a132 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -17,14 +17,18 @@ #include #include +#ifdef HAVE_DIRENT_H +# include // for opendir(), struct DIR +#endif + #ifdef HAVE_UNISTD_H - #include +# include #endif #include "BoxTime.h" #include "FileStream.h" #include "Test.h" -#include "Utils.h" +#include "Utils.h" // for ObjectExists_* (object_exists_t) int num_tests_selected = 0; int num_failures = 0; @@ -35,10 +39,19 @@ std::string first_fail_file; std::string current_test_name; std::list run_only_named_tests; std::map s_test_status; +box_time_t current_test_start; -bool setUp(const char* function_name) +bool setUp(const std::string& function_name, const std::string& specialisation) { - current_test_name = function_name; + std::ostringstream specialised_name; + specialised_name << function_name; + + if(!specialisation.empty()) + { + specialised_name << "(" << specialisation << ")"; + } + + current_test_name = specialised_name.str(); if (!run_only_named_tests.empty()) { @@ -48,7 +61,7 @@ bool setUp(const char* function_name) i = run_only_named_tests.begin(); i != run_only_named_tests.end(); i++) { - if (*i == current_test_name) + if (*i == function_name || *i == current_test_name) { run_this_test = true; break; @@ -62,9 +75,10 @@ bool setUp(const char* function_name) } } - printf("\n\n== %s ==\n", function_name); + printf("\n\n== %s ==\n", current_test_name.c_str()); num_tests_selected++; old_failure_count = num_failures; + current_test_start = GetCurrentBoxTime(); if (original_working_dir == "") { @@ -83,7 +97,8 @@ bool setUp(const char* function_name) } } -#ifdef _MSC_VER + // We need to do something more complex than "rm -rf testfiles" to clean up the mess and + // prepare for the next test, in a way that works on Windows (without rm -rf) as well. DIR* pDir = opendir("testfiles"); if(!pDir) { @@ -101,18 +116,22 @@ bool setUp(const char* function_name) StartsWith("notifyran", filename) || StartsWith("notifyscript.tag", filename) || StartsWith("restore", filename) || + filename == "bbackupd-cache" || filename == "bbackupd-data" || + filename == "store" || filename == "syncallowscript.control" || StartsWith("syncallowscript.notifyran.", filename) || filename == "test2.downloaded" || - EndsWith("testfile", filename)) + EndsWith("testfile", filename) || + EndsWith(".qdbm", filename)) { - std::string filepath = std::string("testfiles\\") + filename; + std::string filepath = std::string("testfiles" DIRECTORY_SEPARATOR) + + filename; - int filetype = ObjectExists(filepath); + object_exists_t filetype = ObjectExists(filepath); if(filetype == ObjectExists_File) { - if(::unlink(filepath.c_str()) != 0) + if(EMU_UNLINK(filepath.c_str()) != 0) { TEST_FAIL_WITH_MESSAGE(BOX_SYS_ERROR_MESSAGE("Failed to delete " "test fixture file: unlink(\"" << filepath << "\")")); @@ -120,6 +139,9 @@ bool setUp(const char* function_name) } else if(filetype == ObjectExists_Dir) { +#ifdef _MSC_VER + // More complex command invocation required to properly encode + // arguments when non-ASCII characters are involved: std::string cmd = "cmd /c rd /s /q " + filepath; WCHAR* wide_cmd = ConvertUtf8ToWideString(cmd.c_str()); if(wide_cmd == NULL) @@ -181,6 +203,11 @@ bool setUp(const char* function_name) CloseHandle(pi.hProcess); CloseHandle(pi.hThread); +#else // !_MSC_VER + // Deleting directories is so much easier on Unix! + std::string cmd = "rm -rf '" + filepath + "'"; + TEST_EQUAL_LINE(0, system(cmd.c_str()), "system() failed: " << cmd); +#endif } else { @@ -190,40 +217,38 @@ bool setUp(const char* function_name) } } closedir(pDir); + FileStream touch("testfiles/accounts.txt", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); -#else - TEST_THAT_THROWONFAIL(system( - "rm -rf testfiles/TestDir* testfiles/0_0 testfiles/0_1 " - "testfiles/0_2 testfiles/accounts.txt " // testfiles/test* .tgz! - "testfiles/file* testfiles/notifyran testfiles/notifyran.* " - "testfiles/notifyscript.tag* " - "testfiles/restore* testfiles/bbackupd-data " - "testfiles/syncallowscript.control " - "testfiles/syncallowscript.notifyran.* " - "testfiles/test2.downloaded" - ) == 0); - TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); -#endif + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-data", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-cache", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/store", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/store/subdir", 0755) == 0); return true; } bool tearDown() { + box_time_t elapsed_time = GetCurrentBoxTime() - current_test_start; + std::ostringstream buf; + buf.setf(std::ios_base::fixed); + buf.precision(1); + buf << " (" << ((float)BoxTimeToMilliSeconds(elapsed_time) / 1000) << " sec)"; + if (num_failures == old_failure_count) { - BOX_NOTICE(current_test_name << " passed"); + BOX_NOTICE(current_test_name << " passed" << buf.str()); s_test_status[current_test_name] = "passed"; return true; } else { - BOX_NOTICE(current_test_name << " failed"); \ + BOX_NOTICE(current_test_name << " failed" << buf.str()); \ s_test_status[current_test_name] = "FAILED"; return false; } @@ -318,46 +343,7 @@ int RunCommand(const std::string& rCommandLine) bool ServerIsAlive(int pid) { - #ifdef WIN32 - - HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, - false, pid); - if (hProcess == NULL) - { - if (GetLastError() != ERROR_INVALID_PARAMETER) - { - BOX_ERROR("Failed to open process " << pid << - ": " << - GetErrorMessage(GetLastError())); - } - return false; - } - - DWORD exitCode; - BOOL result = GetExitCodeProcess(hProcess, &exitCode); - CloseHandle(hProcess); - - if (result == 0) - { - BOX_ERROR("Failed to get exit code for process " << - pid << ": " << - GetErrorMessage(GetLastError())) - return false; - } - - if (exitCode == STILL_ACTIVE) - { - return true; - } - - return false; - - #else // !WIN32 - - if(pid == 0) return false; - return ::kill(pid, 0) != -1; - - #endif // WIN32 + return process_is_running(pid); } int ReadPidFile(const char *pidFile) @@ -382,12 +368,23 @@ int ReadPidFile(const char *pidFile) return pid; } +#ifdef WIN32 +HANDLE sTestChildDaemonJobObject = INVALID_HANDLE_VALUE; +#endif + int LaunchServer(const std::string& rCommandLine, const char *pidFile) { BOX_INFO("Starting server: " << rCommandLine); #ifdef WIN32 + // Use a Windows "Job Object" as a container for all our child + // processes. The test runner will create this job object when + // it starts, and close the handle (killing any running daemons) + // when it exits. This is the best way to avoid daemons hanging + // around and causing subsequent tests to fail, and/or the test + // runner to hang waiting for a daemon that will never terminate. + PROCESS_INFORMATION procInfo; STARTUPINFO startInfo; @@ -419,10 +416,18 @@ int LaunchServer(const std::string& rCommandLine, const char *pidFile) free(tempCmd); TEST_THAT_OR(result != 0, - BOX_LOG_WIN_ERROR("Launch failed: " << rCommandLine); + BOX_LOG_WIN_ERROR("Failed to CreateProcess: " << rCommandLine); return -1; ); + if(sTestChildDaemonJobObject != INVALID_HANDLE_VALUE) + { + if(!AssignProcessToJobObject(sTestChildDaemonJobObject, procInfo.hProcess)) + { + BOX_LOG_WIN_WARNING("Failed to add child process to job object"); + } + } + CloseHandle(procInfo.hProcess); CloseHandle(procInfo.hThread); @@ -550,7 +555,7 @@ void TestRemoteProcessMemLeaksFunc(const char *filename, } // Delete it - ::unlink(filename); + EMU_UNLINK(filename); } #endif } @@ -638,3 +643,32 @@ std::auto_ptr load_config_file(const std::string& config_file, return config; } +bool test_equal_lists(const std::vector& expected_items, + const std::vector& actual_items) +{ + bool all_match = (expected_items.size() == actual_items.size()); + + for(size_t i = 0; i < std::max(expected_items.size(), actual_items.size()); i++) + { + const std::string& expected = (i < expected_items.size()) ? expected_items[i] : "None"; + const std::string& actual = (i < actual_items.size()) ? actual_items[i] : "None"; + TEST_EQUAL_LINE(expected, actual, "Item " << i); + all_match &= (expected == actual); + } + + return all_match; +} + +bool test_equal_maps(const str_map_t& expected_attrs, const str_map_t& actual_attrs) +{ + str_map_diff_t differences = compare_str_maps(expected_attrs, actual_attrs); + for(str_map_diff_t::iterator i = differences.begin(); i != differences.end(); i++) + { + const std::string& name = i->first; + const std::string& expected = i->second.first; + const std::string& actual = i->second.second; + TEST_EQUAL_LINE(expected, actual, "Wrong value for attribute " << name); + } + + return differences.empty(); +} diff --git a/lib/common/Test.h b/lib/common/Test.h index 4b5cef616..2fc230537 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -10,12 +10,25 @@ #ifndef TEST__H #define TEST__H +// for printf: +#include + #include #include #include #include "Configuration.h" +#ifndef TEST_EXECUTABLE +# ifdef _MSC_VER + // Our CMakeFiles compile tests to different executable filenames, + // e.g. test_common.exe instead of _test.exe. + #define TEST_EXECUTABLE BOX_MODULE ".exe" +# else + #define TEST_EXECUTABLE "./_test" +# endif +#endif // TEST_EXECUTABLE + #ifdef WIN32 #define BBACKUPCTL "..\\..\\bin\\bbackupctl\\bbackupctl.exe" #define BBACKUPD "..\\..\\bin\\bbackupd\\bbackupd.exe" @@ -38,13 +51,23 @@ extern int num_tests_selected; extern int old_failure_count; extern std::string first_fail_file; extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; +extern bool bbackupd_args_overridden, bbstored_args_overridden; extern std::list run_only_named_tests; extern std::string current_test_name; extern std::map s_test_status; +#ifdef WIN32 +extern HANDLE sTestChildDaemonJobObject; +#endif + //! Simplifies calling setUp() with the current function name in each test. #define SETUP() \ - if (!setUp(__FUNCTION__)) return true; \ + if (!setUp(__FUNCTION__, "")) return true; \ + try \ + { // left open for TEARDOWN() + +#define SETUP_SPECIALISED(specialisation) \ + if (!setUp(__FUNCTION__, specialisation)) return true; \ try \ { // left open for TEARDOWN() @@ -53,17 +76,17 @@ extern std::map s_test_status; } \ catch (BoxException &e) \ { \ - BOX_NOTICE(__FUNCTION__ << " errored: " << e.what()); \ + BOX_NOTICE(current_test_name << " errored: " << e.what()); \ num_failures++; \ tearDown(); \ - s_test_status[__FUNCTION__] = "ERRORED"; \ + s_test_status[current_test_name] = "ERRORED"; \ return false; \ } //! End the current test. Only use within a test function, because it just returns false! #define FAIL { \ std::ostringstream os; \ - os << "failed at " << __FUNCTION__ << ":" << __LINE__; \ + os << "failed at " << current_test_name << ":" << __LINE__; \ s_test_status[current_test_name] = os.str(); \ return fail(); \ } @@ -167,16 +190,12 @@ extern std::map s_test_status; \ if(_exp_str != _found_str) \ { \ - std::ostringstream _ossl; \ - _ossl << _line; \ - std::string _line_str = _ossl.str(); \ - printf("Expected <%s> but found <%s> in <%s>\n", \ - _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ - \ std::ostringstream _oss3; \ - _oss3 << #_found << " != " << #_expected << " in " << _line; \ - \ - TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + _oss3 << #_found << " != " << #_expected << ": " \ + "expected <" << _exp_str << "> " \ + "but found <" << _found_str << "> " \ + "in <" << _line << ">"; \ + TEST_FAIL_WITH_MESSAGE(_oss3.str()); \ } \ } @@ -202,7 +221,7 @@ extern std::map s_test_status; TEST_EQUAL_LINE(expected, actual.substr(0, std::string(expected).size()), actual); //! Sets up (cleans up) test environment at the start of every test. -bool setUp(const char* function_name); +bool setUp(const std::string& function_name, const std::string& specialisation); //! Checks account for errors and shuts down daemons at end of every test. bool tearDown(); @@ -243,14 +262,10 @@ void safe_sleep(int seconds); std::auto_ptr load_config_file(const std::string& config_file, const ConfigurationVerify& verify); -#ifndef TEST_EXECUTABLE -# ifdef _MSC_VER - // Our CMakeFiles compile tests to different executable filenames, - // e.g. test_common.exe instead of _test.exe. - #define TEST_EXECUTABLE BOX_MODULE ".exe" -# else - #define TEST_EXECUTABLE "./_test" -# endif -#endif // TEST_EXECUTABLE +bool test_equal_lists(const std::vector& expected_items, + const std::vector& actual_items); + +typedef std::map str_map_t; +bool test_equal_maps(const str_map_t& expected_attrs, const str_map_t& actual_attrs); #endif // TEST__H diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index 0915f29a7..a5f47ccaf 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -15,11 +15,6 @@ #include -#ifdef HAVE_EXECINFO_H - #include - #include -#endif - #ifdef HAVE_CXXABI_H #include #endif @@ -28,6 +23,18 @@ #include #endif +#ifdef HAVE_EXECINFO_H + #include +#endif + +#ifdef HAVE_SIGNAL_H + #include +#endif + +#ifdef WIN32 +# include +#endif + #ifdef NEED_BOX_VERSION_H # include "BoxVersion.h" #endif @@ -107,39 +114,69 @@ bool EndsWith(const std::string& suffix, const std::string& haystack) haystack.substr(haystack.size() - suffix.size()) == suffix; } -std::string RemovePrefix(const std::string& prefix, const std::string& haystack) +std::string RemovePrefix(const std::string& prefix, const std::string& haystack, + bool force) { if(StartsWith(prefix, haystack)) { return haystack.substr(prefix.size()); } + else if(force) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "String '" << haystack << "' was expected to start with prefix " + "'" << prefix << "', but did not."); + } else { return ""; } } -std::string RemoveSuffix(const std::string& suffix, const std::string& haystack) +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack, + bool force) { if(EndsWith(suffix, haystack)) { return haystack.substr(0, haystack.size() - suffix.size()); } + else if(force) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "String '" << haystack << "' was expected to end with suffix " + "'" << suffix << "', but did not."); + } else { return ""; } } +// The backtrace routines are used by DebugMemLeakFinder, so we need to disable memory leak +// tracking during them, otherwise we could end up with infinite recursion. +#include "MemLeakFindOff.h" + +const Log::Category BACKTRACE("Backtrace"); + static std::string demangle(const std::string& mangled_name) { std::string demangled_name = mangled_name; - - #ifdef HAVE_CXXABI_H char buffer[1024]; + +#if defined WIN32 + if(UnDecorateSymbolName(mangled_name.c_str(), buffer, sizeof(buffer), + UNDNAME_COMPLETE)) + { + demangled_name = buffer; + } + else + { + BOX_LOG_WIN_ERROR("UnDecorateSymbolName failed"); + } +#elif defined HAVE_CXXABI_H int status; size_t length = sizeof(buffer); - + char* result = abi::__cxa_demangle(mangled_name.c_str(), buffer, &length, &status); @@ -174,56 +211,109 @@ static std::string demangle(const std::string& mangled_name) "with unknown error " << status << ": " << mangled_name); } - #endif // HAVE_CXXABI_H +#endif // HAVE_CXXABI_H return demangled_name; } -void DumpStackBacktrace() +void DumpStackBacktrace(const std::string& filename) { -#ifdef HAVE_EXECINFO_H - void *array[20]; + const int max_length = 20; + void *array[max_length]; + +#if defined WIN32 + size_t size = CaptureStackBackTrace(0, max_length, array, NULL); +#elif defined HAVE_EXECINFO_H size_t size = backtrace(array, 20); - BOX_TRACE("Obtained " << size << " stack frames."); +#else + BOX_TRACE("Backtrace support was not compiled in"); + return; +#endif + + // Instead of calling BOX_TRACE, we call Logging::Log directly in order to pass filename + // as the source file. This allows exception backtraces to be turned on and off by file, + // instead of all of them originating in Utils.cpp. + std::ostringstream output; + output << "Obtained " << size << " stack frames."; + Logging::Log(Log::TRACE, filename, 0, // line + __FUNCTION__, BACKTRACE, output.str()); + + DumpStackBacktrace(filename, size, array); +} + +void DumpStackBacktrace(const std::string& filename, size_t size, void * const * array) +{ +#if defined WIN32 + HANDLE hProcess = GetCurrentProcess(); + // SymInitialize was called in mainhelper_init_win32() + DWORD64 displacement; + char symbol_info_buf[sizeof(SYMBOL_INFO) + 256]; + PSYMBOL_INFO pInfo = (SYMBOL_INFO *)symbol_info_buf; + pInfo->MaxNameLen = 256; + pInfo->SizeOfStruct = sizeof(SYMBOL_INFO); +#endif for(size_t i = 0; i < size; i++) { std::ostringstream output; output << "Stack frame " << i << ": "; - #ifdef HAVE_DLADDR - Dl_info info; - int result = dladdr(array[i], &info); - - if(result == 0) - { - BOX_LOG_SYS_WARNING("Failed to resolve " - "backtrace address " << array[i]); - output << "unresolved address " << array[i]; - } - else if(info.dli_sname == NULL) - { - output << "unknown address " << array[i]; - } - else - { - uint64_t diff = (uint64_t) array[i]; - diff -= (uint64_t) info.dli_saddr; - output << demangle(info.dli_sname) << "+" << - (void *)diff; - } - #else - output << "address " << array[i]; - #endif // HAVE_DLADDR +#if defined WIN32 + if(!SymFromAddr(hProcess, (DWORD64)array[i], &displacement, pInfo)) +#elif defined HAVE_DLADDR + Dl_info info; + int result = dladdr(array[i], &info); + if(result == 0) +#endif + { + BOX_LOG_NATIVE_WARNING("Failed to resolve " + "backtrace address " << array[i]); + output << "unresolved address " << array[i]; + continue; + } + const char* mangled_name = NULL; + void* start_addr; +#if defined WIN32 + mangled_name = &(pInfo->Name[0]); + start_addr = (void *)(pInfo->Address); +#elif defined HAVE_DLADDR + mangled_name = info.dli_sname; + start_addr = info.dli_saddr; +#else + output << "address " << array[i]; BOX_TRACE(output.str()); + continue; +#endif + + if(mangled_name == NULL) + { + output << "unknown address " << array[i]; + } + else + { + uint64_t diff = (uint64_t) array[i]; + diff -= (uint64_t) start_addr; + output << demangle(mangled_name) << "+" << + (void *)diff; + } + + // Instead of calling BOX_TRACE, we call Logging::Log directly in order to pass + // filename as the source file. This allows exception backtraces to be turned on + // and off by file, instead of all of them originating in Utils.cpp. + Logging::Log(Log::TRACE, filename, 0, // line + __FUNCTION__, BACKTRACE, output.str()); + + // We don't really care about frames after main() (e.g. CRT startup), and sometimes + // they contain invalid data (not function pointers) such as 0x7. + if(mangled_name != NULL && strcmp(mangled_name, "main") == 0) + { + break; + } } -#else // !HAVE_EXECINFO_H - BOX_TRACE("Backtrace support was not compiled in"); -#endif // HAVE_EXECINFO_H } - +#include "MemLeakFindOn.h" // -------------------------------------------------------------------------- // @@ -249,20 +339,20 @@ bool FileExists(const std::string& rFilename, int64_t *pFileSize, } } - // is it a file? - if((st.st_mode & S_IFDIR) == 0) + // is it a file? + if(!S_ISDIR(st.st_mode)) { - if(TreatLinksAsNotExisting && ((st.st_mode & S_IFLNK) != 0)) + if(TreatLinksAsNotExisting && S_ISLNK(st.st_mode)) { return false; } - + // Yes. Tell caller the size? if(pFileSize != 0) { *pFileSize = st.st_size; } - + return true; } else @@ -271,6 +361,42 @@ bool FileExists(const std::string& rFilename, int64_t *pFileSize, } } + +std::string GetTempDirPath() +{ +#ifdef WIN32 + char buffer[PATH_MAX+1]; + int len = GetTempPath(sizeof(buffer), buffer); + if(len > sizeof(buffer)) + { + THROW_EXCEPTION(CommonException, TempDirPathTooLong); + } + if(len == 0) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Failed to get temporary directory path"); + } + return std::string(buffer); +#else + const char* result = getenv("TMPDIR"); + if(result == NULL) + { + result = getenv("TEMP"); + } + if(result == NULL) + { + result = getenv("TMP"); + } + if(result == NULL) + { + BOX_WARNING("TMPDIR/TEMP/TMP not set, falling back to /tmp"); + result = "/tmp"; + } + return std::string(result); +#endif +} + + // -------------------------------------------------------------------------- // // Function @@ -279,7 +405,7 @@ bool FileExists(const std::string& rFilename, int64_t *pFileSize, // Created: 23/11/03 // // -------------------------------------------------------------------------- -int ObjectExists(const std::string& rFilename) +object_exists_t ObjectExists(const std::string& rFilename) { EMU_STRUCT_STAT st; if(EMU_STAT(rFilename.c_str(), &st) != 0) @@ -331,7 +457,7 @@ std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, bool MachineReadable) { std::ostringstream result; - + if (MachineReadable) { @@ -353,14 +479,14 @@ std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, bar[l] = ' '; } bar[sizeof(bar)-1] = '\0'; - + result << std::fixed << std::setw(10) << Blocks << " blocks, " << std::setw(10) << HumanReadableSize(Bytes) << ", " << std::setw(3) << std::setprecision(0) << ((Bytes*100)/Max) << "% |" << bar << "|"; } - + return result.str(); } @@ -381,3 +507,78 @@ std::string FormatUsageLineStart(const std::string& rName, return result.str(); } +std::map compare_str_maps(const str_map_t& expected, + const str_map_t& actual) +{ + str_map_diff_t differences; + + // First check that every key in the expected map is present in the actual map, + // with the correct value. + for(str_map_t::const_iterator i = expected.begin(); i != expected.end(); i++) + { + std::string name = i->first; + std::string value = i->second; + str_map_t::const_iterator found = actual.find(name); + if(found == actual.end()) + { + differences[name] = str_pair_t(value, ""); + } + else if(found->second != value) + { + differences[name] = str_pair_t(value, found->second); + } + } + + // Now try the other way around: check that every key in the actual map is present + // in the expected map. We don't need to check values here, because if the key is + // present in both maps but with different values, the first pass above will + // already have recorded it. + for(str_map_t::const_iterator i = actual.begin(); i != actual.end(); i++) + { + std::string name = i->first; + std::string value = i->second; + str_map_t::const_iterator found = expected.find(name); + if(found == expected.end()) + { + differences[name] = str_pair_t("", value); + } + } + + return differences; +} + +bool process_is_running(int pid) +{ +#ifdef WIN32 + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, + false, pid); + if (hProcess == NULL) + { + if (GetLastError() != ERROR_INVALID_PARAMETER) + { + BOX_LOG_WIN_ERROR("Failed to open process " << pid); + } + return false; + } + + DWORD exitCode; + BOOL result = GetExitCodeProcess(hProcess, &exitCode); + CloseHandle(hProcess); + + if (result == 0) + { + BOX_LOG_WIN_ERROR("Failed to get exit code for process " << pid); + return false; + } + + if (exitCode == STILL_ACTIVE) + { + return true; + } + + return false; +#else // !WIN32 + if(pid == 0) return false; + return ::kill(pid, 0) != -1; +#endif // WIN32 +} diff --git a/lib/common/Utils.h b/lib/common/Utils.h index d306ce1ca..87d0350ab 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -10,39 +10,48 @@ #ifndef UTILS__H #define UTILS__H +#include #include #include -#include "MemLeakFindOn.h" - std::string GetBoxBackupVersion(); void SplitString(std::string String, char SplitOn, std::vector &rOutput); bool StartsWith(const std::string& prefix, const std::string& haystack); bool EndsWith(const std::string& prefix, const std::string& haystack); -std::string RemovePrefix(const std::string& prefix, const std::string& haystack); -std::string RemoveSuffix(const std::string& suffix, const std::string& haystack); +std::string RemovePrefix(const std::string& prefix, const std::string& haystack, + bool force = true); +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack, + bool force = true); -void DumpStackBacktrace(); +void DumpStackBacktrace(const std::string& filename); +void DumpStackBacktrace(const std::string& filename, size_t size, void * const * array); +std::string GetTempDirPath(); bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); -enum +typedef enum { + ObjectExists_Unknown = -1, ObjectExists_NoObject = 0, ObjectExists_File = 1, ObjectExists_Dir = 2 -}; -int ObjectExists(const std::string& rFilename); +} object_exists_t; + +object_exists_t ObjectExists(const std::string& rFilename); std::string HumanReadableSize(int64_t Bytes); std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, bool MachineReadable); std::string FormatUsageLineStart(const std::string& rName, bool MachineReadable); -std::string BoxGetTemporaryDirectoryName(); +bool process_is_running(int pid); + +typedef std::pair str_pair_t; +typedef std::map str_map_t; +typedef std::map str_map_diff_t; +str_map_diff_t compare_str_maps(const str_map_t& expected, const str_map_t& actual); -#include "MemLeakFindOff.h" #endif // UTILS__H diff --git a/lib/common/WaitForEvent.cpp b/lib/common/WaitForEvent.cpp index 5646bfbf3..b0422cd46 100644 --- a/lib/common/WaitForEvent.cpp +++ b/lib/common/WaitForEvent.cpp @@ -141,7 +141,8 @@ void *WaitForEvent::Wait() if(mpPollInfo == 0) { // Yes... - mpPollInfo = (struct pollfd *)::malloc((sizeof(struct pollfd) * mItems.size()) + 4); + mpPollInfo = (EMU_STRUCT_POLLFD *)::malloc( + (sizeof(EMU_STRUCT_POLLFD) * mItems.size()) + 4); if(mpPollInfo == 0) { throw std::bad_alloc(); diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h index a80761efd..4077748bd 100644 --- a/lib/common/WaitForEvent.h +++ b/lib/common/WaitForEvent.h @@ -141,7 +141,7 @@ class WaitForEvent #else int mTimeout; std::vector mItems; - struct pollfd *mpPollInfo; + EMU_STRUCT_POLLFD *mpPollInfo; #endif }; diff --git a/lib/common/ZeroStream.h b/lib/common/ZeroStream.h index f91221b02..e3b10b34b 100644 --- a/lib/common/ZeroStream.h +++ b/lib/common/ZeroStream.h @@ -19,15 +19,17 @@ class ZeroStream : public IOStream public: ZeroStream(IOStream::pos_type mSize); - + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); - + virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index bddaa94a3..44959dad2 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -160,12 +160,13 @@ for(my $e = 0; $e <= $#exception; $e++) { if($exception[$e] ne '') { - print CPP "\t\tcase ".$exception[$e].': return "'.$exception[$e].'";'."\n"; + print CPP "\t\tcase ".$exception[$e].': return "'.$class.'Exception'. + '('.$exception[$e].')";'."\n"; } } print CPP <<__E; - default: return "Unknown"; + default: return "${class}Exception(Unknown)"; } } __E diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h index 7d6b25019..0d4a83ad2 100644 --- a/lib/compress/CompressStream.h +++ b/lib/compress/CompressStream.h @@ -35,6 +35,8 @@ class CompressStream : public IOStream virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite); virtual void Close(); virtual bool StreamDataLeft(); diff --git a/lib/crypto/MD5Digest.h b/lib/crypto/MD5Digest.h index 1be01ea93..a83ffb002 100644 --- a/lib/crypto/MD5Digest.h +++ b/lib/crypto/MD5Digest.h @@ -21,7 +21,7 @@ // Created: 8/12/03 // // -------------------------------------------------------------------------- -class MD5Digest +class MD5Digest { public: MD5Digest(); @@ -45,13 +45,122 @@ class MD5Digest }; int CopyDigestTo(uint8_t *to); - bool DigestMatches(uint8_t *pCompareWith) const; private: - MD5_CTX md5; + MD5_CTX md5; uint8_t mDigest[MD5_DIGEST_LENGTH]; }; +class MD5DigestStream : public IOStream +{ +private: + MD5Digest mDigest; + MD5DigestStream(const MD5DigestStream &rToCopy); /* forbidden */ + MD5DigestStream& operator=(const MD5DigestStream &rToCopy); /* forbidden */ + bool mClosed; + +public: + MD5DigestStream() + : mClosed(false) + { } + + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual pos_type BytesLeftToRead() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + mDigest.Add(pBuffer, NBytes); + } + virtual void Write(const std::string& rBuffer, + int Timeout = IOStream::TimeOutInfinite) + { + mDigest.Add(rBuffer); + } + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite) { } + virtual pos_type GetPosition() const + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Seek(pos_type Offset, int SeekType) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Close() + { + mDigest.Finish(); + mClosed = true; + } + + // Has all data that can be read been read? + virtual bool StreamDataLeft() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + // Has the stream been closed (writing not possible) + virtual bool StreamClosed() + { + return mClosed; + } + + // Utility functions + bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, + int Timeout = IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + IOStream::pos_type CopyStreamTo(IOStream &rCopyTo, + int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + void Flush(int Timeout = IOStream::TimeOutInfinite) + { } + static int ConvertSeekTypeToOSWhence(int SeekType) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual std::string ToString() const + { + return "MD5DigestStream"; + } + + std::string DigestAsString() + { + // You can only get a digest when the stream has been closed, because this + // finalises the underlying MD5Digest object. + ASSERT(mClosed); + return mDigest.DigestAsString(); + } + uint8_t *DigestAsData(int *pLength = 0) + { + // You can only get a digest when the stream has been closed, because this + // finalises the underlying MD5Digest object. + ASSERT(mClosed); + return mDigest.DigestAsData(pLength); + } + int CopyDigestTo(uint8_t *to) + { + // You can only get a digest when the stream has been closed, because this + // finalises the underlying MD5Digest object. + ASSERT(mClosed); + return mDigest.CopyDigestTo(to); + } + bool DigestMatches(uint8_t *pCompareWith) const + { + // You can only get a digest when the stream has been closed, because this + // finalises the underlying MD5Digest object. + ASSERT(mClosed); + return mDigest.DigestMatches(pCompareWith); + } +}; + #endif // MD5DIGEST_H diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt index c9b3f940b..47dd7a822 100644 --- a/lib/httpserver/HTTPException.txt +++ b/lib/httpserver/HTTPException.txt @@ -14,4 +14,16 @@ RequestNotInitialised 10 BadResponse 11 ResponseReadFailed 12 NoStreamConfigured 13 -RequestFailedUnexpectedly 14 The request was expected to succeed, but it failed. +RequestFailedUnexpectedly 14 The request was expected to succeed, but it failed +ContentLengthAlreadySet 15 Tried to send a request without content, but the Content-Length header is already set +WrongContentLength 16 There was too much or not enough data in the request content stream +ParameterNotFound 17 An expected parameter was not found in the request +DuplicateParameter 18 A parameter was unexpectedly duplicated in the request +AuthenticationFailed 19 The request could not be authenticated +S3SimulatorError 20 An unspecified error occurred in the S3Simulator +ConditionalRequestConflict 21 A SimpleDb conditional request failed because the item has different attributes than expected +UnexpectedResponseData 22 The response does not have the expected XML structure +SimpleDBItemNotFound 23 The requested item does not exist on the server +FileNotFound 24 The requested file or directory does not exist on the server +RequestTimedOut 25 The client took too long to send the request +ResponseTimedOut 26 The client took too long to read the response diff --git a/lib/httpserver/HTTPHeaders.cpp b/lib/httpserver/HTTPHeaders.cpp new file mode 100644 index 000000000..289c140e1 --- /dev/null +++ b/lib/httpserver/HTTPHeaders.cpp @@ -0,0 +1,326 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPHeaders +// Purpose: Utility class to decode HTTP headers +// Created: 16/8/2015 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "HTTPHeaders.h" +#include "IOStreamGetLine.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPHeaders::ReadFromStream(IOStreamGetLine &rGetLine, +// int Timeout); +// Purpose: Read headers from a stream into internal storage. +// Created: 2015-08-22 +// +// -------------------------------------------------------------------------- +void HTTPHeaders::ReadFromStream(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + std::string currentLine; + + try + { + currentLine = rGetLine.GetLine(false /* no preprocess */, Timeout); + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Client disconnected while sending headers"); + } + else if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + THROW_EXCEPTION(HTTPException, RequestTimedOut); + } + else + { + throw; + } + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + ParseHeaderLine(header); + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPHeaders::ParseHeaderLine +// Purpose: Splits a line into name and value, and adds it to +// this header set. +// Created: 2015-08-22 +// +// -------------------------------------------------------------------------- +void HTTPHeaders::ParseHeaderLine(const std::string& rLine) +{ + // Find where the : is in the line + std::string::size_type colon = rLine.find(':'); + if(colon == std::string::npos || colon == 0 || + colon > rLine.size() - 2) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "Invalid header line: " << rLine); + } + + std::string name = rLine.substr(0, colon); + std::string value = rLine.substr(colon + 2); + AddHeader(name, value); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPHeaders::GetHeader(const std::string& name, +// const std::string* pValueOut) +// Purpose: Get the value of a single header, with the specified +// name, into the std::string pointed to by pValueOut. +// Returns true if the header exists, and false if it +// does not (in which case *pValueOut is not modified). +// Certain headers are stored in specific fields, e.g. +// content-length and host, but this should be done +// transparently to callers of AddHeader and GetHeader. +// Created: 2016-03-12 +// +// -------------------------------------------------------------------------- +bool HTTPHeaders::GetHeader(const std::string& rName, std::string* pValueOut) const +{ + const std::string name = ToLowerCase(rName); + + // Remember to change AddHeader() and GetHeader() together for each + // item in this list! + if(name == "content-length") + { + // Convert number to string. + std::ostringstream out; + out << mContentLength; + *pValueOut = out.str(); + } + else if(name == "content-type") + { + *pValueOut = mContentType; + } + else if(name == "connection") + { + // TODO FIXME: not all values of the Connection header can be + // stored and retrieved at the moment. + *pValueOut = mKeepAlive ? "keep-alive" : "close"; + } + else if (name == "host") + { + std::ostringstream out; + out << mHostName; + if(mHostPort != DEFAULT_PORT) + { + out << ":" << mHostPort; + } + *pValueOut = out.str(); + } + else + { + // All other headers are stored in mExtraHeaders. + + for (std::vector
::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == name) + { + *pValueOut = i->second; + return true; + } + } + + // Not found in mExtraHeaders. + return false; + } + + // For all except the else case above (searching mExtraHeaders), we must have + // found a value, as there will always be one. + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPHeaders::AddHeader(const std::string& name, +// const std::string& value) +// Purpose: Add a single header, with the specified name and +// value, to the internal list of headers. Certain +// headers are stored in specific fields, e.g. +// content-length and host, but this should be done +// transparently to callers of AddHeader and GetHeader. +// Created: 2015-08-22 +// +// -------------------------------------------------------------------------- +void HTTPHeaders::AddHeader(const std::string& rName, const std::string& value) +{ + std::string name = ToLowerCase(rName); + + // Remember to change AddHeader() and GetHeader() together for each + // item in this list! + if(name == "content-length") + { + // Decode number + long len = ::strtol(value.c_str(), NULL, 10); + // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(name == "content-type") + { + // Store rest of string as content type + mContentType = value; + } + else if(name == "connection") + { + // Connection header, what is required? + if(::strcasecmp(value.c_str(), "close") == 0) + { + mKeepAlive = false; + } + else if(::strcasecmp(value.c_str(), "keep-alive") == 0) + { + mKeepAlive = true; + } + // else don't understand, just assume default for protocol version + } + else if (name == "host") + { + // Store host header + mHostName = value; + + // Is there a port number to split off? + std::string::size_type colon = mHostName.find_first_of(':'); + if(colon != std::string::npos) + { + // There's a port in the string... attempt to turn it into an int + mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); + + // Truncate the string to just the hostname + mHostName = mHostName.substr(0, colon); + } + } + else + { + mExtraHeaders.push_back(Header(name, value)); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPHeaders::WriteTo(IOStream& rOutput, int Timeout) +// Purpose: Write all headers to the supplied stream. +// Created: 2015-08-22 +// +// -------------------------------------------------------------------------- +void HTTPHeaders::WriteTo(IOStream& rOutput, int Timeout) const +{ + std::ostringstream oss; + + if (mContentLength != -1) + { + oss << "Content-Length: " << mContentLength << "\r\n"; + } + + if (mContentType != "") + { + oss << "Content-Type: " << mContentType << "\r\n"; + } + + if (mHostName != "") + { + oss << "Host: " << GetHostNameWithPort() << "\r\n"; + } + + if (mKeepAlive) + { + oss << "Connection: keep-alive\r\n"; + } + else + { + oss << "Connection: close\r\n"; + } + + for (std::vector
::const_iterator i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + oss << i->first << ": " << i->second << "\r\n"; + } + + rOutput.Write(oss.str(), Timeout); +} + +std::string HTTPHeaders::GetHostNameWithPort() const +{ + + if (mHostPort != 80) + { + std::ostringstream oss; + oss << mHostName << ":" << mHostPort; + return oss.str(); + } + else + { + return mHostName; + } +} + diff --git a/lib/httpserver/HTTPHeaders.h b/lib/httpserver/HTTPHeaders.h new file mode 100644 index 000000000..b965ac465 --- /dev/null +++ b/lib/httpserver/HTTPHeaders.h @@ -0,0 +1,109 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPHeaders.h +// Purpose: Utility class to decode HTTP headers +// Created: 16/8/2015 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPHEADERS__H +#define HTTPHEADERS__H + +#include +#include + +#include "autogen_HTTPException.h" +#include "IOStream.h" + +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPHeaders +// Purpose: Utility class to decode HTTP headers +// Created: 16/8/2015 +// +// -------------------------------------------------------------------------- +class HTTPHeaders +{ +public: + enum { + DEFAULT_PORT = 80, + UNKNOWN_CONTENT_LENGTH = -1, + }; + + HTTPHeaders() + : mKeepAlive(false), + mHostPort(DEFAULT_PORT), // default if not specified: if you change this, + // remember to change GetHeader("host") as well. + mContentLength(UNKNOWN_CONTENT_LENGTH) + { } + virtual ~HTTPHeaders() { } + // copying is fine + + void ReadFromStream(IOStreamGetLine &rGetLine, int Timeout); + void ParseHeaderLine(const std::string& line); + void AddHeader(const std::string& name, const std::string& value); + void WriteTo(IOStream& rOutput, int Timeout) const; + typedef std::pair Header; + bool GetHeader(const std::string& name, std::string* pValueOut) const; + std::string GetHeaderValue(const std::string& name, bool required = true) const + { + std::string value; + if (GetHeader(name, &value)) + { + return value; + } + + if(required) + { + THROW_EXCEPTION_MESSAGE(CommonException, ConfigNoKey, + "Expected header was not present: " << name); + } + else + { + return ""; + } + } + const std::vector
GetExtraHeaders() const { return mExtraHeaders; } + void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + bool IsKeepAlive() const {return mKeepAlive;} + void SetContentType(const std::string& rContentType) + { + mContentType = rContentType; + } + const std::string& GetContentType() const { return mContentType; } + void SetContentLength(int64_t ContentLength) { mContentLength = ContentLength; } + int64_t GetContentLength() const { return mContentLength; } + const std::string &GetHostName() const {return mHostName;} + const int GetHostPort() const {return mHostPort;} + std::string GetHostNameWithPort() const; + void SetHostName(const std::string& rHostName) + { + AddHeader("host", rHostName); + } + +private: + bool mKeepAlive; + std::string mContentType; + std::string mHostName; + int mHostPort; + int64_t mContentLength; // only used when reading response from stream + std::vector
mExtraHeaders; + + std::string ToLowerCase(const std::string& input) const + { + std::string output = input; + for (std::string::iterator c = output.begin(); + c != output.end(); c++) + { + *c = tolower(*c); + } + return output; + } +}; + +#endif // HTTPHEADERS__H + diff --git a/lib/httpserver/HTTPQueryDecoder.cpp b/lib/httpserver/HTTPQueryDecoder.cpp index c49ac2cef..ff30644e7 100644 --- a/lib/httpserver/HTTPQueryDecoder.cpp +++ b/lib/httpserver/HTTPQueryDecoder.cpp @@ -9,7 +9,9 @@ #include "Box.h" -#include +#include // for std::isalnum +#include +#include #include "HTTPQueryDecoder.h" @@ -157,3 +159,41 @@ void HTTPQueryDecoder::Finish() } +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::URLEncode() +// Purpose: URL-encode a value according to Amazon's rules, +// which are similar to RFC 3986. +// Created: 2015-11-15 +// +// -------------------------------------------------------------------------- + +std::string HTTPQueryDecoder::URLEncode(const std::string& value) +{ + std::ostringstream out; + for(std::string::const_iterator i = value.begin(); i != value.end(); i++) + { + // Do not URL encode any of the unreserved characters that RFC 3986 + // defines: A-Z, a-z, 0-9, hyphen ( - ), underscore ( _ ), period ( . ), + // and tilde ( ~ ). + if(std::isalnum(*i) || *i == '-' || *i == '_' || *i == '.' || *i == '~') + { + out << *i; + } + else + { + // Percent encode all other characters with %XY, where X and Y are + // hex characters 0-9 and uppercase A-F. + out << "%" << + std::hex << + std::uppercase << + std::internal << + std::setw(2) << + std::setfill('0') << + (int)(unsigned char)(*i) << + std::dec; + } + } + return out.str(); +} diff --git a/lib/httpserver/HTTPQueryDecoder.h b/lib/httpserver/HTTPQueryDecoder.h index ca5afe7e6..254dbc64e 100644 --- a/lib/httpserver/HTTPQueryDecoder.h +++ b/lib/httpserver/HTTPQueryDecoder.h @@ -25,14 +25,16 @@ class HTTPQueryDecoder public: HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto); ~HTTPQueryDecoder(); + private: // no copying HTTPQueryDecoder(const HTTPQueryDecoder &); - HTTPQueryDecoder &operator=(const HTTPQueryDecoder &); -public: + HTTPQueryDecoder& operator=(const HTTPQueryDecoder &); +public: void DecodeChunk(const char *pQueryString, int QueryStringSize); void Finish(); + static std::string URLEncode(const std::string& value); private: HTTPRequest::Query_t &mrDecodeInto; diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp index a94d96b0e..65096d2d7 100644 --- a/lib/httpserver/HTTPRequest.cpp +++ b/lib/httpserver/HTTPRequest.cpp @@ -22,6 +22,8 @@ #include "IOStream.h" #include "IOStreamGetLine.h" #include "Logging.h" +#include "PartialReadStream.h" +#include "ReadGatherStream.h" #include "MemLeakFindOn.h" @@ -42,11 +44,8 @@ // -------------------------------------------------------------------------- HTTPRequest::HTTPRequest() : mMethod(Method_UNINITIALISED), - mHostPort(80), // default if not specified mHTTPVersion(0), - mContentLength(-1), mpCookies(0), - mClientKeepAliveRequested(false), mExpectContinue(false), mpStreamToReadFrom(NULL) { @@ -65,11 +64,8 @@ HTTPRequest::HTTPRequest() HTTPRequest::HTTPRequest(enum Method method, const std::string& rURI) : mMethod(method), mRequestURI(rURI), - mHostPort(80), // default if not specified mHTTPVersion(HTTPVersion_1_1), - mContentLength(-1), mpCookies(0), - mClientKeepAliveRequested(false), mExpectContinue(false), mpStreamToReadFrom(NULL) { @@ -115,6 +111,7 @@ std::string HTTPRequest::GetMethodName() const case Method_HEAD: return "HEAD"; case Method_POST: return "POST"; case Method_PUT: return "PUT"; + case Method_DELETE: return "DELETE"; default: std::ostringstream oss; oss << "unknown-" << mMethod; @@ -122,6 +119,24 @@ std::string HTTPRequest::GetMethodName() const }; } +std::string HTTPRequest::GetRequestURI(bool with_parameters_for_get_request) const +{ + if(!with_parameters_for_get_request) + { + return mRequestURI; + } + + std::ostringstream query_string; + for(Query_t::const_iterator i = mQuery.begin(); i != mQuery.end(); i++) + { + query_string << ((i == mQuery.begin()) ? "?" : "&"); + query_string << HTTPQueryDecoder::URLEncode(i->first) << "=" << + HTTPQueryDecoder::URLEncode(i->second); + } + + return mRequestURI + query_string.str(); +} + // -------------------------------------------------------------------------- // // Function @@ -143,11 +158,39 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Read the first line, which is of a different format to the rest of the lines std::string requestLine; - if(!rGetLine.GetLine(requestLine, false /* no preprocessing */, Timeout)) + while(true) { - // Didn't get the request line, probably end of connection which had been kept alive - return false; + try + { + requestLine = rGetLine.GetLine(false /* no preprocessing */, Timeout); + break; + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + // Didn't get the request line, probably end of connection which + // had been kept alive + return false; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + THROW_EXCEPTION(HTTPException, RequestTimedOut); + } + else + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unexpected error while reading request: " << + e.what()); + } + } } + BOX_TRACE("Request line: " << requestLine); // Check the method @@ -178,8 +221,14 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) { mMethod = Method_PUT; } + else if (mHttpVerb == "DELETE") + { + mMethod = Method_DELETE; + } else { + BOX_WARNING("Received HTTP request with unrecognised method: " << + mHttpVerb); mMethod = Method_UNKNOWN; } } @@ -291,7 +340,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // If HTTP 1.1 or greater, assume keep-alive if(mHTTPVersion >= HTTPVersion_1_1) { - mClientKeepAliveRequested = true; + mHeaders.SetKeepAlive(true); } // Decode query string? @@ -303,7 +352,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) } // Now parse the headers - ParseHeaders(rGetLine, Timeout); + mHeaders.ReadFromStream(rGetLine, Timeout); std::string expected; if(GetHeader("Expect", &expected)) @@ -314,20 +363,36 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) } } + const std::string& cookies = mHeaders.GetHeaderValue("cookie", false); + // false => not required, returns "" if header is not present. + if(!cookies.empty()) + { + ParseCookies(cookies); + } + // Parse form data? - if(mMethod == Method_POST && mContentLength >= 0) + int64_t contentLength = mHeaders.GetContentLength(); + if(mMethod == Method_POST && contentLength >= 0) { // Too long? Don't allow people to be nasty by sending lots of data - if(mContentLength > MAX_CONTENT_SIZE) + if(contentLength > MAX_CONTENT_SIZE) { - THROW_EXCEPTION(HTTPException, POSTContentTooLong); + THROW_EXCEPTION_MESSAGE(HTTPException, POSTContentTooLong, + "Client tried to upload " << contentLength << " bytes of " + "content, but our maximum supported size is " << + MAX_CONTENT_SIZE); } // Some data in the request to follow, parsing it bit by bit HTTPQueryDecoder decoder(mQuery); + // Don't forget any data left in the GetLine object int fromBuffer = rGetLine.GetSizeOfBufferedData(); - if(fromBuffer > mContentLength) fromBuffer = mContentLength; + if(fromBuffer > contentLength) + { + fromBuffer = contentLength; + } + if(fromBuffer > 0) { BOX_TRACE("Decoding " << fromBuffer << " bytes of " @@ -336,8 +401,9 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // And tell the getline object to ignore the data we just used rGetLine.IgnoreBufferedData(fromBuffer); } + // Then read any more data, as required - int bytesToGo = mContentLength - fromBuffer; + int bytesToGo = contentLength - fromBuffer; while(bytesToGo > 0) { char buf[4096]; @@ -357,12 +423,12 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Finish off decoder.Finish(); } - else if (mContentLength > 0) + else { IOStream::pos_type bytesToCopy = rGetLine.GetSizeOfBufferedData(); - if (bytesToCopy > mContentLength) + if (contentLength != -1 && bytesToCopy > contentLength) { - bytesToCopy = mContentLength; + bytesToCopy = contentLength; } Write(rGetLine.GetBufferedData(), bytesToCopy); SetForReading(); @@ -372,36 +438,100 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) return true; } -void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) +void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo, int Timeout) { + // TODO FIXME: POST requests (above) do not set mpStreamToReadFrom, would we + // ever want to call ReadContent() on them? I hope not! + ASSERT(mpStreamToReadFrom != NULL); + Seek(0, SeekType_Absolute); - - CopyStreamTo(rStreamToWriteTo); + + // Copy any data that we've already buffered. + CopyStreamTo(rStreamToWriteTo, Timeout); IOStream::pos_type bytesCopied = GetSize(); - while (bytesCopied < mContentLength) + // Copy the data stream, but only upto the content-length. + int64_t contentLength = mHeaders.GetContentLength(); + if(contentLength == -1) { - char buffer[1024]; - IOStream::pos_type bytesToCopy = sizeof(buffer); - if (bytesToCopy > mContentLength - bytesCopied) + // There is no content-length, so copy all of it. Include the buffered + // data (already copied above) in the final content-length, which we + // update in the HTTPRequest headers. + contentLength = bytesCopied; + + // If there is a stream to read from, then copy its contents too. + if(mpStreamToReadFrom != NULL) { - bytesToCopy = mContentLength - bytesCopied; + contentLength += + mpStreamToReadFrom->CopyStreamTo(rStreamToWriteTo, + Timeout); } - bytesToCopy = mpStreamToReadFrom->Read(buffer, bytesToCopy); - rStreamToWriteTo.Write(buffer, bytesToCopy); - bytesCopied += bytesToCopy; + mHeaders.SetContentLength(contentLength); + } + else + { + // Subtract the amount of data already buffered (and already copied above) + // from the total content-length, to get the amount that we are allowed + // and expected to read from the stream. This will leave the stream + // positioned ready for the next request, or EOF, as the client decides. + PartialReadStream partial(*mpStreamToReadFrom, contentLength - + bytesCopied); + partial.CopyStreamTo(rStreamToWriteTo, Timeout); + + // In case of a timeout or error, PartialReadStream::CopyStreamTo + // should have thrown an exception, so this is just defensive, to + // ensure that the source stream is properly positioned to read + // from again, and the destination received the correct number of + // bytes. + ASSERT(!partial.StreamDataLeft()); } } + // -------------------------------------------------------------------------- // // Function -// Name: HTTPRequest::Send(IOStream &, int) -// Purpose: Write the request to an IOStream using HTTP. +// Name: HTTPRequest::Send(IOStream &, int, bool) +// Purpose: Write a request with NO CONTENT to an IOStream using +// HTTP. If you want to send a request WITH content, +// such as a PUT or POST request, use SendWithStream() +// instead. // Created: 03/01/09 // // -------------------------------------------------------------------------- -bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) + +void HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) +{ + if(mHeaders.GetContentLength() > 0) + { + THROW_EXCEPTION(HTTPException, ContentLengthAlreadySet); + } + + if(GetSize() != 0) + { + THROW_EXCEPTION_MESSAGE(HTTPException, WrongContentLength, + "Tried to send a request without content, but there is data " + "in the request buffer waiting to be sent.") + } + + mHeaders.SetContentLength(0); + SendHeaders(rStream, Timeout, ExpectContinue); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::SendHeaders(IOStream &, int, bool) +// Purpose: Write the start of a request to an IOStream using +// HTTP. If you want to send a request WITH content, +// but you can't wait for a response, use this followed +// by sending your stream directly to the socket. +// Created: 2015-10-08 +// +// -------------------------------------------------------------------------- + +void HTTPRequest::SendHeaders(IOStream &rStream, int Timeout, bool ExpectContinue) { switch (mMethod) { @@ -409,18 +539,19 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) THROW_EXCEPTION(HTTPException, RequestNotInitialised); break; case Method_UNKNOWN: THROW_EXCEPTION(HTTPException, BadRequest); break; - case Method_GET: - rStream.Write("GET"); break; - case Method_HEAD: - rStream.Write("HEAD"); break; - case Method_POST: - rStream.Write("POST"); break; - case Method_PUT: - rStream.Write("PUT"); break; + default: + rStream.Write(GetMethodName()); } rStream.Write(" "); rStream.Write(mRequestURI.c_str()); + for(Query_t::iterator i = mQuery.begin(); i != mQuery.end(); i++) + { + rStream.Write( + ((i == mQuery.begin()) ? "?" : "&") + + HTTPQueryDecoder::URLEncode(i->first) + "=" + + HTTPQueryDecoder::URLEncode(i->second)); + } rStream.Write(" "); switch (mHTTPVersion) @@ -433,31 +564,8 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) "Unsupported HTTP version: " << mHTTPVersion); } - rStream.Write("\n"); - std::ostringstream oss; - - if (mContentLength != -1) - { - oss << "Content-Length: " << mContentLength << "\n"; - } - - if (mContentType != "") - { - oss << "Content-Type: " << mContentType << "\n"; - } - - if (mHostName != "") - { - if (mHostPort != 80) - { - oss << "Host: " << mHostName << ":" << mHostPort << - "\n"; - } - else - { - oss << "Host: " << mHostName << "\n"; - } - } + rStream.Write("\r\n"); + mHeaders.WriteTo(rStream, Timeout); if (mpCookies) { @@ -465,207 +573,104 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) "Cookie support not implemented yet"); } - if (mClientKeepAliveRequested) - { - oss << "Connection: keep-alive\n"; - } - else - { - oss << "Connection: close\n"; - } - - for (std::vector
::iterator i = mExtraHeaders.begin(); - i != mExtraHeaders.end(); i++) - { - oss << i->first << ": " << i->second << "\n"; - } - if (ExpectContinue) { - oss << "Expect: 100-continue\n"; + rStream.Write("Expect: 100-continue\r\n"); } - rStream.Write(oss.str().c_str()); - rStream.Write("\n"); - - return true; + rStream.Write("\r\n"); } -void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, - IOStream* pStreamToSend, HTTPResponse& rResponse) -{ - IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); - if (size != IOStream::SizeOfStreamUnknown) - { - mContentLength = size; - } - - Send(rStreamToSendTo, Timeout, true); - - rResponse.Receive(rStreamToSendTo, Timeout); - if (rResponse.GetResponseCode() != 100) - { - // bad response, abort now - return; - } - - pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); - - // receive the final response - rResponse.Receive(rStreamToSendTo, Timeout); -} // -------------------------------------------------------------------------- // // Function -// Name: HTTPRequest::ParseHeaders(IOStreamGetLine &, int) -// Purpose: Private. Parse the headers of the request -// Created: 26/3/04 +// Name: HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, +// int Timeout, IOStream* pStreamToSend, +// HTTPResponse& rResponse) +// Purpose: Write a request WITH CONTENT to an IOStream using +// HTTP. If you want to send a request WITHOUT content, +// such as a GET or DELETE request, use Send() instead. +// Because this is interactive (it uses 100 Continue +// responses) it can only be sent to a SocketStream. +// Created: 03/01/09 // // -------------------------------------------------------------------------- -void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) + +IOStream::pos_type HTTPRequest::SendWithStream(SocketStream &rStreamToSendTo, + int Timeout, IOStream* pStreamToSend, HTTPResponse& rResponse) { - std::string header; - bool haveHeader = false; - while(true) + SendHeaders(rStreamToSendTo, Timeout, true); // ExpectContinue + + rResponse.Receive(rStreamToSendTo, Timeout); + if (rResponse.GetResponseCode() != 100) { - if(rGetLine.IsEOF()) - { - // Header terminates unexpectedly - THROW_EXCEPTION(HTTPException, BadRequest) - } + // bad response, abort now + return 0; + } - std::string currentLine; - if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) - { - // Timeout - THROW_EXCEPTION(HTTPException, RequestReadFailed) - } + IOStream::pos_type bytes_sent = 0; - // Is this a continuation of the previous line? - bool processHeader = haveHeader; - if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + if(mHeaders.GetContentLength() == -1) + { + // We don't know how long the stream is, so just send it all. + // Including any data buffered in the HTTPRequest. + CopyStreamTo(rStreamToSendTo, Timeout); + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); + } + else + { + // Check that the length of the stream is correct, and ensure + // that we don't send too much without realising. + ReadGatherStream gather(false); // don't delete anything + + // Send any data buffered in the HTTPRequest first. + gather.AddBlock(gather.AddComponent(this), GetSize()); + + // And the remaining bytes should be read from the supplied stream. + gather.AddBlock(gather.AddComponent(pStreamToSend), + mHeaders.GetContentLength() - GetSize()); + + bytes_sent = gather.CopyStreamTo(rStreamToSendTo, Timeout); + + if(pStreamToSend->StreamDataLeft()) { - // A continuation, don't process anything yet - processHeader = false; + THROW_EXCEPTION_MESSAGE(HTTPException, WrongContentLength, + "Expected to send " << mHeaders.GetContentLength() << + " bytes, but there is still unsent data left in the " + "stream"); } - //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - // Parse the header -- this will actually process the header - // from the previous run around the loop. - if(processHeader) + if(gather.StreamDataLeft()) { - // Find where the : is in the line - const char *h = header.c_str(); - int p = 0; - while(h[p] != '\0' && h[p] != ':') - { - ++p; - } - // Skip white space - int dataStart = p + 1; - while(h[dataStart] == ' ' || h[dataStart] == '\t') - { - ++dataStart; - } - - std::string header_name(ToLowerCase(std::string(h, - p))); - - if (header_name == "content-length") - { - // Decode number - long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK - if(len < 0) len = 0; - // Store - mContentLength = len; - } - else if (header_name == "content-type") - { - // Store rest of string as content type - mContentType = h + dataStart; - } - else if (header_name == "host") - { - // Store host header - mHostName = h + dataStart; - - // Is there a port number to split off? - std::string::size_type colon = mHostName.find_first_of(':'); - if(colon != std::string::npos) - { - // There's a port in the string... attempt to turn it into an int - mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); - - // Truncate the string to just the hostname - mHostName = mHostName.substr(0, colon); - - BOX_TRACE("Host: header, hostname = " << - "'" << mHostName << "', host " - "port = " << mHostPort); - } - } - else if (header_name == "cookie") - { - // Parse cookies - ParseCookies(header, dataStart); - } - else if (header_name == "connection") - { - // Connection header, what is required? - const char *v = h + dataStart; - if(::strcasecmp(v, "close") == 0) - { - mClientKeepAliveRequested = false; - } - else if(::strcasecmp(v, "keep-alive") == 0) - { - mClientKeepAliveRequested = true; - } - // else don't understand, just assume default for protocol version - } - else - { - mExtraHeaders.push_back(Header(header_name, - h + dataStart)); - } - - // Unset have header flag, as it's now been processed - haveHeader = false; + THROW_EXCEPTION_MESSAGE(HTTPException, WrongContentLength, + "Expected to send " << mHeaders.GetContentLength() << + " bytes, but there was not enough data in the stream"); } + } - // Store the chunk of header the for next time round - if(haveHeader) - { - header += currentLine; - } - else - { - header = currentLine; - haveHeader = true; - } + // We don't support keep-alive, so we must shutdown the write side of the stream + // to signal to the other end that we have no more data to send. + ASSERT(!GetClientKeepAliveRequested()); + rStreamToSendTo.Shutdown(false, true); // !read, write - // End of headers? - if(currentLine.empty()) - { - // All done! - break; - } - } + // receive the final response + rResponse.Receive(rStreamToSendTo, Timeout); + return bytes_sent; } // -------------------------------------------------------------------------- // // Function -// Name: HTTPRequest::ParseCookies(const std::string &, int) +// Name: HTTPRequest::ParseCookies(const std::string &) // Purpose: Parse the cookie header // Created: 20/8/04 // // -------------------------------------------------------------------------- -void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) +void HTTPRequest::ParseCookies(const std::string &rCookieString) { - const char *data = rHeader.c_str() + DataStarts; + const char *data = rCookieString.c_str(); const char *pos = data; const char *itemStart = pos; std::string name; diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h index 16c4d16c5..deba9cbee 100644 --- a/lib/httpserver/HTTPRequest.h +++ b/lib/httpserver/HTTPRequest.h @@ -14,6 +14,8 @@ #include #include "CollectInBufferStream.h" +#include "HTTPHeaders.h" +#include "SocketStream.h" class HTTPResponse; class IOStream; @@ -41,16 +43,49 @@ class HTTPRequest : public CollectInBufferStream Method_GET = 1, Method_HEAD = 2, Method_POST = 3, - Method_PUT = 4 + Method_PUT = 4, + Method_DELETE = 5 }; HTTPRequest(); HTTPRequest(enum Method method, const std::string& rURI); ~HTTPRequest(); -private: - // no copying - HTTPRequest(const HTTPRequest &); - HTTPRequest &operator=(const HTTPRequest &); + + HTTPRequest(const HTTPRequest &to_copy) + : mMethod(to_copy.mMethod), + mRequestURI(to_copy.mRequestURI), + mQueryString(to_copy.mQueryString), + mHTTPVersion(to_copy.mHTTPVersion), + mQuery(to_copy.mQuery), + // it's not safe to copy this, as it may be consumed or destroyed: + mpCookies(NULL), + mHeaders(to_copy.mHeaders), + mExpectContinue(to_copy.mExpectContinue), + // it's not safe to copy this, as it may be consumed or destroyed: + mpStreamToReadFrom(NULL), + mHttpVerb(to_copy.mHttpVerb) + // If you ever add members, be sure to update this list too! + { } + + HTTPRequest &operator=(const HTTPRequest &to_copy) + { + mMethod = to_copy.mMethod; + mRequestURI = to_copy.mRequestURI; + mQueryString = to_copy.mQueryString; + mHTTPVersion = to_copy.mHTTPVersion; + mQuery = to_copy.mQuery; + // it's not safe to copy this; as it may be modified or destroyed: + mpCookies = NULL; + mHeaders = to_copy.mHeaders; + mExpectContinue = to_copy.mExpectContinue; + // it's not safe to copy this; as it may be consumed or destroyed: + mpStreamToReadFrom = NULL; + mHttpVerb = to_copy.mHttpVerb; + // If you ever add members, be sure to update this list too! + + return *this; + } + public: typedef std::multimap Query_t; typedef Query_t::value_type QueryEn_t; @@ -65,60 +100,68 @@ class HTTPRequest : public CollectInBufferStream }; bool Receive(IOStreamGetLine &rGetLine, int Timeout); - bool Send(IOStream &rStream, int Timeout, bool ExpectContinue = false); - void SendWithStream(IOStream &rStreamToSendTo, int Timeout, + void SendHeaders(IOStream &rStream, int Timeout, bool ExpectContinue = false); + void Send(IOStream &rStream, int Timeout, bool ExpectContinue = false); + IOStream::pos_type SendWithStream(SocketStream &rStreamToSendTo, int Timeout, IOStream* pStreamToSend, HTTPResponse& rResponse); - void ReadContent(IOStream& rStreamToWriteTo); + void ReadContent(IOStream& rStreamToWriteTo, int Timeout); typedef std::map CookieJar_t; - // -------------------------------------------------------------------------- - // - // Function - // Name: HTTPResponse::Get*() - // Purpose: Various Get accessors - // Created: 26/3/04 - // - // -------------------------------------------------------------------------- enum Method GetMethod() const {return mMethod;} std::string GetMethodName() const; - const std::string &GetRequestURI() const {return mRequestURI;} + std::string GetRequestURI(bool with_parameters_for_get_request = false) const; - // Note: the HTTPRequest generates and parses the Host: header - // Do not attempt to set one yourself with AddHeader(). - const std::string &GetHostName() const {return mHostName;} + const std::string &GetHostName() const {return mHeaders.GetHostName();} void SetHostName(const std::string& rHostName) { - mHostName = rHostName; + mHeaders.SetHostName(rHostName); } - - const int GetHostPort() const {return mHostPort;} + const int GetHostPort() const {return mHeaders.GetHostPort();} const std::string &GetQueryString() const {return mQueryString;} int GetHTTPVersion() const {return mHTTPVersion;} const Query_t &GetQuery() const {return mQuery;} - int GetContentLength() const {return mContentLength;} - const std::string &GetContentType() const {return mContentType;} + void AddParameter(const std::string& name, const std::string& value) + { + mQuery.insert(QueryEn_t(name, value)); + } + void SetParameter(const std::string& name, const std::string& value) + { + mQuery.erase(name); + mQuery.insert(QueryEn_t(name, value)); + } + void RemoveParameter(const std::string& name) + { + mQuery.erase(name); + } + std::string GetParameterString(const std::string& name, + const std::string& default_value) + { + return GetParameterString(name, default_value, false); // !required + } + std::string GetParameterString(const std::string& name) + { + return GetParameterString(name, "", true); // required + } + const Query_t GetParameters() const + { + return mQuery; + } + + int GetContentLength() const {return mHeaders.GetContentLength();} + const std::string &GetContentType() const {return mHeaders.GetContentType();} const CookieJar_t *GetCookies() const {return mpCookies;} // WARNING: May return NULL bool GetCookie(const char *CookieName, std::string &rValueOut) const; const std::string &GetCookie(const char *CookieName) const; bool GetHeader(const std::string& rName, std::string* pValueOut) const { - std::string header = ToLowerCase(rName); - - for (std::vector
::const_iterator - i = mExtraHeaders.begin(); - i != mExtraHeaders.end(); i++) - { - if (i->first == header) - { - *pValueOut = i->second; - return true; - } - } - - return false; + return mHeaders.GetHeader(rName, pValueOut); + } + void AddHeader(const std::string& rName, const std::string& rValue) + { + mHeaders.AddHeader(rName, rValue); } - std::vector
GetHeaders() { return mExtraHeaders; } + const HTTPHeaders& GetHeaders() const { return mHeaders; } // -------------------------------------------------------------------------- // @@ -129,65 +172,61 @@ class HTTPRequest : public CollectInBufferStream // Created: 22/12/04 // // -------------------------------------------------------------------------- - bool GetClientKeepAliveRequested() const {return mClientKeepAliveRequested;} + bool GetClientKeepAliveRequested() const {return mHeaders.IsKeepAlive();} void SetClientKeepAliveRequested(bool keepAlive) { - mClientKeepAliveRequested = keepAlive; + mHeaders.SetKeepAlive(keepAlive); } - void AddHeader(const std::string& rName, const std::string& rValue) + bool IsExpectingContinue() const { return mExpectContinue; } + + // This is not supposed to be an API, but the S3Simulator needs to be able to + // associate a data stream with an HTTPRequest when handling it in-process. + void SetDataStream(IOStream* pStreamToReadFrom) { - mExtraHeaders.push_back(Header(ToLowerCase(rName), rValue)); + ASSERT(!mpStreamToReadFrom); + mpStreamToReadFrom = pStreamToReadFrom; } - bool IsExpectingContinue() const { return mExpectContinue; } - const char* GetVerb() const + +private: + std::string GetParameterString(const std::string& name, + const std::string& default_value, bool required) { - if (!mHttpVerb.empty()) + Query_t::iterator i = mQuery.find(name); + if(i == mQuery.end()) { - return mHttpVerb.c_str(); + if(required) + { + THROW_EXCEPTION_MESSAGE(HTTPException, ParameterNotFound, + name); + } + else + { + return default_value; + } } - switch (mMethod) + const std::string& value(i->second); + i++; + if(i != mQuery.end() && i->first == name) { - case Method_UNINITIALISED: return "Uninitialized"; - case Method_UNKNOWN: return "Unknown"; - case Method_GET: return "GET"; - case Method_HEAD: return "HEAD"; - case Method_POST: return "POST"; - case Method_PUT: return "PUT"; + THROW_EXCEPTION_MESSAGE(HTTPException, DuplicateParameter, name); } - return "Bad"; + return value; } - -private: - void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); - void ParseCookies(const std::string &rHeader, int DataStarts); + + void ParseCookies(const std::string &rCookieString); enum Method mMethod; std::string mRequestURI; - std::string mHostName; - int mHostPort; std::string mQueryString; int mHTTPVersion; Query_t mQuery; - int mContentLength; - std::string mContentType; CookieJar_t *mpCookies; - bool mClientKeepAliveRequested; - std::vector
mExtraHeaders; + HTTPHeaders mHeaders; bool mExpectContinue; IOStream* mpStreamToReadFrom; std::string mHttpVerb; - - std::string ToLowerCase(const std::string& rInput) const - { - std::string output = rInput; - for (std::string::iterator c = output.begin(); - c != output.end(); c++) - { - *c = tolower(*c); - } - return output; - } + // If you ever add members, be sure to update the copy constructor too! }; #endif // HTTPREQUEST__H diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp index c56f286fb..86e88ba49 100644 --- a/lib/httpserver/HTTPResponse.cpp +++ b/lib/httpserver/HTTPResponse.cpp @@ -33,8 +33,6 @@ std::string HTTPResponse::msDefaultURIPrefix; HTTPResponse::HTTPResponse(IOStream* pStreamToSendTo) : mResponseCode(HTTPResponse::Code_NoContent), mResponseIsDynamicContent(true), - mKeepAlive(false), - mContentLength(-1), mpStreamToSendTo(pStreamToSendTo) { } @@ -51,13 +49,43 @@ HTTPResponse::HTTPResponse(IOStream* pStreamToSendTo) HTTPResponse::HTTPResponse() : mResponseCode(HTTPResponse::Code_NoContent), mResponseIsDynamicContent(true), - mKeepAlive(false), - mContentLength(-1), mpStreamToSendTo(NULL) { } +// allow copying, but be very careful with the response stream, +// you can only read it once! (this class doesn't police it). +HTTPResponse::HTTPResponse(const HTTPResponse& rOther) +: mResponseCode(rOther.mResponseCode), + mResponseIsDynamicContent(rOther.mResponseIsDynamicContent), + mpStreamToSendTo(rOther.mpStreamToSendTo), + mHeaders(rOther.mHeaders) +{ + Write(rOther.GetBuffer(), rOther.GetSize()); + if(rOther.IsSetForReading()) + { + SetForReading(); + } +} + + +HTTPResponse &HTTPResponse::operator=(const HTTPResponse &rOther) +{ + Reset(); + Write(rOther.GetBuffer(), rOther.GetSize()); + mResponseCode = rOther.mResponseCode; + mResponseIsDynamicContent = rOther.mResponseIsDynamicContent; + mHeaders = rOther.mHeaders; + mpStreamToSendTo = rOther.mpStreamToSendTo; + if(rOther.IsSetForReading()) + { + SetForReading(); + } + return *this; +} + + // -------------------------------------------------------------------------- // // Function @@ -90,10 +118,11 @@ const char *HTTPResponse::ResponseCodeToString(int ResponseCode) case Code_Found: return "302 Found"; break; case Code_NotModified: return "304 Not Modified"; break; case Code_TemporaryRedirect: return "307 Temporary Redirect"; break; - case Code_MethodNotAllowed: return "400 Method Not Allowed"; break; + case Code_BadRequest: return "400 Bad Request"; break; case Code_Unauthorized: return "401 Unauthorized"; break; case Code_Forbidden: return "403 Forbidden"; break; case Code_NotFound: return "404 Not Found"; break; + case Code_Conflict: return "409 Conflict"; break; case Code_InternalServerError: return "500 Internal Server Error"; break; case Code_NotImplemented: return "501 Not Implemented"; break; default: @@ -119,20 +148,6 @@ void HTTPResponse::SetResponseCode(int Code) } -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::SetContentType(const char *) -// Purpose: Set content type -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -void HTTPResponse::SetContentType(const char *ContentType) -{ - mContentType = ContentType; -} - - // -------------------------------------------------------------------------- // // Function @@ -141,352 +156,174 @@ void HTTPResponse::SetContentType(const char *ContentType) // Created: 26/3/2004 // // -------------------------------------------------------------------------- -void HTTPResponse::Send(bool OmitContent) +void HTTPResponse::Send(int Timeout) { if (!mpStreamToSendTo) { THROW_EXCEPTION(HTTPException, NoStreamConfigured); } - if (GetSize() != 0 && mContentType.empty()) + if (GetSize() != 0 && mHeaders.GetContentType().empty()) { THROW_EXCEPTION(HTTPException, NoContentTypeSet); } // Build and send header { - std::string header("HTTP/1.1 "); - header += ResponseCodeToString(mResponseCode); - header += "\r\nContent-Type: "; - header += mContentType; - header += "\r\nContent-Length: "; - { - char len[32]; - ::sprintf(len, "%d", OmitContent?(0):(GetSize())); - header += len; - } - // Extra headers... - for(std::vector >::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i) - { - header += "\r\n"; - header += i->first + ": " + i->second; - } - // NOTE: a line ending must be included here in all cases + std::ostringstream header; + header << "HTTP/1.1 "; + header << ResponseCodeToString(mResponseCode); + header << "\r\n"; + mpStreamToSendTo->Write(header.str(), Timeout); + // Control whether the response is cached if(mResponseIsDynamicContent) { // dynamic is private and can't be cached - header += "\r\nCache-Control: no-cache, private"; + mHeaders.AddHeader("Cache-Control", "no-cache, private"); } else { // static is allowed to be cached for a day - header += "\r\nCache-Control: max-age=86400"; + mHeaders.AddHeader("Cache-Control", "max-age=86400"); } - if(mKeepAlive) - { - header += "\r\nConnection: keep-alive\r\n\r\n"; - } - else - { - header += "\r\nConnection: close\r\n\r\n"; - } + // Write to stream + mHeaders.WriteTo(*mpStreamToSendTo, Timeout); // NOTE: header ends with blank line in all cases - - // Write to stream - mpStreamToSendTo->Write(header.c_str(), header.size()); + mpStreamToSendTo->Write(std::string("\r\n"), Timeout); } // Send content - if(!OmitContent) - { - mpStreamToSendTo->Write(GetBuffer(), GetSize()); - } + SetForReading(); + CopyStreamTo(*mpStreamToSendTo, Timeout); } -void HTTPResponse::SendContinue() +void HTTPResponse::SendContinue(int Timeout) { - mpStreamToSendTo->Write("HTTP/1.1 100 Continue\r\n"); + mpStreamToSendTo->Write(std::string("HTTP/1.1 100 Continue\r\n"), Timeout); } -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::ParseHeaders(IOStreamGetLine &, int) -// Purpose: Private. Parse the headers of the response -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +void HTTPResponse::Receive(IOStream& rStream, int Timeout) { - std::string header; - bool haveHeader = false; - while(true) - { - if(rGetLine.IsEOF()) - { - // Header terminates unexpectedly - THROW_EXCEPTION(HTTPException, BadRequest) - } + IOStreamGetLine rGetLine(rStream); - std::string currentLine; - if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) - { - // Timeout - THROW_EXCEPTION(HTTPException, RequestReadFailed) - } + if(rGetLine.IsEOF()) + { + // Connection terminated unexpectedly + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server closed the connection without sending a response"); + } - // Is this a continuation of the previous line? - bool processHeader = haveHeader; - if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + std::string statusLine; + while(true) + { + try { - // A continuation, don't process anything yet - processHeader = false; + statusLine = rGetLine.GetLine(false /* no preprocess */, Timeout); + break; } - //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - - // Parse the header -- this will actually process the header - // from the previous run around the loop. - if(processHeader) + catch(BoxException &e) { - // Find where the : is in the line - const char *h = header.c_str(); - int p = 0; - while(h[p] != '\0' && h[p] != ':') + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) { - ++p; + // try again + continue; } - // Skip white space - int dataStart = p + 1; - while(h[dataStart] == ' ' || h[dataStart] == '\t') + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) { - ++dataStart; + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "Server disconnected before sending status line"); } - - if(p == sizeof("Content-Length")-1 - && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + else if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) { - // Decode number - long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK - if(len < 0) len = 0; - // Store - mContentLength = len; - } - else if(p == sizeof("Content-Type")-1 - && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) - { - // Store rest of string as content type - mContentType = h + dataStart; - } - else if(p == sizeof("Cookie")-1 - && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) - { - THROW_EXCEPTION(HTTPException, NotImplemented); - /* - // Parse cookies - ParseCookies(header, dataStart); - */ - } - else if(p == sizeof("Connection")-1 - && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) - { - // Connection header, what is required? - const char *v = h + dataStart; - if(::strcasecmp(v, "close") == 0) - { - mKeepAlive = false; - } - else if(::strcasecmp(v, "keep-alive") == 0) - { - mKeepAlive = true; - } - // else don't understand, just assume default for protocol version + THROW_EXCEPTION_MESSAGE(HTTPException, ResponseTimedOut, + "Server took too long to send the status line"); } else { - std::string headerName = header.substr(0, p); - AddHeader(headerName, h + dataStart); + throw; } - - // Unset have header flag, as it's now been processed - haveHeader = false; - } - - // Store the chunk of header the for next time round - if(haveHeader) - { - header += currentLine; - } - else - { - header = currentLine; - haveHeader = true; - } - - // End of headers? - if(currentLine.empty()) - { - // All done! - break; } } -} - -void HTTPResponse::Receive(IOStream& rStream, int Timeout) -{ - IOStreamGetLine rGetLine(rStream); - if(rGetLine.IsEOF()) - { - // Connection terminated unexpectedly - THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, - "HTTP server closed the connection without sending a response"); - } - - std::string statusLine; - if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) - { - // Timeout - THROW_EXCEPTION_MESSAGE(HTTPException, ResponseReadFailed, - "Failed to get a response from the HTTP server within the timeout"); - } - - if (statusLine.substr(0, 7) != "HTTP/1." || statusLine[8] != ' ') + if(statusLine.substr(0, 7) != "HTTP/1." || statusLine[8] != ' ') { THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, "HTTP server sent an invalid HTTP status line: " << statusLine); } - if (statusLine[5] == '1' && statusLine[7] == '1') + if(statusLine[5] == '1' && statusLine[7] == '1') { // HTTP/1.1 default is to keep alive - mKeepAlive = true; + mHeaders.SetKeepAlive(true); } // Decode the status code long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); // returns zero in error case, this is OK - if (status < 0) status = 0; + if(status < 0) status = 0; // Store mResponseCode = status; // 100 Continue responses have no headers, terminating newline, or body - if (status == 100) + if(status == 100) { return; } - ParseHeaders(rGetLine, Timeout); + mHeaders.ReadFromStream(rGetLine, Timeout); + int remaining_bytes = mHeaders.GetContentLength(); // push back whatever bytes we have left // rGetLine.DetachFile(); - if (mContentLength > 0) + if(remaining_bytes == -1 || remaining_bytes > 0) { - if (mContentLength < rGetLine.GetSizeOfBufferedData()) + if(remaining_bytes != -1 && + remaining_bytes < rGetLine.GetSizeOfBufferedData()) { // very small response, not good! - THROW_EXCEPTION(HTTPException, NotImplemented); + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server sent a very small response: " << + mHeaders.GetContentLength() << " bytes"); } - mContentLength -= rGetLine.GetSizeOfBufferedData(); + if(remaining_bytes > 0) + { + remaining_bytes -= rGetLine.GetSizeOfBufferedData(); + } Write(rGetLine.GetBufferedData(), rGetLine.GetSizeOfBufferedData()); } - while (mContentLength != 0) // could be -1 as well + while(remaining_bytes != 0) // could be -1 as well { char buffer[4096]; int readSize = sizeof(buffer); - if (mContentLength > 0 && mContentLength < readSize) + + if(remaining_bytes > 0 && remaining_bytes < readSize) { - readSize = mContentLength; + readSize = remaining_bytes; } + readSize = rStream.Read(buffer, readSize, Timeout); - if (readSize == 0) + if(readSize == 0) { break; } - mContentLength -= readSize; + Write(buffer, readSize); + if(remaining_bytes > 0) + { + remaining_bytes -= readSize; + } } SetForReading(); } -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::AddHeader(const char *) -// Purpose: Add header, given entire line -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -/* -void HTTPResponse::AddHeader(const char *EntireHeaderLine) -{ - mExtraHeaders.push_back(std::string(EntireHeaderLine)); -} -*/ - -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::AddHeader(const std::string &) -// Purpose: Add header, given entire line -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -/* -void HTTPResponse::AddHeader(const std::string &rEntireHeaderLine) -{ - mExtraHeaders.push_back(rEntireHeaderLine); -} -*/ - -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::AddHeader(const char *, const char *) -// Purpose: Add header, given header name and it's value -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -void HTTPResponse::AddHeader(const char *pHeader, const char *pValue) -{ - mExtraHeaders.push_back(Header(pHeader, pValue)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::AddHeader(const char *, const std::string &) -// Purpose: Add header, given header name and it's value -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -void HTTPResponse::AddHeader(const char *pHeader, const std::string &rValue) -{ - mExtraHeaders.push_back(Header(pHeader, rValue)); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: HTTPResponse::AddHeader(const std::string &, const std::string &) -// Purpose: Add header, given header name and it's value -// Created: 26/3/04 -// -// -------------------------------------------------------------------------- -void HTTPResponse::AddHeader(const std::string &rHeader, const std::string &rValue) -{ - mExtraHeaders.push_back(Header(rHeader, rValue)); -} - // -------------------------------------------------------------------------- // @@ -520,7 +357,7 @@ void HTTPResponse::SetCookie(const char *Name, const char *Value, const char *Pa h += "; Version=1; Path="; h += Path; - mExtraHeaders.push_back(Header("Set-Cookie", h)); + AddHeader("Set-Cookie", h); } @@ -536,7 +373,7 @@ void HTTPResponse::SetCookie(const char *Name, const char *Value, const char *Pa void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) { if(mResponseCode != HTTPResponse::Code_NoContent - || !mContentType.empty() + || !mHeaders.GetContentType().empty() || GetSize() != 0) { THROW_EXCEPTION(HTTPException, CannotSetRedirectIfReponseHasData) @@ -549,10 +386,10 @@ void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) std::string header; if(IsLocalURI) header += msDefaultURIPrefix; header += RedirectTo; - mExtraHeaders.push_back(Header("Location", header)); + mHeaders.AddHeader("location", header); // Set up some default content - mContentType = "text/html"; + mHeaders.SetContentType("text/html"); #define REDIRECT_HTML_1 "Redirection\n

Redirect to content

\n" Write(REDIRECT_HTML_1, sizeof(REDIRECT_HTML_1) - 1); @@ -573,8 +410,8 @@ void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) void HTTPResponse::SetAsNotFound(const char *URI) { if(mResponseCode != HTTPResponse::Code_NoContent - || mExtraHeaders.size() != 0 - || !mContentType.empty() + || !mHeaders.GetExtraHeaders().empty() + || !mHeaders.GetContentType().empty() || GetSize() != 0) { THROW_EXCEPTION(HTTPException, CannotSetNotFoundIfReponseHasData) @@ -584,7 +421,7 @@ void HTTPResponse::SetAsNotFound(const char *URI) mResponseCode = Code_NotFound; // Set data - mContentType = "text/html"; + mHeaders.SetContentType("text/html"); #define NOT_FOUND_HTML_1 "404 Not Found\n

404 Not Found

\n

The URI " #define NOT_FOUND_HTML_2 " was not found on this server.

\n" Write(NOT_FOUND_HTML_1, sizeof(NOT_FOUND_HTML_1) - 1); diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h index f39825d99..cbc21ea8d 100644 --- a/lib/httpserver/HTTPResponse.h +++ b/lib/httpserver/HTTPResponse.h @@ -14,6 +14,7 @@ #include #include "CollectInBufferStream.h" +#include "HTTPHeaders.h" class IOStreamGetLine; @@ -34,81 +35,47 @@ class HTTPResponse : public CollectInBufferStream // allow copying, but be very careful with the response stream, // you can only read it once! (this class doesn't police it). - HTTPResponse(const HTTPResponse& rOther) - : mResponseCode(rOther.mResponseCode), - mResponseIsDynamicContent(rOther.mResponseIsDynamicContent), - mKeepAlive(rOther.mKeepAlive), - mContentType(rOther.mContentType), - mExtraHeaders(rOther.mExtraHeaders), - mContentLength(rOther.mContentLength), - mpStreamToSendTo(rOther.mpStreamToSendTo) - { - Write(rOther.GetBuffer(), rOther.GetSize()); - } - - HTTPResponse &operator=(const HTTPResponse &rOther) - { - Reset(); - Write(rOther.GetBuffer(), rOther.GetSize()); - mResponseCode = rOther.mResponseCode; - mResponseIsDynamicContent = rOther.mResponseIsDynamicContent; - mKeepAlive = rOther.mKeepAlive; - mContentType = rOther.mContentType; - mExtraHeaders = rOther.mExtraHeaders; - mContentLength = rOther.mContentLength; - mpStreamToSendTo = rOther.mpStreamToSendTo; - return *this; - } - - typedef std::pair Header; + HTTPResponse(const HTTPResponse& rOther); + HTTPResponse &operator=(const HTTPResponse &rOther); void SetResponseCode(int Code); int GetResponseCode() const { return mResponseCode; } - void SetContentType(const char *ContentType); - const std::string& GetContentType() { return mContentType; } - int64_t GetContentLength() { return mContentLength; } + void SetContentType(const char *ContentType) + { + mHeaders.SetContentType(ContentType); + } + const std::string& GetContentType() { return mHeaders.GetContentType(); } + int64_t GetContentLength() { return mHeaders.GetContentLength(); } void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); void SetAsNotFound(const char *URI); - void Send(bool OmitContent = false); - void SendContinue(); + void Send(int Timeout = IOStream::TimeOutInfinite); + void SendContinue(int Timeout = IOStream::TimeOutInfinite); void Receive(IOStream& rStream, int Timeout = IOStream::TimeOutInfinite); - // void AddHeader(const char *EntireHeaderLine); - // void AddHeader(const std::string &rEntireHeaderLine); - void AddHeader(const char *Header, const char *Value); - void AddHeader(const char *Header, const std::string &rValue); - void AddHeader(const std::string &rHeader, const std::string &rValue); - bool GetHeader(const std::string& rName, std::string* pValueOut) const + bool GetHeader(const std::string& name, std::string* pValueOut) const { - for (std::vector
::const_iterator - i = mExtraHeaders.begin(); - i != mExtraHeaders.end(); i++) - { - if (i->first == rName) - { - *pValueOut = i->second; - return true; - } - } - return false; + return mHeaders.GetHeader(name, pValueOut); } - std::string GetHeaderValue(const std::string& rName) + std::string GetHeaderValue(const std::string& name) { - std::string value; - if (!GetHeader(rName, &value)) - { - THROW_EXCEPTION(CommonException, ConfigNoKey); - } - return value; + return mHeaders.GetHeaderValue(name); } + void AddHeader(const std::string& name, const std::string& value) + { + mHeaders.AddHeader(name, value); + } + HTTPHeaders& GetHeaders() { return mHeaders; } // Set dynamic content flag, default is content is dynamic void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} // Set keep alive control, default is to mark as to be closed - void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} - bool IsKeepAlive() {return mKeepAlive;} + void SetKeepAlive(bool KeepAlive) + { + mHeaders.SetKeepAlive(KeepAlive); + } + bool IsKeepAlive() {return mHeaders.IsKeepAlive();} void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); @@ -117,18 +84,23 @@ class HTTPResponse : public CollectInBufferStream Code_OK = 200, Code_NoContent = 204, Code_MovedPermanently = 301, - Code_Found = 302, // redirection + Code_Found = 302, // redirection Code_NotModified = 304, Code_TemporaryRedirect = 307, - Code_MethodNotAllowed = 400, + Code_BadRequest = 400, Code_Unauthorized = 401, Code_Forbidden = 403, Code_NotFound = 404, + Code_Conflict = 409, Code_InternalServerError = 500, Code_NotImplemented = 501 }; static const char *ResponseCodeToString(int ResponseCode); + const char *ResponseCodeString() const + { + return ResponseCodeToString(mResponseCode); + } void WriteStringDefang(const char *String, unsigned int StringLen); void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} @@ -159,18 +131,36 @@ class HTTPResponse : public CollectInBufferStream msDefaultURIPrefix = rPrefix; } + // Update Content-Length from current buffer size. + void SetForReading() + { + CollectInBufferStream::SetForReading(); + // If the ContentLength is unknown, set it to the size of the response. + // But if the user has specifically set it to something, then leave it + // alone. + if(mHeaders.GetContentLength() == HTTPHeaders::UNKNOWN_CONTENT_LENGTH) + { + mHeaders.SetContentLength(GetSize()); + } + } + + // Clear all state for reading again + void Reset() + { + CollectInBufferStream::Reset(); + mHeaders = HTTPHeaders(); + mResponseCode = HTTPResponse::Code_NoContent; + mResponseIsDynamicContent = true; + mpStreamToSendTo = NULL; + } + private: int mResponseCode; bool mResponseIsDynamicContent; - bool mKeepAlive; - std::string mContentType; - std::vector
mExtraHeaders; - int64_t mContentLength; // only used when reading response from stream IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream static std::string msDefaultURIPrefix; - - void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); + HTTPHeaders mHeaders; }; #endif // HTTPRESPONSE__H diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp index a2daed993..abb22f276 100644 --- a/lib/httpserver/HTTPServer.cpp +++ b/lib/httpserver/HTTPServer.cpp @@ -127,17 +127,17 @@ void HTTPServer::Run() // -------------------------------------------------------------------------- // // Function -// Name: HTTPServer::Connection(SocketStream &) +// Name: HTTPServer::Connection(SocketStream &) // Purpose: As interface, handle connection // Created: 26/3/04 // // -------------------------------------------------------------------------- void HTTPServer::Connection(std::auto_ptr apConn) { - // Create a get line object to use + // Create an IOStreamGetLine object to help read the request in. IOStreamGetLine getLine(*apConn); - // Notify dervived claases + // Notify derived classes HTTPConnectionOpening(); bool handleRequests = true; @@ -190,9 +190,9 @@ void HTTPServer::Connection(std::auto_ptr apConn) // Stop now handleRequests = false; } - - // Send the response (omit any content if this is a HEAD method request) - response.Send(request.GetMethod() == HTTPRequest::Method_HEAD); + + // Send the response + response.Send(mTimeout); } // Notify derived classes @@ -214,17 +214,16 @@ void HTTPServer::SendInternalErrorResponse(const std::string& rErrorMsg, { #define ERROR_HTML_1 "Internal Server Error\n" \ "

Internal Server Error

\n" \ - "

An error, type " - #define ERROR_HTML_2 " occured when processing the request.

" \ - "

Please try again later.

" \ + "

An error occurred while processing the request:

\n
"
+	#define ERROR_HTML_2 "
\n

Please try again later.

" \ "\n\n" // Generate the error page - // rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); rResponse.SetContentType("text/html"); - rResponse.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); - rResponse.IOStream::Write(rErrorMsg.c_str()); - rResponse.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); + rResponse.Write(ERROR_HTML_1); + rResponse.Write(rErrorMsg); + rResponse.Write(ERROR_HTML_2); } diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h index 8ac1ff83c..a52e1cfa6 100644 --- a/lib/httpserver/HTTPServer.h +++ b/lib/httpserver/HTTPServer.h @@ -27,15 +27,15 @@ class HTTPResponse; class HTTPServer : public ServerStream { public: - HTTPServer(int Timeout = 60000); - // default timeout leaves 1 minute for clients to get a second request in. + HTTPServer(int Timeout = 600000); + // default timeout leaves a little while for clients to get a second request in. ~HTTPServer(); private: // no copying HTTPServer(const HTTPServer &); HTTPServer &operator=(const HTTPServer &); -public: +public: int GetTimeout() const {return mTimeout;} // -------------------------------------------------------------------------- @@ -56,7 +56,6 @@ class HTTPServer : public ServerStream protected: void SendInternalErrorResponse(const std::string& rErrorMsg, HTTPResponse& rResponse); - int GetTimeout() { return mTimeout; } private: int mTimeout; // Timeout for read operations diff --git a/lib/httpserver/HTTPTest.cpp b/lib/httpserver/HTTPTest.cpp new file mode 100644 index 000000000..4e8016a75 --- /dev/null +++ b/lib/httpserver/HTTPTest.cpp @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPTest.cpp +// Purpose: Amazon S3 simulator start/stop functions +// Created: 14/11/2016 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "ServerControl.h" +#include "Utils.h" + +static int s3simulator_pid; + +bool StartSimulator() +{ + s3simulator_pid = StartDaemon(s3simulator_pid, + "../../bin/s3simulator/s3simulator " + bbstored_args + + " testfiles/s3simulator.conf", "testfiles/s3simulator.pid"); + return s3simulator_pid != 0; +} + +bool StopSimulator() +{ + bool result = StopDaemon(s3simulator_pid, "testfiles/s3simulator.pid", + "s3simulator.memleaks", true); + s3simulator_pid = 0; + return result; +} + +bool kill_simulator_if_running() +{ + bool success = true; + + if(FileExists("testfiles/s3simulator.pid")) + { + TEST_THAT_OR(KillServer("testfiles/s3simulator.pid", true), success = false); + } + + return success; +} + diff --git a/lib/httpserver/HTTPTest.h b/lib/httpserver/HTTPTest.h new file mode 100644 index 000000000..f4c8488bd --- /dev/null +++ b/lib/httpserver/HTTPTest.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPTest.h +// Purpose: Amazon S3 simulator start/stop functions +// Created: 14/11/2016 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPTEST__H +#define HTTPTEST__H + +bool StartSimulator(); +bool StopSimulator(); +bool kill_simulator_if_running(); + +#endif // HTTPTEST__H + diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp index 218140665..8b663f380 100644 --- a/lib/httpserver/S3Client.cpp +++ b/lib/httpserver/S3Client.cpp @@ -14,6 +14,9 @@ // #include // #include +#include +#include +#include #include #include "HTTPRequest.h" @@ -28,19 +31,146 @@ #include "MemLeakFindOn.h" +using boost::property_tree::ptree; + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::ListBucket(const std::string& prefix, +// const std::string& delimiter, +// std::vector* p_contents_out, +// std::vector* p_common_prefixes_out, +// bool* p_truncated_out, int max_keys, +// const std::string& marker) +// Purpose: Retrieve a list of objects in a bucket, with a +// common prefix, optionally starting from a specified +// marker, up to some limit. The entries, and common +// prefixes of entries containing the specified +// delimiter, will be appended to p_contents_out and +// p_common_prefixes_out. Returns the number of items +// appended (p_contents_out + p_common_prefixes_out), +// which may be 0 if there is nothing left to iterate +// over, or no matching files in the bucket. +// Created: 18/03/2016 +// +// -------------------------------------------------------------------------- + +int S3Client::ListBucket(std::vector* p_contents_out, + std::vector* p_common_prefixes_out, + const std::string& prefix, const std::string& delimiter, + bool* p_truncated_out, int max_keys, const std::string& marker) +{ + HTTPRequest request(HTTPRequest::Method_GET, "/"); + request.SetParameter("delimiter", delimiter); + request.SetParameter("prefix", prefix); + request.SetParameter("marker", marker); + if(max_keys != -1) + { + std::ostringstream max_keys_stream; + max_keys_stream << max_keys; + request.SetParameter("max-keys", max_keys_stream.str()); + } + + HTTPResponse response = FinishAndSendRequest(request); + CheckResponse(response, "Failed to list files in bucket"); + ASSERT(response.GetResponseCode() == HTTPResponse::Code_OK); + + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + std::auto_ptr ap_response_stream( + new std::istringstream(response_data)); + + ptree response_tree; + read_xml(*ap_response_stream, response_tree, + boost::property_tree::xml_parser::trim_whitespace); + + if(response_tree.begin()->first != "ListBucketResult") + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "Failed to list files in bucket: unexpected root element in " + "response: " << response_tree.begin()->first); + } + + if(++(response_tree.begin()) != response_tree.end()) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "Failed to list files in bucket: multiple root elements in " + "response: " << (++(response_tree.begin()))->first); + } + + ptree result = response_tree.get_child("ListBucketResult"); + ASSERT(result.get("Delimiter") == delimiter); + ASSERT(result.get("Prefix") == prefix); + ASSERT(result.get("Marker") == marker); + + std::string truncated = result.get("IsTruncated"); + ASSERT(truncated == "true" || truncated == "false"); + if(p_truncated_out) + { + *p_truncated_out = (truncated == "true"); + } + + int num_results = 0; + + // Iterate over all the children of the ListBucketResult, looking for + // nodes called "Contents", and examine them. + BOOST_FOREACH(ptree::value_type &v, result) + { + if(v.first == "Contents") + { + std::string name = v.second.get("Key"); + std::string etag = v.second.get("ETag"); + std::string size = v.second.get("Size"); + const char * size_end_ptr; + int64_t size_int = box_strtoui64(size.c_str(), &size_end_ptr, 10); + if(*size_end_ptr != 0) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "Failed to list files in bucket: bad size in " + "contents: " << size); + } + + p_contents_out->push_back(BucketEntry(name, etag, size_int)); + num_results++; + } + } + + ptree common_prefixes = result.get_child("CommonPrefixes"); + BOOST_FOREACH(ptree::value_type &v, common_prefixes) + { + if(v.first == "Prefix") + { + p_common_prefixes_out->push_back(v.second.data()); + num_results++; + } + } + + return num_results; +} + // -------------------------------------------------------------------------- // // Function -// Name: S3Client::GetObject(const std::string& rObjectURI) +// Name: S3Client::GetObject(const std::string& rObjectURI, +// const std::string& MD5Checksum) // Purpose: Retrieve the object with the specified URI (key) -// from your S3 bucket. +// from your S3 bucket. If you supply an MD5 checksum, +// then it is assumed that you already have the file +// data with that checksum, and if the file version on +// the server is the same, then you will get a 304 +// Not Modified response instead of a 200 OK, and no +// file data. // Created: 09/01/2009 // // -------------------------------------------------------------------------- -HTTPResponse S3Client::GetObject(const std::string& rObjectURI) +HTTPResponse S3Client::GetObject(const std::string& rObjectURI, + const std::string& MD5Checksum) { - return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI); + return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI, + NULL, // pStreamToSend + NULL, // pStreamContentType + MD5Checksum); } // -------------------------------------------------------------------------- @@ -59,7 +189,22 @@ HTTPResponse S3Client::HeadObject(const std::string& rObjectURI) } -HTTPResponse HeadObject(const std::string& rObjectURI); +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::DeleteObject(const std::string& rObjectURI) +// Purpose: Delete the object with the specified URI (key) from +// your S3 bucket. +// Created: 27/01/2016 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::DeleteObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_DELETE, rObjectURI); +} + + // -------------------------------------------------------------------------- // // Function @@ -100,11 +245,35 @@ HTTPResponse S3Client::PutObject(const std::string& rObjectURI, HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, const std::string& rRequestURI, IOStream* pStreamToSend, - const char* pStreamContentType) + const char* pStreamContentType, const std::string& MD5Checksum) { + // It's very unlikely that you want to request a URI from Amazon S3 servers + // that doesn't start with a /. Very very unlikely. + ASSERT(rRequestURI[0] == '/'); HTTPRequest request(Method, rRequestURI); - request.SetHostName(mHostName); - + return FinishAndSendRequest(request, pStreamToSend, pStreamContentType, MD5Checksum); +} + +HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest request, IOStream* pStreamToSend, + const char* pStreamContentType, const std::string& MD5Checksum) +{ + std::string virtual_host_name; + + if(!mVirtualHostName.empty()) + { + virtual_host_name = mVirtualHostName; + } + else + { + virtual_host_name = mHostName; + } + + bool with_parameters_for_get_request = ( + request.GetMethod() == HTTPRequest::Method_GET || + request.GetMethod() == HTTPRequest::Method_HEAD); + BOX_TRACE("S3Client: " << mHostName << " > " << request.GetMethodName() << + " " << request.GetRequestURI(with_parameters_for_get_request)); + std::ostringstream date; time_t tt = time(NULL); struct tm *tp = gmtime(&tt); @@ -131,21 +300,28 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, request.AddHeader("Content-Type", pStreamContentType); } + if (!MD5Checksum.empty()) + { + request.AddHeader("If-None-Match", + std::string("\"") + MD5Checksum + "\""); + } + + request.SetHostName(virtual_host_name); std::string s3suffix = ".s3.amazonaws.com"; std::string bucket; - if (mHostName.size() > s3suffix.size()) + if (virtual_host_name.size() > s3suffix.size()) { - std::string suffix = mHostName.substr(mHostName.size() - + std::string suffix = virtual_host_name.substr(virtual_host_name.size() - s3suffix.size(), s3suffix.size()); if (suffix == s3suffix) { - bucket = mHostName.substr(0, mHostName.size() - + bucket = virtual_host_name.substr(0, virtual_host_name.size() - s3suffix.size()); } } std::ostringstream data; - data << request.GetVerb() << "\n"; + data << request.GetMethodName() << "\n"; data << "\n"; /* Content-MD5 */ data << request.GetContentType() << "\n"; data << date.str() << "\n"; @@ -176,20 +352,27 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, } request.AddHeader("Authorization", auth_code); - + HTTPResponse response; + if (mpSimulator) { if (pStreamToSend) { - pStreamToSend->CopyStreamTo(request); + request.SetDataStream(pStreamToSend); } request.SetForReading(); - CollectInBufferStream response_buffer; - HTTPResponse response(&response_buffer); - mpSimulator->Handle(request, response); - return response; + + // TODO FIXME: HTTPServer::Connection does some post-processing on every + // response to determine whether Connection: keep-alive is possible. + // We should do that here too, but currently our HTTP implementation + // doesn't support chunked encoding, so it's disabled there, so we don't + // do it here either. + + // We are definitely finished writing to the HTTPResponse, so leave it + // ready for reading back. + response.SetForReading(); } else { @@ -201,7 +384,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, mapClientSocket->Open(Socket::TypeINET, mHostName, mPort); } - return SendRequest(request, pStreamToSend, + response = SendRequest(request, pStreamToSend, pStreamContentType); } catch (ConnectionException &ce) @@ -212,7 +395,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, // try to reconnect, just once mapClientSocket->Open(Socket::TypeINET, mHostName, mPort); - return SendRequest(request, pStreamToSend, + response = SendRequest(request, pStreamToSend, pStreamContentType); } else @@ -221,7 +404,20 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, throw; } } + + // No need to call response.SetForReading() because HTTPResponse::Receive() + // already did that. } + + // It's not valid to have a keep-alive response if the length isn't known. + // S3Simulator should really check this, but depending on how it's called above, + // it might be possible to bypass that check, so this is a double-check. + ASSERT(response.GetContentLength() >= 0 || !response.IsKeepAlive()); + + BOX_TRACE("S3Client: " << mHostName << " < " << response.GetResponseCode() << + ": " << response.GetContentLength() << " bytes") + + return response; } // -------------------------------------------------------------------------- @@ -247,11 +443,24 @@ HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, if (pStreamToSend) { - rRequest.SendWithStream(*mapClientSocket, mNetworkTimeout, - pStreamToSend, response); + try + { + rRequest.SendWithStream(*mapClientSocket, mNetworkTimeout, + pStreamToSend, response); + } + catch(BoxException &e) + { + // If we encounter a read error from the stream while sending, then + // the client socket is unsafe (because we have sent a request, and + // possibly some data) so we need to close it. + mapClientSocket.reset(); + throw; + } } else { + // No stream, so it's always safe to enable keep-alive + rRequest.SetClientKeepAliveRequested(true); rRequest.Send(*mapClientSocket, mNetworkTimeout); response.Receive(*mapClientSocket, mNetworkTimeout); } @@ -273,20 +482,33 @@ HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, // // Function // Name: S3Client::CheckResponse(HTTPResponse&, -// std::string& message) +// std::string& message) // Purpose: Check the status code of an Amazon S3 response, and -// throw an exception with a useful message (including -// the supplied message) if it's not a 200 OK response. +// throw an exception with a useful message (including +// the supplied message) if it's not a 200 OK response +// (or 204 if ExpectNoContent is true). // Created: 26/07/2015 // // -------------------------------------------------------------------------- -void S3Client::CheckResponse(const HTTPResponse& response, const std::string& message) const +void S3Client::CheckResponse(const HTTPResponse& response, const std::string& message, + bool ExpectNoContent) const { - if(response.GetResponseCode() != HTTPResponse::Code_OK) + // Throw a different exception type (FileNotFound) for 404 responses, since this makes + // debugging easier (makes the actual cause more obvious). + if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + THROW_EXCEPTION_MESSAGE(HTTPException, FileNotFound, + message << ": " << response.ResponseCodeString()); + } + else if(response.GetResponseCode() != + (ExpectNoContent ? HTTPResponse::Code_NoContent : HTTPResponse::Code_OK)) { + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); THROW_EXCEPTION_MESSAGE(HTTPException, RequestFailedUnexpectedly, - message); + message << ": " << response.ResponseCodeString() << ":\n" << + response_data); } } diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h index 4cbb4b961..114ef9c6f 100644 --- a/lib/httpserver/S3Client.h +++ b/lib/httpserver/S3Client.h @@ -13,6 +13,7 @@ #include #include +#include "Configuration.h" #include "HTTPRequest.h" #include "SocketStream.h" @@ -31,36 +32,78 @@ class IOStream; class S3Client { public: + // Constructor with a simulator: S3Client(HTTPServer* pSimulator, const std::string& rHostName, const std::string& rAccessKey, const std::string& rSecretKey) : mpSimulator(pSimulator), mHostName(rHostName), mAccessKey(rAccessKey), mSecretKey(rSecretKey), - mNetworkTimeout(30000) + mNetworkTimeout(600000) { } - - S3Client(std::string HostName, int Port, const std::string& rAccessKey, - const std::string& rSecretKey) + + // Constructor with a specific hostname, virtualhost name, port etc: + S3Client(const std::string& HostName, int Port, const std::string& rAccessKey, + const std::string& rSecretKey, const std::string& VirtualHostName = "") : mpSimulator(NULL), mHostName(HostName), mPort(Port), + mVirtualHostName(VirtualHostName), mAccessKey(rAccessKey), mSecretKey(rSecretKey), - mNetworkTimeout(30000) + mNetworkTimeout(600000) { } - - HTTPResponse GetObject(const std::string& rObjectURI); + + // Constructor with a Configuration (file): + S3Client(const Configuration& s3config) + : mpSimulator(NULL), + mHostName(s3config.GetKeyValue("HostName")), + mPort(s3config.GetKeyValueInt("Port")), + mVirtualHostName(s3config.GetKeyValue("S3VirtualHostName")), + mAccessKey(s3config.GetKeyValue("AccessKey")), + mSecretKey(s3config.GetKeyValue("SecretKey")), + mNetworkTimeout(600000) + { } + + class BucketEntry { + public: + BucketEntry(const std::string& name, const std::string& etag, + int64_t size) + : mName(name), + mEtag(etag), + mSize(size) + { } + const std::string& name() const { return mName; } + const std::string& etag() const { return mEtag; } + const int64_t size() const { return mSize; } + private: + std::string mName, mEtag; + int64_t mSize; + }; + + int ListBucket(std::vector* p_contents_out, + std::vector* p_common_prefixes_out, + const std::string& prefix = "", const std::string& delimiter = "/", + bool* p_truncated_out = NULL, int max_keys = -1, + const std::string& marker = ""); + HTTPResponse GetObject(const std::string& rObjectURI, + const std::string& MD5Checksum = ""); HTTPResponse HeadObject(const std::string& rObjectURI); HTTPResponse PutObject(const std::string& rObjectURI, IOStream& rStreamToSend, const char* pContentType = NULL); - void CheckResponse(const HTTPResponse& response, const std::string& message) const; + HTTPResponse DeleteObject(const std::string& rObjectURI); + void CheckResponse(const HTTPResponse& response, const std::string& message, + bool ExpectNoContent = false) const; int GetNetworkTimeout() const { return mNetworkTimeout; } private: HTTPServer* mpSimulator; + // mHostName is the network address that we will connect to (e.g. localhost): std::string mHostName; int mPort; + // mVirtualHostName is the Host header that we will send, e.g. + // "quotes.s3.amazonaws.com". If empty, mHostName will be used as a default. + std::string mVirtualHostName; std::auto_ptr mapClientSocket; std::string mAccessKey, mSecretKey; int mNetworkTimeout; // milliseconds @@ -68,7 +111,11 @@ class S3Client HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, const std::string& rRequestURI, IOStream* pStreamToSend = NULL, - const char* pStreamContentType = NULL); + const char* pStreamContentType = NULL, + const std::string& MD5Checksum = ""); + HTTPResponse FinishAndSendRequest(HTTPRequest request, + IOStream* pStreamToSend = NULL, const char* pStreamContentType = NULL, + const std::string& MD5Checksum = ""); HTTPResponse SendRequest(HTTPRequest& rRequest, IOStream* pStreamToSend = NULL, const char* pStreamContentType = NULL); diff --git a/lib/httpserver/S3Simulator.cpp b/lib/httpserver/S3Simulator.cpp index df8910d75..2f8450274 100644 --- a/lib/httpserver/S3Simulator.cpp +++ b/lib/httpserver/S3Simulator.cpp @@ -1,7 +1,7 @@ // -------------------------------------------------------------------------- // // File -// Name: S3Client.cpp +// Name: S3Simulator.cpp // Purpose: Amazon S3 client helper implementation class // Created: 09/01/2009 // @@ -9,25 +9,123 @@ #include "Box.h" +#ifdef HAVE_DIRENT_H +# include +#endif + +#include + #include +#include #include +#include -// #include -// #include - +#include #include +#include "Database.h" #include "HTTPRequest.h" #include "HTTPResponse.h" +#include "HTTPQueryDecoder.h" #include "autogen_HTTPException.h" #include "IOStream.h" #include "Logging.h" +#include "MD5Digest.h" #include "S3Simulator.h" +#include "Utils.h" // for ObjectExists_* (object_exists_t) #include "decode.h" #include "encode.h" #include "MemLeakFindOn.h" +#define PTREE_DOMAIN_NEXT_ID_SEQ "next_id_seq" +#define PTREE_ITEM_NAME "name" +#define PTREE_ITEM_ATTRIBUTES "attributes" + +using boost::property_tree::ptree; + +ptree XmlStringToPtree(const std::string& string) +{ + ptree pt; + std::istringstream stream(string); + read_xml(stream, pt, boost::property_tree::xml_parser::trim_whitespace); + return pt; +} + +std::string PtreeToXmlString(const ptree& pt) +{ + std::ostringstream buf; +// The original direct instantiation of xml_writer_settings stopped in 1.55: +// http://www.pcl-users.org/problem-getting-PCL-1-7-1-on-osx-10-9-td4035213.html +// The arguments to xml_writer_make_settings were changed backwards-incompatibly in Boost 1.56: +// https://github.com/boostorg/property_tree/commit/8af8b6bf3b65fa59792d849b526678f176d87132 +#if BOOST_VERSION >= 105600 + auto settings = boost::property_tree::xml_writer_make_settings('\t', 1); +#elif BOOST_VERSION >= 105500 + auto settings = boost::property_tree::xml_writer_make_settings('\t', 1); +#else + boost::property_tree::xml_writer_settings settings('\t', 1); +#endif + write_xml(buf, pt, settings); + return buf.str(); +} + + +S3Simulator::S3Simulator() +// Increase timeout to 5 minutes, from HTTPServer default of 1 minute, +// to help with debugging. +: HTTPServer(300000) +{ } + + +SimpleDBSimulator::SimpleDBSimulator() +: mDomainsFile("testfiles/domains.qdbm"), + mItemsFile("testfiles/items.qdbm") +{ + Open(DP_OWRITER | DP_OCREAT); +} + +// Open the database file +void SimpleDBSimulator::Open(int mode) +{ + mpDomains = dpopen(mDomainsFile.c_str(), mode, 0); + if(!mpDomains) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + BOX_DBM_MESSAGE("Failed to open domains database: " << mDomainsFile)); + } + + mpItems = dpopen(mItemsFile.c_str(), mode, 0); + if(!mpItems) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + BOX_DBM_MESSAGE("Failed to open items database: " << mItemsFile)); + } +} + +SimpleDBSimulator::~SimpleDBSimulator() +{ + Close(); +} + +void SimpleDBSimulator::Close() +{ + if(mpDomains && !dpclose(mpDomains)) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + BOX_DBM_MESSAGE("Failed to close domains database")); + } + mpDomains = NULL; + + if(mpItems && !dpclose(mpItems)) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + BOX_DBM_MESSAGE("Failed to close items database")); + } + mpItems = NULL; +} + + // -------------------------------------------------------------------------- // // Function @@ -76,6 +174,58 @@ const ConfigurationVerify* S3Simulator::GetConfigVerify() const return &verify; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::GetSortedQueryString(HTTPRequest&) +// Purpose: Returns a query string made from the supplied +// request, with the parameters sorted into alphabetical +// order as required by Amazon authentication. See +// http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/HMACAuth.html +// Created: 2015-11-15 +// +// -------------------------------------------------------------------------- + +std::string S3Simulator::GetSortedQueryString(const HTTPRequest& request) +{ + std::vector param_names; + std::map param_values; + + const HTTPRequest::Query_t& params(request.GetQuery()); + for(HTTPRequest::Query_t::const_iterator i = params.begin(); + i != params.end(); i++) + { + // We don't want to include the Signature parameter in the sorted query + // string, because the client didn't either when computing the signature! + if(i->first != "Signature") + { + param_names.push_back(i->first); + // This algorithm only supports non-repeated parameters, so + // assert that we don't already have a parameter with this name. + ASSERT(param_values.find(i->first) == param_values.end()); + param_values[i->first] = i->second; + } + } + + std::sort(param_names.begin(), param_names.end()); + std::ostringstream out; + + for(std::vector::iterator i = param_names.begin(); + i != param_names.end(); i++) + { + if(i != param_names.begin()) + { + out << "&"; + } + out << HTTPQueryDecoder::URLEncode(*i) << "=" << + HTTPQueryDecoder::URLEncode(param_values[*i]); + } + + return out.str(); +} + + // -------------------------------------------------------------------------- // // Function @@ -94,101 +244,197 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); rResponse.SetContentType("text/plain"); + bool is_simpledb = false; + if(rRequest.GetHostName() == SIMPLEDB_SIMULATOR_HOST) + { + is_simpledb = true; + } + + std::string bucket_name; + try { const Configuration& rConfig(GetConfiguration()); std::string access_key = rConfig.GetKeyValue("AccessKey"); std::string secret_key = rConfig.GetKeyValue("SecretKey"); + std::ostringstream buffer_to_sign; + buffer_to_sign << rRequest.GetMethodName() << "\n"; - std::string md5, date, bucket; - rRequest.GetHeader("content-md5", &md5); - rRequest.GetHeader("date", &date); + if(is_simpledb) + { + if(rRequest.GetParameterString("AWSAccessKeyId") != access_key) + { + THROW_EXCEPTION_MESSAGE(HTTPException, AuthenticationFailed, + "Unknown AWSAccessKeyId: " << + rRequest.GetParameterString("AWSAccessKeyId")); + } - std::string host = rRequest.GetHostName(); - std::string s3suffix = ".s3.amazonaws.com"; - if (host.size() > s3suffix.size()) + buffer_to_sign << rRequest.GetHostName() << "\n"; + buffer_to_sign << rRequest.GetRequestURI() << "\n"; + buffer_to_sign << GetSortedQueryString(rRequest); + } + else { - std::string suffix = host.substr(host.size() - - s3suffix.size(), s3suffix.size()); - if (suffix == s3suffix) + std::string md5, date; + rRequest.GetHeader("content-md5", &md5); + rRequest.GetHeader("date", &date); + + std::string host = rRequest.GetHostName(); + std::string s3suffix = ".s3.amazonaws.com"; + if (host.size() > s3suffix.size()) { - bucket = host.substr(0, host.size() - + std::string suffix = host.substr(host.size() - s3suffix.size(), s3suffix.size()); + + if (suffix == s3suffix) + { + bucket_name = host.substr(0, host.size() - s3suffix.size()); + } } - } - std::ostringstream data; - data << rRequest.GetVerb() << "\n"; - data << md5 << "\n"; - data << rRequest.GetContentType() << "\n"; - data << date << "\n"; + buffer_to_sign << md5 << "\n"; + buffer_to_sign << rRequest.GetContentType() << "\n"; + buffer_to_sign << date << "\n"; - // header names are already in lower case, i.e. canonical form + // header names are already in lower case, i.e. canonical form + std::vector headers = + rRequest.GetHeaders().GetExtraHeaders(); + std::sort(headers.begin(), headers.end()); - std::vector headers = rRequest.GetHeaders(); - std::sort(headers.begin(), headers.end()); + for (std::vector::iterator + i = headers.begin(); i != headers.end(); i++) + { + if (i->first.substr(0, 5) == "x-amz") + { + buffer_to_sign << i->first << ":" << + i->second << "\n"; + } + } - for (std::vector::iterator - i = headers.begin(); i != headers.end(); i++) - { - if (i->first.substr(0, 5) == "x-amz") + if (! bucket_name.empty()) { - data << i->first << ":" << i->second << "\n"; + buffer_to_sign << "/" << bucket_name; } - } - if (! bucket.empty()) - { - data << "/" << bucket; + buffer_to_sign << rRequest.GetRequestURI(); } - data << rRequest.GetRequestURI(); - std::string data_string = data.str(); + std::string string_to_sign = buffer_to_sign.str(); unsigned char digest_buffer[EVP_MAX_MD_SIZE]; unsigned int digest_size = sizeof(digest_buffer); - /* unsigned char* mac = */ HMAC(EVP_sha1(), + /* unsigned char* mac = */ HMAC( + is_simpledb ? EVP_sha256() : EVP_sha1(), secret_key.c_str(), secret_key.size(), - (const unsigned char*)data_string.c_str(), - data_string.size(), digest_buffer, &digest_size); - std::string digest((const char *)digest_buffer, digest_size); + (const unsigned char*)string_to_sign.c_str(), + string_to_sign.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + std::string expected_auth, actual_auth; base64::encoder encoder; - std::string expectedAuth = "AWS " + access_key + ":" + - encoder.encode(digest); - if (expectedAuth[expectedAuth.size() - 1] == '\n') + if(is_simpledb) + { + expected_auth = encoder.encode(digest); + actual_auth = rRequest.GetParameterString("Signature"); + } + else + { + expected_auth = "AWS " + access_key + ":" + encoder.encode(digest); + + if(!rRequest.GetHeader("authorization", &actual_auth)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, + AuthenticationFailed, "Missing Authorization header"); + } + } + + // The Base64 encoder tends to add a newline onto the end of the encoded + // string, which we don't want, so remove it here. + if(expected_auth[expected_auth.size() - 1] == '\n') { - expectedAuth = expectedAuth.substr(0, - expectedAuth.size() - 1); + expected_auth = expected_auth.substr(0, expected_auth.size() - 1); } - std::string actualAuth; - if (!rRequest.GetHeader("authorization", &actualAuth) || - actualAuth != expectedAuth) + if(actual_auth != expected_auth) { - rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); - SendInternalErrorResponse("Authentication Failed", - rResponse); + THROW_EXCEPTION_MESSAGE(HTTPException, + AuthenticationFailed, "Authentication code mismatch: " << + "expected " << expected_auth << " but received " << + actual_auth); + } + + if(is_simpledb && rRequest.GetMethod() == HTTPRequest::Method_GET) + { + HandleSimpleDBGet(rRequest, rResponse); + } + else if(is_simpledb) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unsupported Amazon SimpleDB Method"); } - else if (rRequest.GetMethod() == HTTPRequest::Method_GET) + else if(rRequest.GetMethod() == HTTPRequest::Method_GET && + (rRequest.GetRequestURI() == "" || + rRequest.GetRequestURI() == "/")) + { + HandleListObjects(bucket_name, rRequest, rResponse); + } + else if(rRequest.GetMethod() == HTTPRequest::Method_GET) { HandleGet(rRequest, rResponse); } - else if (rRequest.GetMethod() == HTTPRequest::Method_PUT) + else if(rRequest.GetMethod() == HTTPRequest::Method_HEAD) + { + HandleHead(rRequest, rResponse); + } + else if(rRequest.GetMethod() == HTTPRequest::Method_PUT) { HandlePut(rRequest, rResponse); } + else if(rRequest.GetMethod() == HTTPRequest::Method_DELETE) + { + HandleDelete(rRequest, rResponse); + } else { - rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed); - SendInternalErrorResponse("Unsupported Method", - rResponse); + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unsupported Amazon S3 Method: " << + rRequest.GetMethodName()); } } - catch (CommonException &ce) + catch (BoxException &ce) { SendInternalErrorResponse(ce.what(), rResponse); + + // Override the default status code 500 for a few specific exceptions. + if(EXCEPTION_IS_TYPE(ce, CommonException, OSFileOpenError)) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if(EXCEPTION_IS_TYPE(ce, CommonException, AccessDenied)) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + else if(EXCEPTION_IS_TYPE(ce, HTTPException, AuthenticationFailed)) + { + rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); + } + else if(EXCEPTION_IS_TYPE(ce, HTTPException, ConditionalRequestConflict)) + { + rResponse.SetResponseCode(HTTPResponse::Code_Conflict); + } + else if(EXCEPTION_IS_TYPE(ce, HTTPException, FileNotFound)) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if(EXCEPTION_IS_TYPE(ce, HTTPException, SimpleDBItemNotFound)) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if(EXCEPTION_IS_TYPE(ce, HTTPException, BadRequest)) + { + rResponse.SetResponseCode(HTTPResponse::Code_BadRequest); + } } catch (std::exception &e) { @@ -199,21 +445,252 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) SendInternalErrorResponse("Unknown exception", rResponse); } - if (rResponse.GetResponseCode() != 200 && + if (rResponse.GetResponseCode() != HTTPResponse::Code_OK && + rResponse.GetResponseCode() != HTTPResponse::Code_NotModified && + rResponse.GetResponseCode() != HTTPResponse::Code_NoContent && rResponse.GetSize() == 0) { - // no error message written, provide a default + // Looks like an error response with no error message specified, + // so write a default one. std::ostringstream s; s << rResponse.GetResponseCode(); SendInternalErrorResponse(s.str().c_str(), rResponse); } BOX_NOTICE(rResponse.GetResponseCode() << " " << rRequest.GetMethodName() << " " << - rRequest.GetRequestURI()); + rRequest.GetRequestURI(true)); // with_parameters_for_get_request return; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleListObjects( +// const std::string& bucket_name, +// HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 list objects request. +// Created: 15/03/2016 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleListObjects(const std::string& bucket_name, + HTTPRequest &request, HTTPResponse &response) +{ + if(bucket_name.empty()) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "A bucket name is required"); + } + + std::string delimiter = request.GetParameterString("delimiter", "/"); + if(delimiter != "/") + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, "Delimiter must be /"); + } + + std::string prefix = request.GetParameterString("prefix", ""); + if(prefix != "" && !EndsWith("/", prefix)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Prefix must be empty, or end with /, but was: '" << prefix << "'"); + } + + std::string marker = request.GetParameterString("marker", ""); + std::string max_keys_str = request.GetParameterString("max-keys", "1000"); + int max_keys; + { + char* p_end; + max_keys = strtol(max_keys_str.c_str(), &p_end, 10); + if(*p_end != 0) + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "max-keys parameter must be an integer: '" << + max_keys_str << "'"); + } + } + + std::string base_path = GetConfiguration().GetKeyValue("StoreDirectory"); + std::string prefixed_path = base_path + "/" + prefix; + if(!EndsWith("/", prefixed_path)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, Internal, + "Directory name must end with '/': " << prefixed_path); + } + RemoveSuffix("/", prefixed_path); + + DIR *p_dir = opendir(prefixed_path.c_str()); + if(p_dir == NULL) + { + THROW_EXCEPTION_MESSAGE(HTTPException, FileNotFound, + "Directory not found: " << prefixed_path); + } + + typedef std::map object_name_to_type_t; + object_name_to_type_t object_name_to_type; + + try + { + for(struct dirent* p_dirent = readdir(p_dir); p_dirent != NULL; + p_dirent = readdir(p_dir)) + { + std::string entry_name(p_dirent->d_name); + if(entry_name == "." || entry_name == "..") + { + continue; + } + + std::string entry_path = prefixed_path + DIRECTORY_SEPARATOR + + entry_name; + + // Prefix must be empty, or end with / + ASSERT(prefix == "" || EndsWith("/", prefix)); + std::string object_name = prefix + entry_name; + +#ifdef HAVE_VALID_DIRENT_D_TYPE + if(p_dirent->d_type == DT_UNKNOWN) +#else + // Always use this branch if we don't have struct dirent.d_type: + if(true) +#endif + { + int entry_type = ObjectExists(entry_path); + if(entry_type == ObjectExists_File) + { + object_name_to_type[object_name] = + ObjectExists_File; + } + else if(entry_type == ObjectExists_Dir) + { + object_name_to_type[object_name] = + ObjectExists_Dir; + } + else + { + continue; + } + } +#ifdef HAVE_VALID_DIRENT_D_TYPE + else if(p_dirent->d_type == DT_REG) + { + object_name_to_type[object_name] = ObjectExists_File; + } + else if(p_dirent->d_type == DT_DIR) + { + object_name_to_type[object_name] = ObjectExists_Dir; + } +#endif // HAVE_VALID_DIRENT_D_TYPE + else + { + continue; + } + } + } + catch(BoxException &e) + { + closedir(p_dir); + throw; + } + + ptree result; + result.add("Name", bucket_name); + result.add("Prefix", prefix); + result.add("Marker", marker); + result.add("Delimiter", delimiter); + + ptree common_prefixes; + + bool truncated = false; + int result_count = 0; + for(object_name_to_type_t::iterator i = object_name_to_type.lower_bound(marker); + i != object_name_to_type.end(); i++) + { + if(result_count >= max_keys) + { + truncated = true; + break; + } + + // Both Contents and CommonPrefixes count towards number of + // elements returned. Each CommonPrefix counts as a single return, + // regardless of the number of files it contains/abbreviates. + result_count++; + + if(i->second == ObjectExists_Dir) + { + common_prefixes.add("Prefix", i->first + delimiter); + continue; + } + + std::string entry_path = base_path + DIRECTORY_SEPARATOR + i->first; + int64_t size; + if(!FileExists(entry_path, &size, true)) // TreatLinksAsNotExisting + { + continue; + } + + ptree contents; + contents.add("Key", i->first); + + std::string digest; + { + std::auto_ptr ap_file; + ap_file.reset(new FileStream(entry_path)); + + MD5DigestStream digester; + ap_file->CopyStreamTo(digester); + ap_file->Seek(0, IOStream::SeekType_Absolute); + digester.Close(); + digest = """ + digester.DigestAsString() + """; + } + contents.add("ETag", digest); + + std::ostringstream size_stream; + size_stream << size; + contents.add("Size", size_stream.str()); + + result.add_child("Contents", contents); + } + + closedir(p_dir); + + result.add("IsTruncated", truncated ? "true" : "false"); + result.add_child("CommonPrefixes", common_prefixes); + + ptree response_tree; + response_tree.add_child("ListBucketResult", result); + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html + response.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); + response.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + response.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + response.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + response.AddHeader("Server", "AmazonS3"); + + response.SetResponseCode(HTTPResponse::Code_OK); + response.Write(PtreeToXmlString(response_tree)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleHead(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 HEAD request, i.e. downloading just +// the headers for an existing object. +// Created: 15/08/2015 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleHead(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + HandleGet(rRequest, rResponse, false); // no content +} + + // -------------------------------------------------------------------------- // // Function @@ -225,40 +702,476 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) // // -------------------------------------------------------------------------- -void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) +void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse, + bool IncludeContent) { std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); path += rRequest.GetRequestURI(); std::auto_ptr apFile; + apFile.reset(new FileStream(path)); + int64_t file_length; - try + std::string digest; { - apFile.reset(new FileStream(path)); + MD5DigestStream digester; + apFile->CopyStreamTo(digester); + file_length = apFile->GetPosition(); + apFile->Seek(0, IOStream::SeekType_Absolute); + digester.Close(); + digest = "\"" + digester.DigestAsString() + "\""; } - catch (CommonException &ce) + + rResponse.SetResponseCode(HTTPResponse::Code_OK); + + // For GET and HEAD requests, we must set the Content-Length. See RFC + // 2616 section 4.4, and the Amazon Simple Storage Service API + // Reference, section "HEAD Object" examples, which set it. Also, our + // S3BackupFileSystem needs it! + // + // There are no examples for 304 Not Modified responses to requests + // with If-None-Match (ETag match) so clients should not depend on + // this, so the S3Simulator should not set Content-Length or ETag, to + // ensure that any code which tries to use these headers will fail. + + std::string if_none_match = rRequest.GetHeaders().GetHeaderValue("if-none-match", + false); // required + if(digest == if_none_match) { - if (ce.GetSubType() == CommonException::OSFileOpenError) - { - rResponse.SetResponseCode(HTTPResponse::Code_NotFound); - } - else if (ce.GetSubType() == CommonException::AccessDenied) + rResponse.SetResponseCode(HTTPResponse::Code_NotModified); + rResponse.GetHeaders().SetContentLength(0); + IncludeContent = false; + } + else + { + rResponse.GetHeaders().SetContentLength(file_length); + rResponse.AddHeader("ETag", digest); + } + + if(IncludeContent) + { + apFile->CopyStreamTo(rResponse); + } + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html + rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("Server", "AmazonS3"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleDelete(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 DELETE request, i.e. deleting an +// existing object from the simulated store. +// Created: 27/01/2016 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleDelete(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); + path += rRequest.GetRequestURI(); + + // I think that DELETE is idempotent. + if(FileExists(path)) + { + if(EMU_UNLINK(path.c_str()) != 0) { - rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + THROW_SYS_FILE_ERROR("Failed to delete file", path, + HTTPException, S3SimulatorError); } - throw; } // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html - apFile->CopyStreamTo(rResponse); + rResponse.SetResponseCode(HTTPResponse::Code_NoContent); + rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); - rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); - rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); rResponse.AddHeader("Server", "AmazonS3"); +} + +ptree SimpleDBSimulator::GetDomainProps(const std::string& domain_name) +{ + char *result = dpget(mpDomains, domain_name.c_str(), domain_name.size(), + 0, -1, 0); + if(result == NULL) + { + THROW_EXCEPTION_MESSAGE(HTTPException, S3SimulatorError, + "Domain does not exist: " << domain_name); + } + std::string domain_props_str = result; + free(result); + return XmlStringToPtree(domain_props_str); +} + +void +SimpleDBSimulator::PutDomainProps(const std::string& domain_name, + const ptree domain_props) +{ + std::string domain_props_str = PtreeToXmlString(domain_props); + ASSERT_DBM_OK( + dpput(mpDomains, domain_name.c_str(), domain_name.size(), + domain_props_str.c_str(), domain_props_str.size(), DP_DOVER), + "Failed to create domain: " << domain_name, mDomainsFile, + HTTPException, S3SimulatorError); +} + +// Generate the key that will be used to store this Item in the mpItems database. +std::string ItemKey(const std::string& domain, std::string item_name) +{ + std::ostringstream key_stream; + key_stream << domain << "." << item_name; + return key_stream.str(); +} + +void SimpleDBSimulator::DeleteItem(const std::string& domain_name, + const std::string& item_name, bool throw_if_not_found) +{ + std::string key = ItemKey(domain_name, item_name); + bool result = dpout(mpItems, key.c_str(), key.size()); + if(!result && throw_if_not_found) + { + THROW_EXCEPTION_MESSAGE(HTTPException, S3SimulatorError, + "DeleteItem: Item does not exist: " << key); + } +} + +void ProcessConditionalRequest(const std::string& domain, HTTPRequest& rRequest, + SimpleDBSimulator& simpledb, bool delete_attributes) +{ + const HTTPRequest::Query_t& params(rRequest.GetParameters()); + + // Get the existing attributes for this item, if it exists. + std::multimap attributes = + simpledb.GetAttributes(domain, + rRequest.GetParameterString("ItemName"), + false); // !throw_if_not_found + + // Iterate over all parameters looking for "Attribute.X.Name" and + // "Attribute.X.Value" attributes, putting them into the + // param_index_to_* maps. Note that we keep the "index" as a string + // here, even though it should be an integer, to avoid needlessly + // converting back and forth. Hence all these maps are keyed on + // strings. + std::map param_index_to_name; + std::map param_index_to_value; + std::map param_index_to_replace; + // At the same time, add all "Expected.X.Name" and "Expected.X.Value" + // attributes to the expected_index_to_* maps. + std::map expected_index_to_name; + std::map expected_index_to_value; + + for(HTTPRequest::Query_t::const_iterator i = params.begin(); + i != params.end(); i++) + { + std::string param_name = i->first; + std::string param_value = i->second; + std::string param_number_type = RemovePrefix("Attribute.", + param_name, false); // !force + std::string expected_number_type = RemovePrefix("Expected.", + param_name, false); // !force + if(!param_number_type.empty()) + { + std::string param_index_name = RemoveSuffix(".Name", + param_number_type, false); // !force + std::string param_index_value = RemoveSuffix(".Value", + param_number_type, false); // !force + std::string param_index_replace = RemoveSuffix(".Replace", + param_number_type, false); // !force + if(!param_index_name.empty()) + { + param_index_to_name[param_index_name] = + param_value; + } + else if(!param_index_value.empty()) + { + param_index_to_value[param_index_value] = + param_value; + } + // Replace mode makes no sense when deleting matching attributes + else if(!param_index_replace.empty() && !delete_attributes) + { + param_index_to_replace[param_index_replace] = + (param_value == "true"); + } + else + { + THROW_EXCEPTION_MESSAGE(HTTPException, + S3SimulatorError, "PutAttributes: " + "Unparsed Attribute parameter: " << + param_name); + } + } + else if(!expected_number_type.empty()) + { + std::string expected_index_name = RemoveSuffix(".Name", + expected_number_type, false); // !force + std::string expected_index_value = RemoveSuffix(".Value", + expected_number_type, false); // !force + if(!expected_index_name.empty()) + { + expected_index_to_name[expected_index_name] = + param_value; + } + else if(!expected_index_value.empty()) + { + expected_index_to_value[expected_index_value] = + param_value; + } + else + { + THROW_EXCEPTION_MESSAGE(HTTPException, + S3SimulatorError, "PutAttributes: " + "Unparsed Expected parameter: " << + param_name); + } + } + } + + typedef std::multimap::iterator mm_iter_t; + + // Iterate over the expected maps, matching up the names and values, putting them + // into the expected_values map, which is easier to work with. + for(std::map::iterator + i = expected_index_to_name.begin(); + i != expected_index_to_name.end(); i++) + { + std::string index = i->first; + std::string attr_name = i->second; + + // pev = pointer to expected value + std::map::iterator pev = + expected_index_to_value.find(index); + + if(pev == expected_index_to_value.end()) + { + THROW_EXCEPTION_MESSAGE(HTTPException, + S3SimulatorError, "PutAttributes: Expected name with no " + "value: " << attr_name); + } + + std::string attr_value = pev->second; + bool matched_an_actual_value = false; + + // Loop over the actual values for this attribute name, looking for one + // that matches the expected value. + std::pair range = attributes.equal_range(attr_name); + + // pov = pointer to original/old/current value + for(mm_iter_t pov = range.first; pov != range.second; pov++) + { + if(pov->second == attr_value) + { + matched_an_actual_value = true; + break; + } + } + + if(!matched_an_actual_value) + { + THROW_EXCEPTION_MESSAGE(HTTPException, + ConditionalRequestConflict, "The value of attribute '" << + attr_name << "' was expected to be '" << attr_value << + "', but no matching value was found"); + } + } + + // Iterate over the attribute maps, matching up the names and values, putting them + // into (or removing them from) the item data XML tree. + for(std::map::iterator + i = param_index_to_name.begin(); + i != param_index_to_name.end(); i++) + { + std::string index = i->first; + std::string attr_name = i->second; + + // pnv = pointer to new value (or value to delete). + std::map::iterator pnv = + param_index_to_value.find(index); + std::string attr_value; + + if(pnv == param_index_to_value.end()) + { + THROW_EXCEPTION_MESSAGE(HTTPException, + S3SimulatorError, "PutAttributes: " + "Attribute name with no value: " << + attr_name); + } + else + { + attr_value = pnv->second; + } + + // If we are deleting values, then look for a matching pair, and if we + // find one, delete it. + if(delete_attributes) + { + bool deleted; + do + { + deleted = false; + std::pair range = + attributes.equal_range(attr_name); + + // Loop over all values for this attribute name + // (attr_name), which must all lie between range->first + // and range->second. + for(mm_iter_t + p_orig_attr = range.first; + p_orig_attr != range.second; + p_orig_attr++) + { + if(p_orig_attr->second == attr_value) + { + attributes.erase(p_orig_attr); + deleted = true; + // The iterator is not valid any more, so + // break out and search again. + break; + } + } + } + while(deleted); + } + else + { + // If the Replace parameter is provided and is "true", then + // delete any existing value, so only the new value inserted + // below will remain in the multimap. + std::map::iterator j = + param_index_to_replace.find(index); + if(j != param_index_to_replace.end() && j->second) + { + attributes.erase(attr_name); + } + + attributes.insert( + std::multimap::value_type( + attr_name, attr_value)); + } + } + + // If there are no attributes provided, and we are deleting attributes, then we + // should delete all of them (if the conditions matched, which we have already + // ascertained above). + if(param_index_to_name.empty() && delete_attributes) + { + attributes.clear(); + } + + // If there are no attributes remaining, then delete the whole item. + if(delete_attributes && attributes.empty()) + { + simpledb.DeleteItem(domain, rRequest.GetParameterString("ItemName")); + } + else + { + // Write the new item data XML tree back to the database, overwriting the + // previous values of all attributes. + simpledb.PutAttributes(domain, rRequest.GetParameterString("ItemName"), + attributes); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleSimpleDBGet(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an Amazon SimpleDB GET request. +// Created: 2015-11-16 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleSimpleDBGet(HTTPRequest &rRequest, HTTPResponse &rResponse, + bool IncludeContent) +{ + SimpleDBSimulator simpledb; + std::string action = rRequest.GetParameterString("Action"); + ptree response_tree; rResponse.SetResponseCode(HTTPResponse::Code_OK); + + std::string domain; + if(action != "ListDomains" && action != "Reset") + { + domain = rRequest.GetParameterString("DomainName"); + } + + if(action == "ListDomains") + { + std::vector domains = simpledb.ListDomains(); + + response_tree.add("ListDomainsResponse.ListDomainsResult", ""); + for(std::vector::iterator i = domains.begin(); + i != domains.end(); i++) + { + response_tree.add( + "ListDomainsResponse.ListDomainsResult.DomainName", *i); + } + } + else if(action == "CreateDomain") + { + simpledb.CreateDomain(domain); + response_tree.add("CreateDomainResponse", ""); + } + else if(action == "PutAttributes") + { + ProcessConditionalRequest(domain, rRequest, simpledb, + false); // delete_attributes + response_tree.add("PutAttributesResponse", ""); + } + else if(action == "GetAttributes") + { + std::multimap attributes = + simpledb.GetAttributes(domain, + rRequest.GetParameterString("ItemName")); + + // Ensure that the root element is present, even if there are no + // individual attributes. + response_tree.add("GetAttributesResponse.GetAttributesResult", ""); + + // Add the attributes to the response tree. + for(std::multimap::iterator + i = attributes.begin(); + i != attributes.end(); i++) + { + ptree attribute; + attribute.add("Name", i->first); + attribute.add("Value", i->second); + response_tree.add_child( + "GetAttributesResponse.GetAttributesResult.Attribute", + attribute); + } + } + else if(action == "DeleteAttributes") + { + ProcessConditionalRequest(domain, rRequest, simpledb, + true); // delete_attributes + response_tree.add("DeleteAttributesResponse", ""); + } + else if(action == "Reset") + { + simpledb.Reset(); + response_tree.add("ResetResponse", ""); + } + else + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + THROW_EXCEPTION_MESSAGE(HTTPException, S3SimulatorError, + "Unsupported SimpleDB Action: " << action); + } + + rResponse.Write(PtreeToXmlString(response_tree)); } + // -------------------------------------------------------------------------- // // Function @@ -272,13 +1185,40 @@ void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) { - std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); - path += rRequest.GetRequestURI(); + std::string base_path = GetConfiguration().GetKeyValue("StoreDirectory"); std::auto_ptr apFile; + // Amazon S3 has no explicit directories or directory creation operation, but we + // are using the filesystem for storage, so we need to ensure that any directories + // used in the file's path actually exist before we can create the file itself. + std::string file_uri = rRequest.GetRequestURI(); + for(std::string::size_type next_slash = file_uri.find('/', 1); + next_slash != std::string::npos; + next_slash = file_uri.find('/', next_slash + 1)) + { + std::string parent_dir_path = base_path + file_uri.substr(0, next_slash); + object_exists_t what_exists = ObjectExists(parent_dir_path); + if(what_exists == ObjectExists_NoObject) + { + // Does not exist, need to create it + mkdir(parent_dir_path.c_str(), 0755); + } + else if(what_exists == ObjectExists_Dir) + { + // Directory already exists, nothing to do + } + else + { + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Cannot create directory: something else already exists " + "with this name: " << parent_dir_path); + } + } + + std::string file_path = base_path + file_uri; try { - apFile.reset(new FileStream(path, O_CREAT | O_WRONLY)); + apFile.reset(new FileStream(file_path, O_CREAT | O_TRUNC | O_RDWR)); } catch (CommonException &ce) { @@ -295,18 +1235,148 @@ void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) if (rRequest.IsExpectingContinue()) { + BOX_TRACE("S3Simulator::HandlePut: sending Continue response"); rResponse.SendContinue(); } - rRequest.ReadContent(*apFile); + rRequest.ReadContent(*apFile, GetTimeout()); + BOX_TRACE("S3Simulator::HandlePut: read request data"); + apFile->Seek(0, IOStream::SeekType_Absolute); + + std::string digest; + { + MD5DigestStream digester; + apFile->CopyStreamTo(digester); + digester.Close(); + digest = "\"" + digester.DigestAsString() + "\""; + } // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7"); rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); - rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.AddHeader("ETag", digest); rResponse.SetContentType(""); rResponse.AddHeader("Server", "AmazonS3"); rResponse.SetResponseCode(HTTPResponse::Code_OK); } + + +std::vector SimpleDBSimulator::ListDomains() +{ + if(!dpiterinit(mpDomains)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, S3SimulatorError, + "Failed to start iterating over domains database"); + } + + std::vector domains; + for(char *key = dpiternext(mpDomains, NULL); + key != NULL; + key = dpiternext(mpDomains, NULL)) + { + domains.push_back(key); + } + + return domains; +} + +void SimpleDBSimulator::Reset() +{ + Close(); + Open(DP_OWRITER | DP_OCREAT | DP_OTRUNC); +} + +void SimpleDBSimulator::CreateDomain(const std::string& domain_name) +{ + char *result = dpget(mpDomains, domain_name.c_str(), domain_name.size(), 0, 0, 0); + if(result != NULL) + { + free(result); + // "CreateDomain is an idempotent operation; running it multiple times + // using the same domain name will not result in an error response." + return; + } + + ptree domain_props; + domain_props.add(PTREE_DOMAIN_NEXT_ID_SEQ, 1); + PutDomainProps(domain_name, domain_props); +} + + +void SimpleDBSimulator::PutAttributes(const std::string& domain_name, + const std::string& item_name, + const std::multimap attributes) +{ + ptree item_data; + + // Iterate over the attribute map, adding names and values to the item data + // structure (XML tree). + for(std::multimap::const_iterator + i = attributes.begin(); + i != attributes.end(); i++) + { + std::string path = PTREE_ITEM_ATTRIBUTES "." + i->first; + item_data.add(path, i->second); + } + + // Generate the key that will be used to store this item data in the + // mpItems database. + std::string key = ItemKey(domain_name, item_name); + std::string value = PtreeToXmlString(item_data); + ASSERT_DBM_OK( + dpput(mpItems, key.c_str(), key.size(), value.c_str(), + value.size(), DP_DOVER), + "PutAttributes: Failed to add item", mItemsFile, + HTTPException, S3SimulatorError); +} + +std::multimap SimpleDBSimulator::GetAttributes( + const std::string& domain_name, + const std::string& item_name, + bool throw_if_not_found) +{ + std::multimap attributes; + std::string key = ItemKey(domain_name, item_name); + char* result = dpget(mpItems, key.c_str(), key.size(), 0, -1, 0); + if(result == NULL) + { + if(throw_if_not_found) + { + THROW_EXCEPTION_MESSAGE(HTTPException, SimpleDBItemNotFound, + "GetAttributes: Item does not exist: " << key); + } + else + { + return attributes; + } + } + + std::string item_data_str = result; + free(result); + ptree item_data = XmlStringToPtree(item_data_str); + + // There might not be any attributes, e.g. if they have all been deleted, + // so we need to check for and handle that situation, as it's not an error. + try + { + // Iterate over the attributes in the item data tree, adding names and values + // to the attributes map. + BOOST_FOREACH(ptree::value_type &v, + item_data.get_child(PTREE_ITEM_ATTRIBUTES)) + { + std::string name = v.first; + std::string value = v.second.data(); + attributes.insert( + std::multimap::value_type(name, + value)); + } + } + catch(boost::property_tree::ptree_bad_path &e) + { + // Do nothing, just don't add any attributes to the list. + } + + return attributes; +} diff --git a/lib/httpserver/S3Simulator.h b/lib/httpserver/S3Simulator.h index eef4f4000..2a887b7c2 100644 --- a/lib/httpserver/S3Simulator.h +++ b/lib/httpserver/S3Simulator.h @@ -10,12 +10,60 @@ #ifndef S3SIMULATOR__H #define S3SIMULATOR__H +#include +#include +#include + #include "HTTPServer.h" +#include "depot.h" class ConfigurationVerify; class HTTPRequest; class HTTPResponse; +#define SIMPLEDB_SIMULATOR_HOST "sdb.localhost" + +// -------------------------------------------------------------------------- +// +// Class +// Name: SimpleDBSimulator +// Purpose: Amazon SimpleDB simulation interface. +// Created: 2015-11-21 +// +// -------------------------------------------------------------------------- +class SimpleDBSimulator +{ +public: + SimpleDBSimulator(); + ~SimpleDBSimulator(); + + std::vector ListDomains(); + void CreateDomain(const std::string& domain_name); + void PutAttributes(const std::string& domain_name, + const std::string& item_name, + const std::multimap attributes); + std::multimap GetAttributes( + const std::string& domain_name, + const std::string& item_name, + bool throw_if_not_found = true); + void DeleteItem(const std::string& domain_name, const std::string& item_name, + bool throw_if_not_found = true); + void Reset(); + +protected: + void Open(int mode); + void Close(); + boost::property_tree::ptree GetDomainProps(const std::string& domain_name); + void PutDomainProps(const std::string& domain_name, + const boost::property_tree::ptree domain_props); + +private: + DEPOT* mpDomains; + DEPOT* mpItems; + std::string mDomainsFile, mItemsFile; +}; + + // -------------------------------------------------------------------------- // // Class @@ -27,15 +75,21 @@ class HTTPResponse; class S3Simulator : public HTTPServer { public: - // Increase timeout to 5 minutes, from HTTPServer default of 1 minute, - // to help with debugging. - S3Simulator() : HTTPServer(300000) { } + S3Simulator(); ~S3Simulator() { } const ConfigurationVerify* GetConfigVerify() const; virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); - virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleListObjects(const std::string& bucket_name, + HTTPRequest &request, HTTPResponse &response); + virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse, + bool IncludeContent = true); virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleHead(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleDelete(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleSimpleDBGet(HTTPRequest &rRequest, HTTPResponse &rResponse, + bool IncludeContent = true); + std::string GetSortedQueryString(const HTTPRequest& request); virtual const char *DaemonName() const { diff --git a/lib/httpserver/SimpleDBClient.cpp b/lib/httpserver/SimpleDBClient.cpp new file mode 100644 index 000000000..1e1113b1c --- /dev/null +++ b/lib/httpserver/SimpleDBClient.cpp @@ -0,0 +1,536 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SimpleDBClient.cpp +// Purpose: Amazon SimpleDB client class +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#include +#include + +#include "HTTPQueryDecoder.h" +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "autogen_HTTPException.h" +#include "SimpleDBClient.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::GenerateQueryString( +// const HTTPRequest& request) +// Purpose: Generates and returns an HTTP query string for the +// parameters of the supplied HTTPRequest, using the +// specific format required for SimpleDB request +// authentication signatures. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +// http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/HMACAuth.html +std::string SimpleDBClient::GenerateQueryString(const HTTPRequest& request) +{ + std::vector param_names; + std::map param_values; + + const HTTPRequest::Query_t& params(request.GetQuery()); + for(HTTPRequest::Query_t::const_iterator i = params.begin(); + i != params.end(); i++) + { + // We don't want to include the Signature parameter in the sorted query + // string, because the client didn't either when computing the signature! + if(i->first != "Signature") + { + param_names.push_back(i->first); + // This algorithm only supports non-repeated parameters, so + // assert that we don't already have a parameter with this name. + ASSERT(param_values.find(i->first) == param_values.end()); + param_values[i->first] = i->second; + } + } + + std::sort(param_names.begin(), param_names.end()); + std::ostringstream out; + + for(std::vector::iterator i = param_names.begin(); + i != param_names.end(); i++) + { + if(i != param_names.begin()) + { + out << "&"; + } + out << HTTPQueryDecoder::URLEncode(*i) << "=" << + HTTPQueryDecoder::URLEncode(param_values[*i]); + } + + return out.str(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::CalculateSimpleDBSignature( +// const HTTPRequest& request) +// Purpose: Calculates and returns an Amazon SimpleDB auth +// signature for the supplied HTTPRequest, based on its +// parameters and the access and secret keys that this +// SimpleDBClient was initialised with. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +// http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/HMACAuth.html +std::string SimpleDBClient::CalculateSimpleDBSignature(const HTTPRequest& request) +{ + // This code is very similar to that in S3Client::FinishAndSendRequest, + // but using EVP_sha256 instead of EVP_sha1. TODO FIXME: factor out the + // common parts. + std::string query_string = GenerateQueryString(request); + if(query_string.empty()) + { + THROW_EXCEPTION_MESSAGE(HTTPException, Internal, + "Failed to get query string for request"); + } + + std::ostringstream buffer_to_sign; + buffer_to_sign << request.GetMethodName() << "\n" << + request.GetHeaders().GetHostNameWithPort() << "\n" << + // The HTTPRequestURI component is the HTTP absolute path component + // of the URI up to, but not including, the query string. If the + // HTTPRequestURI is empty, use a forward slash ( / ). + request.GetRequestURI() << "\n" << + query_string; + + // Thanks to https://gist.github.com/tsupo/112188: + unsigned int digest_size; + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + std::string string_to_sign = buffer_to_sign.str(); + + HMAC(EVP_sha256(), + mSecretKey.c_str(), mSecretKey.size(), + (const unsigned char *)string_to_sign.c_str(), string_to_sign.size(), + digest_buffer, &digest_size); + + base64::encoder encoder; + std::string digest((const char *)digest_buffer, digest_size); + std::string auth_code = encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + return auth_code; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::SendAndReceive(HTTPRequest& request, +// HTTPResponse& response, int expected_status_code) +// Purpose: Private method. Sends the supplied HTTPRequest to +// Amazon SimpleDB, and stores the response in the +// supplied HTTPResponse. Since SimpleDB responses are +// usually XML formatted, you may prefer to use +// SendAndReceiveXML() instead, which parses the reply +// XML for you. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +void SimpleDBClient::SendAndReceive(HTTPRequest& request, HTTPResponse& response, + int expected_status_code) +{ + SocketStream sock; + sock.Open(Socket::TypeINET, mHostName, mPort ? mPort : 80); + + // Send() throws exceptions if anything goes wrong. + request.Send(sock, mTimeout); + + // Reset the response in case it has been used before. + response.Reset(); + response.Receive(sock, mTimeout); + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + if(response.GetResponseCode() != expected_status_code) + { + if(response.GetResponseCode() == HTTPResponse::Code_NotFound) + { + THROW_EXCEPTION_MESSAGE(HTTPException, SimpleDBItemNotFound, + "Expected a " << expected_status_code << " response but " + "received a " << response.GetResponseCode() << " " + "instead: " << response_data); + } + else if(response.GetResponseCode() == HTTPResponse::Code_Conflict) + { + THROW_EXCEPTION_MESSAGE(HTTPException, ConditionalRequestConflict, + "Expected a " << expected_status_code << " response but " + "received a " << response.GetResponseCode() << " " + "instead: " << response_data); + } + else + { + THROW_EXCEPTION_MESSAGE(HTTPException, RequestFailedUnexpectedly, + "Expected a " << expected_status_code << " response but " + "received a " << response.GetResponseCode() << " " + "instead: " << response_data); + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::SendAndReceive(HTTPRequest& request, +// ptree& response_tree, +// const std::string& expected_root_element) +// Purpose: Private method. Sends the supplied HTTPRequest to +// Amazon SimpleDB, parses the response as XML into +// the supplied Boost property_tree, and checks that +// the root element is what you expected. The root +// element is part of the command specification, so +// this ensures that the command was processed +// correctly by SimpleDB (or the simulator). +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +void SimpleDBClient::SendAndReceiveXML(HTTPRequest& request, ptree& response_tree, + const std::string& expected_root_element) +{ + HTTPResponse response; + SendAndReceive(request, response); + + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + std::auto_ptr ap_response_stream( + new std::istringstream(response_data)); + read_xml(*ap_response_stream, response_tree, + boost::property_tree::xml_parser::trim_whitespace); + + if(response_tree.begin()->first != expected_root_element) + { + THROW_EXCEPTION_MESSAGE(HTTPException, UnexpectedResponseData, + "Expected response to start with <" << expected_root_element << + "> but found <" << response_tree.begin()->first << "> instead"); + } + + ASSERT(++(response_tree.begin()) == response_tree.end()); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::StartRequest( +// HTTPRequest::Method method, +// const std::string& action) +// Purpose: Private method. Initialises an HTTPRequest for the +// specified method (usually HTTP GET) and action +// (e.g. CreateDomain, PutAttributes). You will need to +// add any additional parameters specific to that +// action to the resulting request, and sign it, before +// calling SendAndReceiveXML() to execute it. The +// request contains a timestamp, and must be executed +// within some time of its creation (15 minutes?) or +// SimpleDB will treat it as expired, and refuse to +// honour it, to prevent replay attacks. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +HTTPRequest SimpleDBClient::StartRequest(HTTPRequest::Method method, + const std::string& action) +{ + HTTPRequest request(method, "/"); + if(!mEndpoint.empty()) + { + request.SetHostName(mEndpoint); + } + else + { + request.SetHostName(mHostName); + } + request.AddParameter("Action", action); + request.AddParameter("AWSAccessKeyId", mAccessKey); + request.AddParameter("SignatureVersion", "2"); + request.AddParameter("SignatureMethod", "HmacSHA256"); + + box_time_t timestamp = mFixedTimestamp; + if(timestamp == 0) + { + timestamp = GetCurrentBoxTime(); + } + + // Generate a timestamp of the format: "2010-01-25T15:01:28-07:00" + // Ideally we'd use GMT as recommended, and end with "Z", but the signed example + // uses -07:00 instead, so we need to support timezones to replicate it (bah!) + // + // http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/HMACAuth.html#AboutTimestamp + // http://www.w3.org/TR/xmlschema-2/#dateTime + // + // If the timestamp is in timezone -07:00, then it's 7 hours later in GMT, + // so we add the offset of -7 * 60 minutes and then use gmtime() to get the time + // components in that timezone. + std::ostringstream buf; + time_t seconds = BoxTimeToSeconds(timestamp) + (mOffsetMinutes * 60); + struct tm tm_now, *tm_ptr = &tm_now; + +#ifdef WIN32 + if((tm_ptr = gmtime(&seconds)) == NULL) +#else + if(gmtime_r(&seconds, &tm_now) == NULL) +#endif + { + THROW_SYS_ERROR("Failed to convert timestamp to components", + CommonException, Internal); + } + + buf << std::setfill('0'); + buf << std::setw(4) << (tm_ptr->tm_year + 1900) << "-" << + std::setw(2) << (tm_ptr->tm_mon + 1) << "-" << + std::setw(2) << (tm_ptr->tm_mday) << "T"; + buf << std::setw(2) << tm_ptr->tm_hour << ":" << + std::setw(2) << tm_ptr->tm_min << ":" << + std::setw(2) << tm_ptr->tm_sec; + + if(mOffsetMinutes) + { + std::div_t rem_quo = std::div(mOffsetMinutes, 60); + int hours = rem_quo.quot; + int mins = rem_quo.rem; + buf << std::showpos << std::internal << std::setw(3) << hours << ":" << + std::noshowpos << std::setw(2) << mins; + } + else + { + buf << "Z"; + } + + request.AddParameter("Timestamp", buf.str()); + request.AddParameter("Version", "2009-04-15"); + return request; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::ListDomains() +// Purpose: Returns a list of domains (a vector of strings) +// which are defined in SimpleDB for the account that +// this client is configured with. This list is global +// to the account, so this function takes no +// parameters. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +SimpleDBClient::list_t SimpleDBClient::ListDomains() +{ + HTTPRequest request = StartRequest(HTTPRequest::Method_GET, "ListDomains"); + request.AddParameter("Signature", CalculateSimpleDBSignature(request)); + + // Send directly to in-process simulator, useful for debugging. + // CollectInBufferStream response_buffer; + // HTTPResponse response(&response_buffer); + HTTPResponse response; + ptree response_tree; + SendAndReceiveXML(request, response_tree, "ListDomainsResponse"); + + list_t domains; + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child("ListDomainsResponse.ListDomainsResult")) + { + domains.push_back(v.second.data()); + } + + return domains; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::CreateDomain() +// Purpose: Creates a new domain in the SimpleDB domain +// namespace for this client's account. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +void SimpleDBClient::CreateDomain(const std::string& domain_name) +{ + HTTPRequest request = StartRequest(HTTPRequest::Method_GET, "CreateDomain"); + request.AddParameter("DomainName", domain_name); + request.AddParameter("Signature", CalculateSimpleDBSignature(request)); + + HTTPResponse response; + ptree response_tree; + SendAndReceiveXML(request, response_tree, "CreateDomainResponse"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::GetAttributes( +// const std::string& domain_name, +// const std::string& item_name, +// bool consistent_read) +// Purpose: Get the attributes of the specified item in the +// specified domain (previously created with +// CreateDomain). +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +SimpleDBClient::str_map_t SimpleDBClient::GetAttributes(const std::string& domain_name, + const std::string& item_name, bool consistent_read) +{ + HTTPRequest request = StartRequest(HTTPRequest::Method_GET, "GetAttributes"); + request.AddParameter("DomainName", domain_name); + request.AddParameter("ItemName", item_name); + if(consistent_read) + { + request.AddParameter("ConsistentRead", "true"); + } + request.AddParameter("Signature", CalculateSimpleDBSignature(request)); + + ptree response_tree; + try + { + SendAndReceiveXML(request, response_tree, "GetAttributesResponse"); + } + catch(HTTPException &e) + { + if(EXCEPTION_IS_TYPE(e, HTTPException, SimpleDBItemNotFound)) + { + THROW_EXCEPTION_MESSAGE(HTTPException, SimpleDBItemNotFound, + "The requested SimpleDB item '" << item_name << "' " + "was not found in domain '" << domain_name << "'"); + } + else + { + throw; + } + } + + str_map_t attributes; + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child( + "GetAttributesResponse.GetAttributesResult")) + { + std::string name = v.second.get("Name"); + std::string value = v.second.get("Value"); + attributes[name] = value; + } + + return attributes; +} + +void SimpleDBClient::AddPutAttributes(HTTPRequest& request, const str_map_t& attributes, + const str_map_t& expected, bool add_required) +{ + int counter = 1; + for(str_map_t::const_iterator i = attributes.begin(); i != attributes.end(); i++) + { + std::ostringstream oss; + oss << "Attribute."; + oss << counter++; + request.AddParameter(oss.str() + ".Name", i->first); + request.AddParameter(oss.str() + ".Value", i->second); + + if(add_required) + { + request.AddParameter(oss.str() + ".Replace", "true"); + } + } + + counter = 1; + for(str_map_t::const_iterator i = expected.begin(); i != expected.end(); i++) + { + std::ostringstream oss; + oss << "Expected."; + oss << counter++; + request.AddParameter(oss.str() + ".Name", i->first); + request.AddParameter(oss.str() + ".Value", i->second); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::PutAttributes( +// const std::string& domain_name, +// const std::string& item_name, +// const SimpleDBClient::str_map_t& attributes, +// const SimpleDBClient::str_map_t& expected) +// Purpose: Create or update an item with the specified name, +// giving it the specified attributes. If the item +// already exists, a new value for any attribute will +// always replace any old value, but existing +// attributes which are not modified are preserved. +// If any expected values are provided, the PUT +// operation is conditional on those values, and will +// throw an exception if the item has different values +// for those attributes. +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +void SimpleDBClient::PutAttributes(const std::string& domain_name, + const std::string& item_name, const SimpleDBClient::str_map_t& attributes, + const SimpleDBClient::str_map_t& expected) +{ + HTTPRequest request = StartRequest(HTTPRequest::Method_GET, "PutAttributes"); + request.AddParameter("DomainName", domain_name); + request.AddParameter("ItemName", item_name); + + AddPutAttributes(request, attributes, expected, true); // add_required + request.AddParameter("Signature", CalculateSimpleDBSignature(request)); + + ptree response_tree; + SendAndReceiveXML(request, response_tree, "PutAttributesResponse"); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: SimpleDBClient::DeleteAttributes( +// const std::string& domain_name, +// const std::string& item_name, +// const SimpleDBClient::str_map_t& attributes) +// Purpose: Deletes one or more attributes associated with the +// item. If all attributes of an item are deleted, the +// item is deleted. If you specify DeleteAttributes +// without attributes or values, all the attributes for +// the item are deleted (and hence the item itself). +// Created: 09/01/2016 +// +// -------------------------------------------------------------------------- +// http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/SDB_API_DeleteAttributes.html + +void SimpleDBClient::DeleteAttributes(const std::string& domain_name, + const std::string& item_name, const SimpleDBClient::str_map_t& attributes, + const SimpleDBClient::str_map_t& expected) +{ + HTTPRequest request = StartRequest(HTTPRequest::Method_GET, "DeleteAttributes"); + request.AddParameter("DomainName", domain_name); + request.AddParameter("ItemName", item_name); + + AddPutAttributes(request, attributes, expected, false); // add_required + request.AddParameter("Signature", CalculateSimpleDBSignature(request)); + + ptree response_tree; + SendAndReceiveXML(request, response_tree, "DeleteAttributesResponse"); +} + diff --git a/lib/httpserver/SimpleDBClient.h b/lib/httpserver/SimpleDBClient.h new file mode 100644 index 000000000..389edefd2 --- /dev/null +++ b/lib/httpserver/SimpleDBClient.h @@ -0,0 +1,112 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SimpleDBClient.h +// Purpose: Amazon SimpleDB client class +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +#ifndef SIMPLEDBCLIENT__H +#define SIMPLEDBCLIENT__H + +#include +#include +#include + +#include + +#include "BoxTime.h" +#include "Configuration.h" +#include "HTTPRequest.h" + +using boost::property_tree::ptree; + +class HTTPResponse; +class HTTPServer; + +// -------------------------------------------------------------------------- +// +// Class +// Name: SimpleDBClient +// Purpose: Amazon S3 client helper implementation class +// Created: 04/01/2016 +// +// -------------------------------------------------------------------------- + +class SimpleDBClient +{ +private: + std::string mHostName, mEndpoint, mAccessKey, mSecretKey; + box_time_t mFixedTimestamp; + int mOffsetMinutes, mPort, mTimeout; + +public: + // Note: endpoint controls the Host: header. If not set, the hostname is used. If + // you want to connect to a particular host and send a different Host: header, + // then set both hostname (to connect to) and endpoint (for the host header), + // otherwise leave endpoint empty. + SimpleDBClient(const std::string& access_key, const std::string& secret_key, + const std::string& hostname = "sdb.eu-west-1.amazonaws.com", int port = 0, + const std::string& endpoint = "", + // Set a default timeout of 300 seconds to make debugging easier + int timeout = 300000) + : mHostName(hostname), + mEndpoint(endpoint), + mAccessKey(access_key), + mSecretKey(secret_key), + mFixedTimestamp(0), + mOffsetMinutes(0), + mPort(port), + mTimeout(timeout) + { } + + SimpleDBClient(const Configuration& s3config) + : mHostName(s3config.GetKeyValue("SimpleDBHostName")), + mEndpoint(s3config.GetKeyValue("SimpleDBEndpoint")), + mAccessKey(s3config.GetKeyValue("AccessKey")), + mSecretKey(s3config.GetKeyValue("SecretKey")), + mFixedTimestamp(0), + mOffsetMinutes(0), + mPort(s3config.GetKeyValueInt("SimpleDBPort")), + // Set a default timeout of 300 seconds to make debugging easier + mTimeout(300000) + { } + + typedef std::vector list_t; + typedef std::map str_map_t; + + list_t ListDomains(); + void CreateDomain(const std::string& domain_name); + str_map_t GetAttributes(const std::string& domain_name, + const std::string& item, bool consistent_read = true); + void PutAttributes(const std::string& domain_name, + const std::string& item_name, const str_map_t& attributes, + const str_map_t& expected = str_map_t()); + void DeleteAttributes(const std::string& domain_name, + const std::string& item_name, const str_map_t& attributes, + const str_map_t& expected = str_map_t()); + + // These shouldn't really be APIs, but exposing them makes it easier to test + // this class. + HTTPRequest StartRequest(HTTPRequest::Method method, const std::string& action); + std::string GenerateQueryString(const HTTPRequest& request); + void SetFixedTimestamp(box_time_t fixed_timestamp, int offset_minutes) + { + mFixedTimestamp = fixed_timestamp; + mOffsetMinutes = offset_minutes; + } + +private: + void SendAndReceive(HTTPRequest& request, HTTPResponse& response, + int expected_status_code = 200); + void SendAndReceiveXML(HTTPRequest& request, ptree& response_tree, + const std::string& expected_root_element); + std::string CalculateSimpleDBSignature(const HTTPRequest& request); + void AddPutAttributes(HTTPRequest& request, const str_map_t& attributes, + const str_map_t& expected, bool add_required); +}; + + +#endif // SIMPLEDBCLIENT__H + diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp index 11c59d629..57dcc1886 100644 --- a/lib/httpserver/cdecode.cpp +++ b/lib/httpserver/cdecode.cpp @@ -12,7 +12,9 @@ extern "C" int base64_decode_value(char value_in) { - static signed const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + // Need to use "signed char" explicitly to work around char being unsigned by default + // on ARM, causing compile errors: https://stackoverflow.com/a/31635045/648162 + static const signed char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; static const char decoding_size = sizeof(decoding); value_in -= 43; if (value_in < 0 || value_in > decoding_size) return -1; diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 72bd8d4e9..ad9fc55aa 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -598,6 +598,11 @@ lstat(const char *file_name, STAT_STRUCT *buf) { #ifdef LINUX_WEIRD_LSTAT lstat_real = (lstat_t*)find_function("__lxstat"); + #elif defined HAVE_LSTAT64 + // If lstat64 is defined, assume that the OS uses it (instead + // of stat) by default now that the 64-bit transition is + // basically complete. + lstat_real = (lstat_t*)find_function("lstat64"); #else lstat_real = (lstat_t*)find_function("lstat"); #endif @@ -642,6 +647,11 @@ stat(const char *file_name, STAT_STRUCT *buf) { #ifdef LINUX_WEIRD_LSTAT stat_real = (lstat_t*)find_function("__xstat"); + #elif defined HAVE_STAT64 + // If stat64 is defined, assume that the OS uses it (instead + // of stat) by default now that the 64-bit transition is + // basically complete. + stat_real = (lstat_t*)find_function("stat64"); #else stat_real = (lstat_t*)find_function("stat"); #endif diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index 4de5f9f26..8a025d884 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -18,11 +18,15 @@ #endif #if defined __NetBSD_Version__ && __NetBSD_Version__ >= 399000800 //3.99.8 vers. -#define FUNC_OPENDIR "__opendir30" -#define FUNC_READDIR "__readdir30" +# define FUNC_OPENDIR "__opendir30" +# define FUNC_READDIR "__readdir30" +#elif defined __APPLE__ +// https://github.com/rust-lang/libc/issues/414#issuecomment-251246205 +# define FUNC_OPENDIR "opendir$INODE64" +# define FUNC_READDIR "readdir$INODE64" #else -#define FUNC_OPENDIR "opendir" -#define FUNC_READDIR "readdir" +# define FUNC_OPENDIR "opendir" +# define FUNC_READDIR "readdir" #endif #include @@ -34,15 +38,15 @@ extern "C" typedef struct dirent *(readdir_t) (DIR *dir); typedef struct dirent *(readdir_t) (DIR *dir); typedef int (closedir_t)(DIR *dir); -#if defined __GNUC__ && __GNUC__ >= 2 +#if defined HAVE___LXSTAT64 || defined HAVE___LXSTAT + // Linux glibc implements the stat function using inline redirection + // to __lxstat(64), so that's what we have to intercept. #define LINUX_WEIRD_LSTAT #define STAT_STRUCT struct stat /* should be stat64 */ - typedef int (lstat_t) (int ver, const char *file_name, - STAT_STRUCT *buf); + typedef int (lstat_t) (int ver, const char *file_name, STAT_STRUCT *buf); #else #define STAT_STRUCT struct stat - typedef int (lstat_t) (const char *file_name, - STAT_STRUCT *buf); + typedef int (lstat_t) (const char *file_name, STAT_STRUCT *buf); #endif } diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index 7b755395c..c027e8881 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -1583,7 +1583,8 @@ bool RaidFileRead::FileExists(int SetNumber, const std::string &rFilename, int64 // Created: 2003/08/20 // // -------------------------------------------------------------------------- -bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector &rOutput) +bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, + DirReadType_t DirReadType, std::vector &rOutput) { // Remove anything in the vector to begin with. rOutput.clear(); diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h index a3c792d08..2399ad3a0 100644 --- a/lib/raidfile/RaidFileRead.h +++ b/lib/raidfile/RaidFileRead.h @@ -56,16 +56,19 @@ class RaidFileRead : public IOStream static bool FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID = 0); static bool DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName); static bool DirectoryExists(int SetNumber, const std::string &rDirName); - enum + typedef enum { DirReadType_FilesOnly = 0, DirReadType_DirsOnly = 1 - }; - static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector &rOutput); + } DirReadType_t; + static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, + DirReadType_t DirReadType, std::vector &rOutput); // Common IOStream interface implementation virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamClosed(); virtual pos_type BytesLeftToRead(); diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 8f95ba657..3bd534c3a 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -57,7 +57,8 @@ RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename) : mSetNumber(SetNumber), mFilename(Filename), mOSFileHandle(-1), // not valid file handle - mRefCount(-1) // unknown refcount + mRefCount(-1), // unknown refcount + mAllowOverwrite(false) // safe default { } @@ -76,7 +77,8 @@ RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename, : mSetNumber(SetNumber), mFilename(Filename), mOSFileHandle(-1), // not valid file handle - mRefCount(refcount) + mRefCount(refcount), + mAllowOverwrite(false) // safe default { // Can't check for zero refcount here, because it's legal // to create a RaidFileWrite to delete an object with zero refcount. @@ -142,7 +144,9 @@ void RaidFileWrite::Open(bool AllowOverwrite) { THROW_EXCEPTION(RaidFileException, AlreadyOpen) } - + + mAllowOverwrite = AllowOverwrite; + // Get disc set RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); @@ -329,7 +333,10 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) THROW_EXCEPTION(RaidFileException, NotOpen) } - if (mRefCount == 0) + // It's allowed to create a file with no references, but you must pass + // AllowOverwrite = false when opening it to assert that it doesn't already + // exist. You cannot modify an existing file with no references. + if(mRefCount == 0 && mAllowOverwrite) { THROW_FILE_ERROR("Attempted to modify object file with " "no references", mTempFilename, RaidFileException, @@ -358,7 +365,7 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) #ifdef WIN32 // need to delete the target first - if(::unlink(renameTo.c_str()) != 0) + if(EMU_UNLINK(renameTo.c_str()) != 0) { DWORD errorNumber = GetLastError(); if (errorNumber != ERROR_FILE_NOT_FOUND) @@ -421,9 +428,9 @@ void RaidFileWrite::Discard() #ifdef WIN32 // On Win32 we must close it first if (::close(mOSFileHandle) != 0 || - ::unlink(writeFilename.c_str()) != 0) + EMU_UNLINK(writeFilename.c_str()) != 0) #else // !WIN32 - if (::unlink(writeFilename.c_str()) != 0 || + if (EMU_UNLINK(writeFilename.c_str()) != 0 || ::close(mOSFileHandle) != 0) #endif // !WIN32 { @@ -520,15 +527,12 @@ void RaidFileWrite::TransformToRaidStorage() // Then open them all for writing (in strict order) try { -#if HAVE_DECL_O_EXLOCK - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe1(stripe1FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe2(stripe2FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> parity(parityFilenameW.c_str()); -#else - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe1(stripe1FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe2(stripe2FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> parity(parityFilenameW.c_str()); -#endif + FileStream stripe1(stripe1FilenameW, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755, + FileStream::EXCLUSIVE); + FileStream stripe2(stripe2FilenameW, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755, + FileStream::EXCLUSIVE); + FileStream parity(parityFilenameW, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755, + FileStream::EXCLUSIVE); // Then... read in data... int bytesRead = -1; @@ -608,10 +612,7 @@ void RaidFileWrite::TransformToRaidStorage() } // Write block - if(::write(parity, parityBuffer, parityWriteSize) != parityWriteSize) - { - THROW_EXCEPTION(RaidFileException, OSError) - } + parity.Write(parityBuffer, parityWriteSize); } // Write stripes @@ -622,10 +623,14 @@ void RaidFileWrite::TransformToRaidStorage() int toWrite = (l == (blocksToDo - 1)) ?(bytesRead - ((blocksToDo-1)*blockSize)) :blockSize; - if(::write(((l&1)==0)?stripe1:stripe2, writeFrom, toWrite) != toWrite) + if((l & 1) == 0) { - THROW_EXCEPTION(RaidFileException, OSError) - } + stripe1.Write(writeFrom, toWrite); + } + else + { + stripe2.Write(writeFrom, toWrite); + } // Next block writeFrom += blockSize; @@ -634,10 +639,12 @@ void RaidFileWrite::TransformToRaidStorage() // Count of blocks done blocksDone += blocksToDo; } + // Error on read? if(bytesRead == -1) { - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to read from temporary RAID file", writeFilename, + RaidFileException, OSError); } // Special case for zero length files @@ -652,13 +659,8 @@ void RaidFileWrite::TransformToRaidStorage() { ASSERT(sizeof(writeFileStat.st_size) <= sizeof(RaidFileRead::FileSizeType)); RaidFileRead::FileSizeType sw = box_hton64(writeFileStat.st_size); - ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0); - if(::write(parity, &sw, sizeof(sw)) != sizeof(sw)) - { - BOX_LOG_SYS_ERROR("Failed to write to file: " << - writeFilename); - THROW_EXCEPTION(RaidFileException, OSError) - } + ASSERT((parity.GetPosition() % blockSize) == 0); + parity.Write(&sw, sizeof(sw)); } // Then close the written files (note in reverse order of opening) @@ -667,10 +669,10 @@ void RaidFileWrite::TransformToRaidStorage() stripe1.Close(); #ifdef WIN32 - // Must delete before renaming + // Must delete any existing file before renaming over it #define CHECK_UNLINK(file) \ { \ - if (::unlink(file) != 0 && errno != ENOENT) \ + if (EMU_UNLINK(file) != 0 && errno != ENOENT) \ { \ THROW_EMU_ERROR("Failed to unlink raidfile " \ "stripe: " << file, RaidFileException, \ @@ -688,29 +690,28 @@ void RaidFileWrite::TransformToRaidStorage() || ::rename(stripe2FilenameW.c_str(), stripe2Filename.c_str()) != 0 || ::rename(parityFilenameW.c_str(), parityFilename.c_str()) != 0) { - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_ERROR("Failed to rename file", RaidFileException, OSError); } // Close the write file writeFile.Close(); // Finally delete the write file - if(::unlink(writeFilename.c_str()) != 0) + if(EMU_UNLINK(writeFilename.c_str()) != 0) { - BOX_LOG_SYS_ERROR("Failed to delete file: " << - writeFilename); - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to delete file", writeFilename, + RaidFileException, OSError); } } catch(...) { // Unlink all the dodgy files - ::unlink(stripe1Filename.c_str()); - ::unlink(stripe2Filename.c_str()); - ::unlink(parityFilename.c_str()); - ::unlink(stripe1FilenameW.c_str()); - ::unlink(stripe2FilenameW.c_str()); - ::unlink(parityFilenameW.c_str()); + EMU_UNLINK(stripe1Filename.c_str()); + EMU_UNLINK(stripe2Filename.c_str()); + EMU_UNLINK(parityFilename.c_str()); + EMU_UNLINK(stripe1FilenameW.c_str()); + EMU_UNLINK(stripe2FilenameW.c_str()); + EMU_UNLINK(parityFilenameW.c_str()); // and send the error on its way throw; @@ -754,7 +755,7 @@ void RaidFileWrite::Delete() // Attempt to delete it bool deletedSomething = false; - if(::unlink(writeFilename.c_str()) == 0) + if(EMU_UNLINK(writeFilename.c_str()) == 0) { deletedSomething = true; } @@ -769,15 +770,15 @@ void RaidFileWrite::Delete() std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 0 % TRANSFORM_NUMBER_DISCS_REQUIRED)); std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 1 % TRANSFORM_NUMBER_DISCS_REQUIRED)); std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 2 % TRANSFORM_NUMBER_DISCS_REQUIRED)); - if(::unlink(stripe1Filename.c_str()) == 0) + if(EMU_UNLINK(stripe1Filename.c_str()) == 0) { deletedSomething = true; } - if(::unlink(stripe2Filename.c_str()) == 0) + if(EMU_UNLINK(stripe2Filename.c_str()) == 0) { deletedSomething = true; } - if(::unlink(parityFilename.c_str()) == 0) + if(EMU_UNLINK(parityFilename.c_str()) == 0) { deletedSomething = true; } diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h index ab9b399ad..380ff9857 100644 --- a/lib/raidfile/RaidFileWrite.h +++ b/lib/raidfile/RaidFileWrite.h @@ -50,6 +50,8 @@ class RaidFileWrite : public IOStream virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); // will exception virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual void Close(); // will discard the file! Use commit instead. @@ -73,6 +75,7 @@ class RaidFileWrite : public IOStream std::string mFilename, mTempFilename; int mOSFileHandle; int mRefCount; + bool mAllowOverwrite; }; #endif // RAIDFILEWRITE__H diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index d3c8441f9..c519d9fa0 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -67,22 +67,18 @@ Daemon::Daemon() : mReloadConfigWanted(false), mTerminateWanted(false), #ifdef WIN32 - mSingleProcess(true), - mRunInForeground(true), + mDaemonize(false), + mForkPerClient(false), mKeepConsoleOpenAfterFork(true), #else - mSingleProcess(false), - mRunInForeground(false), + mDaemonize(true), + mForkPerClient(true), mKeepConsoleOpenAfterFork(false), #endif mHaveConfigFile(false), mLogFileLevel(Log::INVALID), mAppName(DaemonName()) { - // In debug builds, switch on assert failure logging to syslog - ASSERT_FAILS_TO_SYSLOG_ON - // And trace goes to syslog too - TRACE_TO_SYSLOG(true) } // -------------------------------------------------------------------------- @@ -112,9 +108,9 @@ Daemon::~Daemon() std::string Daemon::GetOptionString() { return std::string("c:" - #ifndef WIN32 +#ifndef WIN32 "DF" - #endif +#endif "hkKo:O:") + Logging::OptionParser::GetOptionString(); } @@ -131,8 +127,8 @@ void Daemon::Usage() " argument is the configuration file, or else the default \n" " [" << GetConfigFileName() << "]\n" #ifndef WIN32 - " -D Debugging mode, do not fork, one process only, one client only\n" - " -F Do not fork into background, but fork to serve multiple clients\n" + " -D Do not daemonize (fork into background)\n" + " -F Do not fork a new process for each client\n" #endif " -k Keep console open after fork, keep writing log messages to it\n" " -K Stop writing log messages to console while daemon is running\n" @@ -166,13 +162,13 @@ int Daemon::ProcessOption(signed int option) #ifndef WIN32 case 'D': { - mSingleProcess = true; + mDaemonize = false; } break; case 'F': { - mRunInForeground = true; + mForkPerClient = false; } break; #endif // !WIN32 @@ -299,11 +295,6 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) mHaveConfigFile = true; } - if (argc > optind && ::strcmp(argv[optind], "SINGLEPROCESS") == 0) - { - mSingleProcess = true; optind++; - } - if (argc > optind) { BOX_FATAL("Unknown parameter on command line: " @@ -447,8 +438,6 @@ int Daemon::Main(const std::string &rConfigFileName) std::string pidFileName; - bool asDaemon = !mSingleProcess && !mRunInForeground; - try { if (!Configure(rConfigFileName)) @@ -490,7 +479,7 @@ int Daemon::Main(const std::string &rConfigFileName) daemonUser.ChangeProcessUser(); } - if(asDaemon) + if(mDaemonize) { // Let's go... Daemonise... switch(::fork()) @@ -568,11 +557,11 @@ int Daemon::Main(const std::string &rConfigFileName) #endif // !WIN32 // Write PID to file - char pid[32]; - - int pidsize = snprintf(pid, sizeof(pid), "%d", (int)getpid()); + std::ostringstream pid_buf; + pid_buf << getpid(); + std::string pid_str = pid_buf.str(); - if(::write(pidFile, pid, pidsize) != pidsize) + if(::write(pidFile, pid_str.c_str(), pid_str.size()) != pid_str.size()) { BOX_LOG_SYS_FATAL("Failed to write PID file: " << pidFileName); @@ -587,7 +576,7 @@ int Daemon::Main(const std::string &rConfigFileName) } #endif // BOX_MEMORY_LEAK_TESTING - if(asDaemon && !mKeepConsoleOpenAfterFork) + if(mDaemonize && !mKeepConsoleOpenAfterFork) { #ifndef WIN32 // Close standard streams @@ -611,10 +600,6 @@ int Daemon::Main(const std::string &rConfigFileName) { ::close(devnull); } - - // And definitely don't try and send anything to those file descriptors - // -- this has in the past sent text to something which isn't expecting it. - TRACE_TO_STDOUT(false); #endif // ! WIN32 Logging::ToConsole(false); } @@ -641,20 +626,6 @@ int Daemon::Main(const std::string &rConfigFileName) return 1; } -#ifdef WIN32 - // Under win32 we must initialise the Winsock library - // before using sockets - - WSADATA info; - - if (WSAStartup(0x0101, &info) == SOCKET_ERROR) - { - // will not run without sockets - BOX_FATAL("Failed to initialise Windows Sockets"); - THROW_EXCEPTION(CommonException, Internal) - } -#endif - int retcode = 0; // Main Daemon running @@ -698,7 +669,7 @@ int Daemon::Main(const std::string &rConfigFileName) } // Delete the PID file - ::unlink(pidFileName.c_str()); + EMU_UNLINK(pidFileName.c_str()); // Log BOX_NOTICE("Terminating daemon"); @@ -726,7 +697,7 @@ int Daemon::Main(const std::string &rConfigFileName) #else // Should clean up here, but it breaks memory leak tests. /* - if(asDaemon) + if(mDaemonize) { // we are running in the child by now, and should not return mapConfiguration.reset(); diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index b53849187..2c978600b 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -70,19 +70,26 @@ class Daemon virtual void EnterChild(); static void SetProcessTitle(const char *format, ...); - void SetRunInForeground(bool foreground) + bool IsForkPerClient() { - mRunInForeground = foreground; + return mForkPerClient; } - void SetSingleProcess(bool value) + void SetForkPerClient(bool fork_per_client) { - mSingleProcess = value; + mForkPerClient = fork_per_client; + } + bool IsDaemonize() + { + return mDaemonize; + } + void SetDaemonize(bool daemonize) + { + mDaemonize = daemonize; } protected: virtual void SetupInInitialProcess(); box_time_t GetLoadedConfigModifiedTime() const; - bool IsSingleProcess() { return mSingleProcess; } virtual std::string GetOptionString(); virtual int ProcessOption(signed int option); void ResetLogFile() @@ -104,8 +111,8 @@ class Daemon box_time_t mLoadedConfigModifiedTime; bool mReloadConfigWanted; bool mTerminateWanted; - bool mSingleProcess; - bool mRunInForeground; + bool mDaemonize; + bool mForkPerClient; bool mKeepConsoleOpenAfterFork; bool mHaveConfigFile; Logging::OptionParser mLogLevel; diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 0adf95439..2ce6f1820 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -118,7 +118,8 @@ void Protocol::Handshake() int bytesRead = mapConn->Read(readInto, bytesToRead, GetTimeout()); if(bytesRead == 0) { - THROW_EXCEPTION(ConnectionException, Protocol_Timeout) + THROW_EXCEPTION_MESSAGE(ConnectionException, Protocol_Timeout, + "Timed out waiting " << GetTimeout() << " ms to read handshake"); } readInto += bytesRead; bytesToRead -= bytesRead; @@ -162,7 +163,8 @@ void Protocol::CheckAndReadHdr(void *hdr) if(!mapConn->ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Protocol_Timeout) + THROW_EXCEPTION_MESSAGE(ConnectionException, Protocol_Timeout, + "Timed out waiting " << mTimeout << " ms to read message header"); } } @@ -205,7 +207,8 @@ std::auto_ptr Protocol::ReceiveInternal() if(!mapConn->ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Protocol_Timeout) + THROW_EXCEPTION_MESSAGE(ConnectionException, Protocol_Timeout, + "Timed out waiting " << mTimeout << " ms to read message contents"); } // Setup ready to read out data from the buffer @@ -763,10 +766,22 @@ void Protocol::SendStream(IOStream &rStream) else { // Fixed size stream, send it all in one go - if(!rStream.CopyStreamTo(*mapConn, GetTimeout(), - 4096 /* slightly larger buffer */)) + try + { + rStream.CopyStreamTo(*mapConn, GetTimeout(), + 4096 /* slightly larger buffer */); + } + catch(CommonException &e) { - THROW_EXCEPTION(ConnectionException, Protocol_TimeOutWhenSendingStream) + if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_TimeOutWhenSendingStream, e.GetMessage()); + } + else + { + throw; + } } } // Make sure everything is written diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h index 2e97ba6af..7a7275216 100644 --- a/lib/server/ProtocolUncertainStream.h +++ b/lib/server/ProtocolUncertainStream.h @@ -35,6 +35,8 @@ class ProtocolUncertainStream : public IOStream virtual pos_type BytesLeftToRead(); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp index f1a718dff..2eecb6493 100644 --- a/lib/server/ServerControl.cpp +++ b/lib/server/ServerControl.cpp @@ -51,17 +51,32 @@ bool SendCommands(const std::string& rCmd) // Wait for the configuration summary std::string configSummary; - if(!getLine.GetLine(configSummary)) + while(true) { - BOX_ERROR("Failed to receive configuration summary from daemon"); - return false; - } + try + { + configSummary = getLine.GetLine(false /* no preprocess */); + break; + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF)) + { + BOX_ERROR("Server rejected the connection"); + } + else + { + BOX_ERROR("Failed to receive configuration summary from daemon: " << + e.what()); + } - // Was the connection rejected by the server? - if(getLine.IsEOF()) - { - BOX_ERROR("Server rejected the connection"); - return false; + return false; + } } // Decode it @@ -92,11 +107,36 @@ bool SendCommands(const std::string& rCmd) connection.Write(cmds.c_str(), cmds.size()); // Read the response - std::string line; bool statusOk = !expectResponse; - while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + while (expectResponse) { + std::string line; + try + { + line = getLine.GetLine(false /* no preprocessing */, + 120000); // 2 minute timeout for tests + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // try again + continue; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, GetLineEOF) || + EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + BOX_WARNING("Disconnected by daemon or timed out waiting for " + "response: " << e.what()); + break; + } + else + { + throw; + } + } + // Is this an OK or error line? if (line == "ok") { @@ -198,6 +238,7 @@ bool KillServer(int pid, bool WaitForProcess) } #endif + BOX_INFO("Waiting for server to die (pid " << pid << ")"); printf("Waiting for server to die (pid %d): ", pid); for (int i = 0; i < 300; i++) @@ -231,7 +272,7 @@ bool KillServer(std::string pid_file, bool WaitForProcess) { FileStream fs(pid_file); IOStreamGetLine getline(fs); - std::string line = getline.GetLine(); + std::string line = getline.GetLine(false); // !preprocess int pid = atoi(line.c_str()); bool status = KillServer(pid, WaitForProcess); TEST_EQUAL_LINE(true, status, std::string("kill(") + pid_file + ")"); @@ -239,7 +280,7 @@ bool KillServer(std::string pid_file, bool WaitForProcess) #ifdef WIN32 if(WaitForProcess) { - int unlink_result = unlink(pid_file.c_str()); + int unlink_result = EMU_UNLINK(pid_file.c_str()); TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); if(unlink_result != 0) { @@ -274,7 +315,7 @@ bool StopDaemon(int current_pid, const std::string& pid_file, TEST_THAT_OR(!ServerIsAlive(current_pid), return false); #ifdef WIN32 - int unlink_result = unlink(pid_file.c_str()); + int unlink_result = EMU_UNLINK(pid_file.c_str()); TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); if(unlink_result != 0) { diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 3f6eed7e8..936b3d313 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -118,10 +118,6 @@ class ServerStream : public Daemon { // Child task, dump leaks to trace, which we make sure is on #ifdef BOX_MEMORY_LEAK_TESTING - #ifndef BOX_RELEASE_BUILD - TRACE_TO_SYSLOG(true); - TRACE_TO_STDOUT(true); - #endif memleakfinder_traceblocksinsection(); #endif @@ -208,7 +204,7 @@ class ServerStream : public Daemon } // unlink anything there - ::unlink(c[1].c_str()); + EMU_UNLINK(c[1].c_str()); psocket->Listen(Socket::TypeUNIX, c[1].c_str()); #endif // WIN32 @@ -258,9 +254,9 @@ class ServerStream : public Daemon // Was there one (there should be...) if(connection.get()) { - // Since this is a template parameter, the if() will be optimised out by the compiler #ifndef WIN32 // no fork on Win32 - if(ForkToHandleRequests && !IsSingleProcess()) + // Since this is a template parameter, the if() will be optimised out by the compiler + if(IsForkPerClient()) { pid_t pid = ::fork(); switch(pid) @@ -306,7 +302,20 @@ class ServerStream : public Daemon #endif // !WIN32 // Just handle in this process SetProcessTitle("handling"); - HandleConnection(connection); + + try + { + HandleConnection(connection); + } + catch(BoxException &e) + { + // When only a single process is handling requests, then don't rethrow the + // exception, since that would kill the entire server process. Instead, + // just log it and keep going. + BOX_ERROR("Failed to process a request in single-process mode: " + "caught exception: " << e.what()); + } + SetProcessTitle("idle"); #ifndef WIN32 } @@ -318,16 +327,17 @@ class ServerStream : public Daemon #ifndef WIN32 // Clean up child processes (if forking daemon) - if(ForkToHandleRequests && !IsSingleProcess()) + if(IsForkPerClient()) { WaitForChildren(); } #endif // !WIN32 } } - catch(...) + catch(std::exception &e) { DeleteSockets(); + // Allow the exception to kill the worker process, if uncaught higher up: throw; } @@ -400,11 +410,7 @@ class ServerStream : public Daemon // depends on the forking model in case someone changes it later. bool WillForkToHandleRequests() { - #ifdef WIN32 - return false; - #else - return ForkToHandleRequests && !IsSingleProcess(); - #endif // WIN32 + return IsForkPerClient(); } private: diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index c9c1773db..44752fcb2 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -52,30 +52,24 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, { // Lookup hostname struct hostent *phost = ::gethostbyname(rName.c_str()); - if(phost != NULL) + if(phost != NULL && phost->h_addr_list[0] != 0) { - if(phost->h_addr_list[0] != 0) - { - sockAddrLen = sizeof(addr.sa_inet); + sockAddrLen = sizeof(addr.sa_inet); #ifdef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN - addr.sa_inet.sin_len = sizeof(addr.sa_inet); + addr.sa_inet.sin_len = sizeof(addr.sa_inet); #endif - addr.sa_inet.sin_family = PF_INET; - addr.sa_inet.sin_port = htons(Port); - addr.sa_inet.sin_addr = *((in_addr*)phost->h_addr_list[0]); - for(unsigned int l = 0; l < sizeof(addr.sa_inet.sin_zero); ++l) - { - addr.sa_inet.sin_zero[l] = 0; - } - } - else + addr.sa_inet.sin_family = PF_INET; + addr.sa_inet.sin_port = htons(Port); + addr.sa_inet.sin_addr = *((in_addr*)phost->h_addr_list[0]); + for(unsigned int l = 0; l < sizeof(addr.sa_inet.sin_zero); ++l) { - THROW_EXCEPTION(ConnectionException, SocketNameLookupError); + addr.sa_inet.sin_zero[l] = 0; } } else { - THROW_EXCEPTION(ConnectionException, SocketNameLookupError); + THROW_SOCKET_ERROR("Failed to resolve hostname: " << rName, + ConnectionException, SocketNameLookupError); } } break; diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index 39fe7e240..36712f075 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -231,7 +231,7 @@ class SocketListen } // poll this socket - struct pollfd p; + EMU_STRUCT_POLLFD p; p.fd = mSocketHandle; p.events = POLLIN; p.revents = 0; diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index edb5e5b89..dd373de71 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -13,12 +13,16 @@ #include #endif -#include #include #include -#ifndef WIN32 +#include + +#ifdef WIN32 + #include // for InetNtop +#else #include + #include // for inet_ntop #endif #ifdef HAVE_UCRED_H @@ -182,13 +186,44 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) } // Connect it + std::string name_pretty; + if(Type == Socket::TypeUNIX) + { + // For a UNIX socket, the path is all we need to show the user: + name_pretty = rName; + } + else + { + // For an IP socket, try to include the resolved IP address as well as the hostname: + std::ostringstream name_oss; + char name_buf[256]; +#ifdef WIN32 + const char* addr_str = InetNtop(sockDomain, &addr.sa_generic, name_buf, + sizeof(name_buf)); +#else + const char* addr_str = inet_ntop(sockDomain, &addr.sa_generic, name_buf, + sizeof(name_buf)); +#endif + name_oss << rName << " ("; + if(addr_str == NULL) + { + name_oss << "failed to convert IP address to string"; + } + else + { + name_oss << addr_str; + } + name_oss << ")"; + name_pretty = name_oss.str(); + } + if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) { // Dispose of the socket try { THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, - BOX_SOCKET_ERROR_MESSAGE(Type, rName, Port, + BOX_SOCKET_ERROR_MESSAGE(Type, name_pretty, Port, "Failed to connect to socket")); } catch(ServerException &e) @@ -228,7 +263,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) if(Timeout != IOStream::TimeOutInfinite) { - struct pollfd p; + EMU_STRUCT_POLLFD p; p.fd = mSocketHandle; p.events = POLLIN; p.revents = 0; @@ -238,15 +273,12 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) // error if(errno == EINTR) { - // Signal. Just return 0 bytes - return 0; + THROW_EXCEPTION(CommonException, SignalReceived); } else { - // Bad! - BOX_LOG_SYS_ERROR("Failed to poll socket"); - THROW_EXCEPTION(ServerException, - SocketPollError) + THROW_SYS_ERROR("Failed to poll socket", ServerException, + SocketPollError); } break; @@ -276,8 +308,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) else { // Other error - BOX_LOG_SYS_ERROR("Failed to read from socket"); - THROW_EXCEPTION(ConnectionException, + THROW_SOCKET_ERROR("Failed to read from socket", ConnectionException, SocketReadError); } } @@ -295,7 +326,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) bool SocketStream::Poll(short Events, int Timeout) { // Wait for data to send. - struct pollfd p; + EMU_STRUCT_POLLFD p; p.fd = GetSocketHandle(); p.events = Events; p.revents = 0; @@ -362,7 +393,7 @@ void SocketStream::Write(const void *pBuffer, int NBytes, int Timeout) { // Error. mWriteClosed = true; // assume can't write again - THROW_SYS_ERROR("Failed to write to socket", + THROW_SOCKET_ERROR("Failed to write to socket", ConnectionException, SocketWriteError); } @@ -566,6 +597,6 @@ void SocketStream::CheckForMissingTimeout(int Timeout) if (Timeout == IOStream::TimeOutInfinite) { BOX_WARNING("Network operation started with no timeout!"); - DumpStackBacktrace(); + OPTIONAL_DO_BACKTRACE; } } diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index fd57af8fc..33864f624 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -50,13 +50,7 @@ class SocketStream : public IOStream virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - - // Why not inherited from IOStream? Never mind, we want to enforce - // supplying a timeout for network operations anyway. - virtual void Write(const std::string& rBuffer, int Timeout) - { - IOStream::Write(rBuffer, Timeout); - } + using IOStream::Write; virtual void Close(); virtual bool StreamDataLeft(); diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h index 3fda98c14..2724bac95 100644 --- a/lib/server/SocketStreamTLS.h +++ b/lib/server/SocketStreamTLS.h @@ -45,6 +45,8 @@ class SocketStreamTLS : public SocketStream virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual void Close(); virtual void Shutdown(bool Read = true, bool Write = true); diff --git a/lib/server/TcpNice.cpp b/lib/server/TcpNice.cpp index 79e91eebf..36fd57358 100644 --- a/lib/server/TcpNice.cpp +++ b/lib/server/TcpNice.cpp @@ -30,6 +30,7 @@ #include "MemLeakFindOn.h" +#ifdef ENABLE_TCP_NICE // -------------------------------------------------------------------------- // // Function @@ -130,15 +131,15 @@ NiceSocketStream::NiceSocketStream(std::auto_ptr apSocket) // -------------------------------------------------------------------------- // // Function -// Name: NiceSocketStream::Write(const void *pBuffer, int NBytes) -// Purpose: Writes bytes to the underlying stream, adjusting window size -// using a TcpNice calculator. +// Name: NiceSocketStream::Write(const void *pBuffer, +// int NBytes, int Timeout) +// Purpose: Writes bytes to the underlying stream, adjusting +// window size using a TcpNice calculator. // Created: 2012/02/11 // // -------------------------------------------------------------------------- -void NiceSocketStream::Write(const void *pBuffer, int NBytes) +void NiceSocketStream::Write(const void *pBuffer, int NBytes, int Timeout) { -#if HAVE_DECL_SO_SNDBUF && HAVE_DECL_TCP_INFO if(mEnabled && mapTimer.get() && mapTimer->HasExpired()) { box_time_t newPeriodStart = GetCurrentBoxTime(); @@ -147,6 +148,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) int rtt = 50; // WAG # if HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT + // The Linux way struct tcp_info info; socklen_t optlen = sizeof(info); if(getsockopt(socket, SOL_TCP, TCP_INFO, &info, &optlen) == -1) @@ -164,6 +166,27 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) { rtt = info.tcpi_rtt; } +# elif HAVE_DECL_IPPROTO_TCP && defined HAVE_DECL_TCP_CONNECTION_INFO + // The OSX way: https://stackoverflow.com/a/40478874/648162 + struct tcp_connection_info info; + socklen_t optlen = sizeof(info); + if(getsockopt(socket, IPPROTO_TCP, TCP_CONNECTION_INFO, &info, &optlen) == -1) + { + BOX_LOG_SYS_WARNING("getsockopt(" << socket << ", IPPROTO_TCP, " + "TCP_CONNECTION_INFO) failed"); + } + else if(optlen < sizeof(info)) + { + BOX_WARNING("getsockopt(" << socket << ", IPPROTO_TCP, " + "TCP_CONNECTION_INFO) return structure size " << optlen << ", " + "expected " << sizeof(info)); + } + else + { + rtt = info.tcpi_rttcur; + } +# else +# error "Don't know how to get current TCP RTT" # endif // HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT int newWindow = mTcpNice.GetNextWindowSize(mBytesWrittenThisPeriod, @@ -191,9 +214,8 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) } mBytesWrittenThisPeriod += NBytes; -#endif // HAVE_DECL_SO_SNDBUF - mapSocket->Write(pBuffer, NBytes); + mapSocket->Write(pBuffer, NBytes, Timeout); } // -------------------------------------------------------------------------- @@ -213,7 +235,6 @@ void NiceSocketStream::SetEnabled(bool enabled) if(!enabled) { StopTimer(); -#if HAVE_DECL_SO_SNDBUF int socket = mapSocket->GetSocketHandle(); int newWindow = 1<<17; if(setsockopt(socket, SOL_SOCKET, SO_SNDBUF, @@ -230,6 +251,7 @@ void NiceSocketStream::SetEnabled(bool enabled) BOX_LOG_SYS_WARNING("getsockopt(" << socket << ", SOL_SOCKET, " "SO_SNDBUF, " << newWindow << ") failed"); } -#endif } } + +#endif // ENABLE_TCP_NICE diff --git a/lib/server/TcpNice.h b/lib/server/TcpNice.h index 4381df42d..215a0bd93 100644 --- a/lib/server/TcpNice.h +++ b/lib/server/TcpNice.h @@ -18,6 +18,16 @@ #include "SocketStream.h" #include "Timer.h" +#if HAVE_DECL_SO_SNDBUF +# if HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT +# define ENABLE_TCP_NICE +# elif HAVE_DECL_IPPROTO_TCP && HAVE_DECL_TCP_CONNECTION_INFO +// https://stackoverflow.com/a/40478874/648162 +# define ENABLE_TCP_NICE +# endif +#endif + +#ifdef ENABLE_TCP_NICE // -------------------------------------------------------------------------- // // Class @@ -128,7 +138,9 @@ class NiceSocketStream : public SocketStream } // This is the only magic - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; // Everything else is delegated to the sink virtual int Read(void *pBuffer, int NBytes, @@ -174,5 +186,6 @@ class NiceSocketStream : public SocketStream NiceSocketStream(const NiceSocketStream &rToCopy) { /* do not call */ } }; +#endif // ENABLE_TCP_NICE #endif // TCPNICE__H diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 5473c690b..e9ac63a13 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -40,18 +40,13 @@ class WinNamedPipeStream : public IOStream int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + using IOStream::Write; + virtual void WriteAllBuffered(); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); - // Why not inherited from IOStream? Never mind, we want to enforce - // supplying a timeout for network operations anyway. - virtual void Write(const std::string& rBuffer, int Timeout) - { - IOStream::Write(rBuffer, Timeout); - } - protected: void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index d6c0e216c..6488499ed 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -118,11 +118,11 @@ while() my ($type,$name,$value) = split /\s+/,$l; if($type eq 'CONSTANT') { - push @{$cmd_constants{$current_cmd}},"$name = $value" + push @{$cmd_constants{$current_cmd}}, [$name, $value]; } else { - push @{$cmd_contents{$current_cmd}},$type,$name; + push @{$cmd_contents{$current_cmd}}, $type, $name; } } else @@ -243,6 +243,7 @@ class $request_base_class class $send_receive_class { public: + virtual ~$send_receive_class() { } virtual void Send(const $message_base_class &rObject) = 0; virtual std::auto_ptr<$message_base_class> Receive() = 0; }; @@ -316,8 +317,11 @@ __E # constants if(exists $cmd_constants{$cmd}) { - print H "\tenum\n\t{\n\t\t"; - print H join(",\n\t\t",@{$cmd_constants{$cmd}}); + print H "\tenum\n\t{\n"; + foreach my $constant (@{$cmd_constants{$cmd}}) + { + print H "\t\t", $constant->[0], " = ", $constant->[1], ",\n"; + } print H "\n\t};\n"; } @@ -331,9 +335,19 @@ __E { $error_message = $cmd; my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); - my $error_type = $cmd_constants{"ErrorType"}; + my %constants_hash; + my $error_type_value = ""; + + foreach my $constant (@{$cmd_constants{$cmd}}) + { + if($constant->[0] eq "ErrorType") + { + $error_type_value = $constant->[0]; + } + } + print H <<__E; - $cmd_class(int SubType) : m$mem_type($error_type), m$mem_subtype(SubType) { } + $cmd_class(int SubType) : m$mem_type($error_type_value), m$mem_subtype(SubType) { } bool IsError(int &rTypeOut, int &rSubTypeOut) const; std::string GetMessage() const { return GetMessage(m$mem_subtype); }; static std::string GetMessage(int subtype); @@ -514,13 +528,13 @@ std::string $cmd_class\::GetMessage(int subtype) switch(subtype) { __E - foreach my $const (@{$cmd_constants{$cmd}}) + foreach my $constant (@{$cmd_constants{$cmd}}) { - next unless $const =~ /^Err_(.*)/; + my $enum_name = $constant->[0]; + next unless $enum_name =~ /^Err_(.*)/; my $shortname = $1; - $const =~ s/ = .*//; print CPP <<__E; - case $const: return "$shortname"; + case $enum_name: return "$shortname"; __E } print CPP <<__E; @@ -581,12 +595,14 @@ public: virtual std::auto_ptr ReceiveStream() = 0; bool GetLastError(int &rTypeOut, int &rSubTypeOut); int GetLastErrorType() { return mLastErrorSubType; } + const std::string& GetLastErrorMessage() { return mLastErrorMessage; } protected: - void SetLastError(int Type, int SubType) + void SetLastError(int Type, int SubType, const std::string& Message) { mLastErrorType = Type; mLastErrorSubType = SubType; + mLastErrorMessage = Message; } std::string mPreviousCommand; std::string mPreviousReply; @@ -595,6 +611,7 @@ private: $client_server_base_class(const $client_server_base_class &rToCopy); /* do not call */ int mLastErrorType; int mLastErrorSubType; + std::string mLastErrorMessage; }; class $replyable_base_class : public virtual $client_server_base_class @@ -620,7 +637,8 @@ __E print CPP <<__E; $client_server_base_class\::$client_server_base_class() : mLastErrorType(Protocol::NoError), - mLastErrorSubType(Protocol::NoError) + mLastErrorSubType(Protocol::NoError), + mLastErrorMessage("no messages received") { } $client_server_base_class\::~$client_server_base_class() @@ -683,7 +701,7 @@ void $callable_base_class\::CheckReply(const std::string& requestCommandName, if(rReply.GetType() == expectedType) { // Correct response, do nothing - SetLastError(Protocol::NoError, Protocol::NoError); + SetLastError(Protocol::NoError, Protocol::NoError, "no error"); } else { @@ -692,7 +710,7 @@ void $callable_base_class\::CheckReply(const std::string& requestCommandName, if(rReply.IsError(type, subType)) { - SetLastError(type, subType); + SetLastError(type, subType, (($error_class&)rReply).GetMessage()); THROW_EXCEPTION_MESSAGE(ConnectionException, Protocol_UnexpectedReply, requestCommandName << " command failed: " @@ -701,12 +719,13 @@ void $callable_base_class\::CheckReply(const std::string& requestCommandName, } else { - SetLastError(Protocol::UnknownError, Protocol::UnknownError); + SetLastError(Protocol::UnknownError, Protocol::UnknownError, + rReply.ToString()); THROW_EXCEPTION_MESSAGE(ConnectionException, Protocol_UnexpectedReply, requestCommandName << " command failed: " - "received unexpected response type " << - rReply.GetType()); + "received unexpected response " << + rReply.ToString()); } } @@ -886,6 +905,13 @@ __E private: $context_class &mrContext; std::auto_ptr<$message_base_class> mapLastReply; + +protected: + $context_class& GetContext() + { + return mrContext; + } + public: virtual std::auto_ptr ReceiveStream() { @@ -1094,16 +1120,18 @@ void $server_or_client_class\::DoServer($context_class &rContext) } catch(BoxException &e) { - // First try a the built-in exception handler + // Give the server a chance to handle the exception itself. It will + // rethrow if it cannot, and we will fall through to the generic + // handler below. preply = HandleException(e); } } catch (...) { - // Fallback in case the exception isn't a BoxException - // or the exception handler fails as well. This path - // throws the exception upwards, killing the process - // that handles the current client. + // Fallback in case the exception isn't a BoxException or the server's + // exception handler fails as well. This path rethrows the exception, and it + // is not caught again, so it kills the process that is serving the current + // client. Send($cmd_classes{$error_message}(-1)); throw; } diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 1f6392d5e..512f51a7a 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -2,9 +2,14 @@ #include "emu.h" +#include +#include // for strlen() + +#include +#include + #ifdef WIN32 -#include #include #include #include @@ -15,7 +20,6 @@ #include #include -#include // message resource definitions for syslog() #include "messages.h" @@ -1141,7 +1145,15 @@ struct dirent *readdir(DIR *dp) NULL, NULL); //den->d_name = (char *)dp->info.name; den->d_name = &tempbuff[0]; - den->d_type = dp->info.dwFileAttributes; + den->win_attrs = dp->info.dwFileAttributes; + if(dp->info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + den->d_type = DT_DIR; + } + else + { + den->d_type = DT_REG; + } } else // FindNextFileW failed { @@ -1222,7 +1234,7 @@ int closedir(DIR *dp) // Created: 25th October 2004 // // -------------------------------------------------------------------------- -int poll (struct pollfd *ufds, unsigned long nfds, int timeout) +int poll(EMU_STRUCT_POLLFD *ufds, unsigned long nfds, int timeout) { try { @@ -1247,7 +1259,7 @@ int poll (struct pollfd *ufds, unsigned long nfds, int timeout) for (unsigned long i = 0; i < nfds; i++) { - struct pollfd* ufd = &(ufds[i]); + EMU_STRUCT_POLLFD* ufd = &(ufds[i]); if (ufd->events & POLLIN) { @@ -1273,7 +1285,7 @@ int poll (struct pollfd *ufds, unsigned long nfds, int timeout) { // int errval = WSAGetLastError(); - struct pollfd* pufd = ufds; + EMU_STRUCT_POLLFD* pufd = ufds; for (unsigned long i = 0; i < nfds; i++) { pufd->revents = POLLERR; @@ -1285,7 +1297,7 @@ int poll (struct pollfd *ufds, unsigned long nfds, int timeout) { for (unsigned long i = 0; i < nfds; i++) { - struct pollfd *ufd = &(ufds[i]); + EMU_STRUCT_POLLFD *ufd = &(ufds[i]); if (FD_ISSET(ufd->fd, &readfd)) { @@ -1519,7 +1531,7 @@ void syslog(int loglevel, const char *frmt, ...) assert(len >= 0); if (len < 0) { - printf("%s\r\n", buffer); + printf(" %s\r\n", buffer); fflush(stdout); return; } @@ -1531,7 +1543,7 @@ void syslog(int loglevel, const char *frmt, ...) if (gSyslogH == INVALID_HANDLE_VALUE) { - printf("%s\r\n", buffer); + printf(" %s\r\n", buffer); fflush(stdout); return; } @@ -2028,3 +2040,31 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) #endif // WIN32 +// MSVC < 12 (2013) does not have strtoull(), and _strtoi64 is signed only (truncates all values +// greater than 1<<63 to _I64_MAX, so we roll our own using std::istringstream +// +uint64_t box_strtoui64(const char *nptr, const char **endptr, int base) +{ + std::istringstream iss((std::string(nptr))); + uint64_t result; + + assert(base == 0 || base == 8 || base == 10 || base == 16); + iss >> std::setbase(base); + iss >> result; + + if(endptr != NULL) + { + if(iss.eof()) + { + *endptr = nptr + strlen(nptr); + } + else + { + assert(iss.tellg() >= 0); + *endptr = nptr + iss.tellg(); + } + } + + return result; +} + diff --git a/lib/win32/emu.h b/lib/win32/emu.h index 91793004a..bdf12aea5 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -2,20 +2,31 @@ #include "emu_winver.h" +#include // for uint64_t +#include // for strtoull() + +#if ! defined EMU_INCLUDE +#define EMU_INCLUDE + #ifdef WIN32 #define EMU_STRUCT_STAT struct emu_stat - #define EMU_STAT emu_stat - #define EMU_FSTAT emu_fstat - #define EMU_LSTAT emu_stat + #define EMU_STRUCT_POLLFD struct emu_pollfd + #define EMU_STAT emu_stat + #define EMU_FSTAT emu_fstat + #define EMU_LSTAT emu_stat + #define EMU_LINK emu_link + #define EMU_UNLINK emu_unlink #else #define EMU_STRUCT_STAT struct stat - #define EMU_STAT ::stat - #define EMU_FSTAT ::fstat - #define EMU_LSTAT ::lstat + #define EMU_STRUCT_POLLFD struct pollfd + #define EMU_STAT ::stat + #define EMU_FSTAT ::fstat + #define EMU_LSTAT ::lstat + #define EMU_LINK ::link + #define EMU_UNLINK ::unlink #endif -#if ! defined EMU_INCLUDE && defined WIN32 -#define EMU_INCLUDE +#ifdef WIN32 // Need feature detection macros below #if defined BOX_CMAKE @@ -213,12 +224,26 @@ inline int strncasecmp(const char *s1, const char *s2, size_t count) #error You must not include the MinGW dirent.h! #endif +// File types for struct dirent.d_type. Not all are supported by our emulated readdir(): +#define DT_UNKNOWN 0 +#define DT_FIFO 1 +#define DT_CHR 2 +#define DT_DIR 4 +#define DT_BLK 6 +#define DT_REG 8 +#define DT_LNK 10 +#define DT_SOCK 12 +#define DT_WHT 14 + struct dirent { char *d_name; - DWORD d_type; // file attributes + int d_type; // emulated UNIX file attributes + DWORD win_attrs; // WIN32_FIND_DATA.dwFileAttributes }; +#define HAVE_VALID_DIRENT_D_TYPE 1 + struct DIR { HANDLE fd; // the HANDLE returned by FindFirstFile @@ -297,7 +322,7 @@ extern "C" inline unsigned int sleep(unsigned int secs) #define SHUT_RD SD_RECEIVE #define SHUT_WR SD_SEND -struct pollfd +EMU_STRUCT_POLLFD { SOCKET fd; short int events; @@ -352,13 +377,15 @@ int emu_rename (const char* pOldName, const char* pNewName); #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) -#define link(oldpath, newpath) emu_link (oldpath, newpath) -#define unlink(file) emu_unlink (file) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) #define getcwd(buffer, size) emu_getcwd (buffer, size) #define rename(oldname, newname) emu_rename (oldname, newname) +// link() and unlink() conflict with Boost if implemented using macros like +// the others above, so I've removed the macros and you need to use EMU_LINK +// and EMU_UNLINK everywhere. + // Not safe to replace stat/fstat/lstat on mingw at least, as struct stat // has a 16-bit st_ino and we need a 64-bit one. // @@ -372,7 +399,7 @@ int emu_rename (const char* pOldName, const char* pNewName); int statfs(const char * name, struct statfs * s); -int poll(struct pollfd *ufds, unsigned long nfds, int timeout); +int poll(EMU_STRUCT_POLLFD *ufds, unsigned long nfds, int timeout); struct iovec { void *iov_base; /* Starting address */ @@ -438,4 +465,11 @@ int console_read(char* pBuffer, size_t BufferSize); #pragma warning(disable:4996) // POSIX name for this item is deprecated #endif // _MSC_VER -#endif // !EMU_INCLUDE && WIN32 +#endif // WIN32 + +// MSVC < 12 (2013) does not have strtoull(), and _strtoi64 is signed only (truncates all values +// greater than 1<<63 to _I64_MAX, so we roll our own using std::istringstream +// +uint64_t box_strtoui64(const char *nptr, const char **endptr, int base); + +#endif // !EMU_INCLUDE diff --git a/lib/win32/emu_winver.h b/lib/win32/emu_winver.h index 92060150f..4a7e90746 100644 --- a/lib/win32/emu_winver.h +++ b/lib/win32/emu_winver.h @@ -11,10 +11,6 @@ // We need WINVER at least 0x0500 to use GetFileSizeEx on Cygwin/MinGW, // and 0x0501 for FindFirstFile(W) for opendir/readdir. -// -// WIN32_WINNT versions 0x0600 (Vista) and higher enable WSAPoll() in -// winsock2.h, whose struct pollfd conflicts with ours below, so for -// now we just set it lower than that, to Windows XP (0x0501). #ifdef WINVER # if WINVER != 0x0501 @@ -26,12 +22,12 @@ #define WINVER 0x0501 #ifdef _WIN32_WINNT -# if _WIN32_WINNT != 0x0501 +# if _WIN32_WINNT != 0x0600 // provoke a redefinition warning to track down the offender -# define _WIN32_WINNT 0x0501 +# define _WIN32_WINNT 0x0600 # error Must include emu.h before setting _WIN32_WINNT # endif #endif -#define _WIN32_WINNT 0x0501 +#define _WIN32_WINNT 0x0600 #endif // _EMU_WINVER_H diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp index af2833a11..0daead22f 100755 --- a/lib/win32/getopt_long.cpp +++ b/lib/win32/getopt_long.cpp @@ -1,546 +1,546 @@ -/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */ -/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ -// Adapted for Box Backup by Chris Wilson - -/* - * Copyright (c) 2002 Todd C. Miller - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * Sponsored in part by the Defense Advanced Research Projects - * Agency (DARPA) and Air Force Research Laboratory, Air Force - * Materiel Command, USAF, under agreement number F39502-99-1-0512. - */ -/*- - * Copyright (c) 2000 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Dieter Baron and Thomas Klausner. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -// #include "Box.h" -#include "emu.h" - -#include -#include -#include -#include -#include - -#include "box_getopt.h" - -#ifdef REPLACE_GETOPT // until end of file - -int opterr = 1; /* if error message should be printed */ -int optind = 1; /* index into parent argv vector */ -int optopt = '?'; /* character checked for validity */ -int optreset; /* reset getopt */ -char *optarg; /* argument associated with option */ - -#define PRINT_ERROR ((opterr) && (*options != ':')) - -#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ -#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ -#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */ - -/* return values */ -#define BADCH (int)'?' -#define BADARG ((*options == ':') ? (int)':' : (int)'?') -#define INORDER (int)1 - -#define EMSG "" - -static void warnx(const char* fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - fprintf(stderr, "\n"); -} - -static int getopt_internal(int, char * const *, const char *, - const struct option *, int *, int); -static int parse_long_options(char * const *, const char *, - const struct option *, int *, int); -static int gcd(int, int); -static void permute_args(int, int, int, char * const *); - -static char *place = EMSG; /* option letter processing */ - -/* XXX: set optreset to 1 rather than these two */ -static int nonopt_start = -1; /* first non option argument (for permute) */ -static int nonopt_end = -1; /* first option after non options (for permute) */ - -/* Error messages */ -static const char recargchar[] = "option requires an argument -- %c"; -static const char recargstring[] = "option requires an argument -- %s"; -static const char ambig[] = "ambiguous option -- %.*s"; -static const char noarg[] = "option doesn't take an argument -- %.*s"; -static const char illoptchar[] = "unknown option -- %c"; -static const char illoptstring[] = "unknown option -- %s"; - -/* - * Compute the greatest common divisor of a and b. - */ -static int -gcd(int a, int b) -{ - int c; - - c = a % b; - while (c != 0) { - a = b; - b = c; - c = a % b; - } - - return (b); -} - -/* - * Exchange the block from nonopt_start to nonopt_end with the block - * from nonopt_end to opt_end (keeping the same order of arguments - * in each block). - */ -static void -permute_args(int panonopt_start, int panonopt_end, int opt_end, - char * const *nargv) -{ - int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; - char *swap; - - /* - * compute lengths of blocks and number and size of cycles - */ - nnonopts = panonopt_end - panonopt_start; - nopts = opt_end - panonopt_end; - ncycle = gcd(nnonopts, nopts); - cyclelen = (opt_end - panonopt_start) / ncycle; - - for (i = 0; i < ncycle; i++) { - cstart = panonopt_end+i; - pos = cstart; - for (j = 0; j < cyclelen; j++) { - if (pos >= panonopt_end) - pos -= nnonopts; - else - pos += nopts; - swap = nargv[pos]; - /* LINTED const cast */ - ((char **) nargv)[pos] = nargv[cstart]; - /* LINTED const cast */ - ((char **)nargv)[cstart] = swap; - } - } -} - -/* - * parse_long_options -- - * Parse long options in argc/argv argument vector. - * Returns -1 if short_too is set and the option does not match long_options. - */ -static int -parse_long_options(char * const *nargv, const char *options, - const struct option *long_options, int *idx, int short_too) -{ - char *current_argv, *has_equal; - size_t current_argv_len; - int i, match; - - current_argv = place; - match = -1; - - optind++; - - if ((has_equal = strchr(current_argv, '=')) != NULL) { - /* argument found (--option=arg) */ - current_argv_len = has_equal - current_argv; - has_equal++; - } else - current_argv_len = strlen(current_argv); - - for (i = 0; long_options[i].name; i++) { - /* find matching long option */ - if (strncmp(current_argv, long_options[i].name, - current_argv_len)) - continue; - - if (strlen(long_options[i].name) == current_argv_len) { - /* exact match */ - match = i; - break; - } - /* - * If this is a known short option, don't allow - * a partial match of a single character. - */ - if (short_too && current_argv_len == 1) - continue; - - if (match == -1) /* partial match */ - match = i; - else { - /* ambiguous abbreviation */ - if (PRINT_ERROR) - warnx(ambig, (int)current_argv_len, - current_argv); - optopt = 0; - return (BADCH); - } - } - if (match != -1) { /* option found */ - if (long_options[match].has_arg == no_argument - && has_equal) { - if (PRINT_ERROR) - warnx(noarg, (int)current_argv_len, - current_argv); - /* - * XXX: GNU sets optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - optopt = long_options[match].val; - else - optopt = 0; - return (BADARG); - } - if (long_options[match].has_arg == required_argument || - long_options[match].has_arg == optional_argument) { - if (has_equal) - optarg = has_equal; - else if (long_options[match].has_arg == - required_argument) { - /* - * optional argument doesn't use next nargv - */ - optarg = nargv[optind++]; - } - } - if ((long_options[match].has_arg == required_argument) - && (optarg == NULL)) { - /* - * Missing argument; leading ':' indicates no error - * should be generated. - */ - if (PRINT_ERROR) - warnx(recargstring, - current_argv); - /* - * XXX: GNU sets optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - optopt = long_options[match].val; - else - optopt = 0; - --optind; - return (BADARG); - } - } else { /* unknown option */ - if (short_too) { - --optind; - return (-1); - } - if (PRINT_ERROR) - warnx(illoptstring, current_argv); - optopt = 0; - return (BADCH); - } - if (idx) - *idx = match; - if (long_options[match].flag) { - *long_options[match].flag = long_options[match].val; - return (0); - } else - return (long_options[match].val); -} - -/* - * getopt_internal -- - * Parse argc/argv argument vector. Called by user level routines. - */ -static int -getopt_internal(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx, int flags) -{ - const char * oli; /* option letter list index */ - int optchar, short_too; - static int posixly_correct = -1; - - if (options == NULL) - return (-1); - - /* - * Disable GNU extensions if POSIXLY_CORRECT is set or options - * string begins with a '+'. - */ - if (posixly_correct == -1) - posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); - if (posixly_correct || *options == '+') - flags &= ~FLAG_PERMUTE; - else if (*options == '-') - flags |= FLAG_ALLARGS; - if (*options == '+' || *options == '-') - options++; - - /* - * XXX Some GNU programs (like cvs) set optind to 0 instead of - * XXX using optreset. Work around this braindamage. - */ - if (optind == 0) - optind = optreset = 1; - - optarg = NULL; - if (optreset) - nonopt_start = nonopt_end = -1; -start: - if (optreset || !*place) { /* update scanning pointer */ - optreset = 0; - if (optind >= nargc) { /* end of argument vector */ - place = EMSG; - if (nonopt_end != -1) { - /* do permutation, if we have to */ - permute_args(nonopt_start, nonopt_end, - optind, nargv); - optind -= nonopt_end - nonopt_start; - } - else if (nonopt_start != -1) { - /* - * If we skipped non-options, set optind - * to the first of them. - */ - optind = nonopt_start; - } - nonopt_start = nonopt_end = -1; - return (-1); - } - if (*(place = nargv[optind]) != '-' || - (place[1] == '\0' && strchr(options, '-') == NULL)) { - place = EMSG; /* found non-option */ - if (flags & FLAG_ALLARGS) { - /* - * GNU extension: - * return non-option as argument to option 1 - */ - optarg = nargv[optind++]; - return (INORDER); - } - if (!(flags & FLAG_PERMUTE)) { - /* - * If no permutation wanted, stop parsing - * at first non-option. - */ - return (-1); - } - /* do permutation */ - if (nonopt_start == -1) - nonopt_start = optind; - else if (nonopt_end != -1) { - permute_args(nonopt_start, nonopt_end, - optind, nargv); - nonopt_start = optind - - (nonopt_end - nonopt_start); - nonopt_end = -1; - } - optind++; - /* process next argument */ - goto start; - } - if (nonopt_start != -1 && nonopt_end == -1) - nonopt_end = optind; - - /* - * If we have "-" do nothing, if "--" we are done. - */ - if (place[1] != '\0' && *++place == '-' && place[1] == '\0') { - optind++; - place = EMSG; - /* - * We found an option (--), so if we skipped - * non-options, we have to permute. - */ - if (nonopt_end != -1) { - permute_args(nonopt_start, nonopt_end, - optind, nargv); - optind -= nonopt_end - nonopt_start; - } - nonopt_start = nonopt_end = -1; - return (-1); - } - } - - /* - * Check long options if: - * 1) we were passed some - * 2) the arg is not just "-" - * 3) either the arg starts with -- we are getopt_long_only() - */ - if (long_options != NULL && place != nargv[optind] && - (*place == '-' || (flags & FLAG_LONGONLY))) { - short_too = 0; - if (*place == '-') - place++; /* --foo long option */ - else if (*place != ':' && strchr(options, *place) != NULL) - short_too = 1; /* could be short option too */ - - optchar = parse_long_options(nargv, options, long_options, - idx, short_too); - if (optchar != -1) { - place = EMSG; - return (optchar); - } - } - - if ((optchar = (int)*place++) == (int)':' || - optchar == (int)'-' && *place != '\0' || - (oli = strchr(options, optchar)) == NULL) { - /* - * If the user specified "-" and '-' isn't listed in - * options, return -1 (non-option) as per POSIX. - * Otherwise, it is an unknown option character (or ':'). - */ - if (optchar == (int)'-' && *place == '\0') - return (-1); - if (!*place) - ++optind; - if (PRINT_ERROR) - warnx(illoptchar, optchar); - optopt = optchar; - return (BADCH); - } - if (long_options != NULL && optchar == 'W' && oli[1] == ';') { - /* -W long-option */ - if (*place) /* no space */ - /* NOTHING */; - else if (++optind >= nargc) { /* no arg */ - place = EMSG; - if (PRINT_ERROR) - warnx(recargchar, optchar); - optopt = optchar; - return (BADARG); - } else /* white space */ - place = nargv[optind]; - optchar = parse_long_options(nargv, options, long_options, - idx, 0); - place = EMSG; - return (optchar); - } - if (*++oli != ':') { /* doesn't take argument */ - if (!*place) - ++optind; - } else { /* takes (optional) argument */ - optarg = NULL; - if (*place) /* no white space */ - optarg = place; - /* XXX: disable test for :: if PC? (GNU doesn't) */ - else if (oli[1] != ':') { /* arg not optional */ - if (++optind >= nargc) { /* no arg */ - place = EMSG; - if (PRINT_ERROR) - warnx(recargchar, optchar); - optopt = optchar; - return (BADARG); - } else - optarg = nargv[optind]; - } else if (!(flags & FLAG_PERMUTE)) { - /* - * If permutation is disabled, we can accept an - * optional arg separated by whitespace so long - * as it does not start with a dash (-). - */ - if (optind + 1 < nargc && *nargv[optind + 1] != '-') - optarg = nargv[++optind]; - } - place = EMSG; - ++optind; - } - /* dump back option letter */ - return (optchar); -} - -/* - * getopt -- - * Parse argc/argv argument vector. - * - * [eventually this will replace the BSD getopt] - */ -int -getopt(int nargc, char * const *nargv, const char *options) -{ - - /* - * We don't pass FLAG_PERMUTE to getopt_internal() since - * the BSD getopt(3) (unlike GNU) has never done this. - * - * Furthermore, since many privileged programs call getopt() - * before dropping privileges it makes sense to keep things - * as simple (and bug-free) as possible. - */ - return (getopt_internal(nargc, nargv, options, NULL, NULL, 0)); -} - -/* - * getopt_long -- - * Parse argc/argv argument vector. - */ -int -getopt_long(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx) -{ - - return (getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE)); -} - -/* - * getopt_long_only -- - * Parse argc/argv argument vector. - */ -int -getopt_long_only(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx) -{ - - return (getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE|FLAG_LONGONLY)); -} - -#endif // REPLACE_GETOPT +/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */ +/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ +// Adapted for Box Backup by Chris Wilson + +/* + * Copyright (c) 2002 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Sponsored in part by the Defense Advanced Research Projects + * Agency (DARPA) and Air Force Research Laboratory, Air Force + * Materiel Command, USAF, under agreement number F39502-99-1-0512. + */ +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +// #include "Box.h" +#include "emu.h" + +#include +#include +#include +#include +#include + +#include "box_getopt.h" + +#if REPLACE_GETOPT // until end of file + +int opterr = 1; /* if error message should be printed */ +int optind = 1; /* index into parent argv vector */ +int optopt = '?'; /* character checked for validity */ +int optreset; /* reset getopt */ +char *optarg; /* argument associated with option */ + +#define PRINT_ERROR ((opterr) && (*options != ':')) + +#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ +#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ +#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */ + +/* return values */ +#define BADCH (int)'?' +#define BADARG ((*options == ':') ? (int)':' : (int)'?') +#define INORDER (int)1 + +#define EMSG "" + +static void warnx(const char* fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, "\n"); +} + +static int getopt_internal(int, char * const *, const char *, + const struct option *, int *, int); +static int parse_long_options(char * const *, const char *, + const struct option *, int *, int); +static int gcd(int, int); +static void permute_args(int, int, int, char * const *); + +static char *place = EMSG; /* option letter processing */ + +/* XXX: set optreset to 1 rather than these two */ +static int nonopt_start = -1; /* first non option argument (for permute) */ +static int nonopt_end = -1; /* first option after non options (for permute) */ + +/* Error messages */ +static const char recargchar[] = "option requires an argument -- %c"; +static const char recargstring[] = "option requires an argument -- %s"; +static const char ambig[] = "ambiguous option -- %.*s"; +static const char noarg[] = "option doesn't take an argument -- %.*s"; +static const char illoptchar[] = "unknown option -- %c"; +static const char illoptstring[] = "unknown option -- %s"; + +/* + * Compute the greatest common divisor of a and b. + */ +static int +gcd(int a, int b) +{ + int c; + + c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + + return (b); +} + +/* + * Exchange the block from nonopt_start to nonopt_end with the block + * from nonopt_end to opt_end (keeping the same order of arguments + * in each block). + */ +static void +permute_args(int panonopt_start, int panonopt_end, int opt_end, + char * const *nargv) +{ + int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; + char *swap; + + /* + * compute lengths of blocks and number and size of cycles + */ + nnonopts = panonopt_end - panonopt_start; + nopts = opt_end - panonopt_end; + ncycle = gcd(nnonopts, nopts); + cyclelen = (opt_end - panonopt_start) / ncycle; + + for (i = 0; i < ncycle; i++) { + cstart = panonopt_end+i; + pos = cstart; + for (j = 0; j < cyclelen; j++) { + if (pos >= panonopt_end) + pos -= nnonopts; + else + pos += nopts; + swap = nargv[pos]; + /* LINTED const cast */ + ((char **) nargv)[pos] = nargv[cstart]; + /* LINTED const cast */ + ((char **)nargv)[cstart] = swap; + } + } +} + +/* + * parse_long_options -- + * Parse long options in argc/argv argument vector. + * Returns -1 if short_too is set and the option does not match long_options. + */ +static int +parse_long_options(char * const *nargv, const char *options, + const struct option *long_options, int *idx, int short_too) +{ + char *current_argv, *has_equal; + size_t current_argv_len; + int i, match; + + current_argv = place; + match = -1; + + optind++; + + if ((has_equal = strchr(current_argv, '=')) != NULL) { + /* argument found (--option=arg) */ + current_argv_len = has_equal - current_argv; + has_equal++; + } else + current_argv_len = strlen(current_argv); + + for (i = 0; long_options[i].name; i++) { + /* find matching long option */ + if (strncmp(current_argv, long_options[i].name, + current_argv_len)) + continue; + + if (strlen(long_options[i].name) == current_argv_len) { + /* exact match */ + match = i; + break; + } + /* + * If this is a known short option, don't allow + * a partial match of a single character. + */ + if (short_too && current_argv_len == 1) + continue; + + if (match == -1) /* partial match */ + match = i; + else { + /* ambiguous abbreviation */ + if (PRINT_ERROR) + warnx(ambig, (int)current_argv_len, + current_argv); + optopt = 0; + return (BADCH); + } + } + if (match != -1) { /* option found */ + if (long_options[match].has_arg == no_argument + && has_equal) { + if (PRINT_ERROR) + warnx(noarg, (int)current_argv_len, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + return (BADARG); + } + if (long_options[match].has_arg == required_argument || + long_options[match].has_arg == optional_argument) { + if (has_equal) + optarg = has_equal; + else if (long_options[match].has_arg == + required_argument) { + /* + * optional argument doesn't use next nargv + */ + optarg = nargv[optind++]; + } + } + if ((long_options[match].has_arg == required_argument) + && (optarg == NULL)) { + /* + * Missing argument; leading ':' indicates no error + * should be generated. + */ + if (PRINT_ERROR) + warnx(recargstring, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + --optind; + return (BADARG); + } + } else { /* unknown option */ + if (short_too) { + --optind; + return (-1); + } + if (PRINT_ERROR) + warnx(illoptstring, current_argv); + optopt = 0; + return (BADCH); + } + if (idx) + *idx = match; + if (long_options[match].flag) { + *long_options[match].flag = long_options[match].val; + return (0); + } else + return (long_options[match].val); +} + +/* + * getopt_internal -- + * Parse argc/argv argument vector. Called by user level routines. + */ +static int +getopt_internal(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx, int flags) +{ + const char * oli; /* option letter list index */ + int optchar, short_too; + static int posixly_correct = -1; + + if (options == NULL) + return (-1); + + /* + * Disable GNU extensions if POSIXLY_CORRECT is set or options + * string begins with a '+'. + */ + if (posixly_correct == -1) + posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); + if (posixly_correct || *options == '+') + flags &= ~FLAG_PERMUTE; + else if (*options == '-') + flags |= FLAG_ALLARGS; + if (*options == '+' || *options == '-') + options++; + + /* + * XXX Some GNU programs (like cvs) set optind to 0 instead of + * XXX using optreset. Work around this braindamage. + */ + if (optind == 0) + optind = optreset = 1; + + optarg = NULL; + if (optreset) + nonopt_start = nonopt_end = -1; +start: + if (optreset || !*place) { /* update scanning pointer */ + optreset = 0; + if (optind >= nargc) { /* end of argument vector */ + place = EMSG; + if (nonopt_end != -1) { + /* do permutation, if we have to */ + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + else if (nonopt_start != -1) { + /* + * If we skipped non-options, set optind + * to the first of them. + */ + optind = nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + if (*(place = nargv[optind]) != '-' || + (place[1] == '\0' && strchr(options, '-') == NULL)) { + place = EMSG; /* found non-option */ + if (flags & FLAG_ALLARGS) { + /* + * GNU extension: + * return non-option as argument to option 1 + */ + optarg = nargv[optind++]; + return (INORDER); + } + if (!(flags & FLAG_PERMUTE)) { + /* + * If no permutation wanted, stop parsing + * at first non-option. + */ + return (-1); + } + /* do permutation */ + if (nonopt_start == -1) + nonopt_start = optind; + else if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + nonopt_start = optind - + (nonopt_end - nonopt_start); + nonopt_end = -1; + } + optind++; + /* process next argument */ + goto start; + } + if (nonopt_start != -1 && nonopt_end == -1) + nonopt_end = optind; + + /* + * If we have "-" do nothing, if "--" we are done. + */ + if (place[1] != '\0' && *++place == '-' && place[1] == '\0') { + optind++; + place = EMSG; + /* + * We found an option (--), so if we skipped + * non-options, we have to permute. + */ + if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + } + + /* + * Check long options if: + * 1) we were passed some + * 2) the arg is not just "-" + * 3) either the arg starts with -- we are getopt_long_only() + */ + if (long_options != NULL && place != nargv[optind] && + (*place == '-' || (flags & FLAG_LONGONLY))) { + short_too = 0; + if (*place == '-') + place++; /* --foo long option */ + else if (*place != ':' && strchr(options, *place) != NULL) + short_too = 1; /* could be short option too */ + + optchar = parse_long_options(nargv, options, long_options, + idx, short_too); + if (optchar != -1) { + place = EMSG; + return (optchar); + } + } + + if ((optchar = (int)*place++) == (int)':' || + (optchar == (int)'-' && *place != '\0') || + (oli = strchr(options, optchar)) == NULL) { + /* + * If the user specified "-" and '-' isn't listed in + * options, return -1 (non-option) as per POSIX. + * Otherwise, it is an unknown option character (or ':'). + */ + if (optchar == (int)'-' && *place == '\0') + return (-1); + if (!*place) + ++optind; + if (PRINT_ERROR) + warnx(illoptchar, optchar); + optopt = optchar; + return (BADCH); + } + if (long_options != NULL && optchar == 'W' && oli[1] == ';') { + /* -W long-option */ + if (*place) /* no space */ + /* NOTHING */; + else if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else /* white space */ + place = nargv[optind]; + optchar = parse_long_options(nargv, options, long_options, + idx, 0); + place = EMSG; + return (optchar); + } + if (*++oli != ':') { /* doesn't take argument */ + if (!*place) + ++optind; + } else { /* takes (optional) argument */ + optarg = NULL; + if (*place) /* no white space */ + optarg = place; + /* XXX: disable test for :: if PC? (GNU doesn't) */ + else if (oli[1] != ':') { /* arg not optional */ + if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else + optarg = nargv[optind]; + } else if (!(flags & FLAG_PERMUTE)) { + /* + * If permutation is disabled, we can accept an + * optional arg separated by whitespace so long + * as it does not start with a dash (-). + */ + if (optind + 1 < nargc && *nargv[optind + 1] != '-') + optarg = nargv[++optind]; + } + place = EMSG; + ++optind; + } + /* dump back option letter */ + return (optchar); +} + +/* + * getopt -- + * Parse argc/argv argument vector. + * + * [eventually this will replace the BSD getopt] + */ +int +getopt(int nargc, char * const *nargv, const char *options) +{ + + /* + * We don't pass FLAG_PERMUTE to getopt_internal() since + * the BSD getopt(3) (unlike GNU) has never done this. + * + * Furthermore, since many privileged programs call getopt() + * before dropping privileges it makes sense to keep things + * as simple (and bug-free) as possible. + */ + return (getopt_internal(nargc, nargv, options, NULL, NULL, 0)); +} + +/* + * getopt_long -- + * Parse argc/argv argument vector. + */ +int +getopt_long(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE)); +} + +/* + * getopt_long_only -- + * Parse argc/argv argument vector. + */ +int +getopt_long_only(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE|FLAG_LONGONLY)); +} + +#endif // REPLACE_GETOPT diff --git a/modules.txt b/modules.txt index 4fd27a2db..09561adbf 100644 --- a/modules.txt +++ b/modules.txt @@ -42,18 +42,20 @@ bin/bbstoreaccounts lib/backupclient bin/bbackupd lib/bbackupd bin/bbackupquery lib/bbackupquery bin/bbackupctl lib/backupclient qdbm lib/bbackupd +bin/s3simulator lib/httpserver -test/backupstore bin/bbstored bin/bbstoreaccounts lib/backupclient lib/raidfile -test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupclient bin/bbackupquery bin/bbackupd bin/bbackupctl -test/backupstorepatch bin/bbstored bin/bbstoreaccounts lib/backupclient +# With CMake, dependency on a binary target is not enough to inherit the header files for +# libraries that it uses, so we need to specify lib targets as well as bin dependencies: +test/backupstore lib/backupclient bin/bbstored bin/bbstoreaccounts bin/s3simulator +test/backupstorefix lib/backupclient bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupd bin/bbackupctl +test/backupstorepatch lib/backupclient bin/bbstored bin/bbstoreaccounts test/backupdiff lib/backupclient -test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/bbackupquery lib/bbackupd lib/bbstored lib/server lib/intercept -bin/s3simulator lib/httpserver -test/s3store lib/backupclient lib/httpserver bin/s3simulator bin/bbstoreaccounts +test/bbackupd lib/bbackupd lib/bbstored lib/bbackupquery bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl +test/s3store lib/backupclient bin/s3simulator bin/bbstoreaccounts # HTTP server system lib/httpserver lib/server -test/httpserver lib/httpserver +test/httpserver lib/httpserver bin/s3simulator # END_IF_DISTRIBUTION diff --git a/runtest.pl.in b/runtest.pl.in index a864336b3..8a2c8befc 100755 --- a/runtest.pl.in +++ b/runtest.pl.in @@ -14,13 +14,15 @@ use lib dirname($0)."/infrastructure"; use BoxPlatform; my %opts; -getopts('acnv', \%opts); +getopts('acnTO:v', \%opts); -# Don't actually run the test, just prepare for it. +my $appveyor_mode = $opts{'a'}; my $cmake_build = $opts{'c'}; +# Don't actually run the test, just prepare for it: my $prepare_only = $opts{'n'}; +my $timestamp_tests = $opts{'T'}; +my $extra_options = $opts{'O'} || ''; my $verbose_build = $opts{'v'}; -my $appveyor_mode = $opts{'a'}; my $test_name = shift @ARGV; my $test_mode = shift @ARGV; @@ -109,7 +111,8 @@ if ($exit_code != 0) { print <<__E; -One or more tests have failed. Please check the following common causes: +One or more tests have failed in $test_mode mode. Please check the following +common causes: * Check that no instances of bbstored or bbackupd are already running on this machine. @@ -187,7 +190,7 @@ sub runtest ); # Our CMake buildsystem doesn't do anything to support testextra files - # (Makfile syntax), so fake it. + # (Makefile syntax), so fake it. if (-r "$test_src_dir/testextra") { open EXTRA, "$test_src_dir/testextra" @@ -240,14 +243,11 @@ sub runtest { push @results,"$t: make failed"; appveyor_test_status($t, "NotRunnable", time() - $start_time, - "pre-test commands failed"); + "make failed"); $exit_code = 2; return; } - my $logfile = "test-$t.log"; - my $test_res; - if($prepare_only) { appveyor_test_status($t, "Skipped", time() - $start_time, @@ -255,6 +255,10 @@ sub runtest return; } + my $test_res; + my $logfile = "test-$t.log"; + my $test_options = ($timestamp_tests ? "-T" : "")." ".$extra_options; + # run it if($cmake_build) { @@ -262,7 +266,7 @@ sub runtest open LOG, ">$logfile" or die "$logfile: $!"; chdir("$base_dir/$test_mode/test/$t"); - open TEE, "$test_dst_exe |" + open TEE, "$test_dst_exe $test_options |" or die "$test_dst_dir/$test_dst_exe: $!"; while (my $line = ) @@ -277,7 +281,7 @@ sub runtest else { chdir($base_dir); - $test_res = system("cd $test_mode/test/$t ; sh t 2>&1 " . + $test_res = system("cd $test_mode/test/$t; sh t $test_options 2>&1 " . "| tee ../../../$logfile"); } diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 6441d66cc..30aa35516 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -12,9 +12,15 @@ #include #include +#ifdef HAVE_PROCESS_H +# include // for getpid() on Windows +#endif + #include "Archive.h" +#include "BackupAccountControl.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" +#include "BackupDaemonConfigVerify.h" #include "BackupProtocol.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" @@ -34,12 +40,15 @@ #include "FileStream.h" #include "HousekeepStoreAccount.h" #include "MemBlockStream.h" +#include "NamedLock.h" // for BOX_LOCK_TYPE_F_SETLK #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" +#include "S3Simulator.h" #include "SSLLib.h" #include "ServerControl.h" +#include "SimpleDBClient.h" #include "Socket.h" #include "SocketStreamTLS.h" #include "StoreStructure.h" @@ -53,12 +62,16 @@ #define ENCFILE_SIZE 2765 // Make some test attributes -#define ATTR1_SIZE 245 -#define ATTR2_SIZE 23 -#define ATTR3_SIZE 122 +#define ATTR1_SIZE 245 +#define ATTR2_SIZE 23 +#define ATTR3_SIZE 122 #define SHORT_TIMEOUT 5000 +#define DEFAULT_BBSTORED_CONFIG_FILE "testfiles/bbstored.conf" +#define DEFAULT_BBACKUPD_CONFIG_FILE "testfiles/bbackupd.conf" +#define DEFAULT_S3_CACHE_DIR "testfiles/bbackupd-cache" + int attr1[ATTR1_SIZE]; int attr2[ATTR2_SIZE]; int attr3[ATTR3_SIZE]; @@ -139,7 +152,35 @@ static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", #define UPLOAD_FILE_TO_MOVE 8 #define UNLINK_IF_EXISTS(filename) \ - if (FileExists(filename)) { TEST_THAT(unlink(filename) == 0); } + if (FileExists(filename)) { TEST_THAT(EMU_UNLINK(filename) == 0); } + +int s3simulator_pid = 0; + +bool StartSimulator() +{ + s3simulator_pid = StartDaemon(s3simulator_pid, + "../../bin/s3simulator/s3simulator " + bbstored_args + + " testfiles/s3simulator.conf", "testfiles/s3simulator.pid"); + return s3simulator_pid != 0; +} + +bool StopSimulator() +{ + bool result = StopDaemon(s3simulator_pid, "testfiles/s3simulator.pid", + "s3simulator.memleaks", true); + s3simulator_pid = 0; + return result; +} + +bool kill_running_daemons() +{ +#ifndef WIN32 + TEST_THAT_OR(::system("test ! -r testfiles/s3simulator.pid || " + "kill `cat testfiles/s3simulator.pid`") == 0, FAIL); + TEST_THAT_OR(::system("rm -f testfiles/s3simulator.pid") == 0, FAIL); +#endif + return true; +} //! Simplifies calling setUp() with the current function name in each test. #define SETUP_TEST_BACKUPSTORE() \ @@ -150,6 +191,45 @@ static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); \ TEST_THAT_OR(create_account(10000, 20000), FAIL); +bool setup_test_backupstore_specialised(const std::string& spec_name, + BackupAccountControl& control) +{ + if (ServerIsAlive(bbstored_pid)) + { + TEST_THAT_OR(StopServer(), FAIL); + } + + ExpectedRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1); + set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + + if(spec_name == "s3") + { + TEST_THAT_OR( + dynamic_cast(control). + CreateAccount("test", 10000, 20000) == 0, FAIL); + } + else if(spec_name == "store") + { + TEST_THAT_OR( + dynamic_cast(control). + CreateAccount(0, 10000, 20000) == 0, FAIL); + } + else + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Don't know how to create accounts for store type: " << + spec_name); + } + + return true; +} + +#define SETUP_TEST_BACKUPSTORE_SPECIALISED(name, control) \ + SETUP_SPECIALISED(name); \ + TEST_THAT_OR(setup_test_backupstore_specialised(name, control), FAIL); \ + try \ + { // left open for TEARDOWN_TEST_BACKUPSTORE_SPECIALISED() + //! Checks account for errors and shuts down daemons at end of every test. bool teardown_test_backupstore() { @@ -158,7 +238,7 @@ bool teardown_test_backupstore() if (FileExists("testfiles/0_0/backup/01234567/info.rf")) { TEST_THAT_OR(check_reference_counts(), status = false); - TEST_THAT_OR(check_account(), status = false); + TEST_EQUAL_OR(0, check_account_for_errors(), status = false); } return status; @@ -170,6 +250,46 @@ bool teardown_test_backupstore() TEST_THAT(teardown_test_backupstore()); \ TEARDOWN(); +//! Checks account for errors and shuts down daemons at end of every test. +bool teardown_test_backupstore_specialised(const std::string& spec_name, + BackupAccountControl& control) +{ + bool status = true; + + BackupFileSystem& fs(control.GetFileSystem()); + TEST_THAT_OR(check_reference_counts( + fs.GetPermanentRefCountDatabase(true)), // ReadOnly + status = false); + TEST_EQUAL_OR(0, check_account_for_errors(fs), status = false); + control.GetFileSystem().ReleaseLock(); + + return status; +} + +#define TEARDOWN_TEST_BACKUPSTORE_SPECIALISED(name, control) \ + if (ServerIsAlive(bbstored_pid)) \ + StopServer(); \ + if(control.GetCurrentFileSystem() != NULL) \ + { \ + control.GetCurrentFileSystem()->ReleaseLock(); \ + } \ + TEST_THAT_OR(teardown_test_backupstore_specialised(name, control), FAIL); \ + } \ + catch (BoxException &e) \ + { \ + BOX_WARNING("Specialised test failed with exception, cleaning up: " << \ + name << ": " << e.what()); \ + if (ServerIsAlive(bbstored_pid)) \ + StopServer(); \ + if(control.GetCurrentFileSystem() != NULL) \ + { \ + control.GetCurrentFileSystem()->ReleaseLock(); \ + } \ + TEST_THAT(teardown_test_backupstore_specialised(name, control)); \ + throw; \ + } \ + TEARDOWN(); + // Nice random data for testing written files class R250 { public: @@ -269,6 +389,54 @@ void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t Fl TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet)); } +std::auto_ptr get_raid_file(int64_t ObjectID) +{ + std::string filename; + StoreStructure::MakeObjectFilename(ObjectID, + "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, + filename, false /* EnsureDirectoryExists */); + return RaidFileRead::Open(0, filename); +} + +int get_disc_usage_in_blocks(bool IsDirectory, int64_t ObjectID, + const std::string& SpecialisationName, BackupAccountControl& control) +{ + if(SpecialisationName == "s3") + { + S3BackupFileSystem& fs + (dynamic_cast(control.GetFileSystem())); + std::string local_path = "testfiles/store" + + (IsDirectory ? fs.GetDirectoryURI(ObjectID) : + fs.GetFileURI(ObjectID)); + int size = TestGetFileSize(local_path); + TEST_LINE(size != -1, "File does not exist: " << local_path); + return fs.GetSizeInBlocks(size); + } + else + { + // TODO: merge get_raid_file() into here + return get_raid_file(ObjectID)->GetDiscUsageInBlocks(); + } +} + +std::auto_ptr get_object_stream(bool IsDirectory, int64_t ObjectID, + const std::string& SpecialisationName, BackupFileSystem& fs) +{ + if(SpecialisationName == "s3") + { + S3BackupFileSystem& s3fs(dynamic_cast(fs)); + std::string local_path = "testfiles/store" + + (IsDirectory ? s3fs.GetDirectoryURI(ObjectID) : + s3fs.GetFileURI(ObjectID)); + return std::auto_ptr(new FileStream(local_path)); + } + else + { + // TODO: merge get_raid_file() into here + return std::auto_ptr(get_raid_file(ObjectID).release()); + } +} + bool test_filename_encoding() { SETUP_TEST_BACKUPSTORE(); @@ -482,7 +650,7 @@ void test_test_file(int t, IOStream &rStream) free(data); in.Close(); - TEST_THAT(unlink("testfiles/test_download") == 0); + TEST_THAT(EMU_UNLINK("testfiles/test_download") == 0); } void assert_everything_deleted(BackupProtocolCallable &protocol, int64_t DirID) @@ -537,7 +705,7 @@ void create_file_in_dir(std::string name, std::string source, int64_t parentId, name_encoded, upload)); int64_t objectId = stored->GetObjectID(); - if (pRefCount) + if(pRefCount) { TEST_EQUAL(objectId, pRefCount->GetLastObjectIDUsed()); TEST_EQUAL(1, pRefCount->GetRefCount(objectId)) @@ -567,7 +735,7 @@ int64_t create_test_data_subdirs(BackupProtocolCallable &protocol, BOX_TRACE("Creating subdirs, depth = " << depth << ", dirid = " << BOX_FORMAT_OBJECTID(subdirid)); - if (pRefCount) + if(pRefCount) { TEST_EQUAL(subdirid, pRefCount->GetLastObjectIDUsed()); TEST_EQUAL(1, pRefCount->GetRefCount(subdirid)) @@ -774,15 +942,6 @@ bool check_files_same(const char *f1, const char *f2) return same; } -std::auto_ptr get_raid_file(int64_t ObjectID) -{ - std::string filename; - StoreStructure::MakeObjectFilename(ObjectID, - "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, - filename, false /* EnsureDirectoryExists */); - return RaidFileRead::Open(0, filename); -} - int64_t create_directory(BackupProtocolCallable& protocol, int64_t parent_dir_id = BACKUPSTORE_ROOT_DIRECTORY_ID); int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, @@ -835,6 +994,7 @@ bool test_temporary_refcount_db_is_independent() bool test_server_housekeeping() { SETUP_TEST_BACKUPSTORE(); + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); int encfile[ENCFILE_SIZE]; { @@ -857,7 +1017,7 @@ bool test_server_housekeeping() 0, false); int root_dir_blocks = get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID)->GetDiscUsageInBlocks(); - TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_files(fs, 0, 0, 0, 1)); TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); @@ -989,7 +1149,7 @@ bool test_server_housekeeping() } int file1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); - TEST_THAT(check_num_files(1, 0, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 0, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, 0, 0, root_dir_blocks, file1_blocks + root_dir_blocks)); @@ -1003,6 +1163,8 @@ bool test_server_housekeeping() ); TEST_EQUAL_LINE(3, patch1_id, "wrong ObjectID for newly uploaded " "patch file"); + // Update expected reference count of this new object + set_refcount(patch1_id, 1); // We need to check the old file's size, because it's been replaced // by a reverse diff, and patch1_id is a complete file, not a diff. @@ -1012,9 +1174,10 @@ bool test_server_housekeeping() // the server code is not smart enough to realise that the file // contents are identical, so it will create an empty patch. - TEST_THAT(check_num_files(1, 1, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 1, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + root_dir_blocks)); + TEST_THAT(check_reference_counts()); // Change the file and upload again, as a patch to the original file. { @@ -1033,26 +1196,29 @@ bool test_server_housekeeping() ); TEST_EQUAL_LINE(4, patch2_id, "wrong ObjectID for newly uploaded " "patch file"); + set_refcount(patch2_id, 1); // How many blocks used by the new file? // We need to check the old file's size, because it's been replaced // by a reverse diff, and patch1_id is a complete file, not a diff. int patch2_blocks = get_raid_file(patch1_id)->GetDiscUsageInBlocks(); - TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 2, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + root_dir_blocks)); + TEST_THAT(check_reference_counts()); // Housekeeping should not change anything just yet protocol.QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); - TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 2, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + root_dir_blocks)); + TEST_THAT(check_reference_counts()); // Upload not as a patch, but as a completely different file. This // marks the previous file as old (because the filename is the same) @@ -1066,31 +1232,34 @@ bool test_server_housekeeping() ); TEST_EQUAL_LINE(5, replaced_id, "wrong ObjectID for newly uploaded " "full file"); + set_refcount(replaced_id, 1); // How many blocks used by the new file? This time we need to check // the new file, because it's not a patch. int replaced_blocks = get_raid_file(replaced_id)->GetDiscUsageInBlocks(); - TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 3, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks + patch1_blocks + patch2_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + root_dir_blocks)); // total + TEST_THAT(check_reference_counts()); // Housekeeping should not change anything just yet protocol.QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); - TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_files(fs, 1, 3, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks + patch1_blocks + patch2_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + root_dir_blocks)); // total + TEST_THAT(check_reference_counts()); // But if we reduce the limits, then it will protocol.QueryFinished(); @@ -1100,12 +1269,17 @@ bool test_server_housekeeping() TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); - TEST_THAT(check_num_files(1, 1, 0, 1)); + // We expect housekeeping to have removed the two oldest versions: + set_refcount(store1objid, 0); + set_refcount(patch1_id, 0); + + TEST_THAT(check_num_files(fs, 1, 1, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + replaced_blocks + root_dir_blocks)); // total + TEST_THAT(check_reference_counts()); // Check that deleting files is accounted for as well protocol.QueryDeleteFile( @@ -1113,12 +1287,13 @@ bool test_server_housekeeping() store1name); // Filename // The old version file is deleted as well! - TEST_THAT(check_num_files(0, 1, 2, 1)); + TEST_THAT(check_num_files(fs, 0, 1, 2, 1)); TEST_THAT(check_num_blocks(protocol, 0, // current file1_blocks, // old replaced_blocks + file1_blocks, // deleted root_dir_blocks, // directories file1_blocks + replaced_blocks + root_dir_blocks)); + TEST_THAT(check_reference_counts()); // Reduce limits again, check that removed files are subtracted from // block counts. @@ -1126,43 +1301,20 @@ bool test_server_housekeeping() TEST_THAT(change_account_limits("0B", "2000B")); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); + set_refcount(store1objid, 0); - TEST_THAT(check_num_files(0, 0, 0, 1)); - TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); + // We expect housekeeping to have removed the two most recent versions + // of the now-deleted file: + set_refcount(patch2_id, 0); + set_refcount(replaced_id, 0); - // Used to not consume the stream - std::auto_ptr upload(new ZeroStream(1000)); - TEST_COMMAND_RETURNS_ERROR(protocol, QueryStoreFile( - BACKUPSTORE_ROOT_DIRECTORY_ID, - 0, - 0, /* use for attr hash too */ - 99999, /* diff from ID */ - uploads[0].name, - upload), - Err_DiffFromFileDoesNotExist); - - // TODO FIXME These tests should not be here, but in - // test_server_commands. But make sure you use a network protocol, - // not a local one, when you move them. - - // Try using GetFile on a directory - { - int64_t subdirid = create_directory(protocol); - TEST_COMMAND_RETURNS_ERROR(protocol, - QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid), - Err_FileDoesNotVerify); - } - - // Try retrieving an object that doesn't exist. That used to return - // BackupProtocolSuccess(NoObject) for no apparent reason. - TEST_COMMAND_RETURNS_ERROR(protocol, QueryGetObject(store1objid + 1), - Err_DoesNotExist); + TEST_THAT(check_num_files(fs, 0, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); + TEST_THAT(check_reference_counts()); // Close the protocol, so we can housekeep the account protocol.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - ExpectedRefCounts.resize(3); // stop test failure in teardown_test_backupstore() TEARDOWN_TEST_BACKUPSTORE(); } @@ -1239,17 +1391,21 @@ int64_t assert_readonly_connection_succeeds(BackupProtocolCallable& protocol) return loginConf->GetClientStoreMarker(); } -bool test_multiple_uploads() +bool test_multiple_uploads(const std::string& specialisation_name, + BackupAccountControl& control) { - SETUP_TEST_BACKUPSTORE(); - TEST_THAT_OR(StartServer(), FAIL); + SETUP_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); - std::auto_ptr apProtocol = - connect_and_login(context); + BackupFileSystem& fs(control.GetFileSystem()); + BackupStoreContext rwContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails + BackupStoreContext roContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails - // TODO FIXME replace protocolReadOnly with apProtocolReadOnly. - BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", - "backup/01234567/", 0, true); // ReadOnly + std::auto_ptr apProtocol( + new BackupProtocolLocal2(rwContext, 0x01234567, false)); // !ReadOnly + std::auto_ptr apProtocolReadOnly( + new BackupProtocolLocal2(roContext, 0x01234567, true)); // ReadOnly // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) for(int l = 0; l < 3; ++l) @@ -1267,19 +1423,19 @@ bool test_multiple_uploads() // Read the dir from the readonly connection (make sure it gets in the cache) // Command - protocolReadOnly.QueryListDirectory( + apProtocolReadOnly->QueryListDirectory( BACKUPSTORE_ROOT_DIRECTORY_ID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */); // Stream - BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), - protocolReadOnly.GetTimeout()); + BackupStoreDirectory dir(apProtocolReadOnly->ReceiveStream(), + apProtocolReadOnly->GetTimeout()); TEST_THAT(dir.GetNumberOfEntries() == 0); // TODO FIXME dedent { - TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_files(fs, 0, 0, 0, 1)); // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); @@ -1320,22 +1476,22 @@ bool test_multiple_uploads() if (t >= 13) expected_num_old_files++; int expected_num_current_files = t + 1 - expected_num_old_files; - TEST_THAT(check_num_files(expected_num_current_files, + TEST_THAT(check_num_files(fs, expected_num_current_files, expected_num_old_files, 0, 1)); apProtocol->QueryFinished(); - protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - apProtocol = connect_and_login(context); - protocolReadOnly.Reopen(); + apProtocolReadOnly->QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); + apProtocol->Reopen(); + apProtocolReadOnly->Reopen(); - TEST_THAT(check_num_files(expected_num_current_files, + TEST_THAT(check_num_files(fs, expected_num_current_files, expected_num_old_files, 0, 1)); } // Add some attributes onto one of them { - TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 3, 3, 0, 1)); std::auto_ptr attrnew( new MemBlockStream(attr3, sizeof(attr3))); std::auto_ptr set(apProtocol->QuerySetReplacementFileAttributes( @@ -1344,14 +1500,14 @@ bool test_multiple_uploads() uploads[UPLOAD_ATTRS_EN].name, attrnew)); TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid); - TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 3, 3, 0, 1)); } apProtocol->QueryFinished(); - protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - apProtocol = connect_and_login(context); - protocolReadOnly.Reopen(); + apProtocolReadOnly->QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); + apProtocol->Reopen(); + apProtocolReadOnly->Reopen(); // Delete one of them (will implicitly delete an old version) { @@ -1359,24 +1515,14 @@ bool test_multiple_uploads() BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); - TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 4, 3, 2, 1)); } -#ifdef _MSC_VER - BOX_TRACE("1"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif - apProtocol->QueryFinished(); - protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - apProtocol = connect_and_login(context); - protocolReadOnly.Reopen(); - -#ifdef _MSC_VER - BOX_TRACE("2"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif + apProtocolReadOnly->QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); + apProtocol->Reopen(); + apProtocolReadOnly->Reopen(); // Check that the block index can be obtained by name even though it's been deleted { @@ -1405,17 +1551,12 @@ bool test_multiple_uploads() test_test_file(t, *filestream); } -#ifdef _MSC_VER - BOX_TRACE("3"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif - { StreamableMemBlock attrtest(attr3, sizeof(attr3)); // Use the read only connection to verify that the directory is as we expect printf("\n\n==== Reading directory using read-only connection\n"); - check_dir_after_uploads(protocolReadOnly, attrtest); + check_dir_after_uploads(*apProtocolReadOnly, attrtest); printf("done.\n\n"); // And on the read/write one check_dir_after_uploads(*apProtocol, attrtest); @@ -1424,11 +1565,6 @@ bool test_multiple_uploads() // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); -#ifdef _MSC_VER - BOX_TRACE("4"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif - // Check diffing and rsync like stuff... // Build a modified file { @@ -1446,41 +1582,18 @@ bool test_multiple_uploads() ::free(buf); } - TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 4, 3, 2, 1)); // Run housekeeping (for which we need to disconnect // ourselves) and check that it doesn't change the numbers // of files - -#ifdef _MSC_VER - BOX_TRACE("5"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif - apProtocol->QueryFinished(); - protocolReadOnly.QueryFinished(); - - std::auto_ptr apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - BackupStoreAccountDatabase::Entry account = - apAccounts->GetEntry(0x1234567); -#ifdef _MSC_VER - BOX_TRACE("6"); - system("dir testfiles\\0_0\\backup\\01234567"); -#endif - TEST_EQUAL(0, run_housekeeping(account)); - - // Also check that bbstoreaccounts doesn't change anything, - // using an external process instead of the internal one. - TEST_THAT_OR(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf check 01234567 fix") == 0, - FAIL); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - apProtocol = connect_and_login(context); - protocolReadOnly.Reopen(); + apProtocolReadOnly->QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); + apProtocol->Reopen(); + apProtocolReadOnly->Reopen(); - TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 4, 3, 2, 1)); { // Fetch the block index for this one @@ -1503,13 +1616,19 @@ bool test_multiple_uploads() NULL, // pointer to DiffTimer impl &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); + // Sent this to a file, so we can check the size, rather than uploading it directly { FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT); patchstream->CopyStreamTo(patch); } + + // Release blockIndexStream to close the RaidFile, so that we can rename over it + blockIndexStream.reset(); + // Make sure the stream is a plausible size for a patch containing only one new block TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING ".patch") < (8*1024)); + // Upload it int64_t patchedID = 0; { @@ -1529,32 +1648,51 @@ bool test_multiple_uploads() set_refcount(patchedID, 1); // Then download it to check it's OK - std::auto_ptr getFile(apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, patchedID)); + std::auto_ptr getFile( + apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); std::auto_ptr filestream(apProtocol->ReceiveStream()); BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", SHORT_TIMEOUT); + // Check it's the same TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod")); - TEST_THAT(check_num_files(UPLOAD_NUM - 4, 4, 2, 1)); + TEST_THAT(check_num_files(fs, UPLOAD_NUM - 4, 4, 2, 1)); } } apProtocol->QueryFinished(); - protocolReadOnly.QueryFinished(); + apProtocolReadOnly->QueryFinished(); - TEARDOWN_TEST_BACKUPSTORE(); + TEARDOWN_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); } -bool test_server_commands() +bool test_server_commands(const std::string& specialisation_name, + BackupAccountControl& control) { - SETUP_TEST_BACKUPSTORE(); + SETUP_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); + + // Write the test file for create_file to upload: + write_test_file(0); + + BackupFileSystem& fs(control.GetFileSystem()); + BackupStoreContext rwContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails + BackupStoreContext roContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails std::auto_ptr apProtocol( - new BackupProtocolLocal2(0x01234567, "test", - "backup/01234567/", 0, false)); + new BackupProtocolLocal2(rwContext, 0x01234567, false)); // !ReadOnly + BackupProtocolLocal2 protocolReadOnly(roContext, 0x01234567, true); // ReadOnly + + // Try retrieving an object that doesn't exist. That used to return + // BackupProtocolSuccess(NoObject) for no apparent reason. + { + TEST_COMMAND_RETURNS_ERROR(*apProtocol, QueryGetObject(2), + Err_DoesNotExist); + } - // Try using GetFile on an object ID that doesn't exist in the directory + // Try using GetFile on an object ID that doesn't exist in the directory. { TEST_COMMAND_RETURNS_ERROR(*apProtocol, QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, @@ -1562,12 +1700,53 @@ bool test_server_commands() Err_DoesNotExistInDirectory); } + // Try uploading a file that doesn't verify. + { + std::auto_ptr upload(new ZeroStream(1000)); + TEST_COMMAND_RETURNS_ERROR(*apProtocol, QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, + 0, /* use for attr hash too */ + 0, /* diff from ID */ + uploads[0].name, + upload), + Err_FileDoesNotVerify); + } + + // TODO FIXME: in the case of S3 stores, we will have sent the request (but no data) before + // the client realises that the stream is invalid, and aborts. The S3 server will receive a + // PUT request for a zero-byte file, and have no idea that it's not a valid file, so it will + // store it. We should send a checksum (if possible) and a content-length (at least) to + // prevent this, and test that no file is stored instead of unlinking it here. Alternatively, + // the server could notice that the client closed the connection and didn't read the 200 OK + // response (but sent back an RST instead), and delete the file that it just created. + if(specialisation_name == "s3") + { + TEST_EQUAL(0, EMU_UNLINK("testfiles/store/subdir/0x2.file")); + } + + // Try uploading a file referencing another file which doesn't exist. + // This used to not consume the stream, leaving it unusable. + { + std::auto_ptr upload(new ZeroStream(1000)); + TEST_COMMAND_RETURNS_ERROR(*apProtocol, QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, + 0, /* use for attr hash too */ + 99999, /* diff from ID */ + uploads[0].name, + upload), + Err_DiffFromFileDoesNotExist); + } + // BLOCK // TODO FIXME dedent this block. { // Create a directory int64_t subdirid = create_directory(*apProtocol); - TEST_THAT(check_num_files(0, 0, 0, 2)); + // Ensure that store info is flushed out to disk, so we can check it: + rwContext.SaveStoreInfo(false); // !AllowDelay + TEST_THAT(check_num_files(fs, 0, 0, 0, 2)); // Try using GetFile on the directory { @@ -1579,10 +1758,7 @@ bool test_server_commands() // Stick a file in it int64_t subdirfileid = create_file(*apProtocol, subdirid); - TEST_THAT(check_num_files(1, 0, 0, 2)); - - BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", - "backup/01234567/", 0, true); // read-only + TEST_THAT(check_num_files(fs, 1, 0, 0, 2)); BOX_TRACE("Checking root directory using read-only connection"); { @@ -1616,10 +1792,12 @@ bool test_server_commands() } // Check that the last entry looks right + TEST_THAT_OR(en != NULL, FAIL); TEST_EQUAL(subdirid, en->GetObjectID()); TEST_THAT(en->GetName() == dirname); TEST_EQUAL(BackupProtocolListDirectory::Flags_Dir, en->GetFlags()); - int64_t actual_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + int64_t actual_size = get_disc_usage_in_blocks(true, // IsDirectory + subdirid, specialisation_name, control); TEST_EQUAL(actual_size, en->GetSizeInBlocks()); TEST_EQUAL(FAKE_MODIFICATION_TIME, en->GetModificationTime()); } @@ -1646,7 +1824,8 @@ bool test_server_commands() TEST_EQUAL(subdirfileid, en->GetObjectID()); TEST_THAT(en->GetName() == uploads[0].name); TEST_EQUAL(BackupProtocolListDirectory::Flags_File, en->GetFlags()); - int64_t actual_size = get_raid_file(subdirfileid)->GetDiscUsageInBlocks(); + int64_t actual_size = get_disc_usage_in_blocks(false, // !IsDirectory + subdirfileid, specialisation_name, control); TEST_EQUAL(actual_size, en->GetSizeInBlocks()); TEST_THAT(en->GetModificationTime() != 0); @@ -1681,14 +1860,35 @@ bool test_server_commands() { std::auto_ptr attrnew( new MemBlockStream(attr2, sizeof(attr2))); - std::auto_ptr changereply(apProtocol->QueryChangeDirAttributes( + std::auto_ptr changereply( + apProtocol->QueryChangeDirAttributes( subdirid, 329483209443598LL, attrnew)); TEST_THAT(changereply->GetObjectID() == subdirid); } - // Check the new attributes + // Check the new attributes using the read-write connection + { + // Command + apProtocol->QueryListDirectory( + subdirid, + 0, // no flags + BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, + true /* get attributes */); + // Stream + BackupStoreDirectory dir(apProtocol->ReceiveStream(), + SHORT_TIMEOUT); + TEST_THAT(dir.GetNumberOfEntries() == 0); + + // Attributes + TEST_THAT(dir.HasAttributes()); + TEST_EQUAL(329483209443598LL, dir.GetAttributesModTime()); + StreamableMemBlock attrtest(attr2, sizeof(attr2)); + TEST_THAT(dir.GetAttributes() == attrtest); + } + + // Check the new attributes using the read-only connection { // Command protocolReadOnly.QueryListDirectory( @@ -1710,7 +1910,7 @@ bool test_server_commands() BackupStoreFilenameClear& oldName(uploads[0].name); int64_t root_file_id = create_file(*apProtocol, BACKUPSTORE_ROOT_DIRECTORY_ID); - TEST_THAT(check_num_files(2, 0, 0, 2)); + TEST_THAT(check_num_files(fs, 2, 0, 0, 2)); // Upload a new version of the file as well, to ensure that the // old version is moved along with the current version. @@ -1720,7 +1920,7 @@ bool test_server_commands() 0, // AttributesHash oldName); set_refcount(root_file_id, 1); - TEST_THAT(check_num_files(2, 1, 0, 2)); + TEST_THAT(check_num_files(fs, 2, 1, 0, 2)); // Check that it's in the root directory (it won't be for long) protocolReadOnly.QueryListDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, @@ -1876,11 +2076,11 @@ bool test_server_commands() set_refcount(subsubdirid, 1); set_refcount(subsubfileid, 1); - TEST_THAT(check_num_files(3, 1, 0, 3)); + TEST_THAT(check_num_files(fs, 3, 1, 0, 3)); apProtocol->QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); apProtocol->Reopen(); protocolReadOnly.Reopen(); @@ -1936,33 +2136,43 @@ bool test_server_commands() } } -//} skip: + TEST_THAT(check_reference_counts( + fs.GetPermanentRefCountDatabase(true))); // ReadOnly // Create some nice recursive directories - TEST_THAT(check_reference_counts()); - write_test_file(1); int64_t dirtodelete; { - std::auto_ptr apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - std::auto_ptr apRefCount( - BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true)); - - - dirtodelete = create_test_data_subdirs(*apProtocol, + BackupStoreRefCountDatabase& rRefCount( + fs.GetPermanentRefCountDatabase(true)); // ReadOnly + dirtodelete = create_test_data_subdirs(*apProtocol, BACKUPSTORE_ROOT_DIRECTORY_ID, - "test_delete", 6 /* depth */, apRefCount.get()); + "test_delete", 6 /* depth */, &rRefCount); } - TEST_THAT(check_reference_counts()); + TEST_THAT(check_reference_counts( + fs.GetPermanentRefCountDatabase(true))); // ReadOnly apProtocol->QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - TEST_THAT(check_reference_counts()); + TEST_THAT(run_housekeeping_and_check_account(control.GetFileSystem())); + TEST_THAT(check_reference_counts( + fs.GetPermanentRefCountDatabase(true))); // ReadOnly + + // Close the refcount database and reopen it, check that the counts are + // still the same. + fs.CloseRefCountDatabase( + &fs.GetPermanentRefCountDatabase(true)); // ReadOnly + + { + BackupStoreRefCountDatabase* pRefCount = + fs.GetCurrentRefCountDatabase(); + TEST_EQUAL((BackupStoreRefCountDatabase *)NULL, pRefCount); + pRefCount = &fs.GetPermanentRefCountDatabase(true); // ReadOnly + TEST_EQUAL(pRefCount, fs.GetCurrentRefCountDatabase()); + TEST_THAT(check_reference_counts(*pRefCount)); + } // And delete them apProtocol->Reopen(); @@ -1976,8 +2186,9 @@ bool test_server_commands() apProtocol->QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - TEST_THAT(check_reference_counts()); + TEST_THAT(run_housekeeping_and_check_account(fs)); + TEST_THAT(check_reference_counts( + fs.GetPermanentRefCountDatabase(true))); // ReadOnly protocolReadOnly.Reopen(); // Get the root dir, checking for deleted items @@ -2019,11 +2230,12 @@ bool test_server_commands() apProtocol->QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(run_housekeeping_and_check_account()); - TEST_THAT(check_reference_counts()); + TEST_THAT(run_housekeeping_and_check_account(fs)); + TEST_THAT(check_reference_counts( + fs.GetPermanentRefCountDatabase(true))); // ReadOnly } - TEARDOWN_TEST_BACKUPSTORE(); + TEARDOWN_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); } int get_object_size(BackupProtocolCallable& protocol, int64_t ObjectID, @@ -2041,33 +2253,32 @@ int get_object_size(BackupProtocolCallable& protocol, int64_t ObjectID, return en->GetSizeInBlocks(); } -bool write_dir(BackupStoreDirectory& dir) +bool test_directory_parent_entry_tracks_directory_size( + const std::string& specialisation_name, BackupAccountControl& control) { - std::string rfn; - StoreStructure::MakeObjectFilename(dir.GetObjectID(), - "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, - rfn, false); // EnsureDirectoryExists - RaidFileWrite rfw(0, rfn); - rfw.Open(true); // AllowOverwrite - dir.WriteToStream(rfw); - rfw.Commit(/* ConvertToRaidNow */ true); - return true; -} + SETUP_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); -bool test_directory_parent_entry_tracks_directory_size() -{ - SETUP_TEST_BACKUPSTORE(); +#ifdef BOX_RELEASE_BUILD + BOX_NOTICE("skipping test: takes too long in release mode"); +#else + // Write the test file for create_file to upload: + write_test_file(0); - BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", - 0, false); - BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", - "backup/01234567/", 0, true); // read only + BackupFileSystem& fs(control.GetFileSystem()); + BackupStoreContext rwContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails + BackupStoreContext roContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails + + BackupProtocolLocal2 protocol(rwContext, 0x01234567, false); // !ReadOnly + BackupProtocolLocal2 protocolReadOnly(roContext, 0x01234567, true); // ReadOnly int64_t subdirid = create_directory(protocol); // Get the root directory cached in the read-only connection, and // test that the initial size is correct. - int old_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + int old_size = get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control); TEST_THAT(old_size > 0); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); @@ -2087,8 +2298,11 @@ bool test_directory_parent_entry_tracks_directory_size() std::ostringstream name; name << "testfile_" << i; last_added_filename = name.str(); + // No need to catch exceptions here, because we do not expect to hit the account + // hard limit, and if we do it should cause the test to fail. last_added_file_id = create_file(protocol, subdirid, name.str()); - new_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + new_size = get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control); } // Check that the root directory entry has been updated @@ -2098,19 +2312,18 @@ bool test_directory_parent_entry_tracks_directory_size() // Now delete an entry, and check that the size is reduced protocol.QueryDeleteFile(subdirid, BackupStoreFilenameClear(last_added_filename)); - ExpectedRefCounts[last_added_file_id] = 0; // Reduce the limits, to remove it permanently from the store protocol.QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(change_account_limits("0B", "20000B")); - TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(change_account_limits(control, "0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account(fs)); set_refcount(last_added_file_id, 0); protocol.Reopen(); protocolReadOnly.Reopen(); - TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); - + TEST_EQUAL(old_size, get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control)); // Check that the entry in the root directory was updated too TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); @@ -2118,52 +2331,45 @@ bool test_directory_parent_entry_tracks_directory_size() // Push the limits back up protocol.QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(change_account_limits("1000B", "20000B")); - TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(change_account_limits(control, "1000B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account(fs)); protocol.Reopen(); protocolReadOnly.Reopen(); // Now modify the root directory to remove its entry for this one - BackupStoreDirectory root(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + BackupStoreDirectory root( + *get_object_stream(true, // IsDirectory + BACKUPSTORE_ROOT_DIRECTORY_ID, // ObjectID + specialisation_name, fs), IOStream::TimeOutInfinite); BackupStoreDirectory::Entry *en = root.FindEntryByID(subdirid); TEST_THAT_OR(en, return false); BackupStoreDirectory::Entry enCopy(*en); root.DeleteEntry(subdirid); - TEST_THAT(write_dir(root)); + fs.PutDirectory(root); - // Add a directory, this should try to push the object size back up, - // which will try to modify the subdir's entry in its parent, which - // no longer exists, which should just log an error instead of - // aborting/segfaulting. - create_directory(protocol, subdirid); + // Add a directory, this should try to push the object size back up, which will try to + // modify the subdir's entry in its parent, which no longer exists, which should return + // an error, but only when the cache is flushed (which could be much later): + TEST_CHECK_THROWS( + create_directory(protocol, subdirid), + ConnectionException, + Protocol_UnexpectedReply); + TEST_PROTOCOL_ERROR_OR(protocol, Err_DoesNotExistInDirectory,); // Repair the error ourselves, as bbstoreaccounts can't. protocol.QueryFinished(); - enCopy.SetSizeInBlocks(get_raid_file(subdirid)->GetDiscUsageInBlocks()); + enCopy.SetSizeInBlocks(get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control)); root.AddEntry(enCopy); - TEST_THAT(write_dir(root)); - - // We also have to remove the entry for lovely_directory created by - // create_directory(), because otherwise we can't create it again. - // (Perhaps it should not have been committed because we failed to - // update the parent, but currently it is.) - BackupStoreDirectory subdir(*get_raid_file(subdirid), - IOStream::TimeOutInfinite); - { - BackupStoreDirectory::Iterator i(subdir); - en = i.FindMatchingClearName( - BackupStoreFilenameClear("lovely_directory")); - } - TEST_THAT_OR(en, return false); + fs.PutDirectory(root); protocol.Reopen(); - protocol.QueryDeleteDirectory(en->GetObjectID()); - set_refcount(en->GetObjectID(), 0); // This should have fixed the error, so we should be able to add the // entry now. This should push the object size back up. int64_t dir2id = create_directory(protocol, subdirid); - TEST_EQUAL(new_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(new_size, get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control)); TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); @@ -2174,22 +2380,25 @@ bool test_directory_parent_entry_tracks_directory_size() // Reduce the limits, to remove it permanently from the store protocol.QueryFinished(); protocolReadOnly.QueryFinished(); - TEST_THAT(change_account_limits("0B", "20000B")); - TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(change_account_limits(control, "0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account(fs)); protocol.Reopen(); protocolReadOnly.Reopen(); // Check that the entry in the root directory was updated - TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(old_size, get_disc_usage_in_blocks(true, subdirid, specialisation_name, + control)); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // Check that bbstoreaccounts check fix will detect and repair when // a directory's parent entry has the wrong size for the directory. - protocol.QueryFinished(); - root.ReadFromStream(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + root.ReadFromStream( + *get_object_stream(true, // IsDirectory + BACKUPSTORE_ROOT_DIRECTORY_ID, // ObjectID + specialisation_name, fs), IOStream::TimeOutInfinite); en = root.FindEntryByID(subdirid); TEST_THAT_OR(en != 0, return false); @@ -2198,7 +2407,7 @@ bool test_directory_parent_entry_tracks_directory_size() // Sleep to ensure that the directory file timestamp changes, so that // the read-only connection will discard its cached copy. safe_sleep(1); - TEST_THAT(write_dir(root)); + fs.PutDirectory(root); TEST_EQUAL(1234, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); @@ -2208,21 +2417,22 @@ bool test_directory_parent_entry_tracks_directory_size() safe_sleep(1); protocolReadOnly.QueryFinished(); - TEST_EQUAL(1, check_account_for_errors()); + TEST_EQUAL(1, check_account_for_errors(fs)); protocolReadOnly.Reopen(); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); protocolReadOnly.QueryFinished(); +#endif // BOX_RELEASE_BUILD - TEARDOWN_TEST_BACKUPSTORE(); + TEARDOWN_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); } bool test_cannot_open_multiple_writable_connections() { SETUP_TEST_BACKUPSTORE(); - // First try a local protocol. This works even on Windows. + // First try a local protocol (makes debugging easier): BackupProtocolLocal2 protocolWritable(0x01234567, "test", "backup/01234567/", 0, false); // Not read-only @@ -2262,7 +2472,7 @@ bool test_cannot_open_multiple_writable_connections() TEARDOWN_TEST_BACKUPSTORE(); } -bool test_encoding() +bool test_file_encoding() { // Now test encoded files // TODO: This test needs to check failure situations as well as everything working, @@ -2365,17 +2575,17 @@ bool test_encoding() size_t file_size = enc.GetPosition(); TEST_EQUAL(file_size, contents.GetSize()); - for(int buffer_size = 1; ; buffer_size <<= 1) + for(size_t buffer_size = 1; ; buffer_size <<= 1) { enc.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::VerifyStream verifier(enc); CollectInBufferStream temp_copy; - BackupStoreFile::VerifyStream verifier(&temp_copy); - enc.CopyStreamTo(verifier, IOStream::TimeOutInfinite, - buffer_size); + verifier.CopyStreamTo(temp_copy, + IOStream::TimeOutInfinite, buffer_size); // The block index is only validated on Close(), which // CopyStreamTo() doesn't do. - verifier.Close(); + verifier.Close(false); // !CloseReadFromStream temp_copy.SetForReading(); TEST_EQUAL(file_size, temp_copy.GetSize()); @@ -2485,13 +2695,17 @@ bool test_symlinks() bool test_store_info() { SETUP_TEST_BACKUPSTORE(); + RaidBackupFileSystem fs(76, "test-info/", 0); { RaidFileWrite::CreateDirectory(0, "test-info"); - BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL); - TEST_CHECK_THROWS(BackupStoreInfo::CreateNew(76, "test-info/", 0, 0, 0), RaidFileException, CannotOverwriteExistingFile); - std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); - TEST_CHECK_THROWS(info->Save(), BackupStoreException, StoreInfoIsReadOnly); + BackupStoreInfo info(76, 3461231233455433LL, 2934852487LL); + fs.PutBackupStoreInfo(info); + } + + { + std::auto_ptr info = fs.GetBackupStoreInfoUncached(); + TEST_CHECK_THROWS(fs.PutBackupStoreInfo(*info), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksUsed(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(1), BackupStoreException, StoreInfoIsReadOnly); @@ -2499,28 +2713,33 @@ bool test_store_info() TEST_CHECK_THROWS(info->AddDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->SetAccountName("hello"), BackupStoreException, StoreInfoIsReadOnly); } + { - std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, false)); - info->ChangeBlocksUsed(8); - info->ChangeBlocksInOldFiles(9); - info->ChangeBlocksInDeletedFiles(10); - info->ChangeBlocksUsed(-1); - info->ChangeBlocksInOldFiles(-4); - info->ChangeBlocksInDeletedFiles(-9); - TEST_CHECK_THROWS(info->ChangeBlocksUsed(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); - TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); - TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); - info->AddDeletedDirectory(2); - info->AddDeletedDirectory(3); - info->AddDeletedDirectory(4); - info->RemovedDeletedDirectory(3); - info->SetAccountName("whee"); - TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList); - info->Save(); + BackupStoreInfo& info(fs.GetBackupStoreInfo(false)); // !ReadOnly + info.ChangeBlocksUsed(8); + info.ChangeBlocksInOldFiles(9); + info.ChangeBlocksInDeletedFiles(10); + info.ChangeBlocksUsed(-1); + info.ChangeBlocksInOldFiles(-4); + info.ChangeBlocksInDeletedFiles(-9); + TEST_CHECK_THROWS(info.ChangeBlocksUsed(-100), + BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); + TEST_CHECK_THROWS(info.ChangeBlocksInOldFiles(-100), + BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); + TEST_CHECK_THROWS(info.ChangeBlocksInDeletedFiles(-100), + BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); + info.AddDeletedDirectory(2); + info.AddDeletedDirectory(3); + info.AddDeletedDirectory(4); + info.RemovedDeletedDirectory(3); + info.SetAccountName("whee"); + TEST_CHECK_THROWS(info.RemovedDeletedDirectory(9), + BackupStoreException, StoreInfoDirNotInList); + fs.PutBackupStoreInfo(info); } { - std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); + std::auto_ptr info = fs.GetBackupStoreInfoUncached(); TEST_THAT(info->GetBlocksUsed() == 7); TEST_THAT(info->GetBlocksInOldFiles() == 5); TEST_THAT(info->GetBlocksInDeletedFiles() == 1); @@ -2576,6 +2795,33 @@ bool test_bbstoreaccounts_create() "10000B 20000B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + // This code is almost exactly the same as tests3store.cpp:check_new_account_info() + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); + std::auto_ptr info = fs.GetBackupStoreInfoUncached(); + TEST_EQUAL(0x01234567, info->GetAccountID()); + TEST_EQUAL(1, info->GetLastObjectIDUsed()); + TEST_EQUAL(2, info->GetBlocksUsed()); + TEST_EQUAL(0, info->GetBlocksInCurrentFiles()); + TEST_EQUAL(0, info->GetBlocksInOldFiles()); + TEST_EQUAL(0, info->GetBlocksInDeletedFiles()); + TEST_EQUAL(2, info->GetBlocksInDirectories()); + TEST_EQUAL(0, info->GetDeletedDirectories().size()); + TEST_EQUAL(10000, info->GetBlocksSoftLimit()); + TEST_EQUAL(20000, info->GetBlocksHardLimit()); + TEST_EQUAL(0, info->GetNumCurrentFiles()); + TEST_EQUAL(0, info->GetNumOldFiles()); + TEST_EQUAL(0, info->GetNumDeletedFiles()); + TEST_EQUAL(1, info->GetNumDirectories()); + TEST_EQUAL(true, info->IsAccountEnabled()); + TEST_EQUAL(true, info->IsReadOnly()); + TEST_EQUAL(0, info->GetClientStoreMarker()); + TEST_EQUAL("", info->GetAccountName()); + + std::auto_ptr root_stream = + get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID); + BackupStoreDirectory root_dir(*root_stream); + TEST_EQUAL(0, root_dir.GetNumberOfEntries()); + TEARDOWN_TEST_BACKUPSTORE(); } @@ -2606,12 +2852,12 @@ bool test_login_with_disabled_account() // make sure something is written to it std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - std::auto_ptr apReferences( + std::auto_ptr apReferences = BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true)); + apAccounts->GetEntry(0x1234567), true); TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, apReferences->GetLastObjectIDUsed()); - TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)); apReferences.reset(); // Test that login fails on a disabled account @@ -2651,7 +2897,7 @@ bool test_login_with_no_refcount_db() // Delete the refcount database and try to log in again. Check that // we're locked out of the account until housekeeping has recreated // the refcount db. - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_EQUAL(0, EMU_UNLINK("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); TEST_CHECK_THROWS(BackupProtocolLocal2 protocolLocal(0x01234567, "test", "backup/01234567/", 0, false), // Not read-only BackupStoreException, CorruptReferenceCountDatabase); @@ -2676,7 +2922,7 @@ bool test_login_with_no_refcount_db() // because housekeeping may fix the refcount database while we're // stepping through. TEST_THAT_THROWONFAIL(StartServer()); - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_EQUAL(0, EMU_UNLINK("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); TEST_CHECK_THROWS(connect_and_login(context), ConnectionException, Protocol_UnexpectedReply); @@ -2751,16 +2997,26 @@ bool test_housekeeping_deletes_files() TEARDOWN_TEST_BACKUPSTORE(); } -bool test_account_limits_respected() +bool test_account_limits_respected(const std::string& specialisation_name, + BackupAccountControl& control) { - SETUP_TEST_BACKUPSTORE(); - TEST_THAT_OR(StartServer(), FAIL); + SETUP_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); + + BackupFileSystem& fs(control.GetFileSystem()); + BackupStoreContext rwContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails + BackupStoreContext roContext(fs, 0x01234567, NULL, // mpHousekeeping + "fake test connection"); // rConnectionDetails // Set a really small hard limit - TEST_THAT_OR(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf setlimit 01234567 " - "2B 2B") == 0, FAIL); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + if(specialisation_name == "s3") + { + control.SetLimit("1B", "1B"); + } + else + { + control.SetLimit("2B", "2B"); + } // Try to upload a file and create a directory, both of which would exceed the // current account limits, and check that each command returns an error. @@ -2768,8 +3024,8 @@ bool test_account_limits_respected() write_test_file(3); // Open a connection to the server - std::auto_ptr apProtocol( - connect_and_login(context)); + std::auto_ptr apProtocol( + new BackupProtocolLocal2(rwContext, 0x01234567, false)); // !ReadOnly BackupStoreFilenameClear fnx("exceed-limit"); int64_t modtime = 0; std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/test3", BACKUPSTORE_ROOT_DIRECTORY_ID, fnx, &modtime)); @@ -2799,7 +3055,7 @@ bool test_account_limits_respected() apProtocol->QueryFinished(); } - TEARDOWN_TEST_BACKUPSTORE(); + TEARDOWN_TEST_BACKUPSTORE_SPECIALISED(specialisation_name, control); } int multi_server() @@ -2873,7 +3129,7 @@ bool test_open_files_with_limited_win32_permissions() return true; } -void compare_backupstoreinfo_values_to_expected +bool compare_backupstoreinfo_values_to_expected ( const std::string& test_phase, const info_StreamFormat_1& expected, @@ -2883,6 +3139,8 @@ void compare_backupstoreinfo_values_to_expected const MemBlockStream& extra_data = MemBlockStream(/* empty */) ) { + int num_failures_initial = num_failures; + TEST_EQUAL_LINE(ntohl(expected.mAccountID), actual.GetAccountID(), test_phase << " AccountID"); #define TEST_INFO_EQUAL(property) \ @@ -2935,16 +3193,19 @@ void compare_backupstoreinfo_values_to_expected TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), actual.GetExtraData().GetBuffer(), extra_data.GetSize()), test_phase << " extra data has wrong contents"); + + return (num_failures == num_failures_initial); } bool test_read_old_backupstoreinfo_files() { SETUP_TEST_BACKUPSTORE(); + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); + // Create an account for the test client - std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, - "backup/01234567/", 0, /* ReadOnly */ false); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), + std::auto_ptr apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_EQUAL_LINE(true, apInfoReadOnly->IsAccountEnabled(), "'bbstoreaccounts create' should have set AccountEnabled flag"); info_StreamFormat_1 info_v1; @@ -2975,16 +3236,22 @@ bool test_read_old_backupstoreinfo_files() rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ false); - compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, - *apInfo, "" /* no name by default */, - true /* enabled by default */); + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, + *apInfoReadOnly, "" /* no name by default */, + true /* enabled by default */)); - apInfo->SetAccountName("bonk"); + BackupStoreInfo* pInfoReadWrite = + &(fs.GetBackupStoreInfo(false, true)); // !ReadOnly, Refresh + TEST_THAT( + compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, + *pInfoReadWrite, "" /* no name by default */, + true /* enabled by default */)); - // Save the info again - apInfo->Save(/* allowOverwrite */ true); + // Save the info again, with a new account name + pInfoReadWrite->SetAccountName("bonk"); + fs.PutBackupStoreInfo(*pInfoReadWrite); // Check that it was saved in the new Archive format std::auto_ptr rfr(RaidFileRead::Open(0, info_filename, 0)); @@ -3000,25 +3267,29 @@ bool test_read_old_backupstoreinfo_files() rfr.reset(); // load it, and check that all values are loaded properly - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ false); - compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", - info_v1, *apInfo, "bonk", true); + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected( + "loaded in v1, resaved in v2", + info_v1, *apInfoReadOnly, "bonk", true)); // Check that the new AccountEnabled flag is saved properly - apInfo->SetAccountEnabled(false); - apInfo->Save(/* allowOverwrite */ true); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ false); - compare_backupstoreinfo_values_to_expected("saved in v2, loaded in v2", - info_v1, *apInfo, "bonk", false /* as modified above */); - apInfo->SetAccountEnabled(true); - apInfo->Save(/* allowOverwrite */ true); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ true); - compare_backupstoreinfo_values_to_expected("resaved in v2 with " - "account enabled", info_v1, *apInfo, "bonk", - true /* as modified above */); + pInfoReadWrite->SetAccountEnabled(false); + fs.PutBackupStoreInfo(*pInfoReadWrite); + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved in v2, loaded in v2", + info_v1, *apInfoReadOnly, "bonk", false /* as modified above */)); + + pInfoReadWrite->SetAccountEnabled(true); + fs.PutBackupStoreInfo(*pInfoReadWrite); + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("resaved in v2 with " + "account enabled", info_v1, *apInfoReadOnly, "bonk", + true /* as modified above */)); // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true @@ -3029,36 +3300,41 @@ bool test_read_old_backupstoreinfo_files() magic = htonl(INFO_MAGIC_VALUE_2); apArchive.reset(new Archive(*rfw, IOStream::TimeOutInfinite)); rfw->Write(&magic, sizeof(magic)); - apArchive->Write(apInfo->GetAccountID()); + apArchive->Write(apInfoReadOnly->GetAccountID()); apArchive->Write(std::string("test")); - apArchive->Write(apInfo->GetClientStoreMarker()); - apArchive->Write(apInfo->GetLastObjectIDUsed()); - apArchive->Write(apInfo->GetBlocksUsed()); - apArchive->Write(apInfo->GetBlocksInCurrentFiles()); - apArchive->Write(apInfo->GetBlocksInOldFiles()); - apArchive->Write(apInfo->GetBlocksInDeletedFiles()); - apArchive->Write(apInfo->GetBlocksInDirectories()); - apArchive->Write(apInfo->GetBlocksSoftLimit()); - apArchive->Write(apInfo->GetBlocksHardLimit()); - apArchive->Write(apInfo->GetNumCurrentFiles()); - apArchive->Write(apInfo->GetNumOldFiles()); - apArchive->Write(apInfo->GetNumDeletedFiles()); - apArchive->Write(apInfo->GetNumDirectories()); - apArchive->Write((int64_t) apInfo->GetDeletedDirectories().size()); - apArchive->Write(apInfo->GetDeletedDirectories()[0]); - apArchive->Write(apInfo->GetDeletedDirectories()[1]); + apArchive->Write(apInfoReadOnly->GetClientStoreMarker()); + apArchive->Write(apInfoReadOnly->GetLastObjectIDUsed()); + apArchive->Write(apInfoReadOnly->GetBlocksUsed()); + apArchive->Write(apInfoReadOnly->GetBlocksInCurrentFiles()); + apArchive->Write(apInfoReadOnly->GetBlocksInOldFiles()); + apArchive->Write(apInfoReadOnly->GetBlocksInDeletedFiles()); + apArchive->Write(apInfoReadOnly->GetBlocksInDirectories()); + apArchive->Write(apInfoReadOnly->GetBlocksSoftLimit()); + apArchive->Write(apInfoReadOnly->GetBlocksHardLimit()); + apArchive->Write(apInfoReadOnly->GetNumCurrentFiles()); + apArchive->Write(apInfoReadOnly->GetNumOldFiles()); + apArchive->Write(apInfoReadOnly->GetNumDeletedFiles()); + apArchive->Write(apInfoReadOnly->GetNumDirectories()); + apArchive->Write((int64_t) apInfoReadOnly->GetDeletedDirectories().size()); + apArchive->Write(apInfoReadOnly->GetDeletedDirectories()[0]); + apArchive->Write(apInfoReadOnly->GetDeletedDirectories()[1]); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ false); - compare_backupstoreinfo_values_to_expected("saved in v2 without " - "AccountEnabled", info_v1, *apInfo, "test", true); + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved in v2 without " + "AccountEnabled", info_v1, *apInfoReadOnly, "test", true)); // Default for missing AccountEnabled should be true + pInfoReadWrite = &(fs.GetBackupStoreInfo(false, true)); // !ReadOnly, Refresh + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved in v2 without " + "AccountEnabled", info_v1, *pInfoReadWrite, "test", true)); + // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". - apInfo->Save(/* allowOverwrite */ true); + fs.PutBackupStoreInfo(*pInfoReadWrite); // Append some extra data after the known account values, to simulate a // new addition to the store format. Check that this extra data is loaded @@ -3077,48 +3353,58 @@ bool test_read_old_backupstoreinfo_files() extra_data.CopyStreamTo(*rfw); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, - /* ReadOnly */ false); - TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_EQUAL_LINE(extra_data.GetSize(), apInfoReadOnly->GetExtraData().GetSize(), "wrong amount of extra data loaded from info file"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), - apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), + apInfoReadOnly->GetExtraData().GetBuffer(), extra_data.GetSize()), "extra data loaded from info file has wrong contents"); + // Save the file and load again, check that the extra data is still there - apInfo->Save(/* allowOverwrite */ true); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); - compare_backupstoreinfo_values_to_expected("saved in future format " - "with extra_data", info_v1, *apInfo, "test", true, extra_data); + pInfoReadWrite = &(fs.GetBackupStoreInfo(false, true)); // !ReadOnly, Refresh + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved in future format " + "with extra_data", info_v1, *pInfoReadWrite, "test", true, + extra_data)); + fs.PutBackupStoreInfo(*pInfoReadWrite); + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved in future format " + "with extra_data", info_v1, *apInfoReadOnly, "test", true, + extra_data)); // Check that the new bbstoreaccounts command sets the flag properly TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 no") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); - TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_EQUAL_LINE(false, apInfoReadOnly->IsAccountEnabled(), "'bbstoreaccounts disabled no' should have reset AccountEnabled flag"); TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 yes") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); - TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_EQUAL_LINE(true, apInfoReadOnly->IsAccountEnabled(), "'bbstoreaccounts disabled yes' should have set AccountEnabled flag"); // Check that BackupStoreInfo::CreateForRegeneration saves all the // expected properties, including any extra data for forward // compatibility extra_data.Seek(0, IOStream::SeekType_Absolute); - apInfo = BackupStoreInfo::CreateForRegeneration( - apInfo->GetAccountID(), "spurtle" /* rAccountName */, - "backup/01234567/" /* rRootDir */, 0 /* DiscSet */, - apInfo->GetLastObjectIDUsed(), - apInfo->GetBlocksUsed(), - apInfo->GetBlocksInCurrentFiles(), - apInfo->GetBlocksInOldFiles(), - apInfo->GetBlocksInDeletedFiles(), - apInfo->GetBlocksInDirectories(), - apInfo->GetBlocksSoftLimit(), - apInfo->GetBlocksHardLimit(), + apInfoReadOnly = BackupStoreInfo::CreateForRegeneration( + apInfoReadOnly->GetAccountID(), "spurtle" /* rAccountName */, + apInfoReadOnly->GetLastObjectIDUsed(), + apInfoReadOnly->GetBlocksUsed(), + apInfoReadOnly->GetBlocksInCurrentFiles(), + apInfoReadOnly->GetBlocksInOldFiles(), + apInfoReadOnly->GetBlocksInDeletedFiles(), + apInfoReadOnly->GetBlocksInDirectories(), + apInfoReadOnly->GetBlocksSoftLimit(), + apInfoReadOnly->GetBlocksHardLimit(), false /* AccountEnabled */, extra_data); // CreateForRegeneration always sets the ClientStoreMarker to 0 @@ -3127,18 +3413,23 @@ bool test_read_old_backupstoreinfo_files() info_v1.mNumberDeletedDirectories = 0; // check that the store info has the correct values in memory - compare_backupstoreinfo_values_to_expected("stored by " - "BackupStoreInfo::CreateForRegeneration", info_v1, *apInfo, - "spurtle", false /* AccountEnabled */, extra_data); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("stored by " + "BackupStoreInfo::CreateForRegeneration", info_v1, + *apInfoReadOnly, "spurtle", false /* AccountEnabled */, + extra_data)); // Save the file and load again, check that the extra data is still there - apInfo->Save(/* allowOverwrite */ true); - apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); - compare_backupstoreinfo_values_to_expected("saved by " - "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, - *apInfo, "spurtle", false /* AccountEnabled */, extra_data); + fs.PutBackupStoreInfo(*apInfoReadOnly); + + apInfoReadOnly = fs.GetBackupStoreInfoUncached(); + TEST_THAT( + compare_backupstoreinfo_values_to_expected("saved by " + "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, + *apInfoReadOnly, "spurtle", false /* AccountEnabled */, + extra_data)); // Delete the account to stop teardown_test_backupstore checking it for errors. - apInfo.reset(); + apInfoReadOnly.reset(); TEST_THAT(delete_account()); TEARDOWN_TEST_BACKUPSTORE(); @@ -3232,6 +3523,142 @@ bool test_read_write_attr_streamformat() TEARDOWN_TEST_BACKUPSTORE(); } +bool test_s3backupfilesystem(Configuration& config, S3BackupAccountControl& s3control) +{ + SETUP_TEST_BACKUPSTORE(); + + // Test that S3BackupFileSystem returns a RevisionID based on the ETag (MD5 + // checksum) of the file. + // rand() is platform-specific, so we can't rely on it to generate files with a + // particular ETag, so we write the file ourselves instead. + { + FileStream fs("testfiles/store/subdir/0.file", O_CREAT | O_WRONLY | O_BINARY); + for(int i = 0; i < 455; i++) + { + char c = (char)i; + fs.Write(&c, 1); + } + } + + const Configuration s3config = config.GetSubConfiguration("S3Store"); + S3Client client(s3config); + HTTPResponse response = client.HeadObject("/subdir/0.file"); + client.CheckResponse(response, "Failed to get file /subdir/0.file"); + + std::string etag = response.GetHeaderValue("etag"); + TEST_EQUAL("\"447baac70b0149224b4f48daedf5266f\"", etag); + + S3BackupFileSystem fs(config, "/subdir/", DEFAULT_S3_CACHE_DIR, client); + int64_t revision_id = 0, expected_id = 0x447baac70b014922; + TEST_THAT(fs.ObjectExists(0, &revision_id)); + TEST_EQUAL(expected_id, revision_id); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +// Test that the S3 backend correctly locks and unlocks the store using SimpleDB. +bool test_simpledb_locking(Configuration& config, S3BackupAccountControl& s3control) +{ + SETUP_TEST_BACKUPSTORE(); + + const Configuration s3config = config.GetSubConfiguration("S3Store"); + S3Client s3client(s3config); + SimpleDBClient client(s3config); + + // There should be no locks at the beginning. In fact the domain should not even + // exist: the client should create it itself. + std::vector expected_domains; + TEST_THAT(test_equal_lists(expected_domains, client.ListDomains())); + + SimpleDBClient::str_map_t expected; + + // Create a client in a scope, so it will be destroyed when the scope ends. + { + S3BackupFileSystem fs(config, "/foo/", DEFAULT_S3_CACHE_DIR, s3client); + + // Check that it hasn't acquired a lock yet. + TEST_CHECK_THROWS( + client.GetAttributes("boxbackup_locks", "localhost/subdir/"), + HTTPException, SimpleDBItemNotFound); + + box_time_t before = GetCurrentBoxTime(); + // If this fails, it will throw an exception: + fs.GetLock(); + box_time_t after = GetCurrentBoxTime(); + + // Check that it has now acquired a lock. + SimpleDBClient::str_map_t attributes = + client.GetAttributes("boxbackup_locks", "localhost/subdir/"); + expected["locked"] = "true"; + + std::ostringstream locker_buf; + locker_buf << fs.GetCurrentUserName() << "@" << fs.GetCurrentHostName() << + "(" << getpid() << ")"; + TEST_EQUAL(locker_buf.str(), fs.GetSimpleDBLockValue()); + expected["locker"] = locker_buf.str(); + + std::ostringstream pid_buf; + pid_buf << getpid(); + expected["pid"] = pid_buf.str(); + + char hostname_buf[1024]; + TEST_EQUAL(0, gethostname(hostname_buf, sizeof(hostname_buf))); + TEST_EQUAL(hostname_buf, fs.GetCurrentHostName()); + expected["hostname"] = hostname_buf; + + TEST_THAT(fs.GetSinceTime() >= before); + TEST_THAT(fs.GetSinceTime() <= after); + std::ostringstream since_buf; + since_buf << fs.GetSinceTime(); + expected["since"] = since_buf.str(); + + TEST_THAT(test_equal_maps(expected, attributes)); + + // Try to acquire another one, check that it fails. + S3BackupFileSystem fs2(config, "/foo/", DEFAULT_S3_CACHE_DIR, s3client); + TEST_CHECK_THROWS( + fs2.GetLock(), + BackupStoreException, CouldNotLockStoreAccount); + + // And that the lock was not disturbed + TEST_THAT(test_equal_maps(expected, attributes)); + } + + // Check that when the S3BackupFileSystem went out of scope, it released the lock + expected["locked"] = ""; + { + SimpleDBClient::str_map_t attributes = + client.GetAttributes("boxbackup_locks", "localhost/subdir/"); + TEST_THAT(test_equal_maps(expected, attributes)); + } + + // And that we can acquire it again: + { + S3BackupFileSystem fs(config, "/foo/", DEFAULT_S3_CACHE_DIR, s3client); + fs.GetLock(); + + expected["locked"] = "true"; + std::ostringstream since_buf; + since_buf << fs.GetSinceTime(); + expected["since"] = since_buf.str(); + + SimpleDBClient::str_map_t attributes = + client.GetAttributes("boxbackup_locks", "localhost/subdir/"); + TEST_THAT(test_equal_maps(expected, attributes)); + } + + // And release it again: + expected["locked"] = ""; + { + SimpleDBClient::str_map_t attributes = + client.GetAttributes("boxbackup_locks", "localhost/subdir/"); + TEST_THAT(test_equal_maps(expected, attributes)); + } + + TEARDOWN_TEST_BACKUPSTORE(); +} + + int test(int argc, const char *argv[]) { TEST_THAT(test_open_files_with_limited_win32_permissions()); @@ -3274,32 +3701,76 @@ int test(int argc, const char *argv[]) for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} } + context.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); + + std::auto_ptr s3config = load_config_file( + DEFAULT_BBACKUPD_CONFIG_FILE, BackupDaemonConfigVerify); + // Use an auto_ptr so we can release it, and thus the lock, before stopping the + // daemon on which locking relies: + std::auto_ptr ap_s3control( + new S3BackupAccountControl(*s3config)); + + std::auto_ptr storeconfig = load_config_file( + DEFAULT_BBSTORED_CONFIG_FILE, BackupConfigFileVerify); + BackupStoreAccountControl storecontrol(*storeconfig, 0x01234567); + + TEST_THAT(kill_running_daemons()); + TEST_THAT(StartSimulator()); + TEST_THAT(test_s3backupfilesystem(*s3config, *ap_s3control)); + TEST_THAT(test_simpledb_locking(*s3config, *ap_s3control)); + + typedef std::map test_specialisation; + test_specialisation specialisations; + specialisations["s3"] = ap_s3control.get(); + specialisations["store"] = &storecontrol; + +#define RUN_SPECIALISED_TEST(name, pControl, function) \ + TEST_THAT(function(name, *(pControl))); + TEST_THAT(test_filename_encoding()); TEST_THAT(test_temporary_refcount_db_is_independent()); TEST_THAT(test_bbstoreaccounts_create()); TEST_THAT(test_bbstoreaccounts_delete()); TEST_THAT(test_backupstore_directory()); - TEST_THAT(test_directory_parent_entry_tracks_directory_size()); + + // Run all tests that take a BackupAccountControl argument twice, once with an + // S3BackupAccountControl and once with a BackupStoreAccountControl. + + for(test_specialisation::iterator i = specialisations.begin(); + i != specialisations.end(); i++) + { + RUN_SPECIALISED_TEST(i->first, i->second, + test_directory_parent_entry_tracks_directory_size); + } + TEST_THAT(test_cannot_open_multiple_writable_connections()); - TEST_THAT(test_encoding()); + TEST_THAT(test_file_encoding()); TEST_THAT(test_symlinks()); TEST_THAT(test_store_info()); - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); - TEST_THAT(test_login_without_account()); TEST_THAT(test_login_with_disabled_account()); TEST_THAT(test_login_with_no_refcount_db()); TEST_THAT(test_server_housekeeping()); - TEST_THAT(test_server_commands()); - TEST_THAT(test_account_limits_respected()); - TEST_THAT(test_multiple_uploads()); + TEST_THAT(test_multiple_uploads("store", *specialisations["store"])); + + for(test_specialisation::iterator i = specialisations.begin(); + i != specialisations.end(); i++) + { + RUN_SPECIALISED_TEST(i->first, i->second, test_server_commands); + RUN_SPECIALISED_TEST(i->first, i->second, test_account_limits_respected); + } + TEST_THAT(test_housekeeping_deletes_files()); TEST_THAT(test_read_write_attr_streamformat()); + // Release lock before shutting down the simulator: + ap_s3control.reset(); + TEST_THAT(StopSimulator()); + return finish_test_suite(); } diff --git a/test/backupstore/testfiles/bbackupd.conf b/test/backupstore/testfiles/bbackupd.conf new file mode 100644 index 000000000..a6a473f59 --- /dev/null +++ b/test/backupstore/testfiles/bbackupd.conf @@ -0,0 +1,71 @@ + +CertificateFile = testfiles/clientCerts.pem +PrivateKeyFile = testfiles/clientPrivKey.pem +TrustedCAsFile = testfiles/clientTrustedCAs.pem + +KeysFile = testfiles/bbackupd.keys + +DataDirectory = testfiles/bbackupd-data + +S3Store +{ + HostName = localhost + S3VirtualHostName = testing.s3.amazonaws.com + + # The S3Simulator requires us to send the correct endpoint (via the Host header) to + # distinguish between S3 and SimpleDB requests, so we cannot leave it at the default, + # empty value. It must be set to exactly this value for SimpleDB requests: + SimpleDBEndpoint = sdb.localhost + SimpleDBHostName = localhost + SimpleDBPort = 22080 + + Port = 22080 + BasePath = /subdir/ + AccessKey = 0PN5J17HBGZHT7JJ3X82 + SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o + CacheDirectory = testfiles/bbackupd-cache +} + +UpdateStoreInterval = 3 +BackupErrorDelay = 10 +MinimumFileAge = 4 +MaxUploadWait = 24 +DeleteRedundantLocationsAfter = 10 + +FileTrackingSizeThreshold = 1024 +DiffingUploadSizeThreshold = 1024 + +MaximumDiffingTime = 3 +KeepAliveTime = 1 + +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log + +CommandSocket = testfiles/bbackupd.sock + +NotifyScript = /usr/bin/perl testfiles/notifyscript.pl +SyncAllowScript = /usr/bin/perl testfiles/syncallowscript.pl + +Server +{ + PidFile = testfiles/bbackupd.pid +} + +BackupLocations +{ + Test1 + { + Path = testfiles/TestDir1 + + ExcludeFile = testfiles/TestDir1/excluded_1 + ExcludeFile = testfiles/TestDir1/excluded_2 + ExcludeFilesRegex = \.excludethis$ + ExcludeFilesRegex = EXCLUDE + AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis + ExcludeDir = testfiles/TestDir1/exclude_dir + ExcludeDir = testfiles/TestDir1/exclude_dir_2 + ExcludeDirsRegex = not_this_dir + AlwaysIncludeDirsRegex = ALWAYSINCLUDE + } +} + diff --git a/test/backupstore/testfiles/s3simulator.conf b/test/backupstore/testfiles/s3simulator.conf new file mode 100644 index 000000000..c9895e9ff --- /dev/null +++ b/test/backupstore/testfiles/s3simulator.conf @@ -0,0 +1,10 @@ +AccessKey = 0PN5J17HBGZHT7JJ3X82 +SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o +StoreDirectory = testfiles/store +AddressPrefix = http://localhost:22080 + +Server +{ + PidFile = testfiles/s3simulator.pid + ListenAddresses = inet:localhost:22080 +} diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp index 38492bd15..9f438db73 100644 --- a/test/backupstorefix/testbackupstorefix.cpp +++ b/test/backupstorefix/testbackupstorefix.cpp @@ -43,7 +43,7 @@ #include "MemLeakFindOn.h" -/* +/* Errors checked: @@ -75,9 +75,12 @@ int discSetNum = 0; std::map nameToID; std::map objectIsDir; -#define RUN_CHECK \ - ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567"); \ - ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567 fix"); +#define RUN_CHECK \ + BOX_INFO("Running bbstoreaccounts to check and then repair the account"); \ + ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf -Utbbstoreaccounts " \ + "-L/FileSystem/Locking=trace check 01234567"); \ + ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf -Utbbstoreaccounts " \ + "-L/FileSystem/Locking=trace check 01234567 fix"); // Get ID of an object given a filename int32_t getID(const char *name) @@ -233,14 +236,14 @@ void test_dir_fixing() int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); */ - dir.AddEntry(fnames[0], 12, 2 /* id */, 1, + dir.AddEntry(fnames[0], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); dir.AddEntry(fnames[1], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); dir.AddEntry(fnames[0], 12, 3 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); dir.AddEntry(fnames[0], 12, 5 /* id */, 1, - BackupStoreDirectory::Entry::Flags_File | + BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2); /* @@ -445,8 +448,21 @@ void check_and_fix_root_dir(dir_en_check after_entries[], check_root_dir_ok(after_entries, after_deps); } +bool compare_store_contents_with_expected(int phase) +{ + BOX_INFO("Running testbackupstorefix.pl to check contents of store (phase " << + phase << ")"); + std::ostringstream cmd; + cmd << PERL_EXECUTABLE " testfiles/testbackupstorefix.pl "; + cmd << ((phase == 6) ? "reroot" : "check") << " " << phase; + return ::system(cmd.str().c_str()) == 0; +} + int test(int argc, const char *argv[]) { + // Enable logging timestamps to help debug race conditions on AppVeyor + Console::SetShowTimeMicros(true); + { MEMLEAKFINDER_NO_LEAKS; fnames[0].SetAsClearFilename("x1"); @@ -463,13 +479,13 @@ int test(int argc, const char *argv[]) BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); // Create an account - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf " "create 01234567 0 10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Run the perl script to create the initial directories - TEST_THAT_ABORTONFAIL(::system(PERL_EXECUTABLE + TEST_THAT_ABORTONFAIL(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl init") == 0); BOX_INFO(" === Test that an entry pointing to a file that doesn't " @@ -546,8 +562,11 @@ int test(int argc, const char *argv[]) BOX_INFO(" === Test that an entry pointing to a directory whose " "raidfile is corrupted doesn't crash"); - // Start the bbstored server - TEST_THAT_OR(StartServer(), return 1); + // Start the bbstored server. Enable logging to help debug if the store is unexpectedly + // locked when we try to check or query it (race conditions): + std::string daemon_args(bbstored_args_overridden ? bbstored_args : + "-kT -Winfo -tbbstored -L/FileSystem/Locking=trace"); + TEST_THAT_OR(StartServer(daemon_args), return 1); // Instead of starting a client, read the file listing file created by // testbackupstorefix.pl and upload them in the correct order, so that the object @@ -560,7 +579,7 @@ int test(int argc, const char *argv[]) BackupProtocolLocal2 client(0x01234567, "test", accountRootDir, discSetNum, false); - for(getline.GetLine(line, true); line != ""; getline.GetLine(line, true)) + for(line = getline.GetLine(true); line != ""; line = getline.GetLine(true)) { std::string full_path = line; ASSERT(StartsWith("testfiles/TestDir1/", full_path)); @@ -671,7 +690,7 @@ int test(int argc, const char *argv[]) char name[256]; while(::fgets(line, sizeof(line), f) != 0) { - TEST_THAT(::sscanf(line, "%x %s %s", &id, + TEST_THAT(::sscanf(line, "%x %s %s", &id, flags, name) == 3); bool isDir = (::strcmp(flags, "-d---") == 0); //TRACE3("%x,%d,%s\n", id, isDir, name); @@ -691,7 +710,7 @@ int test(int argc, const char *argv[]) } { // Add a spurious file - RaidFileWrite random(discSetNum, + RaidFileWrite random(discSetNum, accountRootDir + "randomfile"); random.Open(); random.Write("test", 4); @@ -702,11 +721,11 @@ int test(int argc, const char *argv[]) RUN_CHECK // Check everything is as it was - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 0") == 0); + TEST_THAT(compare_store_contents_with_expected(0)); + // Check the random file doesn't exist { - TEST_THAT(!RaidFileRead::FileExists(discSetNum, + TEST_THAT(!RaidFileRead::FileExists(discSetNum, accountRootDir + "01/randomfile")); } @@ -714,20 +733,13 @@ int test(int argc, const char *argv[]) BOX_INFO(" === Delete an entry for an object from dir, change that " "object to be a patch, check it's deleted"); { - // Temporarily stop the server, so it doesn't repair the refcount error. Except - // on win32, where hard-killing the server can leave a lockfile in place, - // breaking the rest of the test. -#ifdef WIN32 - // Wait for the server to finish housekeeping first, by getting a lock on - // the account. - std::auto_ptr apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - BackupStoreAccounts acc(*apAccounts); - NamedLock lock; - acc.LockAccount(0x1234567, lock); -#else - TEST_THAT(StopServer()); -#endif + // Wait for the server to finish housekeeping (if any) and then lock the account + // before damaging it, to prevent housekeeping from repairing the damage in the + // background. We could just stop the server, but on Windows that can leave the + // account half-cleaned and always leaves a PID file lying around, which breaks + // the rest of the test, so we do it this way on all platforms instead. + RaidBackupFileSystem fs(0x01234567, accountRootDir, 0); // discSet + fs.GetLock(); // Open dir and find entry int64_t delID = getID("Test1/cannes/ict/metegoguered/oats"); @@ -783,9 +795,8 @@ int test(int argc, const char *argv[]) // ERROR: BlocksInCurrentFiles changed from 228 to 226 // ERROR: NumCurrentFiles changed from 114 to 113 // WARNING: Reference count of object 0x44 changed from 1 to 0 -#ifdef WIN32 - lock.ReleaseLock(); -#endif + fs.ReleaseLock(); + TEST_EQUAL(5, check_account_for_errors()); { std::auto_ptr usage = @@ -799,16 +810,8 @@ int test(int argc, const char *argv[]) TEST_EQUAL(usage->GetBlocksInDirectories(), 56); } - // Start the server again, so testbackupstorefix.pl can run bbackupquery which - // connects to it. Except on win32, where we didn't stop it earlier. -#ifndef WIN32 - TEST_THAT(StartServer()); -#endif - // Check - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 1") - == 0); + TEST_THAT(compare_store_contents_with_expected(1)); // Check the modified file doesn't exist TEST_THAT(!RaidFileRead::FileExists(discSetNum, fn)); @@ -894,8 +897,8 @@ int test(int argc, const char *argv[]) } // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 2") == 0); + TEST_THAT(compare_store_contents_with_expected(2)); + { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); @@ -954,8 +957,8 @@ int test(int argc, const char *argv[]) RUN_CHECK // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 3") == 0); + TEST_THAT(compare_store_contents_with_expected(3)); + { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); @@ -971,22 +974,36 @@ int test(int argc, const char *argv[]) RUN_CHECK // Check everything is where it is predicted to be - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 4") == 0); + TEST_THAT(compare_store_contents_with_expected(4)); // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Corrupt file and dir"); - // File - CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", - 33, "34i729834298349283479233472983sdfhasgs"); - // Dir - CorruptObject("Test1/cannes/imulatrougge/foreomizes",23, - "dsf32489sdnadf897fd2hjkesdfmnbsdfcsfoisufio2iofe2hdfkjhsf"); + { + // Increase log level for locking errors to help debug random failures on AppVeyor + LogLevelOverrideByFileGuard increase_lock_logging("", // rFileName + BackupFileSystem::LOCKING.ToString(), Log::TRACE); // NewLevel + increase_lock_logging.Install(); + + // Wait for the server to finish housekeeping (if any) and then lock the account + // before damaging it, to avoid a race condition where bbstored runs housekeeping + // after we disconnect, extremely slowly on AppVeyor, and this causes the second + // bbstoreaccounts check command to time out waiting for a lock. + RaidBackupFileSystem fs(0x01234567, accountRootDir, 0); // discSet + fs.GetLock(); + + // File + CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", 33, + "34i729834298349283479233472983sdfhasgs"); + // Dir + CorruptObject("Test1/cannes/imulatrougge/foreomizes", 23, + "dsf32489sdnadf897fd2hjkesdfmnbsdfcsfoisufio2iofe2hdfkjhsf"); + } + // Fix it RUN_CHECK + // Check everything is where it should be - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl check 5") == 0); + TEST_THAT(compare_store_contents_with_expected(5)); // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Overwrite root with a file"); @@ -997,22 +1014,16 @@ int test(int argc, const char *argv[]) r->CopyStreamTo(w); w.Commit(true /* convert now */); } + // Fix it RUN_CHECK - // Check everything is where it should be - TEST_THAT(::system(PERL_EXECUTABLE - " testfiles/testbackupstorefix.pl reroot 6") == 0); + // Check everything is where it should be + TEST_THAT(compare_store_contents_with_expected(6)); // --------------------------------------------------------- // Stop server - TEST_THAT(KillServer(bbstored_pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif + TEST_THAT(StopServer()); return 0; } diff --git a/test/backupstorepatch/testbackupstorepatch.cpp b/test/backupstorepatch/testbackupstorepatch.cpp index 46f278ad1..74dc9dd9b 100644 --- a/test/backupstorepatch/testbackupstorepatch.cpp +++ b/test/backupstorepatch/testbackupstorepatch.cpp @@ -13,11 +13,13 @@ #include #include -#include "autogen_BackupProtocol.h" +#include "BackupAccountControl.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" +#include "BackupProtocol.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" @@ -39,6 +41,7 @@ #include "Socket.h" #include "SocketStreamTLS.h" #include "StoreStructure.h" +#include "StoreTestUtils.h" // for run_housekeeping() #include "TLSContext.h" #include "Test.h" @@ -51,19 +54,20 @@ typedef struct bool IsCompletelyDifferent; bool HasBeenDeleted; int64_t DepNewer, DepOlder; + int64_t CurrentSizeInBlocks; } file_info; file_info test_files[] = { -// ChPnt, Insert, Delete, ID, IsCDf, BeenDel - {0, 0, 0, 0, false, false}, // 0 dummy first entry - {32000, 2087, 0, 0, false, false}, // 1 +// ChPnt, Insert, Delete, ID, IsCDf, BeenDel + {0, 0, 0, 0, false, false}, // 0 dummy first entry + {32000, 2087, 0, 0, false, false}, // 1 {1000, 1998, 2976, 0, false, false}, // 2 - {27800, 0, 288, 0, false, false}, // 3 - {3208, 1087, 98, 0, false, false}, // 4 - {56000, 23087, 98, 0, false, false}, // 5 - {0, 98765, 9999999,0, false, false}, // 6 completely different, make a break in the storage - {9899, 9887, 2, 0, false, false}, // 7 + {27800, 0, 288, 0, false, false}, // 3 + {3208, 1087, 98, 0, false, false}, // 4 - this entry is deleted from middle of patch chain on r=1 + {56000, 23087, 98, 0, false, false}, // 5 + {0, 98765, 9999999,0, false, false}, // 6 completely different, make a break in the storage + {9899, 9887, 2, 0, false, false}, // 7 {12984, 12345, 1234, 0, false, false}, // 8 {1209, 29885, 3498, 0, false, false} // 9 }; @@ -75,6 +79,7 @@ int test_file_remove_order[] = {0, 2, 3, 5, 8, 1, 4, -1}; #define FIRST_FILE_SIZE (64*1024+3) #define BUFFER_SIZE (256*1024) #define SHORT_TIMEOUT 5000 +#define HOUSEKEEPING_IN_PROCESS // Chunk of memory to use for copying files, etc static void *buffer = 0; @@ -171,8 +176,6 @@ bool files_identical(const char *file1, const char *file2) return true; } - - void create_test_files() { // Create first file @@ -337,7 +340,16 @@ int test(int argc, const char *argv[]) } RaidFileDiscSet rfd(rcontroller.GetDiscSet(discSet)); - int pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", + std::string errs; + std::auto_ptr config( + Configuration::LoadAndVerify("testfiles/bbstored.conf", + &BackupConfigFileVerify, errs)); + TEST_EQUAL(0, errs.size()); + + BackupStoreAccountControl control(*config, 0x01234567); + BackupFileSystem& filesystem(control.GetFileSystem()); + + int pid = LaunchServer(BBSTORED " -c testfiles/bbstored.conf " + bbstored_args, "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid > 0) @@ -451,6 +463,21 @@ int test(int argc, const char *argv[]) { TEST_THAT(en->GetDependsNewer() == 0); TEST_THAT(en->GetDependsOlder() == 0); + bool found = false; + + for(int tfi = 0; tfi < NUMBER_FILES; tfi++) + { + if(test_files[tfi].IDOnServer == en->GetObjectID()) + { + found = true; + test_files[tfi].CurrentSizeInBlocks = + en->GetSizeInBlocks(); + break; + } + } + + TEST_LINE(found, "Unexpected file found on server: " << + en->GetObjectID()); } } @@ -475,6 +502,19 @@ int test(int argc, const char *argv[]) test_files[f].DepOlder = older; } +#ifdef HOUSEKEEPING_IN_PROCESS + // Kill store server + TEST_THAT(KillServer(pid)); + TEST_THAT(!ServerIsAlive(pid)); + + #ifndef WIN32 + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif +#else + // We need to leave the bbstored process running, so that we can connect to it + // and retrieve directory listings from it. +#endif // HOUSEKEEPING_IN_PROCESS + // Check the stuff on the server int deleteIndex = 0; while(true) @@ -482,23 +522,41 @@ int test(int argc, const char *argv[]) // Load up the root directory BackupStoreDirectory dir; { + // Take a lock before actually reading files from disk, + // to avoid them changing under our feet. + filesystem.GetLock(); + std::auto_ptr dirStream(RaidFileRead::Open(0, "backup/01234567/o01")); dir.ReadFromStream(*dirStream, SHORT_TIMEOUT); dir.Dump(0, true); + + // Find the test_files entry for the file that was just deleted: + int just_deleted = deleteIndex == 0 ? -1 : test_file_remove_order[deleteIndex - 1]; + file_info* p_just_deleted; + if(just_deleted == 0 || just_deleted == -1) + { + p_just_deleted = NULL; + } + else + { + p_just_deleted = test_files + just_deleted; + } // Check that dependency info is correct - for(unsigned int f = 0; f < NUMBER_FILES; ++f) + for(unsigned int f = 1; f < NUMBER_FILES; ++f) { //TRACE1("t f = %d\n", f); BackupStoreDirectory::Entry *en = dir.FindEntryByID(test_files[f].IDOnServer); if(en == 0) { - TEST_THAT(test_files[f].HasBeenDeleted); + TEST_LINE(test_files[f].HasBeenDeleted, + "Test file " << f << " (id " << + BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") was unexpectedly deleted by housekeeping"); // check that unreferenced // object was removed by // housekeeping std::string filenameOut; - int startDisc = 0; StoreStructure::MakeObjectFilename( test_files[f].IDOnServer, storeRootDir, discSet, @@ -514,25 +572,84 @@ int test(int argc, const char *argv[]) } else { - TEST_THAT(!test_files[f].HasBeenDeleted); + TEST_LINE(!test_files[f].HasBeenDeleted, + "Test file " << f << " (id " << + BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") was unexpectedly not deleted by housekeeping"); TEST_THAT(en->GetDependsNewer() == test_files[f].DepNewer); - TEST_THAT(en->GetDependsOlder() == test_files[f].DepOlder); + TEST_EQUAL_LINE(test_files[f].DepOlder, en->GetDependsOlder(), + "Test file " << f << " (id " << + BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") has different dependencies than " + "expected after housekeeping"); // Test that size is plausible if(en->GetDependsNewer() == 0) { // Should be a full file - TEST_THAT(en->GetSizeInBlocks() > 40); + TEST_LINE(en->GetSizeInBlocks() > 40, + "Test file " << f << " (id " << + BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") was smaller than expected: " + "wanted a full file with >40 blocks, " + "but found " << en->GetSizeInBlocks()); } else { // Should be a patch - TEST_THAT(en->GetSizeInBlocks() < 40); + TEST_LINE(en->GetSizeInBlocks() < 40, + "Test file " << f << " (id " << + BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") was larger than expected: " + "wanted a patch file with <40 blocks, " + "but found " << en->GetSizeInBlocks()); } } + + // All the files that we've deleted so far should have had + // HasBeenDeleted set to true. + if(test_files[f].HasBeenDeleted) + { + TEST_LINE(en == NULL, "File " << f << " should have been " + "deleted by this point") + } + else if(en == 0) + { + TEST_FAIL_WITH_MESSAGE("File " << f << " has been unexpectedly " + "deleted, cannot check its size"); + } + // If the file that was just deleted was a patch that this file depended on, + // then it should have been merged with this file, which should have made this + // file larger. But that might not translate to a larger number of blocks. + else if(test_files[just_deleted].DepOlder == test_files[f].IDOnServer) + { + TEST_LINE(en->GetSizeInBlocks() >= test_files[f].CurrentSizeInBlocks, + "File " << f << " has been merged with an older patch, " + "so it should be larger than its previous size of " << + test_files[f].CurrentSizeInBlocks << " blocks, but it is " << + en->GetSizeInBlocks() << " blocks now"); + } + else + { + // This file should not have changed in size. + TEST_EQUAL_LINE(test_files[f].CurrentSizeInBlocks, en->GetSizeInBlocks(), + "File " << f << " unexpectedly changed size"); + } + + if(en != 0) + { + // Update test_files to record new size for next pass: + test_files[f].CurrentSizeInBlocks = en->GetSizeInBlocks(); + } } + + filesystem.ReleaseLock(); } - // Open a connection to the server (need to do this each time, otherwise housekeeping won't delete files) +#ifdef HOUSEKEEPING_IN_PROCESS + BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", 0, true); +#else + // Open a connection to the server (need to do this each time, otherwise + // housekeeping won't run on Windows, and thus won't delete any files). SocketStreamTLS *pConn = new SocketStreamTLS; std::auto_ptr apConn(pConn); pConn->Open(context, Socket::TypeINET, "localhost", @@ -543,11 +660,16 @@ int test(int argc, const char *argv[]) TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); protocol.QueryLogin(0x01234567, 0); } +#endif - // Pull all the files down, and check that they match the files on disc + // Pull all the files down, and check that they (still) match the files + // that we uploaded earlier. for(unsigned int f = 0; f < NUMBER_FILES; ++f) { - ::printf("r=%d, f=%d\n", deleteIndex, f); + ::printf("r=%d, f=%d, id=%08llx, blocks=%d, deleted=%s\n", deleteIndex, f, + (long long)test_files[f].IDOnServer, + test_files[f].CurrentSizeInBlocks, + test_files[f].HasBeenDeleted ? "true" : "false"); // Might have been deleted if(test_files[f].HasBeenDeleted) @@ -559,22 +681,32 @@ int test(int argc, const char *argv[]) char filename[64], filename_fetched[64]; ::sprintf(filename, "testfiles/%d.test", f); ::sprintf(filename_fetched, "testfiles/%d.test.fetched", f); - ::unlink(filename_fetched); + EMU_UNLINK(filename_fetched); // Fetch the file + try { std::auto_ptr getobj(protocol.QueryGetFile( BackupProtocolListDirectory::RootDirectory, test_files[f].IDOnServer)); TEST_THAT(getobj->GetObjectID() == test_files[f].IDOnServer); - // BLOCK - { - // Get stream - std::auto_ptr filestream(protocol.ReceiveStream()); - // Get and decode - BackupStoreFile::DecodeFile(*filestream, filename_fetched, SHORT_TIMEOUT); - } } + catch(ConnectionException &e) + { + TEST_FAIL_WITH_MESSAGE("Failed to get test file " << f << + " (id " << BOX_FORMAT_OBJECTID(test_files[f].IDOnServer) << + ") from server: " << e.what()); + continue; + } + + // BLOCK + { + // Get stream + std::auto_ptr filestream(protocol.ReceiveStream()); + // Get and decode + BackupStoreFile::DecodeFile(*filestream, filename_fetched, SHORT_TIMEOUT); + } + // Test for identicalness TEST_THAT(files_identical(filename_fetched, filename)); @@ -587,9 +719,13 @@ int test(int argc, const char *argv[]) } } - // Close the connection + // Close the connection protocol.QueryFinished(); + // Take a lock before modifying the directory + filesystem.GetLock(); + filesystem.GetDirectory(BackupProtocolListDirectory::RootDirectory, dir); + // Mark one of the elements as deleted if(test_file_remove_order[deleteIndex] == -1) { @@ -600,30 +736,45 @@ int test(int argc, const char *argv[]) // Modify the entry BackupStoreDirectory::Entry *pentry = dir.FindEntryByID(test_files[todel].IDOnServer); - TEST_THAT(pentry != 0); + TEST_LINE_OR(pentry != 0, "Cannot delete test file " << todel << " (id " << + BOX_FORMAT_OBJECTID(test_files[todel].IDOnServer) << "): not found on server", + break); + pentry->AddFlags(BackupStoreDirectory::Entry::Flags_RemoveASAP); - // Save it back - { - RaidFileWrite writedir(0, "backup/01234567/o01"); - writedir.Open(true /* overwrite */); - dir.WriteToStream(writedir); - writedir.Commit(true); - } + filesystem.PutDirectory(dir); - // Get the revision number of the root directory, before housekeeping makes any changes. + // Get the revision number of the root directory, before we release + // the lock (and therefore before housekeeping makes any changes). int64_t first_revision = 0; - RaidFileRead::FileExists(0, "backup/01234567/o01", &first_revision); + TEST_THAT(filesystem.ObjectExists(BackupProtocolListDirectory::RootDirectory, + &first_revision)); + filesystem.ReleaseLock(); + +#ifdef HOUSEKEEPING_IN_PROCESS + // Housekeeping wants to open both a temporary and a permanent refcount DB, + // and after committing the temporary one, it becomes the permanent one and + // not ReadOnly, and the BackupFileSystem does not allow opening another + // temporary refcount DB if the permanent one is open for writing (with good + // reason), so we need to close it here so that housekeeping can open it + // again, read-only, on the second and subsequent passes. + if(filesystem.GetCurrentRefCountDatabase() != NULL) + { + filesystem.CloseRefCountDatabase(filesystem.GetCurrentRefCountDatabase()); + } -#ifdef WIN32 + TEST_EQUAL_LINE(0, run_housekeeping(filesystem), + "Housekeeping detected errors in account"); +#else +# ifdef WIN32 // Cannot signal bbstored to do housekeeping now, and we don't need to, as we will // wait up to 32 seconds and detect automatically when it has finished. -#else +# else // Send the server a restart signal, so it does // housekeeping immediately, and wait for it to happen // Wait for old connections to terminate ::sleep(1); ::kill(pid, SIGHUP); -#endif +# endif // WIN32 // Wait for changes to be written back to the root directory. for(int secs_remaining = 32; secs_remaining >= 0; secs_remaining--) @@ -634,19 +785,42 @@ int test(int argc, const char *argv[]) ::fflush(stdout); // Early end? - if(!TestFileExists("testfiles/0_0/backup/01234567/write.lock")) + try { - int64_t revid = 0; - RaidFileRead::FileExists(0, "backup/01234567/o01", &revid); - if(revid != first_revision) + filesystem.GetLock(); + int64_t current_revision = 0; + TEST_THAT(filesystem.ObjectExists(BackupProtocolListDirectory::RootDirectory, + ¤t_revision)); + filesystem.ReleaseLock(); + + if(current_revision != first_revision) { + // Root directory has changed, and housekeeping is + // not running right now (as we have a lock), so it + // must have run already. break; } } + catch(BackupStoreException &e) + { + if(EXCEPTION_IS_TYPE(e, BackupStoreException, + CouldNotLockStoreAccount)) + { + // Housekeeping must still be running. Do nothing, + // hopefully after another few seconds it will have + // finished. + } + else + { + // Unexpected exception + throw; + } + } TEST_LINE(secs_remaining != 0, "No changes detected to root directory after 32 seconds"); } ::printf("\n"); +#endif // HOUSEKEEPING_IN_PROCESS // Flag for test test_files[todel].HasBeenDeleted = true; @@ -664,7 +838,10 @@ int test(int argc, const char *argv[]) } if(z < (int)NUMBER_FILES) test_files[z].DepOlder = test_files[todel].DepOlder; } - + +#ifdef HOUSEKEEPING_IN_PROCESS + // We already killed the bbstored process earlier +#else // Kill store server TEST_THAT(KillServer(pid)); TEST_THAT(!ServerIsAlive(pid)); @@ -672,6 +849,7 @@ int test(int argc, const char *argv[]) #ifndef WIN32 TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif +#endif // HOUSEKEEPING_IN_PROCESS } ::free(buffer); diff --git a/test/basicserver/TestCommands.cpp b/test/basicserver/TestCommands.cpp index bdbdffeb8..51b46545f 100644 --- a/test/basicserver/TestCommands.cpp +++ b/test/basicserver/TestCommands.cpp @@ -106,3 +106,16 @@ std::auto_ptr TestProtocolString::DoCommand(TestProtocolRep return std::auto_ptr(new TestProtocolString(mTest)); } +std::auto_ptr TestProtocolDeliberateError::DoCommand( + TestProtocolReplyable &rProtocol, TestContext &rContext) const +{ + return std::auto_ptr(new TestProtocolError( + TestProtocolError::ErrorType, + TestProtocolError::Err_DeliberateError)); +} + +std::auto_ptr TestProtocolUnexpectedError::DoCommand( + TestProtocolReplyable &rProtocol, TestContext &rContext) const +{ + THROW_EXCEPTION_MESSAGE(CommonException, Internal, "unexpected error"); +} diff --git a/test/basicserver/TestProtocol.txt b/test/basicserver/TestProtocol.txt index 5bca9f49b..87d9f7cc0 100644 --- a/test/basicserver/TestProtocol.txt +++ b/test/basicserver/TestProtocol.txt @@ -9,6 +9,8 @@ BEGIN_OBJECTS Error 0 IsError(Type,SubType) Reply int32 Type int32 SubType + CONSTANT ErrorType 1000 + CONSTANT Err_DeliberateError 1 Hello 1 Command(Hello) Reply int32 Number32 @@ -40,3 +42,6 @@ SendStream 8 Command(GetStream) StreamWithCommand String 9 Command(String) Reply string Test +DeliberateError 10 Command(String) + +UnexpectedError 11 Command(String) diff --git a/test/basicserver/testbasicserver.cpp b/test/basicserver/testbasicserver.cpp index 6f2def54a..1ff1e37a4 100644 --- a/test/basicserver/testbasicserver.cpp +++ b/test/basicserver/testbasicserver.cpp @@ -10,6 +10,10 @@ #include "Box.h" +#ifdef HAVE_SIGNAL_H +# include +#endif + #include #include @@ -33,8 +37,8 @@ #define SERVER_LISTEN_PORT 2003 // in ms -#define COMMS_READ_TIMEOUT 4 -#define COMMS_SERVER_WAIT_BEFORE_REPLYING 40 +#define COMMS_READ_TIMEOUT 4 +#define COMMS_SERVER_WAIT_BEFORE_REPLYING 1000 // Use a longer timeout to give Srv2TestConversations time to write 20 MB to each of // three child processes before starting to read it back again, without the children // timing out and aborting. @@ -67,9 +71,10 @@ void testservers_pause_before_reply() #ifdef WIN32 Sleep(COMMS_SERVER_WAIT_BEFORE_REPLYING); #else + int64_t nsec = COMMS_SERVER_WAIT_BEFORE_REPLYING * 1000LL * 1000; // convert to ns struct timespec t; - t.tv_sec = 0; - t.tv_nsec = COMMS_SERVER_WAIT_BEFORE_REPLYING * 1000 * 1000; // convert to ns + t.tv_sec = nsec / NANO_SEC_IN_SEC; + t.tv_nsec = nsec % NANO_SEC_IN_SEC; ::nanosleep(&t, NULL); #endif } @@ -84,22 +89,22 @@ void testservers_connection(SocketStream &rStream) if(typeid(rStream) == typeid(SocketStreamTLS)) { // need to wait for some data before sending stuff, otherwise timeout test doesn't work - std::string line; - while(!getline.GetLine(line)) - ; + std::string line = getline.GetLine(false); // !preprocess SocketStreamTLS &rtls = (SocketStreamTLS&)rStream; std::string line1("CONNECTED:"); line1 += rtls.GetPeerCommonName(); line1 += '\n'; + + // Reply after a short delay, to allow the client to test timing out + // in GetLine(): testservers_pause_before_reply(); + rStream.Write(line1.c_str(), line1.size()); } while(!getline.IsEOF()) { - std::string line; - while(!getline.GetLine(line)) - ; + std::string line = getline.GetLine(false); // !preprocess if(line == "QUIT") { break; @@ -313,20 +318,25 @@ void Srv2TestConversations(const std::vector &conns) { getline[c] = new IOStreamGetLine(*conns[c]); - bool hadTimeout = false; if(typeid(*conns[c]) == typeid(SocketStreamTLS)) { SocketStreamTLS *ptls = (SocketStreamTLS *)conns[c]; - printf("Connected to '%s'\n", ptls->GetPeerCommonName().c_str()); + BOX_INFO("Connected to '" << ptls->GetPeerCommonName() << "'"); // Send some data, any data, to get the first response. conns[c]->Write("Hello\n", 6); - std::string line1; - while(!getline[c]->GetLine(line1, false, COMMS_READ_TIMEOUT)) - hadTimeout = true; - TEST_THAT(line1 == "CONNECTED:CLIENT"); - TEST_THAT(hadTimeout) + // First read should timeout, while server sleeps in + // testservers_pause_before_reply(): + TEST_CHECK_THROWS(getline[c]->GetLine(false, + COMMS_SERVER_WAIT_BEFORE_REPLYING * 0.5), + CommonException, IOStreamTimedOut); + + // Second read should not timeout, because we should have waited + // COMMS_SERVER_WAIT_BEFORE_REPLYING * 1.5 + std::string line1 = getline[c]->GetLine(false, + COMMS_SERVER_WAIT_BEFORE_REPLYING); + TEST_EQUAL(line1, "CONNECTED:CLIENT"); } } @@ -338,8 +348,32 @@ void Srv2TestConversations(const std::vector &conns) conns[c]->Write(tosend[q], strlen(tosend[q])); std::string rep; bool hadTimeout = false; - while(!getline[c]->GetLine(rep, false, COMMS_READ_TIMEOUT)) - hadTimeout = true; + while(true) + { + try + { + // COMMS_READ_TIMEOUT is very short, so we will get a lot of these: + HideSpecificExceptionGuard guard(CommonException::ExceptionType, + CommonException::IOStreamTimedOut); + rep = getline[c]->GetLine(false, COMMS_READ_TIMEOUT); + break; + } + catch(BoxException &e) + { + if(EXCEPTION_IS_TYPE(e, CommonException, IOStreamTimedOut)) + { + hadTimeout = true; + } + else if(EXCEPTION_IS_TYPE(e, CommonException, SignalReceived)) + { + // just try again + } + else + { + throw; + } + } + } TEST_EQUAL_LINE(rep, recieve[q], "Line " << q); TEST_LINE(hadTimeout, "Line " << q) } @@ -452,6 +486,14 @@ void TestStreamReceive(TestProtocolClient &protocol, int value, bool uncertainst int test(int argc, const char *argv[]) { + // Timing information is very useful for debugging race conditions during this test, + // so enable it unconditionally: + Console::SetShowTime(true); + if(Logging::GetConsole().GetLevel() < Log::INFO) + { + Logging::FilterConsole(Log::INFO); + } + // Server launching stuff if(argc >= 2) { @@ -494,6 +536,11 @@ int test(int argc, const char *argv[]) } } +#ifndef WIN32 + // Don't die quietly if the server dies while we're communicating with it + signal(SIGPIPE, SIG_IGN); +#endif + //printf("SKIPPING TESTS------------------------\n"); //goto protocolserver; @@ -514,7 +561,7 @@ int test(int argc, const char *argv[]) // Move the config file over #ifdef WIN32 - TEST_THAT(::unlink("testfiles" + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "srv1.conf") != -1); #endif @@ -729,13 +776,13 @@ int test(int argc, const char *argv[]) TEST_THAT(reply->GetValuePlusOne() == 810); } - // Streams, twice, both uncertain and certain sizes + BOX_INFO("Streams, twice, both uncertain and certain sizes"); TestStreamReceive(protocol, 374, false); TestStreamReceive(protocol, 23983, true); TestStreamReceive(protocol, 12098, false); TestStreamReceive(protocol, 4342, true); - // Try to send a stream + BOX_INFO("Try to send a stream"); { std::auto_ptr s(new CollectInBufferStream()); @@ -747,13 +794,14 @@ int test(int argc, const char *argv[]) TEST_THAT(reply->GetStartingValue() == sizeof(buf)); } - // Lots of simple queries + BOX_INFO("Lots of simple queries"); for(int q = 0; q < 514; q++) { std::auto_ptr reply(protocol.QuerySimple(q)); TEST_THAT(reply->GetValuePlusOne() == (q+1)); } - // Send a list of strings to it + + BOX_INFO("Send a list of strings"); { std::vector strings; strings.push_back(std::string("test1")); @@ -763,7 +811,7 @@ int test(int argc, const char *argv[]) TEST_THAT(reply->GetNumberOfStrings() == 3); } - // And another + BOX_INFO("Send some mixed data"); { std::auto_ptr reply(protocol.QueryHello(41,87,11,std::string("pingu"))); TEST_THAT(reply->GetNumber32() == 12); @@ -771,11 +819,61 @@ int test(int argc, const char *argv[]) TEST_THAT(reply->GetNumber8() == 22); TEST_THAT(reply->GetText() == "Hello world!"); } - - // Quit query to finish - protocol.QueryQuit(); - - // Kill it + + BOX_INFO("Try to trigger an expected error"); + { + TEST_CHECK_THROWS(protocol.QueryDeliberateError(), + ConnectionException, Protocol_UnexpectedReply); + int type, subtype; + protocol.GetLastError(type, subtype); + TEST_EQUAL(TestProtocolError::ErrorType, type); + TEST_EQUAL(TestProtocolError::Err_DeliberateError, subtype); + } + + BOX_INFO("Try to trigger an unexpected error"); + // ... by throwing an exception that isn't caught and handled by + // HandleException: + { + TEST_CHECK_THROWS(protocol.QueryUnexpectedError(), + ConnectionException, Protocol_UnexpectedReply); + int type, subtype; + protocol.GetLastError(type, subtype); + TEST_EQUAL(TestProtocolError::ErrorType, type); + TEST_EQUAL(-1, subtype); + } + + // The unexpected exception should kill the server child process that we + // connected to (except on Windows where the server does not fork a child), + // so we cannot communicate with it any more: + BOX_INFO("Try to end protocol (should fail as server is already dead)"); + { + bool didthrow = false; + HideExceptionMessageGuard hide; + try + { + protocol.QueryQuit(); + } + catch(ConnectionException &e) + { + if(e.GetSubType() == ConnectionException::SocketReadError || + e.GetSubType() == ConnectionException::SocketWriteError) + { + didthrow = true; + } + else + { + TEST_FAIL_WITH_MESSAGE("Caught unexpected exception: " << + e.what()); + throw; + } + } + if(!didthrow) + { + TEST_FAIL_WITH_MESSAGE("Didn't throw expected exception"); + } + } + + // Kill the main server process: TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index cc602f228..1e5e3af18 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -9,7 +9,7 @@ #include "Box.h" -// do not include MinGW's dirent.h on Win32, +// do not include MinGW's dirent.h on Win32, // as we override some of it in lib/win32. #ifndef WIN32 @@ -70,7 +70,6 @@ #include "Configuration.h" #include "FileModificationTime.h" #include "FileStream.h" -#include "intercept.h" #include "IOStreamGetLine.h" #include "LocalProcessStream.h" #include "MemBlockStream.h" @@ -107,7 +106,7 @@ void wait_for_backup_operation(const char* message) bool readxattr_into_map(const char *filename, std::map &rOutput) { rOutput.clear(); - + ssize_t xattrNamesBufferSize = llistxattr(filename, NULL, 0); if(xattrNamesBufferSize < 0) { @@ -131,7 +130,7 @@ bool readxattr_into_map(const char *filename, std::map char *xattrDataBuffer = 0; int xattrDataBufferSize = 0; // note: will leak these buffers if a read error occurs. (test code, so doesn't matter) - + ssize_t ns = llistxattr(filename, xattrNamesBuffer, xattrNamesBufferSize); if(ns < 0) { @@ -145,9 +144,9 @@ bool readxattr_into_map(const char *filename, std::map { // Store size of name int xattrNameSize = strlen(xattrName); - + bool ok = true; - + ssize_t dataSize = lgetxattr(filename, xattrName, NULL, 0); if(dataSize < 0) { @@ -213,7 +212,7 @@ bool readxattr_into_map(const char *filename, std::map } // Got the data in the buffer } - + // Store in map if(ok) { @@ -224,7 +223,7 @@ bool readxattr_into_map(const char *filename, std::map xattrName += xattrNameSize + 1; } } - + if(xattrNamesBuffer != 0) ::free(xattrNamesBuffer); if(xattrDataBuffer != 0) ::free(xattrDataBuffer); } @@ -247,12 +246,12 @@ bool write_xattr_test(const char *filename, const char *attrName, unsigned int l { char data[1024]; if(length > sizeof(data)) length = sizeof(data); - + if(::fread(data, length, 1, xattrTestDataHandle) != 1) { return false; } - + if(::lsetxattr(filename, attrName, data, length, 0) != 0) { if(pNotSupported != 0) @@ -302,7 +301,7 @@ bool attrmatch(const char *f1, const char *f2) TEST_FAIL_WITH_MESSAGE("No symlinks on win32!") #else if((s2.st_mode & S_IFMT) != S_IFLNK) return false; - + char p1[PATH_MAX], p2[PATH_MAX]; int p1l = ::readlink(f1, p1, PATH_MAX); int p2l = ::readlink(f2, p2, PATH_MAX); @@ -427,7 +426,7 @@ bool kill_running_daemons() return success; } -bool setup_test_bbackupd(BackupDaemon& bbackupd, bool do_unpack_files = true, +bool prepare_test_with_client_daemon(BackupDaemon& bbackupd, bool do_unpack_files = true, bool do_start_bbstored = true) { Timers::Cleanup(false); // don't throw exception if not initialised @@ -486,13 +485,13 @@ bool setup_test_bbackupd(BackupDaemon& bbackupd, bool do_unpack_files = true, #define SETUP_WITHOUT_FILES() \ SETUP_TEST_BBACKUPD(); \ BackupDaemon bbackupd; \ - TEST_THAT_OR(setup_test_bbackupd(bbackupd, false), FAIL); \ + TEST_THAT_OR(prepare_test_with_client_daemon(bbackupd, false), FAIL); \ TEST_THAT_OR(::mkdir("testfiles/TestDir1", 0755) == 0, FAIL); #define SETUP_WITH_BBSTORED() \ SETUP_TEST_BBACKUPD(); \ BackupDaemon bbackupd; \ - TEST_THAT_OR(setup_test_bbackupd(bbackupd), FAIL); + TEST_THAT_OR(prepare_test_with_client_daemon(bbackupd), FAIL); #define TEARDOWN_TEST_BBACKUPD() \ TEST_THAT(bbackupd_pid == 0 || StopClient()); \ @@ -518,7 +517,7 @@ bool test_basics() void *te = ::memchr(t2.GetBuffer(), 't', t2.GetSize() - 3); TEST_THAT(te == 0 || ::memcmp(te, "test", 4) != 0); #endif - + BackupClientFileAttributes t3; { Logger::LevelGuard(Logging::GetConsole(), Log::ERROR); @@ -531,7 +530,7 @@ bool test_basics() fclose(f); f = fopen("testfiles/test2_n", "w"); fclose(f); - + // Apply attributes to these new files t1.WriteAttributes("testfiles/test1_n"); #ifdef WIN32 @@ -555,7 +554,7 @@ bool test_basics() TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_n")); TEST_THAT(attrmatch("testfiles/test2", "testfiles/test2_n")); #endif - + // Check encryption, and recovery from encryption // First, check that two attributes taken from the same thing have different encrypted values (think IV) BackupClientFileAttributes t1b; @@ -593,27 +592,27 @@ bool test_basics() // Write more attributes TEST_THAT(write_xattr_test("testfiles/test1", "user.attr_2", 947)); TEST_THAT(write_xattr_test("testfiles/test1", "user.sadfohij39998.3hj", 123)); - + // Read file attributes x1.ReadAttributes("testfiles/test1"); - + // Write file attributes FILE *f = fopen("testfiles/test1_nx", "w"); fclose(f); x1.WriteAttributes("testfiles/test1_nx"); - + // Compare to see if xattr copied TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_nx")); - + // Add more attributes to a file x2.ReadAttributes("testfiles/test1"); TEST_THAT(write_xattr_test("testfiles/test1", "user.328989sj..sdf", 23)); - + // Read them again, and check that the Compare() function detects that they're different x3.ReadAttributes("testfiles/test1"); TEST_THAT(x1.Compare(x2, true, true)); TEST_THAT(!x1.Compare(x3, true, true)); - + // Change the value of one of them, leaving the size the same. TEST_THAT(write_xattr_test("testfiles/test1", "user.328989sj..sdf", 23)); x4.ReadAttributes("testfiles/test1"); @@ -632,12 +631,12 @@ int64_t GetDirID(BackupProtocolCallable &protocol, const char *name, int64_t InD BackupProtocolListDirectory::Flags_Dir, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* want attributes */); - + // Retrieve the directory from the stream following BackupStoreDirectory dir; std::auto_ptr dirstream(protocol.ReceiveStream()); dir.ReadFromStream(*dirstream, protocol.GetTimeout()); - + BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; int64_t dirid = 0; @@ -676,7 +675,7 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); - + // Test the restoration TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", /* remote */ @@ -692,13 +691,13 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) } exit(0); break; - + case -1: { printf("Fork failed\n"); exit(1); } - + default: { // Wait until a resume file is written, then terminate the child @@ -708,7 +707,7 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) int64_t resumesize = 0; if(FileExists("testfiles/restore-interrupt.boxbackupresume", &resumesize) && resumesize > 16) { - // It's done something. Terminate it. + // It's done something. Terminate it. ::kill(pid, SIGTERM); break; } @@ -720,11 +719,11 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) // child has finished anyway. return; } - + // Give up timeslot so as not to hog the processor ::sleep(0); } - + // Just wait until the child has completed int status = 0; waitpid(pid, &status, 0); @@ -740,7 +739,7 @@ bool set_file_time(const char* filename, FILETIME creationTime, HANDLE handle = openfile(filename, O_RDWR, 0); TEST_THAT(handle != INVALID_HANDLE_VALUE); if (handle == INVALID_HANDLE_VALUE) return false; - + BOOL success = SetFileTime(handle, &creationTime, &lastAccessTime, &lastModTime); TEST_THAT(success); @@ -750,10 +749,6 @@ bool set_file_time(const char* filename, FILETIME creationTime, } #endif -void intercept_setup_delay(const char *filename, unsigned int delay_after, - int delay_ms, int syscall_to_delay); -bool intercept_triggered(); - int64_t SearchDir(BackupStoreDirectory& rDir, const std::string& rChildName) { @@ -785,7 +780,7 @@ int start_internal_daemon() // ensure that no child processes end up running tests! int own_pid = getpid(); BOX_TRACE("Test PID is " << own_pid); - + // this is a quick hack to allow passing some options to the daemon const char* argv[] = { "dummy", @@ -806,9 +801,9 @@ int start_internal_daemon() } spDaemon = NULL; // to ensure not used by parent - + TEST_EQUAL_LINE(0, result, "Daemon exit code"); - + // ensure that no child processes end up running tests! if (getpid() != own_pid) { @@ -818,10 +813,10 @@ int start_internal_daemon() } TEST_THAT(TestFileExists("testfiles/bbackupd.pid")); - + printf("Waiting for backup daemon to start: "); int pid = -1; - + for (int i = 0; i < 30; i++) { printf("."); @@ -836,9 +831,9 @@ int start_internal_daemon() if (pid > 0) { break; - } + } } - + printf(" done.\n"); fflush(stdout); @@ -854,120 +849,6 @@ bool stop_internal_daemon(int pid) return killed_server; } -static struct dirent readdir_test_dirent; -static int readdir_test_counter = 0; -static int readdir_stop_time = 0; -static char stat_hook_filename[512]; - -// First test hook, during the directory scanning stage, returns empty. -// (Where is this stage? I can't find it, so I switched from using -// readdir_test_hook_1 to readdir_test_hook_2 in intercept tests.) -// This will not match the directory on the store, so a sync will start. -// We set up the next intercept for the same directory by passing NULL. - -extern "C" struct dirent *readdir_test_hook_2(DIR *dir); - -#ifdef LINUX_WEIRD_LSTAT -extern "C" int lstat_test_hook(int ver, const char *file_name, struct stat *buf); -#else -extern "C" int lstat_test_hook(const char *file_name, struct stat *buf); -#endif - -extern "C" struct dirent *readdir_test_hook_1(DIR *dir) -{ -#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE - intercept_setup_readdir_hook(NULL, readdir_test_hook_2); -#endif - return NULL; -} - -// Second test hook, called by BackupClientDirectoryRecord::SyncDirectory, -// keeps returning new filenames until the timer expires, then disables the -// intercept. - -extern "C" struct dirent *readdir_test_hook_2(DIR *dir) -{ - if (spDaemon->IsTerminateWanted()) - { - // force daemon to crash, right now - return NULL; - } - - time_t time_now = time(NULL); - - if (time_now >= readdir_stop_time) - { -#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE - BOX_NOTICE("Cancelling readdir hook at " << time_now); - intercept_setup_readdir_hook(NULL, NULL); - intercept_setup_lstat_hook (NULL, NULL); - // we will not be called again. -#else - BOX_NOTICE("Failed to cancel readdir hook at " << time_now); -#endif - } - else - { - BOX_TRACE("readdir hook still active at " << time_now << ", " - "waiting for " << readdir_stop_time); - } - - // fill in the struct dirent appropriately - memset(&readdir_test_dirent, 0, sizeof(readdir_test_dirent)); - - #ifdef HAVE_STRUCT_DIRENT_D_INO - readdir_test_dirent.d_ino = ++readdir_test_counter; - #endif - - snprintf(readdir_test_dirent.d_name, - sizeof(readdir_test_dirent.d_name), - "test.%d", readdir_test_counter); - BOX_TRACE("readdir hook returning " << readdir_test_dirent.d_name); - - // ensure that when bbackupd stats the file, it gets the - // right answer - snprintf(stat_hook_filename, sizeof(stat_hook_filename), - "testfiles/TestDir1/spacetest/d1/test.%d", - readdir_test_counter); - -#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE - intercept_setup_lstat_hook(stat_hook_filename, lstat_test_hook); -#endif - - // sleep a bit to reduce the number of dirents returned - if (time_now < readdir_stop_time) - { - ::safe_sleep(1); - } - - return &readdir_test_dirent; -} - -#ifdef LINUX_WEIRD_LSTAT -extern "C" int lstat_test_hook(int ver, const char *file_name, struct stat *buf) -#else -extern "C" int lstat_test_hook(const char *file_name, struct stat *buf) -#endif -{ - // TRACE1("lstat hook triggered for %s", file_name); - memset(buf, 0, sizeof(*buf)); - buf->st_mode = S_IFREG; - return 0; -} - -// Simulate a symlink that is on a different device than the file -// that it points to. -int lstat_test_post_hook(int old_ret, const char *file_name, struct stat *buf) -{ - BOX_TRACE("lstat post hook triggered for " << file_name); - if (old_ret == 0 && - strcmp(file_name, "testfiles/symlink-to-TestDir1") == 0) - { - buf->st_dev ^= 0xFFFF; - } - return old_ret; -} - bool test_entry_deleted(BackupStoreDirectory& rDir, const std::string& rName) { @@ -1184,7 +1065,7 @@ bool test_readdirectory_on_nonexistent_dir() { std::auto_ptr client = connect_and_login( sTlsContext, 0 /* read-write */); - + { Logger::LevelGuard(Logging::GetConsole(), Log::ERROR); TEST_CHECK_THROWS(ReadDirectory(*client, 0x12345678), @@ -1269,7 +1150,7 @@ bool test_getobject_on_nonexistent_file() last_message); } } - + TEARDOWN_TEST_BBACKUPD(); } @@ -1381,7 +1262,7 @@ bool test_backup_disappearing_directory() dirname, // dirname, attrStream)); clientContext.CloseAnyOpenConnection(); - + // Object ID for later creation int64_t oid = dirCreate->GetObjectID(); BackupClientDirectoryRecordHooked record(oid, "Test1"); @@ -1449,6 +1330,7 @@ class KeepAliveBackupProtocolLocal : public BackupProtocolLocal2 mNumKeepAlivesReceived++; return response; } + using BackupProtocolLocal2::Query; }; bool test_ssl_keepalives() @@ -1458,7 +1340,7 @@ bool test_ssl_keepalives() KeepAliveBackupProtocolLocal connection(0x01234567, "test", "backup/01234567/", 0, false); MockBackupDaemon bbackupd(connection); - TEST_THAT_OR(setup_test_bbackupd(bbackupd), FAIL); + TEST_THAT_OR(prepare_test_with_client_daemon(bbackupd), FAIL); // Test that sending a keepalive actually works, when the timeout has expired, // but doesn't send anything at the beginning: @@ -1475,7 +1357,7 @@ bool test_ssl_keepalives() bbackupd, // rProgressNotifier false, // TcpNiceMode connection); // rClient - + // Set the timeout to 1 second context.SetKeepAliveTime(1); @@ -1561,25 +1443,25 @@ bool test_backup_hardlinked_files() TEST_COMPARE(Compare_Same); // Create some hard links. First in the same directory: - TEST_THAT(link("testfiles/TestDir1/x1/dsfdsfs98.fd", + TEST_THAT(EMU_LINK("testfiles/TestDir1/x1/dsfdsfs98.fd", "testfiles/TestDir1/x1/hardlink1") == 0); bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); // Now in a different directory TEST_THAT(mkdir("testfiles/TestDir1/x2", 0755) == 0); - TEST_THAT(link("testfiles/TestDir1/x1/dsfdsfs98.fd", + TEST_THAT(EMU_LINK("testfiles/TestDir1/x1/dsfdsfs98.fd", "testfiles/TestDir1/x2/hardlink2") == 0); bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); // Now delete one of them - TEST_THAT(unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); // And another. - TEST_THAT(unlink("testfiles/TestDir1/x1/hardlink1") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/hardlink1") == 0); bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); @@ -1592,6 +1474,8 @@ bool test_backup_pauses_when_store_is_full() unpack_files("spacetest1", "testfiles/TestDir1"); TEST_THAT_OR(StartClient(), FAIL); + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); + // TODO FIXME dedent { // wait for files to be uploaded @@ -1604,7 +1488,7 @@ bool test_backup_pauses_when_store_is_full() { std::auto_ptr client = connect_and_login(sTlsContext, 0 /* read-write */); - TEST_THAT(check_num_files(5, 0, 0, 9)); + TEST_THAT(check_num_files(fs, 5, 0, 0, 9)); TEST_THAT(check_num_blocks(*client, 10, 0, 0, 18, 28)); client->QueryFinished(); } @@ -1619,7 +1503,7 @@ bool test_backup_pauses_when_store_is_full() unpack_files("spacetest2", "testfiles/TestDir1"); // Delete a file and a directory - TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/spacetest/f1") == 0); #ifdef WIN32 TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\spacetest\\d7") == 0); #else @@ -1712,7 +1596,7 @@ bool test_backup_pauses_when_store_is_full() TEST_EQUAL(SearchDir(*spacetest_dir, "f1"), 0); TEST_EQUAL(SearchDir(*spacetest_dir, "d7"), 0); - TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_files(fs, 4, 0, 0, 8)); TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); client->QueryFinished(); } @@ -1732,9 +1616,11 @@ bool test_bbackupd_exclusions() { SETUP_WITHOUT_FILES(); + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); + TEST_THAT(unpack_files("spacetest1", "testfiles/TestDir1")); // Delete a file and a directory - TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/spacetest/f1") == 0); #ifdef WIN32 TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\spacetest\\d7") == 0); @@ -1756,7 +1642,7 @@ bool test_bbackupd_exclusions() { std::auto_ptr client = connect_and_login(sTlsContext, 0 /* read-write */); - TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_files(fs, 4, 0, 0, 8)); TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); client->QueryFinished(); } @@ -1781,7 +1667,7 @@ bool test_bbackupd_exclusions() { std::auto_ptr client = connect_and_login(sTlsContext, 0 /* read-write */); - TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_files(fs, 4, 0, 0, 8)); TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); client->QueryFinished(); } @@ -1819,7 +1705,7 @@ bool test_bbackupd_exclusions() // housekeeping the next time it runs. We hold onto the client // context (and hence an open connection) to stop it from // running for now. - + // But we can't do that on Windows, because bbstored only // support one simultaneous connection. So we have to hope that // housekeeping has run recently enough that it doesn't need to @@ -1836,7 +1722,7 @@ bool test_bbackupd_exclusions() std::auto_ptr client = connect_and_login(sTlsContext, BackupProtocolLogin::Flags_ReadOnly); - + std::auto_ptr rootDir = ReadDirectory(*client); @@ -1884,7 +1770,7 @@ bool test_bbackupd_exclusions() // server which are not deleted, they use 2 blocks // each, the rest is directories and 2 deleted files // (f2 and d3/d4/f5) - TEST_THAT(check_num_files(2, 0, 2, 8)); + TEST_THAT(check_num_files(fs, 2, 0, 2, 8)); TEST_THAT(check_num_blocks(*client, 4, 0, 4, 16, 24)); // Log out. @@ -1925,7 +1811,7 @@ bool test_bbackupd_exclusions() // f2, d3, d3/d4 and d3/d4/f5 have been removed. // The files were counted as deleted files before, the // deleted directories just as directories. - TEST_THAT(check_num_files(2, 0, 0, 6)); + TEST_THAT(check_num_files(fs, 2, 0, 0, 6)); TEST_THAT(check_num_blocks(*client, 4, 0, 0, 12, 16)); // Log out. @@ -1934,7 +1820,7 @@ bool test_bbackupd_exclusions() // Need 22 blocks free to upload everything TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf setlimit 01234567 0B 22B") + "testfiles/bbstored.conf setlimit 01234567 0B 22B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); @@ -1943,8 +1829,8 @@ bool test_bbackupd_exclusions() bbackupd.RunSyncNow(); TEST_THAT(!bbackupd.StorageLimitExceeded()); - // Check that the contents of the store are the same - // as the contents of the disc + // Check that the contents of the store are the same + // as the contents of the disc TEST_COMPARE(Compare_Same, "-c testfiles/bbackupd-exclude.conf"); BOX_TRACE("done."); @@ -1953,7 +1839,7 @@ bool test_bbackupd_exclusions() std::auto_ptr client = connect_and_login(sTlsContext, 0 /* read-write */); - TEST_THAT(check_num_files(4, 0, 0, 7)); + TEST_THAT(check_num_files(fs, 4, 0, 0, 7)); TEST_THAT(check_num_blocks(*client, 8, 0, 0, 14, 22)); // d2/f6, d6/d8 and d6/d8/f7 are new @@ -2005,7 +1891,7 @@ bool test_bbackupd_responds_to_connection_failure() FAIL); const char* control_string = "whee!\n"; - TEST_THAT(write(fd, control_string, + TEST_THAT(write(fd, control_string, strlen(control_string)) == (int)strlen(control_string)); close(fd); @@ -2055,7 +1941,7 @@ bool test_bbackupd_responds_to_connection_failure() MockBackupProtocolLocal client(0x01234567, "test", "backup/01234567/", 0, false); MockBackupDaemon bbackupd(client); - TEST_THAT_OR(setup_test_bbackupd(bbackupd, false, false), FAIL); + TEST_THAT_OR(prepare_test_with_client_daemon(bbackupd, false, false), FAIL); TEST_THAT(::system("rm -f testfiles/notifyran.store-full.*") == 0); std::auto_ptr apClientContext; @@ -2154,8 +2040,7 @@ bool test_absolute_symlinks_not_followed_during_restore() // check that the original file was not overwritten FileStream fs(SYM_DIR "/a/subdir/content"); IOStreamGetLine gl(fs); - std::string line; - TEST_THAT(gl.GetLine(line)); + std::string line = gl.GetLine(false); TEST_THAT(line != "before"); TEST_EQUAL("after", line); @@ -2195,7 +2080,7 @@ bool test_initially_missing_locations_are_not_forgotten() TEST_THAT(!TestFileExists("testfiles/notifyran.backup-ok.1")); TEST_THAT( TestFileExists("testfiles/notifyran.backup-finish.1")); TEST_THAT(!TestFileExists("testfiles/notifyran.backup-finish.2")); - + // Did it actually get created? Should not have been! TEST_THAT_OR(!search_for_file("Test2"), FAIL); } @@ -2334,20 +2219,20 @@ bool test_unicode_filenames_can_be_backed_up() // TODO FIXME dedent { // We have no guarantee that a random Unicode string can be - // represented in the user's character set, so we go the - // other way, taking three random characters from the + // represented in the user's character set, so we go the + // other way, taking three random characters from the // character set and converting them to Unicode. Unless the // console codepage is CP_UTF8, in which case our random // characters are not valid, so we use the UTF8 version // of them instead. // - // We hope that these characters are valid in most - // character sets, but they probably are not in multibyte - // character sets such as Shift-JIS, GB2312, etc. This test - // will probably fail if your system locale is set to + // We hope that these characters are valid in most + // character sets, but they probably are not in multibyte + // character sets such as Shift-JIS, GB2312, etc. This test + // will probably fail if your system locale is set to // Chinese, Japanese, etc. where one of these character // sets is used by default. You can check the character - // set for your system in Control Panel -> Regional + // set for your system in Control Panel -> Regional // Options -> General -> Language Settings -> Set Default // (System Locale). Because bbackupquery converts from // system locale to UTF-8 via the console code page @@ -2355,7 +2240,7 @@ bool test_unicode_filenames_can_be_backed_up() // they must also be valid in your code page (850 for // Western Europe). // - // In ISO-8859-1 (Danish locale) they are three Danish + // In ISO-8859-1 (Danish locale) they are three Danish // accented characters, which are supported in code page // 850. Depending on your locale, YYMV (your yak may vomit). @@ -2403,7 +2288,7 @@ bool test_unicode_filenames_can_be_backed_up() TEST_THAT(ConvertUtf8ToConsole(filename.c_str(), consoleFileName)); - // test that bbackupd will let us lcd into the local + // test that bbackupd will let us lcd into the local // directory using a relative path, and back out TEST_THAT(bbackupquery("\"lcd testfiles/TestDir1/" + systemDirName + "\" \"lcd ..\"")); @@ -2447,7 +2332,7 @@ bool test_unicode_filenames_can_be_backed_up() int64_t testDirId = SearchDir(*dir, dirname.c_str()); TEST_THAT(testDirId != 0); dir = ReadDirectory(*client, testDirId); - + TEST_THAT(SearchDir(*dir, filename.c_str()) != 0); // Log out client->QueryFinished(); @@ -2459,17 +2344,16 @@ bool test_unicode_filenames_can_be_backed_up() "-q \"list Test1\" quit"; pid_t bbackupquery_pid; std::auto_ptr queryout; - queryout = LocalProcessStream(command.c_str(), + queryout = LocalProcessStream(command.c_str(), bbackupquery_pid); TEST_THAT(queryout.get() != NULL); TEST_THAT(bbackupquery_pid != -1); IOStreamGetLine reader(*queryout); - std::string line; bool found = false; while (!reader.IsEOF()) { - TEST_THAT(reader.GetLine(line)); + std::string line = reader.GetLine(false); // !preprocess if (line.find(consoleDirName) != std::string::npos) { found = true; @@ -2485,7 +2369,7 @@ bool test_unicode_filenames_can_be_backed_up() // the file in console encoding command = BBACKUPQUERY " -c testfiles/bbackupd.conf " "-Wwarning \"list Test1/" + systemDirName + "\" quit"; - queryout = LocalProcessStream(command.c_str(), + queryout = LocalProcessStream(command.c_str(), bbackupquery_pid); TEST_THAT(queryout.get() != NULL); TEST_THAT(bbackupquery_pid != -1); @@ -2494,7 +2378,7 @@ bool test_unicode_filenames_can_be_backed_up() found = false; while (!reader2.IsEOF()) { - TEST_THAT(reader2.GetLine(line)); + std::string line = reader2.GetLine(false); // !preprocess if (line.find(consoleFileName) != std::string::npos) { found = true; @@ -2519,30 +2403,30 @@ bool test_unicode_filenames_can_be_backed_up() TEST_COMPARE(Compare_Same, "", "-cEQ Test1/" + systemDirName + " testfiles/restore-" + systemDirName); - std::string fileToUnlink = "testfiles/restore-" + + std::string fileToUnlink = "testfiles/restore-" + dirname + "/" + filename; - TEST_THAT(::unlink(fileToUnlink.c_str()) == 0); + TEST_THAT(EMU_UNLINK(fileToUnlink.c_str()) == 0); // Check that bbackupquery can get the file when given // on the command line in system encoding. TEST_THAT(bbackupquery("\"get Test1/" + systemDirName + "/" + - systemFileName + " " + "testfiles/restore-" + + systemFileName + " " + "testfiles/restore-" + systemDirName + "/" + systemFileName + "\"")); // And after changing directory to a relative path TEST_THAT(bbackupquery( "\"lcd testfiles\" " - "\"cd Test1/" + systemDirName + "\" " + + "\"cd Test1/" + systemDirName + "\" " + "\"get " + systemFileName + "\"")); // cannot overwrite a file that exists, so delete it std::string tmp = "testfiles/" + filename; - TEST_THAT(::unlink(tmp.c_str()) == 0); + TEST_THAT(EMU_UNLINK(tmp.c_str()) == 0); // And after changing directory to an absolute path TEST_THAT(bbackupquery( "\"lcd " + cwd + "/testfiles\" " - "\"cd Test1/" + systemDirName + "\" " + + "\"cd Test1/" + systemDirName + "\" " + "\"get " + systemFileName + "\"")); // Compare to make sure it was restored properly. The Get @@ -2555,7 +2439,7 @@ bool test_unicode_filenames_can_be_backed_up() TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); } #endif // WIN32 - + TEARDOWN_TEST_BBACKUPD(); } @@ -2571,18 +2455,18 @@ bool test_sync_allow_script_can_pause_backup() // we now have 3 seconds before bbackupd // runs the SyncAllowScript again. - const char* sync_control_file = "testfiles" + const char* sync_control_file = "testfiles" DIRECTORY_SEPARATOR "syncallowscript.control"; - int fd = open(sync_control_file, + int fd = open(sync_control_file, O_CREAT | O_EXCL | O_WRONLY, 0700); if (fd <= 0) { perror(sync_control_file); } TEST_THAT(fd > 0); - + const char* control_string = "10\n"; - TEST_THAT(write(fd, control_string, + TEST_THAT(write(fd, control_string, strlen(control_string)) == (int)strlen(control_string)); close(fd); @@ -2608,22 +2492,22 @@ bool test_sync_allow_script_can_pause_backup() // 5 seconds (normally about 3 seconds) wait_for_operation(1, "2 seconds before next run"); - TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR "syncallowscript.notifyran.1", &st) != 0); wait_for_operation(4, "2 seconds after run"); - TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR "syncallowscript.notifyran.1", &st) == 0); - TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR "syncallowscript.notifyran.2", &st) != 0); // next poll should happen within the next // 10 seconds (normally about 8 seconds) wait_for_operation(6, "2 seconds before next run"); - TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR "syncallowscript.notifyran.2", &st) != 0); wait_for_operation(4, "2 seconds after run"); - TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR "syncallowscript.notifyran.2", &st) == 0); // bbackupquery compare might take a while @@ -2633,7 +2517,7 @@ bool test_sync_allow_script_can_pause_backup() // check that no backup has run (compare fails) TEST_COMPARE(Compare_Different); - TEST_THAT(unlink(sync_control_file) == 0); + TEST_THAT(EMU_UNLINK(sync_control_file) == 0); wait_for_sync_start(); long end_time = time(NULL); long wait_time = end_time - start_time + 2; @@ -2671,13 +2555,13 @@ bool test_delete_update_and_symlink_files() // TODO FIXME dedent { // Delete a file - TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); #ifndef WIN32 // New symlink TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/symlink-to-dir") == 0); - #endif + #endif // Update a file (will be uploaded as a diff) { @@ -2685,7 +2569,7 @@ bool test_delete_update_and_symlink_files() // threshold in the bbackupd.conf file TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); - + // Add a bit to the end FILE *f = ::fopen("testfiles/TestDir1/f45.df", "a"); TEST_THAT(f != 0); @@ -2710,14 +2594,23 @@ bool test_delete_update_and_symlink_files() TEARDOWN_TEST_BBACKUPD(); } -// Check that store errors are reported neatly. This test uses an independent -// daemon to check the daemon's backup loop delay, so it's easier to debug -// with the command: ./t -VTttest -e test_store_error_reporting -// --bbackupd-args=-kTtbbackupd +// Check that store errors are reported neatly. bool test_store_error_reporting() { SETUP_WITH_BBSTORED(); - TEST_THAT(StartClient()); + + // Temporarily enable timestamp logging, to help debug race conditions causing + // test failures: + Logger::LevelGuard temporary_verbosity(Logging::GetConsole(), Log::TRACE); + Console::SettingsGuard save_old_settings; + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + + // Start the bbackupd client, with timestamp logging enabled too: + std::string daemon_args(bbackupd_args_overridden ? bbackupd_args : + "-kU -Wnotice -tbbackupd"); + TEST_THAT_OR(StartClient("testfiles/bbackupd.conf", daemon_args), FAIL); + wait_for_sync_end(); // TODO FIXME dedent @@ -2741,8 +2634,8 @@ bool test_store_error_reporting() // Lock scope { - NamedLock writeLock; - acc.LockAccount(0x01234567, writeLock); + RaidBackupFileSystem fs(0x01234567, "backup/01234567/", 0); + fs.GetLock(); TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", "testfiles/0_0/backup/01234567/info.rf.bak") == 0); @@ -2754,7 +2647,7 @@ bool test_store_error_reporting() // Create a file to trigger an upload { - int fd1 = open("testfiles/TestDir1/force-upload", + int fd1 = open("testfiles/TestDir1/force-upload", O_CREAT | O_EXCL | O_WRONLY, 0700); TEST_THAT(fd1 > 0); TEST_THAT(write(fd1, "just do it", 10) == 10); @@ -2770,7 +2663,7 @@ bool test_store_error_reporting() // snapshot mode, check that it automatically syncs after // an error, without waiting for another sync command. TEST_THAT(StopClient()); - TEST_THAT(StartClient("testfiles/bbackupd-snapshot.conf")); + TEST_THAT(StartClient("testfiles/bbackupd-snapshot.conf", daemon_args)); sync_and_wait(); // Check that the error was reported once more @@ -2798,7 +2691,7 @@ bool test_store_error_reporting() // Set a tag for the notify script to distinguish from // previous runs. { - int fd1 = open("testfiles/notifyscript.tag", + int fd1 = open("testfiles/notifyscript.tag", O_CREAT | O_EXCL | O_WRONLY, 0700); TEST_THAT(fd1 > 0); TEST_THAT(write(fd1, "wait-snapshot", 13) == 13); @@ -2818,15 +2711,17 @@ bool test_store_error_reporting() // Should not have backed up, should still get errors TEST_COMPARE(Compare_Different); - // wait another 2 seconds, bbackup should have run - wait_for_operation(2, "bbackupd to recover"); + // Wait another 4 seconds, bbackup should have run. Ideally 2 seconds would be + // enough, and it usually is, but sometimes Travis is heavily loaded and the backup + // takes ~4 seconds to run! + wait_for_operation(4, "bbackupd to recover"); TEST_THAT(TestFileExists("testfiles/" "notifyran.backup-start.wait-snapshot.1")); - + // Check that it did get uploaded, and we have no more errors TEST_COMPARE(Compare_Same); - TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); + TEST_THAT(EMU_UNLINK("testfiles/notifyscript.tag") == 0); // Stop the snapshot bbackupd TEST_THAT(StopClient()); @@ -2841,7 +2736,7 @@ bool test_store_error_reporting() // Modify a file to trigger an upload { - int fd1 = open("testfiles/TestDir1/force-upload", + int fd1 = open("testfiles/TestDir1/force-upload", O_WRONLY, 0700); TEST_THAT(fd1 > 0); TEST_THAT(write(fd1, "and again", 9) == 9); @@ -2849,7 +2744,7 @@ bool test_store_error_reporting() } // Restart bbackupd in automatic mode - TEST_THAT_OR(StartClient(), FAIL); + TEST_THAT_OR(StartClient("testfiles/bbackupd.conf", daemon_args), FAIL); sync_and_wait(); // Fix the store again @@ -2873,7 +2768,7 @@ bool test_store_error_reporting() // Set a tag for the notify script to distinguish from // previous runs. { - int fd1 = open("testfiles/notifyscript.tag", + int fd1 = open("testfiles/notifyscript.tag", O_CREAT | O_EXCL | O_WRONLY, 0700); TEST_THAT(fd1 > 0); TEST_THAT(write(fd1, "wait-automatic", 14) == 14); @@ -2893,15 +2788,15 @@ bool test_store_error_reporting() // Should not have backed up, should still get errors TEST_COMPARE(Compare_Different); - // wait another 2 seconds, bbackup should have run - wait_for_operation(2, "bbackupd to recover"); + // wait another 4 seconds, bbackup should have run + wait_for_operation(4, "bbackupd to recover"); TEST_THAT(TestFileExists("testfiles/" "notifyran.backup-start.wait-automatic.1")); - + // Check that it did get uploaded, and we have no more errors TEST_COMPARE(Compare_Same); - TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); + TEST_THAT(EMU_UNLINK("testfiles/notifyscript.tag") == 0); } TEARDOWN_TEST_BBACKUPD(); @@ -2915,7 +2810,7 @@ bool test_change_file_to_symlink_and_back() // New symlink TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/symlink-to-dir") == 0); - #endif + #endif bbackupd.RunSyncNow(); @@ -2925,7 +2820,7 @@ bool test_change_file_to_symlink_and_back() // Replace symlink with directory, add new directory. #ifndef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/symlink-to-dir") + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/symlink-to-dir") == 0); #endif @@ -2938,8 +2833,8 @@ bool test_change_file_to_symlink_and_back() // avoid deletion by the housekeeping process later #ifndef WIN32 - TEST_THAT(::symlink("does-not-exist", - "testfiles/TestDir1/x1/dir-to-file/contents") + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/x1/dir-to-file/contents") == 0); #endif @@ -2950,34 +2845,34 @@ bool test_change_file_to_symlink_and_back() // And the inverse, replace a directory with a file/symlink #ifndef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/dir-to-file" "/contents") == 0); #endif TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0); #ifndef WIN32 - TEST_THAT(::symlink("does-not-exist", + TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0); #endif wait_for_operation(5, "files to be old enough"); bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - + // And then, put it back to how it was before. BOX_INFO("Replace symlink with directory (which was a symlink)"); #ifndef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/x1" + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1" "/dir-to-file") == 0); #endif - TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", + TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0); #ifndef WIN32 - TEST_THAT(::symlink("does-not-exist", + TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file/contents2") == 0); #endif @@ -2986,20 +2881,20 @@ bool test_change_file_to_symlink_and_back() bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - // And finally, put it back to how it was before + // And finally, put it back to how it was before // it was put back to how it was before - // This gets lots of nasty things in the store with + // This gets lots of nasty things in the store with // directories over other old directories. #ifndef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/dir-to-file" "/contents2") == 0); #endif TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0); #ifndef WIN32 - TEST_THAT(::symlink("does-not-exist", + TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0); #endif @@ -3020,7 +2915,7 @@ bool test_file_rename_tracking() { // rename an untracked file over an existing untracked file BOX_INFO("Rename over existing untracked file"); - int fd1 = open("testfiles/TestDir1/untracked-1", + int fd1 = open("testfiles/TestDir1/untracked-1", O_CREAT | O_EXCL | O_WRONLY, 0700); int fd2 = open("testfiles/TestDir1/untracked-2", O_CREAT | O_EXCL | O_WRONLY, 0700); @@ -3040,11 +2935,11 @@ bool test_file_rename_tracking() TEST_COMPARE(Compare_Same); #ifdef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/untracked-2") + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/untracked-2") == 0); #endif - TEST_THAT(::rename("testfiles/TestDir1/untracked-1", + TEST_THAT(::rename("testfiles/TestDir1/untracked-1", "testfiles/TestDir1/untracked-2") == 0); TEST_THAT(!TestFileExists("testfiles/TestDir1/untracked-1")); TEST_THAT( TestFileExists("testfiles/TestDir1/untracked-2")); @@ -3078,7 +2973,7 @@ bool test_file_rename_tracking() TEST_COMPARE(Compare_Same); #ifdef WIN32 - TEST_THAT(::unlink("testfiles/TestDir1/tracked-2") + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/tracked-2") == 0); #endif @@ -3093,7 +2988,7 @@ bool test_file_rename_tracking() // case which went wrong: rename a tracked file // over a deleted file BOX_INFO("Rename an existing file over a deleted file"); - TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); @@ -3126,11 +3021,11 @@ bool test_upload_very_old_files() #ifndef WIN32 ::chmod("testfiles/TestDir1/sub23/dhsfdss/blf.h", 0415); #endif - + // Wait and test bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - + // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); @@ -3149,7 +3044,7 @@ bool test_upload_very_old_files() 0777) == 0); #endif - FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", + FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", "w+"); if (f == 0) @@ -3170,7 +3065,7 @@ bool test_upload_very_old_files() BoxTimeToTimeval(SecondsToBoxTime( (time_t)(365*24*60*60)), times[1]); times[0] = times[1]; - TEST_THAT(::utimes("testfiles/TestDir1/sub23/rand.h", + TEST_THAT(::utimes("testfiles/TestDir1/sub23/rand.h", times) == 0); } @@ -3193,8 +3088,8 @@ bool test_excluded_files_are_not_backed_up() BackupProtocolLocal2 client(0x01234567, "test", "backup/01234567/", 0, false); MockBackupDaemon bbackupd(client); - - TEST_THAT_OR(setup_test_bbackupd(bbackupd, + + TEST_THAT_OR(prepare_test_with_client_daemon(bbackupd, true, // do_unpack_files false // do_start_bbstored ), FAIL); @@ -3222,14 +3117,14 @@ bool test_excluded_files_are_not_backed_up() BackupProtocolLogin::Flags_ReadOnly); */ BackupProtocolCallable* pClient = &client; - + std::auto_ptr dir = ReadDirectory(*pClient); int64_t testDirId = SearchDir(*dir, "Test1"); TEST_THAT(testDirId != 0); dir = ReadDirectory(*pClient, testDirId); - + TEST_THAT(!SearchDir(*dir, "excluded_1")); TEST_THAT(!SearchDir(*dir, "excluded_2")); TEST_THAT(!SearchDir(*dir, "exclude_dir")); @@ -3284,7 +3179,7 @@ bool test_read_error_reporting() TEST_THAT(::mkdir("testfiles/TestDir1/sub23" "/read-fail-test-dir", 0000) == 0); int fd = ::open("testfiles/TestDir1" - "/read-fail-test-file", + "/read-fail-test-file", O_CREAT | O_WRONLY, 0000); TEST_THAT(fd != -1); ::close(fd); @@ -3302,7 +3197,7 @@ bool test_read_error_reporting() // Check that the error was only reported once TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2")); - // Set permissions on file and dir to stop + // Set permissions on file and dir to stop // errors in the future TEST_THAT(::chmod("testfiles/TestDir1/sub23" "/read-fail-test-dir", 0770) == 0); @@ -3322,7 +3217,7 @@ bool test_continuously_updated_file() // TODO FIXME dedent { - // Make sure everything happens at the same point in the + // Make sure everything happens at the same point in the // sync cycle: wait until exactly the start of a sync wait_for_sync_start(); @@ -3341,7 +3236,7 @@ bool test_continuously_updated_file() fclose(f); safe_sleep(1); } - + // Check there's a difference int compareReturnValue = ::system("perl testfiles/" "extcheck1.pl"); @@ -3395,7 +3290,7 @@ bool test_delete_dir_change_attribute() #endif TEST_COMPARE(Compare_Different); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); } @@ -3414,7 +3309,7 @@ bool test_restore_files_and_directories() int64_t restoredirid = 0; { // connect and log in - std::auto_ptr client = + std::auto_ptr client = connect_and_login(sTlsContext, BackupProtocolLogin::Flags_ReadOnly); @@ -3431,7 +3326,7 @@ bool test_restore_files_and_directories() false /* restore deleted */, false /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_Complete); // On Win32 we can't open another connection @@ -3444,7 +3339,7 @@ bool test_restore_files_and_directories() false /* restore deleted */, false /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_TargetExists); // Find ID of the deleted directory @@ -3459,7 +3354,7 @@ bool test_restore_files_and_directories() true /* restore deleted */, false /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_Complete); // Make sure you can't restore to a nonexistant path @@ -3472,12 +3367,12 @@ bool test_restore_files_and_directories() Log::FATAL); TEST_THAT(BackupClientRestore(*client, restoredirid, "Test1", - "testfiles/no-such-path/subdir", - true /* print progress dots */, + "testfiles/no-such-path/subdir", + true /* print progress dots */, true /* restore deleted */, false /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_TargetPathNotFound); } @@ -3511,7 +3406,7 @@ bool test_compare_detects_attribute_changes() TEST_RETURN(exit_status, 0); TEST_COMPARE(Compare_Different); - + // set it back, expect no failures exit_status = ::system("attrib -r " "testfiles\\TestDir1\\f1.dat"); @@ -3523,9 +3418,9 @@ bool test_compare_detects_attribute_changes() const char* testfile = "testfiles\\TestDir1\\f1.dat"; HANDLE handle = openfile(testfile, O_RDWR, 0); TEST_THAT(handle != INVALID_HANDLE_VALUE); - + FILETIME creationTime, lastModTime, lastAccessTime; - TEST_THAT(GetFileTime(handle, &creationTime, &lastAccessTime, + TEST_THAT(GetFileTime(handle, &creationTime, &lastAccessTime, &lastModTime) != 0); TEST_THAT(CloseHandle(handle)); @@ -3607,7 +3502,7 @@ bool test_rename_operations() // TODO FIXME dedent { BOX_INFO("Rename directory"); - TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", + TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", "testfiles/TestDir1/renamed-dir") == 0); bbackupd.RunSyncNow(); @@ -3617,9 +3512,9 @@ bool test_rename_operations() TEST_COMPARE(Compare_Same, "", "-acqQ"); // Rename some files -- one under the threshold, others above - TEST_THAT(rename("testfiles/TestDir1/df324", + TEST_THAT(rename("testfiles/TestDir1/df324", "testfiles/TestDir1/df324-ren") == 0); - TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", + TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", "testfiles/TestDir1/find2perl-ren") == 0); bbackupd.RunSyncNow(); @@ -3647,8 +3542,8 @@ bool test_sync_files_with_timestamps_in_future() fclose(f); // and then move the time forwards! struct timeval times[2]; - BoxTimeToTimeval(GetCurrentBoxTime() + - SecondsToBoxTime((time_t)(365*24*60*60)), + BoxTimeToTimeval(GetCurrentBoxTime() + + SecondsToBoxTime((time_t)(365*24*60*60)), times[1]); times[0] = times[1]; TEST_THAT(::utimes("testfiles/TestDir1/sub23/" @@ -3667,8 +3562,16 @@ bool test_sync_files_with_timestamps_in_future() // Check change of store marker pauses daemon bool test_changing_client_store_marker_pauses_daemon() { + // Debugging this test requires INFO level logging + Logger::LevelGuard increase_to_info(Logging::GetConsole(), Log::INFO); + SETUP_WITH_BBSTORED(); - TEST_THAT(StartClient()); + + // Start the bbackupd client. Enable logging to help debug race + // conditions causing test failure: + std::string daemon_args(bbackupd_args_overridden ? bbackupd_args : + "-kT -Wnotice -tbbackupd"); + TEST_THAT_OR(StartClient("testfiles/bbackupd.conf", daemon_args), FAIL); // Wait for the client to upload all current files. We also time // approximately how long a sync takes. @@ -3677,7 +3580,7 @@ bool test_changing_client_store_marker_pauses_daemon() box_time_t sync_time = GetCurrentBoxTime() - sync_start_time; BOX_INFO("Sync takes " << BOX_FORMAT_MICROSECONDS(sync_time)); - // Time how long a compare takes. On NetBSD it's 3 seconds, and that + // Time how long a compare takes. On NetBSD it's 3 seconds, and that // interferes with test timing unless we account for it. box_time_t compare_start_time = GetCurrentBoxTime(); // There should be no differences right now (yet). @@ -3691,7 +3594,7 @@ bool test_changing_client_store_marker_pauses_daemon() // TODO FIXME dedent { - // Then... connect to the server, and change the + // Then... connect to the server, and change the // client store marker. See what that does! { bool done = false; @@ -3707,16 +3610,16 @@ bool test_changing_client_store_marker_pauses_daemon() // it should have changed std::auto_ptr loginConf(protocol->QueryLogin(0x01234567, 0)); TEST_THAT(loginConf->GetClientStoreMarker() != 0); - + // Change it to something else BOX_INFO("Changing client store marker " "from " << loginConf->GetClientStoreMarker() << " to 12"); protocol->QuerySetClientStoreMarker(12); - + // Success! done = true; - + // Log out protocol->QueryFinished(); } @@ -3735,7 +3638,7 @@ bool test_changing_client_store_marker_pauses_daemon() TEST_THAT(done); } - // Make a change to a file, to detect whether or not + // Make a change to a file, to detect whether or not // it's hanging around waiting to retry. { FILE *f = ::fopen("testfiles/TestDir1/fileaftermarker", "w"); @@ -3818,21 +3721,21 @@ bool test_interrupted_restore_can_be_recovered() // rather than doing anything TEST_THAT(BackupClientRestore(*client, restoredirid, "Test1", "testfiles/restore-interrupt", - true /* print progress dots */, - false /* restore deleted */, - false /* undelete after */, + true /* print progress dots */, + false /* restore deleted */, + false /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_ResumePossible); // Then resume it TEST_THAT(BackupClientRestore(*client, restoredirid, "Test1", "testfiles/restore-interrupt", - true /* print progress dots */, - false /* restore deleted */, - false /* undelete after */, + true /* print progress dots */, + false /* restore deleted */, + false /* undelete after */, true /* resume */, - false /* keep going */) + false /* keep going */) == Restore_Complete); client->QueryFinished(); @@ -3851,7 +3754,7 @@ bool assert_x1_deleted_or_not(bool expected_deleted) { std::auto_ptr client = connect_and_login(sTlsContext, 0 /* read-write */); - + std::auto_ptr dir = ReadDirectory(*client); int64_t testDirId = SearchDir(*dir, "Test1"); TEST_THAT_OR(testDirId != 0, return false); @@ -3873,7 +3776,7 @@ bool test_restore_deleted_files() bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - TEST_THAT(::unlink("testfiles/TestDir1/f1.dat") == 0); + TEST_THAT(EMU_UNLINK("testfiles/TestDir1/f1.dat") == 0); #ifdef WIN32 TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\x1") == 0); #else @@ -3903,11 +3806,11 @@ bool test_restore_deleted_files() // Do restore and undelete TEST_THAT(BackupClientRestore(*client, deldirid, "Test1", "testfiles/restore-Test1-x1-2", - true /* print progress dots */, - true /* deleted files */, + true /* print progress dots */, + true /* deleted files */, true /* undelete after */, false /* resume */, - false /* keep going */) + false /* keep going */) == Restore_Complete); client->QueryFinished(); @@ -3917,7 +3820,7 @@ bool test_restore_deleted_files() TEST_COMPARE(Compare_Same, "", "-cEQ Test1/x1 " "testfiles/restore-Test1-x1-2"); } - + // Final check on notifications TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2")); @@ -3947,7 +3850,7 @@ bool test_locked_file_behaviour() TEST_THAT_OR(handle != INVALID_HANDLE_VALUE, FAIL); { - // this sync should try to back up the file, + // this sync should try to back up the file, // and fail, because it's locked bbackupd.RunSyncNowWithExceptionHandling(); TEST_THAT(TestFileExists("testfiles/" diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index cba40fe73..5f82764f6 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -13,46 +13,66 @@ #include #include -#include "Test.h" +#ifdef HAVE_SIGNAL_H +# include +#endif + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_SYS_TYPES_H +# include +#endif + +#ifdef HAVE_SYS_WAIT_H +# include +#endif + +#include "Archive.h" +#include "CollectInBufferStream.h" +#include "CommonException.h" #include "Configuration.h" +#include "Conversion.h" +#include "ExcludeList.h" #include "FdGetLine.h" -#include "Guards.h" #include "FileStream.h" +#include "Guards.h" #include "InvisibleTempFileStream.h" #include "IOStreamGetLine.h" +#include "Logging.h" +#include "MemBlockStream.h" #include "NamedLock.h" +#include "PartialReadStream.h" #include "ReadGatherStream.h" -#include "MemBlockStream.h" -#include "ExcludeList.h" -#include "CommonException.h" -#include "Conversion.h" -#include "autogen_ConversionException.h" -#include "CollectInBufferStream.h" -#include "Archive.h" +#include "Test.h" #include "Timer.h" -#include "Logging.h" #include "ZeroStream.h" -#include "PartialReadStream.h" +#include "autogen_ConversionException.h" #include "MemLeakFindOn.h" using namespace BoxConvert; -void test_conversions() +bool test_conversions() { + SETUP(); + TEST_THAT((Convert(std::string("32"))) == 32); TEST_THAT((Convert("42")) == 42); TEST_THAT((Convert("-42")) == -42); TEST_CHECK_THROWS((Convert("500")), ConversionException, IntOverflowInConvertFromString); TEST_CHECK_THROWS((Convert("pants")), ConversionException, BadStringRepresentationOfInt); TEST_CHECK_THROWS((Convert("")), ConversionException, CannotConvertEmptyStringToInt); - + std::string a(Convert(63)); TEST_THAT(a == "63"); std::string b(Convert(-3473463)); TEST_THAT(b == "-3473463"); std::string c(Convert(344)); TEST_THAT(c == "344"); + + TEARDOWN(); } ConfigurationVerifyKey verifykeys1_1_1[] = @@ -69,7 +89,7 @@ ConfigurationVerifyKey verifykeys1_1_2[] = }; -ConfigurationVerify verifysub1_1[] = +ConfigurationVerify verifysub1_1[] = { { "*", @@ -94,13 +114,13 @@ ConfigurationVerifyKey verifykeys1_1[] = ConfigurationVerifyKey("string2", ConfigTest_Exists | ConfigTest_LastEntry) }; -ConfigurationVerifyKey verifykeys1_2[] = +ConfigurationVerifyKey verifykeys1_2[] = { ConfigurationVerifyKey("carrots", ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("string", ConfigTest_Exists | ConfigTest_LastEntry) }; -ConfigurationVerify verifysub1[] = +ConfigurationVerify verifysub1[] = { { "test1", @@ -127,7 +147,7 @@ ConfigurationVerifyKey verifykeys1[] = ConfigurationVerifyKey("BoolTrue2", ConfigTest_IsBool), ConfigurationVerifyKey("BoolFalse1", ConfigTest_IsBool), ConfigurationVerifyKey("BoolFalse2", ConfigTest_IsBool), - ConfigurationVerifyKey("TOPlevel", + ConfigurationVerifyKey("TOPlevel", ConfigTest_LastEntry | ConfigTest_Exists) }; @@ -147,12 +167,12 @@ class TestLogger : public Logger Log::Level mTargetLevel; public: - TestLogger(Log::Level targetLevel) + TestLogger(Log::Level targetLevel) : mTriggered(false), mTargetLevel(targetLevel) - { + { Logging::Add(this); } - ~TestLogger() + ~TestLogger() { Logging::Remove(this); } @@ -175,106 +195,212 @@ class TestLogger : public Logger virtual void SetProgramName(const std::string& rProgramName) { } }; -int test(int argc, const char *argv[]) +// Test PartialReadStream and ReadGatherStream handling of files over 2GB (refs #2) +bool test_stream_large_files() { - // Test PartialReadStream and ReadGatherStream handling of files - // over 2GB (refs #2) + SETUP(); + + char buffer[8]; + + ZeroStream zero(0x80000003); + zero.Seek(0x7ffffffe, IOStream::SeekType_Absolute); + TEST_THAT(zero.GetPosition() == 0x7ffffffe); + TEST_THAT(zero.Read(buffer, 8) == 5); + TEST_THAT(zero.GetPosition() == 0x80000003); + TEST_THAT(zero.Read(buffer, 8) == 0); + zero.Seek(0, IOStream::SeekType_Absolute); + TEST_THAT(zero.GetPosition() == 0); + + char* buffer2 = new char [0x1000000]; + TEST_THAT(buffer2 != NULL); + + PartialReadStream part(zero, 0x80000002); + for (int i = 0; i < 0x80; i++) { - char buffer[8]; + int read = part.Read(buffer2, 0x1000000); + TEST_THAT(read == 0x1000000); + } + TEST_THAT(part.Read(buffer, 8) == 2); + TEST_THAT(part.Read(buffer, 8) == 0); - ZeroStream zero(0x80000003); - zero.Seek(0x7ffffffe, IOStream::SeekType_Absolute); - TEST_THAT(zero.GetPosition() == 0x7ffffffe); - TEST_THAT(zero.Read(buffer, 8) == 5); - TEST_THAT(zero.GetPosition() == 0x80000003); - TEST_THAT(zero.Read(buffer, 8) == 0); - zero.Seek(0, IOStream::SeekType_Absolute); - TEST_THAT(zero.GetPosition() == 0); + delete [] buffer2; - char* buffer2 = new char [0x1000000]; - TEST_THAT(buffer2 != NULL); + ReadGatherStream gather(false); + zero.Seek(0, IOStream::SeekType_Absolute); + int component = gather.AddComponent(&zero); + gather.AddBlock(component, 0x80000002); + TEST_THAT(gather.Read(buffer, 8) == 8); - PartialReadStream part(zero, 0x80000002); - for (int i = 0; i < 0x80; i++) - { - int read = part.Read(buffer2, 0x1000000); - TEST_THAT(read == 0x1000000); - } - TEST_THAT(part.Read(buffer, 8) == 2); - TEST_THAT(part.Read(buffer, 8) == 0); + TEARDOWN(); +} + +// Test self-deleting temporary file streams +bool test_invisible_temp_file_stream() +{ + SETUP(); - delete [] buffer2; + std::string tempfile("testfiles/tempfile"); + TEST_CHECK_THROWS(InvisibleTempFileStream fs(tempfile.c_str()), + CommonException, OSFileOpenError); + InvisibleTempFileStream fs(tempfile.c_str(), O_CREAT); - ReadGatherStream gather(false); - zero.Seek(0, IOStream::SeekType_Absolute); - int component = gather.AddComponent(&zero); - gather.AddBlock(component, 0x80000002); - TEST_THAT(gather.Read(buffer, 8) == 8); - } +#ifdef WIN32 + // file is still visible under Windows + TEST_THAT(TestFileExists(tempfile.c_str())); - // Test self-deleting temporary file streams - { - std::string tempfile("testfiles/tempfile"); - TEST_CHECK_THROWS(InvisibleTempFileStream fs(tempfile.c_str()), - CommonException, OSFileOpenError); - InvisibleTempFileStream fs(tempfile.c_str(), O_CREAT); + // opening it again should work + InvisibleTempFileStream fs2(tempfile.c_str()); + TEST_THAT(TestFileExists(tempfile.c_str())); - #ifdef WIN32 - // file is still visible under Windows - TEST_THAT(TestFileExists(tempfile.c_str())); + // opening it to create should work + InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); + TEST_THAT(TestFileExists(tempfile.c_str())); - // opening it again should work - InvisibleTempFileStream fs2(tempfile.c_str()); - TEST_THAT(TestFileExists(tempfile.c_str())); + // opening it to create exclusively should fail + TEST_CHECK_THROWS(InvisibleTempFileStream fs4(tempfile.c_str(), + O_CREAT | O_EXCL), CommonException, OSFileOpenError); - // opening it to create should work - InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); - TEST_THAT(TestFileExists(tempfile.c_str())); + fs2.Close(); +#else + // file is not visible under Unix + TEST_THAT(!TestFileExists(tempfile.c_str())); - // opening it to create exclusively should fail - TEST_CHECK_THROWS(InvisibleTempFileStream fs4(tempfile.c_str(), - O_CREAT | O_EXCL), CommonException, OSFileOpenError); + // opening it again should fail + TEST_CHECK_THROWS(InvisibleTempFileStream fs2(tempfile.c_str()), + CommonException, OSFileOpenError); - fs2.Close(); - #else - // file is not visible under Unix - TEST_THAT(!TestFileExists(tempfile.c_str())); + // opening it to create should work + InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); + TEST_THAT(!TestFileExists(tempfile.c_str())); - // opening it again should fail - TEST_CHECK_THROWS(InvisibleTempFileStream fs2(tempfile.c_str()), - CommonException, OSFileOpenError); + // opening it to create exclusively should work + InvisibleTempFileStream fs4(tempfile.c_str(), O_CREAT | O_EXCL); + TEST_THAT(!TestFileExists(tempfile.c_str())); - // opening it to create should work - InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); - TEST_THAT(!TestFileExists(tempfile.c_str())); + fs4.Close(); +#endif - // opening it to create exclusively should work - InvisibleTempFileStream fs4(tempfile.c_str(), O_CREAT | O_EXCL); - TEST_THAT(!TestFileExists(tempfile.c_str())); + fs.Close(); + fs3.Close(); - fs4.Close(); - #endif + // now that it's closed, it should be invisible on all platforms + TEST_THAT(!TestFileExists(tempfile.c_str())); - fs.Close(); - fs3.Close(); + TEARDOWN(); +} - // now that it's closed, it should be invisible on all platforms - TEST_THAT(!TestFileExists(tempfile.c_str())); - } +// Test that named locks work as expected +bool test_named_locks() +{ + SETUP(); - // Test that named locks work as expected { NamedLock lock1; - TEST_THAT(lock1.TryAndGetLock("testfiles/locktest")); + // Try and get a lock on a name in a directory which doesn't exist + TEST_CHECK_THROWS(lock1.TryAndGetLock( + "testfiles" + DIRECTORY_SEPARATOR "non-exist" + DIRECTORY_SEPARATOR "lock"), + CommonException, OSFileOpenError); + + // And a more reasonable request + TEST_THAT(lock1.TryAndGetLock( + "testfiles" DIRECTORY_SEPARATOR "lock1") == true); + + // Try to lock something using the same lock + TEST_CHECK_THROWS( + lock1.TryAndGetLock( + "testfiles" + DIRECTORY_SEPARATOR "non-exist" + DIRECTORY_SEPARATOR "lock2"), + CommonException, NamedLockAlreadyLockingSomething); + } + + { + // Check that it unlocked when it went out of scope + NamedLock lock3; + TEST_THAT(lock3.TryAndGetLock( + "testfiles" DIRECTORY_SEPARATOR "lock1") == true); + } + + { + // And unlocking works + NamedLock lock4; + TEST_CHECK_THROWS(lock4.ReleaseLock(), CommonException, + NamedLockNotHeld); + TEST_THAT(lock4.TryAndGetLock( + "testfiles" DIRECTORY_SEPARATOR "lock4") == true); + lock4.ReleaseLock(); + NamedLock lock5; + TEST_THAT(lock5.TryAndGetLock( + "testfiles" DIRECTORY_SEPARATOR "lock4") == true); + // And can reuse it + TEST_THAT(lock4.TryAndGetLock( + "testfiles" DIRECTORY_SEPARATOR "lock5") == true); + } + +#ifndef WIN32 // Windows locking is tested differently, below + { + // Test that named locks are actually exclusive! + int child_pid = fork(); + if(child_pid == 0) + { + // This is the child process. Run ourselves with a special argument + // which will lock the lockfile until killed + TEST_THAT(execl(TEST_EXECUTABLE, TEST_EXECUTABLE, "lockwait", NULL) == 0); + } + else + { + sleep(1); + } + // With a lock held, we should not be able to acquire another. TEST_THAT(!NamedLock().TryAndGetLock("testfiles/locktest")); + + kill(child_pid, SIGTERM); + waitpid(child_pid, NULL, 0); } + { - // But with the lock released, we should be able to. + // But with the lock released (by killing the child process), we should be able to + // acquire it here: TEST_THAT(NamedLock().TryAndGetLock("testfiles/locktest")); } +#endif // !WIN32 + + // Test that double-acquiring locks fails on platforms with non-re-entrant locks + // (BOX_LOCK_TYPE_O_EXLOCK and BOX_LOCK_TYPE_WIN32) and raises an exception when the outer + // lock is unlocked (and the error is discovered) on other platforms. + { + NamedLock lock1; + TEST_THAT(lock1.TryAndGetLock("testfiles" DIRECTORY_SEPARATOR "lock")); + + // And again on that name. This works on platforms that have + // non-reentrant file locks: Windows and O_EXLOCK. + NamedLock lock2; +#if defined BOX_LOCK_TYPE_F_SETLK + // This lock type is reentrant, unfortunately. It appears that we can lock the same + // file again, and we only detect the problem when we unlock and then try to delete + // the lockfile for the first lock, when we discover that we've already deleted it, + // which means that we made a mistake: + TEST_THAT(lock2.TryAndGetLock("testfiles" DIRECTORY_SEPARATOR "lock")); + lock2.ReleaseLock(); + TEST_CHECK_THROWS(lock1.ReleaseLock(), CommonException, OSFileError); +#else + // These lock types are non-reentrant, so any attempt to lock them again should fail: + TEST_THAT(!lock2.TryAndGetLock("testfiles" DIRECTORY_SEPARATOR "lock")); +#endif + } + + TEARDOWN(); +} + +#ifdef BOX_MEMORY_LEAK_TESTING +// Test that memory leak detection doesn't crash +bool test_memory_leak_detection() +{ + SETUP(); - // Test that memory leak detection doesn't crash { char *test = new char[1024]; delete [] test; @@ -282,7 +408,6 @@ int test(int argc, const char *argv[]) delete s; } -#ifdef BOX_MEMORY_LEAK_TESTING { Timers::Cleanup(); @@ -304,11 +429,18 @@ int test(int argc, const char *argv[]) Timers::Init(); } + + TEARDOWN(); +} #endif // BOX_MEMORY_LEAK_TESTING +bool test_timers() +{ + SETUP(); + // test main() initialises timers for us, so uninitialise them Timers::Cleanup(); - + // Check that using timer methods without initialisation // throws an assertion failure. Can only do this in debug mode #ifndef BOX_RELEASE_BUILD @@ -326,10 +458,10 @@ int test(int argc, const char *argv[]) // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); } #endif - + // Check that we can initialise the timers Timers::Init(); - + // Check that double initialisation throws an exception #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Init(), CommonException, @@ -338,7 +470,7 @@ int test(int argc, const char *argv[]) // Check that we can clean up the timers Timers::Cleanup(); - + // Check that double cleanup throws an exception #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, @@ -405,6 +537,13 @@ int test(int argc, const char *argv[]) // Leave timers initialised for rest of test. // Test main() will cleanup after test finishes. + TEARDOWN(); +} + +bool test_getline() +{ + SETUP(); + static const char *testfilelines[] = { "First line", @@ -435,7 +574,7 @@ int test(int argc, const char *argv[]) FileHandleGuard file("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt"); FdGetLine getline(file); - + int l = 0; while(testfilelines[l] != 0) { @@ -452,9 +591,9 @@ int test(int argc, const char *argv[]) { FileHandleGuard file("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt"); - FILE *file2 = fopen("testfiles" DIRECTORY_SEPARATOR + FILE *file2 = fopen("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt", "r"); - TEST_THAT_ABORTONFAIL(file2 != 0); + TEST_THAT(file2 != 0); FdGetLine getline(file); char ll[512]; @@ -474,43 +613,40 @@ int test(int argc, const char *argv[]) } TEST_THAT(getline.IsEOF()); TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); - + fclose(file2); } // Then the IOStream version of get line, seeing as we're here... { - FileStream file("testfiles" DIRECTORY_SEPARATOR + FileStream file("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt", O_RDONLY); IOStreamGetLine getline(file); - + int l = 0; while(testfilelines[l] != 0) { TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, true)) - { - // skip line - } + std::string line = getline.GetLine(true); TEST_EQUAL(testfilelines[l], line); l++; } TEST_THAT(getline.IsEOF()); std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); + TEST_CHECK_THROWS(getline.GetLine(false), CommonException, GetLineEOF); } + // and again without pre-processing { - FileStream file("testfiles" DIRECTORY_SEPARATOR + FileStream file("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt", O_RDONLY); IOStreamGetLine getline(file); - FILE *file2 = fopen("testfiles" DIRECTORY_SEPARATOR + FILE *file2 = fopen("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt", "r"); - TEST_THAT_ABORTONFAIL(file2 != 0); + TEST_THAT(file2 != 0); char ll[512]; - + while(!feof(file2)) { fgets(ll, sizeof(ll), file2); @@ -522,25 +658,30 @@ int test(int argc, const char *argv[]) ll[e] = '\0'; TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, false)) - ; + std::string line = getline.GetLine(false); TEST_EQUAL(ll, line); } TEST_THAT(getline.IsEOF()); std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); - + TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); + fclose(file2); } + TEARDOWN(); +} + +bool test_configuration() +{ + SETUP(); + // Doesn't exist { std::string errMsg; TEST_CHECK_THROWS(std::auto_ptr pconfig( Configuration::LoadAndVerify( - "testfiles" DIRECTORY_SEPARATOR "DOESNTEXIST", - &verify, errMsg)), + "testfiles" DIRECTORY_SEPARATOR "DOESNTEXIST", + &verify, errMsg)), CommonException, OSFileOpenError); } @@ -549,13 +690,14 @@ int test(int argc, const char *argv[]) std::string errMsg; std::auto_ptr pconfig( Configuration::LoadAndVerify( - "testfiles" DIRECTORY_SEPARATOR "config1.txt", + "testfiles" DIRECTORY_SEPARATOR "config1.txt", &verify, errMsg)); if(!errMsg.empty()) { printf("UNEXPECTED error msg is:\n------\n%s------\n", errMsg.c_str()); } - TEST_THAT_ABORTONFAIL(pconfig.get() != 0); + + TEST_THAT(pconfig.get() != 0); TEST_THAT(errMsg.empty()); TEST_THAT(pconfig->KeyExists("TOPlevel")); TEST_THAT(pconfig->GetKeyValue("TOPlevel") == "value"); @@ -591,7 +733,7 @@ int test(int argc, const char *argv[]) const Configuration &sub1_3 = sub1.GetSubConfiguration("subconfig3"); TEST_THAT(sub1_3.GetKeyValueInt("carrots") == 050); TEST_THAT(sub1_3.GetKeyValue("terrible") == "absolutely"); - } + } static const char *file[][2] = { @@ -657,7 +799,7 @@ int test(int argc, const char *argv[]) errMsg = errMsg.substr(0, errMsg.size() > 0 ? errMsg.size() - 1 : 0); TEST_EQUAL_LINE(file[l][1], errMsg, file[l][0]); } - + // Check that multivalues happen as expected // (single value in a multivalue already checked) { @@ -676,7 +818,7 @@ int test(int argc, const char *argv[]) TEST_THAT(pconfig->GetKeyValue("MultiValue") == expectedvalue); } - // Check boolean values + // Check boolean values { std::string errMsg; std::auto_ptr pconfig( @@ -690,251 +832,302 @@ int test(int argc, const char *argv[]) TEST_THAT(pconfig->GetKeyValueBool("BoolFalse1") == false); TEST_THAT(pconfig->GetKeyValueBool("BoolFalse2") == false); } - - // Test named locks - { - NamedLock lock1; - // Try and get a lock on a name in a directory which doesn't exist - TEST_CHECK_THROWS(lock1.TryAndGetLock( - "testfiles" - DIRECTORY_SEPARATOR "non-exist" - DIRECTORY_SEPARATOR "lock"), - CommonException, OSFileError); - // And a more resonable request - TEST_THAT(lock1.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock1") == true); + TEARDOWN(); +} - // Try to lock something using the same lock - TEST_CHECK_THROWS( - lock1.TryAndGetLock( - "testfiles" - DIRECTORY_SEPARATOR "non-exist" - DIRECTORY_SEPARATOR "lock2"), - CommonException, NamedLockAlreadyLockingSomething); -#if defined(HAVE_FLOCK) || HAVE_DECL_O_EXLOCK - // And again on that name - NamedLock lock2; - TEST_THAT(lock2.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock1") == false); -#endif - } - { - // Check that it unlocked when it went out of scope - NamedLock lock3; - TEST_THAT(lock3.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock1") == true); - } +// Test the ReadGatherStream +bool test_read_gather_stream() +{ + SETUP(); + + #define GATHER_DATA1 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + #define GATHER_DATA2 "ZYZWVUTSRQPOMNOLKJIHGFEDCBA9876543210zyxwvutsrqpomno" + + // Make two streams + MemBlockStream s1(GATHER_DATA1, sizeof(GATHER_DATA1)); + MemBlockStream s2(GATHER_DATA2, sizeof(GATHER_DATA2)); + + // And a gather stream + ReadGatherStream gather(false /* no deletion */); + + // Add the streams + int s1_c = gather.AddComponent(&s1); + int s2_c = gather.AddComponent(&s2); + TEST_THAT(s1_c == 0); + TEST_THAT(s2_c == 1); + + // Set up some blocks + gather.AddBlock(s1_c, 11); + gather.AddBlock(s1_c, 2); + gather.AddBlock(s1_c, 8, true, 2); + gather.AddBlock(s2_c, 20); + gather.AddBlock(s1_c, 20); + gather.AddBlock(s2_c, 25); + gather.AddBlock(s1_c, 10, true, 0); + #define GATHER_RESULT "0123456789abc23456789ZYZWVUTSRQPOMNOLKJIHabcdefghijklmnopqrstGFEDCBA9876543210zyxwvuts0123456789" + + // Read them in... + char buffer[1024]; + unsigned int r = 0; + while(r < sizeof(GATHER_RESULT) - 1) { - // And unlocking works - NamedLock lock4; - TEST_CHECK_THROWS(lock4.ReleaseLock(), CommonException, - NamedLockNotHeld); - TEST_THAT(lock4.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock4") == true); - lock4.ReleaseLock(); - NamedLock lock5; - TEST_THAT(lock5.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock4") == true); - // And can reuse it - TEST_THAT(lock4.TryAndGetLock( - "testfiles" DIRECTORY_SEPARATOR "lock5") == true); - } + int s = gather.Read(buffer + r, 7); + r += s; - // Test the ReadGatherStream - { - #define GATHER_DATA1 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - #define GATHER_DATA2 "ZYZWVUTSRQPOMNOLKJIHGFEDCBA9876543210zyxwvutsrqpomno" - - // Make two streams - MemBlockStream s1(GATHER_DATA1, sizeof(GATHER_DATA1)); - MemBlockStream s2(GATHER_DATA2, sizeof(GATHER_DATA2)); - - // And a gather stream - ReadGatherStream gather(false /* no deletion */); - - // Add the streams - int s1_c = gather.AddComponent(&s1); - int s2_c = gather.AddComponent(&s2); - TEST_THAT(s1_c == 0); - TEST_THAT(s2_c == 1); - - // Set up some blocks - gather.AddBlock(s1_c, 11); - gather.AddBlock(s1_c, 2); - gather.AddBlock(s1_c, 8, true, 2); - gather.AddBlock(s2_c, 20); - gather.AddBlock(s1_c, 20); - gather.AddBlock(s2_c, 25); - gather.AddBlock(s1_c, 10, true, 0); - #define GATHER_RESULT "0123456789abc23456789ZYZWVUTSRQPOMNOLKJIHabcdefghijklmnopqrstGFEDCBA9876543210zyxwvuts0123456789" - - // Read them in... - char buffer[1024]; - unsigned int r = 0; - while(r < sizeof(GATHER_RESULT) - 1) + TEST_THAT(gather.GetPosition() == r); + if(r < sizeof(GATHER_RESULT) - 1) { - int s = gather.Read(buffer + r, 7); - r += s; - - TEST_THAT(gather.GetPosition() == r); - if(r < sizeof(GATHER_RESULT) - 1) - { - TEST_THAT(gather.StreamDataLeft()); - TEST_THAT(static_cast(gather.BytesLeftToRead()) == sizeof(GATHER_RESULT) - 1 - r); - } - else - { - TEST_THAT(!gather.StreamDataLeft()); - TEST_THAT(gather.BytesLeftToRead() == 0); - } + TEST_THAT(gather.StreamDataLeft()); + TEST_THAT(static_cast(gather.BytesLeftToRead()) == sizeof(GATHER_RESULT) - 1 - r); } - TEST_THAT(r == sizeof(GATHER_RESULT) - 1); - TEST_THAT(::memcmp(buffer, GATHER_RESULT, sizeof(GATHER_RESULT) - 1) == 0); - } - - // Test ExcludeList - { - ExcludeList elist; - // Check assumption - TEST_THAT(Configuration::MultiValueSeparator == '\x01'); - // Add definite entries - elist.AddDefiniteEntries(std::string("\x01")); - elist.AddDefiniteEntries(std::string("")); - elist.AddDefiniteEntries(std::string("Definite1\x01/dir/DefNumberTwo\x01\x01ThingDefThree")); - elist.AddDefiniteEntries(std::string("AnotherDef")); - TEST_THAT(elist.SizeOfDefiniteList() == 4); - - // Add regex entries - #ifdef HAVE_REGEX_SUPPORT + else { - HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); - elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")); - elist.AddRegexEntries(std::string("")); - TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression); - TEST_THAT(elist.SizeOfRegexList() == 3); + TEST_THAT(!gather.StreamDataLeft()); + TEST_THAT(gather.BytesLeftToRead() == 0); } - #else - TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")), CommonException, RegexNotSupportedOnThisPlatform); - TEST_THAT(elist.SizeOfRegexList() == 0); - #endif - - #ifdef WIN32 - #define CASE_SENSITIVE false - #else - #define CASE_SENSITIVE true - #endif - - // Try some matches! - TEST_THAT(elist.IsExcluded(std::string("Definite1")) == true); - TEST_THAT(elist.IsExcluded(std::string("/dir/DefNumberTwo")) == true); - TEST_THAT(elist.IsExcluded(std::string("ThingDefThree")) == true); - TEST_THAT(elist.IsExcluded(std::string("AnotherDef")) == true); - TEST_THAT(elist.IsExcluded(std::string("dir/DefNumberTwo")) == false); - - // Try some case insensitive matches, - // that should pass on Win32 and fail elsewhere - TEST_THAT(elist.IsExcluded("DEFINITe1") - == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded("/Dir/DefNumberTwo") - == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded("thingdefthree") - == !CASE_SENSITIVE); - - #ifdef HAVE_REGEX_SUPPORT - TEST_THAT(elist.IsExcluded(std::string("b.reg")) == true); - TEST_THAT(elist.IsExcluded(std::string("B.reg")) == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded(std::string("b.Reg")) == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded(std::string("e.reg")) == false); - TEST_THAT(elist.IsExcluded(std::string("e.Reg")) == false); - TEST_THAT(elist.IsExcluded(std::string("DEfinite1")) == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded(std::string("DEXCLUDEfinite1")) == true); - TEST_THAT(elist.IsExcluded(std::string("DEfinitexclude1")) == !CASE_SENSITIVE); - TEST_THAT(elist.IsExcluded(std::string("exclude")) == true); - TEST_THAT(elist.IsExcluded(std::string("ExcludE")) == !CASE_SENSITIVE); - #endif - - #undef CASE_SENSITIVE - - TestLogger logger(Log::WARNING); - TEST_THAT(!logger.IsTriggered()); - elist.AddDefiniteEntries(std::string("/foo")); - TEST_THAT(!logger.IsTriggered()); - elist.AddDefiniteEntries(std::string("/foo/")); - TEST_THAT(logger.IsTriggered()); - logger.Reset(); - elist.AddDefiniteEntries(std::string("/foo" - DIRECTORY_SEPARATOR)); - TEST_THAT(logger.IsTriggered()); - logger.Reset(); - elist.AddDefiniteEntries(std::string("/foo" - DIRECTORY_SEPARATOR "bar\x01/foo")); - TEST_THAT(!logger.IsTriggered()); - elist.AddDefiniteEntries(std::string("/foo" - DIRECTORY_SEPARATOR "bar\x01/foo" - DIRECTORY_SEPARATOR)); - TEST_THAT(logger.IsTriggered()); } - test_conversions(); + TEST_THAT(r == sizeof(GATHER_RESULT) - 1); + TEST_THAT(::memcmp(buffer, GATHER_RESULT, sizeof(GATHER_RESULT) - 1) == 0); - // test that we can use Archive and CollectInBufferStream - // to read and write arbitrary types to a memory buffer + TEARDOWN(); +} +// Test ExcludeList +bool test_exclude_list() +{ + SETUP(); + + ExcludeList elist; + // Check assumption + TEST_THAT(Configuration::MultiValueSeparator == '\x01'); + // Add definite entries + elist.AddDefiniteEntries(std::string("\x01")); + elist.AddDefiniteEntries(std::string("")); + elist.AddDefiniteEntries(std::string("Definite1\x01/dir/DefNumberTwo\x01\x01ThingDefThree")); + elist.AddDefiniteEntries(std::string("AnotherDef")); + TEST_THAT(elist.SizeOfDefiniteList() == 4); + + // Add regex entries + #ifdef HAVE_REGEX_SUPPORT { - CollectInBufferStream buffer; + HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); + elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")); + elist.AddRegexEntries(std::string("")); + TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression); + TEST_THAT(elist.SizeOfRegexList() == 3); + } + #else + TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")), CommonException, RegexNotSupportedOnThisPlatform); + TEST_THAT(elist.SizeOfRegexList() == 0); + #endif + + #ifdef WIN32 + #define CASE_SENSITIVE false + #else + #define CASE_SENSITIVE true + #endif + + // Try some matches! + TEST_THAT(elist.IsExcluded(std::string("Definite1")) == true); + TEST_THAT(elist.IsExcluded(std::string("/dir/DefNumberTwo")) == true); + TEST_THAT(elist.IsExcluded(std::string("ThingDefThree")) == true); + TEST_THAT(elist.IsExcluded(std::string("AnotherDef")) == true); + TEST_THAT(elist.IsExcluded(std::string("dir/DefNumberTwo")) == false); + + // Try some case insensitive matches, + // that should pass on Win32 and fail elsewhere + TEST_THAT(elist.IsExcluded("DEFINITe1") + == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded("/Dir/DefNumberTwo") + == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded("thingdefthree") + == !CASE_SENSITIVE); + + #ifdef HAVE_REGEX_SUPPORT + TEST_THAT(elist.IsExcluded(std::string("b.reg")) == true); + TEST_THAT(elist.IsExcluded(std::string("B.reg")) == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded(std::string("b.Reg")) == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded(std::string("e.reg")) == false); + TEST_THAT(elist.IsExcluded(std::string("e.Reg")) == false); + TEST_THAT(elist.IsExcluded(std::string("DEfinite1")) == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded(std::string("DEXCLUDEfinite1")) == true); + TEST_THAT(elist.IsExcluded(std::string("DEfinitexclude1")) == !CASE_SENSITIVE); + TEST_THAT(elist.IsExcluded(std::string("exclude")) == true); + TEST_THAT(elist.IsExcluded(std::string("ExcludE")) == !CASE_SENSITIVE); + #endif + + #undef CASE_SENSITIVE + + TestLogger logger(Log::WARNING); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo")); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo/")); + TEST_THAT(logger.IsTriggered()); + logger.Reset(); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR)); + TEST_THAT(logger.IsTriggered()); + logger.Reset(); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR "bar\x01/foo")); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR "bar\x01/foo" + DIRECTORY_SEPARATOR)); + TEST_THAT(logger.IsTriggered()); + + TEARDOWN(); +} + +// Test that we can use Archive and CollectInBufferStream +// to read and write arbitrary types to a memory buffer +bool test_archive() +{ + SETUP(); + + CollectInBufferStream buffer; + ASSERT(buffer.GetPosition() == 0); + + { + Archive archive(buffer, 0); ASSERT(buffer.GetPosition() == 0); - { - Archive archive(buffer, 0); - ASSERT(buffer.GetPosition() == 0); - - archive.Write((bool) true); - archive.Write((bool) false); - archive.Write((int) 0x12345678); - archive.Write((int) 0x87654321); - archive.Write((int64_t) 0x0badfeedcafebabeLL); - archive.Write((uint64_t) 0xfeedfacedeadf00dLL); - archive.Write((uint8_t) 0x01); - archive.Write((uint8_t) 0xfe); - archive.Write(std::string("hello world!")); - archive.Write(std::string("goodbye cruel world!")); - } + archive.Write((bool) true); + archive.Write((bool) false); + archive.Write((int) 0x12345678); + archive.Write((int) 0x87654321); + archive.Write((int64_t) 0x0badfeedcafebabeLL); + archive.Write((uint64_t) 0xfeedfacedeadf00dLL); + archive.Write((uint8_t) 0x01); + archive.Write((uint8_t) 0xfe); + archive.Write(std::string("hello world!")); + archive.Write(std::string("goodbye cruel world!")); + } + + CollectInBufferStream buf2; + buf2.Write(buffer.GetBuffer(), buffer.GetSize()); + TEST_THAT(buf2.GetPosition() == buffer.GetSize()); - CollectInBufferStream buf2; - buf2.Write(buffer.GetBuffer(), buffer.GetSize()); - TEST_THAT(buf2.GetPosition() == buffer.GetSize()); + buf2.SetForReading(); + TEST_THAT(buf2.GetPosition() == 0); - buf2.SetForReading(); + { + Archive archive(buf2, 0); TEST_THAT(buf2.GetPosition() == 0); - { - Archive archive(buf2, 0); - TEST_THAT(buf2.GetPosition() == 0); + bool b; + archive.Read(b); TEST_THAT(b == true); + archive.Read(b); TEST_THAT(b == false); - bool b; - archive.Read(b); TEST_THAT(b == true); - archive.Read(b); TEST_THAT(b == false); + int i; + archive.Read(i); TEST_THAT(i == 0x12345678); + archive.Read(i); TEST_THAT((unsigned int)i == 0x87654321); - int i; - archive.Read(i); TEST_THAT(i == 0x12345678); - archive.Read(i); TEST_THAT((unsigned int)i == 0x87654321); + uint64_t i64; + archive.Read(i64); TEST_THAT(i64 == 0x0badfeedcafebabeLL); + archive.Read(i64); TEST_THAT(i64 == 0xfeedfacedeadf00dLL); - uint64_t i64; - archive.Read(i64); TEST_THAT(i64 == 0x0badfeedcafebabeLL); - archive.Read(i64); TEST_THAT(i64 == 0xfeedfacedeadf00dLL); + uint8_t i8; + archive.Read(i8); TEST_THAT(i8 == 0x01); + archive.Read(i8); TEST_THAT(i8 == 0xfe); - uint8_t i8; - archive.Read(i8); TEST_THAT(i8 == 0x01); - archive.Read(i8); TEST_THAT(i8 == 0xfe); + std::string s; + archive.Read(s); TEST_THAT(s == "hello world!"); + archive.Read(s); TEST_THAT(s == "goodbye cruel world!"); - std::string s; - archive.Read(s); TEST_THAT(s == "hello world!"); - archive.Read(s); TEST_THAT(s == "goodbye cruel world!"); + TEST_THAT(!buf2.StreamDataLeft()); + } - TEST_THAT(!buf2.StreamDataLeft()); - } + TEARDOWN(); +} + +// Test that box_strtoui64 works properly +bool test_box_strtoui64() +{ + SETUP(); + + TEST_EQUAL(1234567890123456, box_strtoui64("1234567890123456", NULL, 10)); + TEST_EQUAL(0x1234567890123456, box_strtoui64("1234567890123456", NULL, 16)); + TEST_EQUAL(0xd9385a13c3842ba0, box_strtoui64("d9385a13c3842ba0", NULL, 16)); + const char *input = "12a34"; + const char *endptr; + TEST_EQUAL(12, box_strtoui64(input, &endptr, 10)); + TEST_EQUAL(input + 2, endptr); + TEST_EQUAL(0x12a34, box_strtoui64(input, &endptr, 16)); + TEST_EQUAL(input + 5, endptr); + + TEARDOWN(); +} + +// Test that RemovePrefix and RemoveSuffix work properly +bool test_remove_prefix_suffix() +{ + SETUP(); + + TEST_EQUAL("food", RemovePrefix("", "food", false)); // !force + TEST_EQUAL("food", RemoveSuffix("", "food", false)); // !force + TEST_EQUAL("", RemovePrefix("d", "food", false)); // !force + TEST_EQUAL("", RemoveSuffix("f", "food", false)); // !force + TEST_EQUAL("", RemovePrefix("dz", "food", false)); // !force + TEST_EQUAL("", RemoveSuffix("fz", "food", false)); // !force + TEST_EQUAL("ood", RemovePrefix("f", "food", false)); // !force + TEST_EQUAL("foo", RemoveSuffix("d", "food", false)); // !force + TEST_EQUAL("od", RemovePrefix("fo", "food", false)); // !force + TEST_EQUAL("fo", RemoveSuffix("od", "food", false)); // !force + TEST_EQUAL("", RemovePrefix("food", "food", false)); // !force + TEST_EQUAL("", RemoveSuffix("food", "food", false)); // !force + TEST_EQUAL("", RemovePrefix("foodz", "food", false)); // !force + TEST_EQUAL("", RemoveSuffix("foodz", "food", false)); // !force + TEST_EQUAL("", RemoveSuffix("zfood", "food", false)); // !force + + TEST_EQUAL("food", RemovePrefix("", "food", true)); // force + TEST_EQUAL("food", RemoveSuffix("", "food", true)); // force + TEST_CHECK_THROWS(RemovePrefix("d", "food", true), CommonException, Internal); // force + TEST_CHECK_THROWS(RemoveSuffix("f", "food", true), CommonException, Internal); // force + TEST_CHECK_THROWS(RemovePrefix("dz", "food", true), CommonException, Internal); // force + TEST_CHECK_THROWS(RemoveSuffix("fz", "food", true), CommonException, Internal); // force + TEST_EQUAL("ood", RemovePrefix("f", "food", true)); // force + TEST_EQUAL("foo", RemoveSuffix("d", "food", true)); // force + TEST_EQUAL("od", RemovePrefix("fo", "food", true)); // force + TEST_EQUAL("fo", RemoveSuffix("od", "food", true)); // force + TEST_CHECK_THROWS(RemovePrefix("foodz", "food", true), CommonException, Internal); // force + TEST_CHECK_THROWS(RemoveSuffix("foodz", "food", true), CommonException, Internal); // force + TEST_CHECK_THROWS(RemoveSuffix("zfood", "food", true), CommonException, Internal); // force + + // Test that force defaults to true: + TEST_CHECK_THROWS(RemovePrefix("d", "food"), CommonException, Internal); + + TEARDOWN(); +} + +int test(int argc, const char *argv[]) +{ + if(argc == 2 && strcmp(argv[1], "lockwait") == 0) + { + NamedLock lock1; + TEST_THAT(lock1.TryAndGetLock("testfiles/locktest")); + sleep(3600); + return 0; } - return 0; + test_stream_large_files(); + test_invisible_temp_file_stream(); + test_named_locks(); +#ifdef BOX_MEMORY_LEAK_TESTING + test_memory_leak_detection(); +#endif + test_timers(); + test_getline(); + test_configuration(); + test_read_gather_stream(); + test_exclude_list(); + test_conversions(); + test_archive(); + test_box_strtoui64(); + test_remove_prefix_suffix(); + + return finish_test_suite(); } diff --git a/test/compress/testcompress.cpp b/test/compress/testcompress.cpp index 76512e6af..1208c25e8 100644 --- a/test/compress/testcompress.cpp +++ b/test/compress/testcompress.cpp @@ -52,6 +52,7 @@ class CopyInToOutStream : public IOStream { buffers[(currentBuffer + 1) & 1].Write(pBuffer, NBytes, Timeout); } + using IOStream::Write; bool StreamDataLeft() { return buffers[currentBuffer].StreamDataLeft() || buffers[(currentBuffer + 1) % 1].GetSize() > 0; diff --git a/test/httpserver/testfiles/dsfdsfs98.fd b/test/httpserver/testfiles/dsfdsfs98.fd new file mode 100755 index 000000000..f19769482 Binary files /dev/null and b/test/httpserver/testfiles/dsfdsfs98.fd differ diff --git a/test/httpserver/testfiles/puppy.jpg b/test/httpserver/testfiles/puppy.jpg new file mode 100644 index 000000000..a326a6a7e --- /dev/null +++ b/test/httpserver/testfiles/puppy.jpg @@ -0,0 +1 @@ +omgpuppies! diff --git a/test/httpserver/testfiles/s3simulator.conf b/test/httpserver/testfiles/s3simulator.conf index 07921560e..40c872e8f 100644 --- a/test/httpserver/testfiles/s3simulator.conf +++ b/test/httpserver/testfiles/s3simulator.conf @@ -1,6 +1,6 @@ AccessKey = 0PN5J17HBGZHT7JJ3X82 SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o -StoreDirectory = testfiles +StoreDirectory = testfiles/store AddressPrefix = http://localhost:1080 Server diff --git a/test/httpserver/testhttpserver.cpp b/test/httpserver/testhttpserver.cpp index 469fa3837..6933893a5 100644 --- a/test/httpserver/testhttpserver.cpp +++ b/test/httpserver/testhttpserver.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -18,32 +19,45 @@ #include #endif +#include +#include +#include +#include #include #include "autogen_HTTPException.h" +#include "HTTPQueryDecoder.h" #include "HTTPRequest.h" #include "HTTPResponse.h" #include "HTTPServer.h" +#include "HTTPTest.h" #include "IOStreamGetLine.h" +#include "MD5Digest.h" #include "S3Client.h" #include "S3Simulator.h" #include "ServerControl.h" +#include "SimpleDBClient.h" #include "Test.h" +#include "ZeroStream.h" #include "decode.h" #include "encode.h" #include "MemLeakFindOn.h" #define SHORT_TIMEOUT 5000 +#define LONG_TIMEOUT 300000 + +using boost::property_tree::ptree; class TestWebServer : public HTTPServer { public: - TestWebServer(); - ~TestWebServer(); + TestWebServer() + : HTTPServer(LONG_TIMEOUT) + { } + ~TestWebServer() { } virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); - }; // Build a nice HTML response, so this can also be tested neatly in a browser @@ -125,23 +139,687 @@ void TestWebServer::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) rResponse.Write(DEFAULT_RESPONSE_2, sizeof(DEFAULT_RESPONSE_2) - 1); } -TestWebServer::TestWebServer() {} -TestWebServer::~TestWebServer() {} +std::vector get_entry_names(const std::vector entries) +{ + std::vector entry_names; + for(std::vector::const_iterator i = entries.begin(); + i != entries.end(); i++) + { + entry_names.push_back(i->name()); + + } + return entry_names; +} -int test(int argc, const char *argv[]) +bool exercise_s3client(S3Client& client) { - if(argc >= 2 && ::strcmp(argv[1], "server") == 0) + int num_failures_initial = num_failures; + + HTTPResponse response = client.GetObject("/photos/puppy.jpg"); + TEST_EQUAL(200, response.GetResponseCode()); + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("omgpuppies!\n", response_data); + TEST_THAT(!response.IsKeepAlive()); + + // make sure that assigning to HTTPResponse does clear stream + response = client.GetObject("/photos/puppy.jpg"); + TEST_EQUAL(200, response.GetResponseCode()); + response_data = std::string((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("omgpuppies!\n", response_data); + TEST_THAT(!response.IsKeepAlive()); + + response = client.GetObject("/nonexist"); + TEST_EQUAL(404, response.GetResponseCode()); + TEST_THAT(!response.IsKeepAlive()); + + FileStream fs("testfiles/dsfdsfs98.fd"); + std::string digest; + { - // Run a server - TestWebServer server; - return server.Main("doesnotexist", argc - 1, argv + 1); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + fs.Seek(0, IOStream::SeekType_Absolute); + digester.Close(); + digest = digester.DigestAsString(); + TEST_EQUAL("dc3b8c5e57e71d31a0a9d7cbeee2e011", digest); } - if(argc >= 2 && ::strcmp(argv[1], "s3server") == 0) + // The destination file should not exist before we upload it: + TEST_THAT(!FileExists("testfiles/store/newfile")); + response = client.PutObject("/newfile", fs); + TEST_EQUAL(200, response.GetResponseCode()); + TEST_THAT(!response.IsKeepAlive()); + TEST_EQUAL("\"" + digest + "\"", response.GetHeaders().GetHeaderValue("etag")); + + // This will fail if the file was created in the wrong place: + TEST_THAT(FileExists("testfiles/store/newfile")); + + response = client.GetObject("/newfile"); + TEST_EQUAL(200, response.GetResponseCode()); + TEST_EQUAL(4269, response.GetSize()); + + fs.Seek(0, IOStream::SeekType_Absolute); + TEST_THAT(fs.CompareWith(response)); + TEST_EQUAL("\"" + digest + "\"", response.GetHeaders().GetHeaderValue("etag")); + + // Test that GET requests set the Content-Length header correctly. + int actual_size = TestGetFileSize("testfiles/dsfdsfs98.fd"); + TEST_THAT(actual_size > 0); + TEST_EQUAL(actual_size, response.GetContentLength()); + + // Try to get it again, with the etag of the existing copy, and check that we get + // a 304 Not Modified response. + response = client.GetObject("/newfile", digest); + TEST_EQUAL(HTTPResponse::Code_NotModified, response.GetResponseCode()); + + // There are no examples for 304 Not Modified responses to requests with + // If-None-Match (ETag match) so clients should not depend on this, so the + // S3Simulator should return 0 instead of the object size and no ETag, to ensure + // that any code which tries to use the Content-Length or ETag of such a response + // will fail. + TEST_EQUAL(0, response.GetContentLength()); + TEST_EQUAL("", response.GetHeaders().GetHeaderValue("etag", false)); // !required + + // Test that HEAD requests set the Content-Length header correctly. We need the + // actual object size, not 0, despite there being no content in the response. + // RFC 2616 section 4.4 says "1.Any response message which "MUST NOT" include a + // message-body (such as ... any response to a HEAD request) is always terminated + // by the first empty line after the header fields, regardless of the + // entity-header fields present in the message... If a Content-Length header field + // (section 14.13) is present, its decimal value in OCTETs represents both the + // entity-length and the transfer-length." + // + // Also the Amazon Simple Storage Service API Reference, section "HEAD Object" + // examples show the Content-Length being returned as non-zero for a HEAD request, + // and ETag being returned too. + + response = client.HeadObject("/newfile"); + TEST_EQUAL(actual_size, response.GetContentLength()); + TEST_EQUAL(200, response.GetResponseCode()); + // We really need the ETag header in response to HEAD requests! + TEST_EQUAL("\"" + digest + "\"", + response.GetHeaders().GetHeaderValue("etag", false)); // !required + // Check that there is NO body. The request should not have been treated as a + // GET request! + ZeroStream empty(0); + TEST_THAT(fs.CompareWith(response)); + + // Replace the file contents with a smaller file, check that it works and that + // the file is truncated at the end of the new data. + CollectInBufferStream test_data; + test_data.Write(std::string("hello")); + test_data.SetForReading(); + response = client.PutObject("/newfile", test_data); + TEST_EQUAL(200, response.GetResponseCode()); + TEST_EQUAL("\"5d41402abc4b2a76b9719d911017c592\"", + response.GetHeaders().GetHeaderValue("etag", false)); // !required + TEST_EQUAL(5, TestGetFileSize("testfiles/store/newfile")); + + // This will fail if the file was created in the wrong place: + TEST_THAT(FileExists("testfiles/store/newfile")); + response = client.DeleteObject("/newfile"); + TEST_EQUAL(HTTPResponse::Code_NoContent, response.GetResponseCode()); + TEST_THAT(!FileExists("testfiles/store/newfile")); + + // Try uploading a file in a subdirectory, which should create it implicitly + // and automatically. + TEST_EQUAL(0, ObjectExists("testfiles/store/sub")); + TEST_THAT(!FileExists("testfiles/store/sub/newfile")); + response = client.PutObject("/sub/newfile", fs); + TEST_EQUAL(200, response.GetResponseCode()); + response = client.GetObject("/sub/newfile"); + TEST_EQUAL(200, response.GetResponseCode()); + TEST_THAT(fs.CompareWith(response)); + response = client.DeleteObject("/sub/newfile"); + TEST_EQUAL(HTTPResponse::Code_NoContent, response.GetResponseCode()); + TEST_THAT(!FileExists("testfiles/store/sub/newfile")); + + // There is no way to explicitly delete a directory either, so we must do that + // ourselves + TEST_THAT(rmdir("testfiles/store/sub") == 0); + + // Test the ListBucket command. + std::vector actual_contents; + std::vector actual_common_prefixes; + TEST_EQUAL(3, client.ListBucket(&actual_contents, &actual_common_prefixes)); + std::vector actual_entry_names = + get_entry_names(actual_contents); + + std::vector expected_contents; + expected_contents.push_back("dsfdsfs98.fd"); + TEST_THAT(test_equal_lists(expected_contents, actual_entry_names)); + + std::vector expected_common_prefixes; + expected_common_prefixes.push_back("photos/"); + expected_common_prefixes.push_back("subdir/"); + TEST_THAT(test_equal_lists(expected_common_prefixes, actual_common_prefixes)); + + // Test that max_keys works. + actual_contents.clear(); + actual_common_prefixes.clear(); + + bool is_truncated; + TEST_EQUAL(2, + client.ListBucket( + &actual_contents, &actual_common_prefixes, + "", // prefix + "/", // delimiter + &is_truncated, + 2)); // max_keys + + TEST_THAT(is_truncated); + expected_contents.clear(); + expected_contents.push_back("dsfdsfs98.fd"); + actual_entry_names = get_entry_names(actual_contents); + TEST_THAT(test_equal_lists(expected_contents, actual_entry_names)); + + expected_common_prefixes.clear(); + expected_common_prefixes.push_back("photos/"); + TEST_THAT(test_equal_lists(expected_common_prefixes, actual_common_prefixes)); + + // Test that marker works. + actual_contents.clear(); + actual_common_prefixes.clear(); + + TEST_EQUAL(2, + client.ListBucket( + &actual_contents, &actual_common_prefixes, + "", // prefix + "/", // delimiter + &is_truncated, + 2, // max_keys + "photos")); // marker + + TEST_THAT(!is_truncated); + expected_contents.clear(); + actual_entry_names = get_entry_names(actual_contents); + TEST_THAT(test_equal_lists(expected_contents, actual_entry_names)); + + expected_common_prefixes.push_back("subdir/"); + TEST_THAT(test_equal_lists(expected_common_prefixes, actual_common_prefixes)); + + // Test is successful if the number of failures has not increased. + return (num_failures == num_failures_initial); +} + + +std::string generate_query_string(const HTTPRequest& request) +{ + std::vector param_names; + std::map param_values; + + const HTTPRequest::Query_t& params(request.GetQuery()); + for(HTTPRequest::Query_t::const_iterator i = params.begin(); + i != params.end(); i++) { - // Run a server - S3Simulator server; - return server.Main("doesnotexist", argc - 1, argv + 1); + // We don't want to include the Signature parameter in the sorted query + // string, because the client didn't either when computing the signature! + if(i->first != "Signature") + { + param_names.push_back(i->first); + // This algorithm only supports non-repeated parameters, so + // assert that we don't already have a parameter with this name. + TEST_LINE_OR(param_values.find(i->first) == param_values.end(), + "Multiple values for parameter '" << i->first << "'", + return ""); + param_values[i->first] = i->second; + } + } + + std::sort(param_names.begin(), param_names.end()); + std::ostringstream out; + + for(std::vector::iterator i = param_names.begin(); + i != param_names.end(); i++) + { + if(i != param_names.begin()) + { + out << "&"; + } + out << HTTPQueryDecoder::URLEncode(*i) << "=" << + HTTPQueryDecoder::URLEncode(param_values[*i]); + } + + return out.str(); +} + + +std::string calculate_s3_signature(const HTTPRequest& request, + const std::string& aws_secret_access_key) +{ + // This code is very similar to that in S3Client::FinishAndSendRequest. + // TODO FIXME: factor out the common parts. + + std::ostringstream buffer_to_sign; + buffer_to_sign << request.GetMethodName() << "\n" << + request.GetHeaders().GetHeaderValue("Content-MD5", + false) << "\n" << // !required + request.GetContentType() << "\n" << + request.GetHeaders().GetHeaderValue("Date", + true) << "\n"; // required + + // TODO FIXME: add support for X-Amz headers (S3 DG page 38) + + std::string bucket; + std::string host_header = request.GetHeaders().GetHeaderValue("Host", + true); // required + std::string s3suffix = ".s3.amazonaws.com"; + if(host_header.size() > s3suffix.size()) + { + std::string suffix = host_header.substr(host_header.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = "/" + host_header.substr(0, host_header.size() - + s3suffix.size()); + } + } + + buffer_to_sign << bucket << request.GetRequestURI(); + + // TODO FIXME: add support for sub-resources. S3 DG page 36. + + // Thanks to https://gist.github.com/tsupo/112188: + unsigned int digest_size; + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + std::string string_to_sign = buffer_to_sign.str(); + + HMAC(EVP_sha1(), + aws_secret_access_key.c_str(), aws_secret_access_key.size(), + (const unsigned char *)string_to_sign.c_str(), string_to_sign.size(), + digest_buffer, &digest_size); + + base64::encoder encoder; + std::string digest((const char *)digest_buffer, digest_size); + std::string auth_code = encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + return auth_code; +} + + +// http://docs.aws.amazon.com/AmazonSimpleDB/latest/DeveloperGuide/HMACAuth.html +std::string calculate_simpledb_signature(const HTTPRequest& request, + const std::string& aws_secret_access_key) +{ + // This code is very similar to that in S3Client::FinishAndSendRequest, + // but using EVP_sha256 instead of EVP_sha1. TODO FIXME: factor out the + // common parts. + std::string query_string = generate_query_string(request); + TEST_THAT_OR(query_string != "", return ""); + + std::ostringstream buffer_to_sign; + buffer_to_sign << request.GetMethodName() << "\n" << + request.GetHeaders().GetHostNameWithPort() << "\n" << + // The HTTPRequestURI component is the HTTP absolute path component + // of the URI up to, but not including, the query string. If the + // HTTPRequestURI is empty, use a forward slash ( / ). + request.GetRequestURI() << "\n" << + query_string; + + // Thanks to https://gist.github.com/tsupo/112188: + unsigned int digest_size; + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + std::string string_to_sign = buffer_to_sign.str(); + + HMAC(EVP_sha256(), + aws_secret_access_key.c_str(), aws_secret_access_key.size(), + (const unsigned char *)string_to_sign.c_str(), string_to_sign.size(), + digest_buffer, &digest_size); + + base64::encoder encoder; + std::string digest((const char *)digest_buffer, digest_size); + std::string auth_code = encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + return auth_code; +} + +bool add_simpledb_signature(HTTPRequest& request, const std::string& aws_secret_access_key) +{ + std::string signature = calculate_simpledb_signature(request, + aws_secret_access_key); + request.SetParameter("Signature", signature); + return !signature.empty(); +} + +bool send_and_receive(HTTPRequest& request, HTTPResponse& response, + int expected_status_code = 200) +{ + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + request.Send(sock, LONG_TIMEOUT); + + response.Reset(); + response.Receive(sock, LONG_TIMEOUT); + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL_LINE(expected_status_code, response.GetResponseCode(), + response_data); + return (response.GetResponseCode() == expected_status_code); +} + +bool parse_xml_response(HTTPResponse& response, ptree& response_tree, + const std::string& expected_root_element) +{ + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + std::auto_ptr ap_response_stream( + new std::istringstream(response_data)); + read_xml(*ap_response_stream, response_tree, + boost::property_tree::xml_parser::trim_whitespace); + + TEST_EQUAL_OR(expected_root_element, response_tree.begin()->first, return false); + TEST_LINE(++(response_tree.begin()) == response_tree.end(), + "There should only be one item in the response tree root"); + + return true; +} + +bool send_and_receive_xml(HTTPRequest& request, ptree& response_tree, + const std::string& expected_root_element) +{ + HTTPResponse response; + TEST_THAT_OR(send_and_receive(request, response), return false); + return parse_xml_response(response, response_tree, expected_root_element); +} + +typedef std::multimap multimap_t; +typedef multimap_t::value_type attr_t; + +std::vector simpledb_list_domains(const std::string& access_key, + const std::string& secret_key) +{ + HTTPRequest request(HTTPRequest::Method_GET, "/"); + request.SetHostName(SIMPLEDB_SIMULATOR_HOST); + request.AddParameter("Action", "ListDomains"); + request.AddParameter("AWSAccessKeyId", access_key); + request.AddParameter("SignatureVersion", "2"); + request.AddParameter("SignatureMethod", "HmacSHA256"); + request.AddParameter("Timestamp", "2010-01-25T15:01:28-07:00"); + request.AddParameter("Version", "2009-04-15"); + + TEST_THAT_OR(add_simpledb_signature(request, secret_key), + return std::vector()); + + ptree response_tree; + TEST_THAT(send_and_receive_xml(request, response_tree, "ListDomainsResponse")); + + std::vector domains; + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child("ListDomainsResponse.ListDomainsResult")) + { + domains.push_back(v.second.data()); + } + + return domains; +} + +HTTPRequest simpledb_get_attributes_request(const std::string& access_key, + const std::string& secret_key) +{ + HTTPRequest request(HTTPRequest::Method_GET, "/"); + request.SetHostName(SIMPLEDB_SIMULATOR_HOST); + request.AddParameter("Action", "GetAttributes"); + request.AddParameter("DomainName", "MyDomain"); + request.AddParameter("ItemName", "JumboFez"); + request.AddParameter("AWSAccessKeyId", access_key); + request.AddParameter("SignatureVersion", "2"); + request.AddParameter("SignatureMethod", "HmacSHA256"); + request.AddParameter("Timestamp", "2010-01-25T15:01:28-07:00"); + request.AddParameter("Version", "2009-04-15"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + return request; +} + +bool simpledb_get_attributes_error(const std::string& access_key, + const std::string& secret_key, int expected_status_code) +{ + HTTPRequest request = simpledb_get_attributes_request(access_key, secret_key); + HTTPResponse response; + TEST_THAT_OR(send_and_receive(request, response, expected_status_code), + return false); + // Nothing else to check: there is no XML + return true; +} + +bool simpledb_get_attributes(const std::string& access_key, const std::string& secret_key, + const multimap_t& const_attributes) +{ + HTTPRequest request = simpledb_get_attributes_request(access_key, secret_key); + + ptree response_tree; + TEST_THAT_OR(send_and_receive_xml(request, response_tree, + "GetAttributesResponse"), return false); + + // Check that all attributes were written correctly + TEST_EQUAL_LINE(const_attributes.size(), + response_tree.get_child("GetAttributesResponse.GetAttributesResult").size(), + "Wrong number of attributes in response"); + + bool all_match = (const_attributes.size() == + response_tree.get_child("GetAttributesResponse.GetAttributesResult").size()); + + multimap_t attributes = const_attributes; + multimap_t::iterator i = attributes.begin(); + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child( + "GetAttributesResponse.GetAttributesResult")) + { + std::string name = v.second.get("Name"); + std::string value = v.second.get("Value"); + if(i == attributes.end()) + { + TEST_EQUAL_LINE("", name, "Unexpected attribute name"); + TEST_EQUAL_LINE("", value, "Unexpected attribute value"); + all_match = false; + } + else + { + TEST_EQUAL_LINE(i->first, name, "Wrong attribute name"); + TEST_EQUAL_LINE(i->second, value, "Wrong attribute value"); + all_match &= (i->first == name); + all_match &= (i->second == value); + i++; + } + } + + return all_match; +} + +#define EXAMPLE_S3_ACCESS_KEY "0PN5J17HBGZHT7JJ3X82" +#define EXAMPLE_S3_SECRET_KEY "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" + +bool test_httpserver() +{ + SETUP(); + + { + FileStream fs("testfiles/dsfdsfs98.fd"); + MD5DigestStream digester; + fs.CopyStreamTo(digester); + fs.Seek(0, IOStream::SeekType_Absolute); + digester.Close(); + std::string digest = digester.DigestAsString(); + TEST_EQUAL("dc3b8c5e57e71d31a0a9d7cbeee2e011", digest); + } + + // Test that HTTPRequest with parameters is encoded correctly + { + HTTPRequest request(HTTPRequest::Method_GET, "/newfile"); + CollectInBufferStream request_buffer; + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite); + request_buffer.SetForReading(); + + std::string request_str((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + const std::string expected_str("GET /newfile HTTP/1.1\r\nConnection: close\r\n\r\n"); + TEST_EQUAL(expected_str, request_str); + + request.AddParameter("foo", "Bar"); + request_buffer.Reset(); + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite); + request_str = std::string((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + TEST_EQUAL("GET /newfile?foo=Bar HTTP/1.1\r\nConnection: close\r\n\r\n", request_str); + + request.AddParameter("foo", "baz"); + request_buffer.Reset(); + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite); + request_str = std::string((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + TEST_EQUAL("GET /newfile?foo=Bar&foo=baz HTTP/1.1\r\nConnection: close\r\n\r\n", request_str); + + request.SetParameter("whee", "bonk"); + request_buffer.Reset(); + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite); + request_str = std::string((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + TEST_EQUAL("GET /newfile?foo=Bar&foo=baz&whee=bonk HTTP/1.1\r\nConnection: close\r\n\r\n", request_str); + + request.SetParameter("foo", "bolt"); + request_buffer.Reset(); + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite); + request_str = std::string((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + TEST_EQUAL("GET /newfile?foo=bolt&whee=bonk HTTP/1.1\r\nConnection: close\r\n\r\n", request_str); + + HTTPRequest newreq = request; + TEST_EQUAL("bolt", newreq.GetParameterString("foo")); + TEST_EQUAL("bonk", newreq.GetParameterString("whee")); + TEST_EQUAL("blue", newreq.GetParameterString("colour", "blue")); + TEST_CHECK_THROWS(newreq.GetParameterString("colour"), HTTPException, + ParameterNotFound); + } + + // Test that HTTPRequest can be written to and read from a stream. + for(int enable_continue = 0; enable_continue < 2; enable_continue++) + { + HTTPRequest request(HTTPRequest::Method_PUT, "/newfile"); + request.SetHostName("quotes.s3.amazonaws.com"); + // Write headers in lower case. + request.AddHeader("date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("authorization", + "AWS " EXAMPLE_S3_ACCESS_KEY ":XtMYZf0hdOo4TdPYQknZk0Lz7rw="); + request.AddHeader("Content-Type", "text/plain"); + request.SetClientKeepAliveRequested(true); + + // First stream just the headers into a CollectInBufferStream, and check the + // exact contents written: + CollectInBufferStream request_buffer; + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite, + (bool)enable_continue); + request_buffer.SetForReading(); + std::string request_str((const char *)request_buffer.GetBuffer(), + request_buffer.GetSize()); + std::string expected_str( + "PUT /newfile HTTP/1.1\r\n" + "Content-Type: text/plain\r\n" + "Host: quotes.s3.amazonaws.com\r\n" + "Connection: keep-alive\r\n" + "date: Wed, 01 Mar 2006 12:00:00 GMT\r\n" + "authorization: AWS " EXAMPLE_S3_ACCESS_KEY ":XtMYZf0hdOo4TdPYQknZk0Lz7rw=\r\n"); + if(enable_continue == 1) + { + expected_str += "Expect: 100-continue\r\n"; + } + TEST_EQUAL(expected_str + "\r\n", request_str); + + // Now stream the entire request into the CollectInBufferStream. Because there + // isn't an HTTP server to respond to us, we can't use SendWithStream, so just + // send the headers and then the content separately: + request_buffer.Reset(); + request.SendHeaders(request_buffer, IOStream::TimeOutInfinite, + (bool)enable_continue); + FileStream fs("testfiles/photos/puppy.jpg"); + fs.CopyStreamTo(request_buffer); + request_buffer.SetForReading(); + + IOStreamGetLine getLine(request_buffer); + HTTPRequest request2; + TEST_THAT(request2.Receive(getLine, IOStream::TimeOutInfinite)); + + TEST_EQUAL(HTTPRequest::Method_PUT, request2.GetMethod()); + TEST_EQUAL("PUT", request2.GetMethodName()); + TEST_EQUAL("/newfile", request2.GetRequestURI()); + TEST_EQUAL("quotes.s3.amazonaws.com", request2.GetHostName()); + TEST_EQUAL(80, request2.GetHostPort()); + TEST_EQUAL("", request2.GetQueryString()); + TEST_EQUAL("text/plain", request2.GetContentType()); + // Content-Length was not known when the stream was sent, so it should + // be unknown in the received stream too (certainly before it has all + // been read!) + TEST_EQUAL(-1, request2.GetContentLength()); + const HTTPHeaders& headers(request2.GetHeaders()); + TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", + headers.GetHeaderValue("Date")); + TEST_EQUAL("AWS " EXAMPLE_S3_ACCESS_KEY ":XtMYZf0hdOo4TdPYQknZk0Lz7rw=", + headers.GetHeaderValue("Authorization")); + TEST_THAT(request2.GetClientKeepAliveRequested()); + TEST_EQUAL((bool)enable_continue, request2.IsExpectingContinue()); + + CollectInBufferStream request_data; + request2.ReadContent(request_data, IOStream::TimeOutInfinite); + TEST_EQUAL(fs.GetPosition(), request_data.GetPosition()); + request_data.SetForReading(); + fs.Seek(0, IOStream::SeekType_Absolute); + TEST_THAT(fs.CompareWith(request_data, IOStream::TimeOutInfinite)); + } + + // Test that HTTPResponse can be written to and read from a stream. + // TODO FIXME: we should stream the response instead of buffering it, on both + // sides (send and receive). + { + // Stream it to a CollectInBufferStream + CollectInBufferStream response_buffer; + + HTTPResponse response(&response_buffer); + FileStream fs("testfiles/photos/puppy.jpg"); + // Write headers in lower case. + response.SetResponseCode(HTTPResponse::Code_OK); + response.AddHeader("date", "Wed, 01 Mar 2006 12:00:00 GMT"); + response.AddHeader("authorization", + "AWS " EXAMPLE_S3_ACCESS_KEY ":XtMYZf0hdOo4TdPYQknZk0Lz7rw="); + response.AddHeader("content-type", "text/perl"); + fs.CopyStreamTo(response); + response.Send(); + response_buffer.SetForReading(); + + HTTPResponse response2; + response2.Receive(response_buffer); + + TEST_EQUAL(200, response2.GetResponseCode()); + TEST_EQUAL("text/perl", response2.GetContentType()); + + // TODO FIXME: Content-Length was not known when the stream was sent, + // so it should be unknown in the received stream too (certainly before + // it has all been read!) This is currently wrong because we read the + // entire response into memory immediately. + TEST_EQUAL(fs.GetPosition(), response2.GetContentLength()); + + HTTPHeaders& headers(response2.GetHeaders()); + TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", + headers.GetHeaderValue("Date")); + TEST_EQUAL("AWS " EXAMPLE_S3_ACCESS_KEY ":XtMYZf0hdOo4TdPYQknZk0Lz7rw=", + headers.GetHeaderValue("Authorization")); + + CollectInBufferStream response_data; + // request2.ReadContent(request_data, IOStream::TimeOutInfinite); + response2.CopyStreamTo(response_data); + TEST_EQUAL(fs.GetPosition(), response_data.GetPosition()); + response_data.SetForReading(); + fs.Seek(0, IOStream::SeekType_Absolute); + TEST_THAT(fs.CompareWith(response_data, IOStream::TimeOutInfinite)); } #ifndef WIN32 @@ -156,11 +834,11 @@ int test(int argc, const char *argv[]) // Run the request script TEST_THAT(::system("perl testfiles/testrequests.pl") == 0); -#ifdef ENABLE_KEEPALIVE_SUPPORT // incomplete, need chunked encoding support #ifndef WIN32 signal(SIGPIPE, SIG_IGN); #endif +#ifdef ENABLE_KEEPALIVE_SUPPORT // incomplete, need chunked encoding support SocketStream sock; sock.Open(Socket::TypeINET, "localhost", 1080); @@ -258,14 +936,26 @@ int test(int argc, const char *argv[]) TEST_THAT(StopDaemon(pid, "testfiles/httpserver.pid", "generic-httpserver.memleaks", true)); - // correct, official signature should succeed, with lower-case header + // Copy testfiles/puppy.jpg to testfiles/store/photos/puppy.jpg + { + TEST_THAT(::mkdir("testfiles/store/photos", 0755) == 0); + FileStream in("testfiles/puppy.jpg", O_RDONLY); + FileStream out("testfiles/store/photos/puppy.jpg", O_CREAT | O_WRONLY); + in.CopyStreamTo(out); + } + + // This is the example from the Amazon S3 Developers Guide, page 31. + // Correct, official signature should succeed, with lower-case headers. { // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); request.SetHostName("johnsmith.s3.amazonaws.com"); request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); + std::string signature = calculate_s3_signature(request, + EXAMPLE_S3_SECRET_KEY); + TEST_EQUAL(signature, "xXjDGYUmKxnwqr5KXNPGldn5LbA="); request.AddHeader("authorization", - "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="); + "AWS " EXAMPLE_S3_ACCESS_KEY ":" + signature); S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); @@ -281,14 +971,14 @@ int test(int argc, const char *argv[]) TEST_EQUAL("omgpuppies!\n", response_data); } - // modified signature should fail + // Modified signature should fail. { // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); request.SetHostName("johnsmith.s3.amazonaws.com"); request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); request.AddHeader("authorization", - "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbB="); + "AWS " EXAMPLE_S3_ACCESS_KEY ":xXjDGYUmKxnwqr5KXNPGldn5LbB="); S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); @@ -304,56 +994,111 @@ int test(int argc, const char *argv[]) TEST_EQUAL("" "Internal Server Error\n" "

Internal Server Error

\n" - "

An error, type Authentication Failed occured " - "when processing the request.

" + "

An error occurred while processing the request:

\n" + "
HTTPException(AuthenticationFailed): "
+			"Authentication code mismatch: expected AWS 0PN5J17HBGZHT7JJ3X82"
+			":xXjDGYUmKxnwqr5KXNPGldn5LbA= but received AWS "
+			"0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbB=
\n" "

Please try again later.

\n" "\n", response_data); } - // S3Client tests with S3Simulator in-process server for debugging + // Copy testfiles/dsfdsfs98.fd to testfiles/store/dsfdsfs98.fd + { + FileStream in("testfiles/dsfdsfs98.fd", O_RDONLY); + FileStream out("testfiles/store/dsfdsfs98.fd", O_CREAT | O_WRONLY); + in.CopyStreamTo(out); + } + + // Tests for the S3Simulator ListBucket implementation { S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); - S3Client client(&simulator, "johnsmith.s3.amazonaws.com", - "0PN5J17HBGZHT7JJ3X82", - "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"); - - HTTPResponse response = client.GetObject("/photos/puppy.jpg"); - TEST_EQUAL(200, response.GetResponseCode()); - std::string response_data((const char *)response.GetBuffer(), - response.GetSize()); - TEST_EQUAL("omgpuppies!\n", response_data); - // make sure that assigning to HTTPResponse does clear stream - response = client.GetObject("/photos/puppy.jpg"); - TEST_EQUAL(200, response.GetResponseCode()); - response_data = std::string((const char *)response.GetBuffer(), - response.GetSize()); - TEST_EQUAL("omgpuppies!\n", response_data); + // List contents of bucket + HTTPRequest request(HTTPRequest::Method_GET, "/"); + request.SetParameter("delimiter", "/"); + request.SetHostName("johnsmith.s3.amazonaws.com"); + request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); + std::string signature = calculate_s3_signature(request, + EXAMPLE_S3_SECRET_KEY); + request.AddHeader("authorization", + "AWS " EXAMPLE_S3_ACCESS_KEY ":" + signature); - response = client.GetObject("/nonexist"); - TEST_EQUAL(404, response.GetResponseCode()); - - FileStream fs("testfiles/testrequests.pl"); - response = client.PutObject("/newfile", fs); - TEST_EQUAL(200, response.GetResponseCode()); + HTTPResponse response; + simulator.Handle(request, response); + TEST_EQUAL(HTTPResponse::Code_OK, response.GetResponseCode()); + std::vector expected_contents; - response = client.GetObject("/newfile"); - TEST_EQUAL(200, response.GetResponseCode()); - TEST_THAT(fs.CompareWith(response)); - TEST_EQUAL(0, ::unlink("testfiles/newfile")); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + ptree response_tree; + TEST_THAT(parse_xml_response(response, response_tree, + "ListBucketResult")); + // A response containing a single item should not be truncated! + TEST_EQUAL("false", + response_tree.get( + "ListBucketResult.IsTruncated")); + + // Iterate over all the children of the ListBucketResult, looking for + // nodes called "Contents", and examine them. + std::vector contents; + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child("ListBucketResult")) + { + if(v.first == "Contents") + { + std::string name = v.second.get("Key"); + contents.push_back(name); + if(name == "dsfdsfs98.fd") + { + TEST_EQUAL(""dc3b8c5e57e71d31a0a9d7cbeee2e011"", + v.second.get("ETag")); + TEST_EQUAL("4269", v.second.get("Size")); + } + } + } + + expected_contents.push_back("dsfdsfs98.fd"); + TEST_THAT(test_equal_lists(expected_contents, contents)); + + int num_common_prefixes = 0; + BOOST_FOREACH(ptree::value_type &v, + response_tree.get_child("ListBucketResult.CommonPrefixes")) + { + num_common_prefixes++; + TEST_EQUAL("Prefix", v.first); + std::string expected_name; + if(num_common_prefixes == 1) + { + expected_name = "photos/"; + } + else + { + expected_name = "subdir/"; + } + TEST_EQUAL_LINE(expected_name, v.second.data(), + "line " << num_common_prefixes); + } + TEST_EQUAL(2, num_common_prefixes); + } } + // Test the S3Simulator's implementation of PUT file uploads { HTTPRequest request(HTTPRequest::Method_PUT, "/newfile"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("authorization", - "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw="); - request.AddHeader("Content-Type", "text/plain"); + // request.AddHeader("Content-Type", "text/plain"); - FileStream fs("testfiles/testrequests.pl"); - fs.CopyStreamTo(request); + std::string signature = calculate_s3_signature(request, + EXAMPLE_S3_SECRET_KEY); + TEST_EQUAL(signature, "XtMYZf0hdOo4TdPYQknZk0Lz7rw="); + request.AddHeader("authorization", "AWS " EXAMPLE_S3_ACCESS_KEY ":" + + signature); + + FileStream fs("testfiles/dsfdsfs98.fd"); + request.SetDataStream(&fs); request.SetForReading(); CollectInBufferStream response_buffer; @@ -369,15 +1114,24 @@ int test(int argc, const char *argv[]) TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); - TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("\"dc3b8c5e57e71d31a0a9d7cbeee2e011\"", response.GetHeaderValue("ETag")); TEST_EQUAL("", response.GetContentType()); TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); TEST_EQUAL(0, response.GetSize()); - FileStream f1("testfiles/testrequests.pl"); - FileStream f2("testfiles/newfile"); + FileStream f1("testfiles/dsfdsfs98.fd"); + FileStream f2("testfiles/store/newfile"); TEST_THAT(f1.CompareWith(f2)); - TEST_EQUAL(0, ::unlink("testfiles/newfile")); + TEST_EQUAL(0, EMU_UNLINK("testfiles/store/newfile")); + } + + // S3Client tests with S3Simulator in-process server for debugging + { + S3Simulator simulator; + simulator.Configure("testfiles/s3simulator.conf"); + S3Client client(&simulator, "johnsmith.s3.amazonaws.com", + EXAMPLE_S3_ACCESS_KEY, EXAMPLE_S3_SECRET_KEY); + TEST_THAT(exercise_s3client(client)); } // Start the S3Simulator server @@ -385,72 +1139,76 @@ int test(int argc, const char *argv[]) "testfiles/s3simulator.pid"); TEST_THAT_OR(pid > 0, return 1); + // This is the example from the Amazon S3 Developers Guide, page 31 { - SocketStream sock; - sock.Open(Socket::TypeINET, "localhost", 1080); + HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); + request.SetHostName("johnsmith.s3.amazonaws.com"); + request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); + request.AddHeader("authorization", + "AWS " EXAMPLE_S3_ACCESS_KEY ":xXjDGYUmKxnwqr5KXNPGldn5LbA="); + HTTPResponse response; + TEST_THAT(send_and_receive(request, response)); + } + + // Test that requests for nonexistent files correctly return a 404 error + { HTTPRequest request(HTTPRequest::Method_GET, "/nonexist"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:0cSX/YPdtXua1aFFpYmH1tc0ajA="); request.SetClientKeepAliveRequested(true); - request.Send(sock, SHORT_TIMEOUT); + + std::string signature = calculate_s3_signature(request, + EXAMPLE_S3_SECRET_KEY); + TEST_EQUAL(signature, "0cSX/YPdtXua1aFFpYmH1tc0ajA="); + request.AddHeader("authorization", "AWS " EXAMPLE_S3_ACCESS_KEY ":" + + signature); HTTPResponse response; - response.Receive(sock, SHORT_TIMEOUT); - std::string value; - TEST_EQUAL(404, response.GetResponseCode()); + TEST_THAT(send_and_receive(request, response, 404)); + TEST_THAT(!response.IsKeepAlive()); } - #ifndef WIN32 // much harder to make files inaccessible on WIN32 +#ifndef WIN32 // much harder to make files inaccessible on WIN32 // Make file inaccessible, should cause server to return a 403 error, // unless of course the test is run as root :) { - SocketStream sock; - sock.Open(Socket::TypeINET, "localhost", 1080); - - TEST_THAT(chmod("testfiles/testrequests.pl", 0) == 0); - HTTPRequest request(HTTPRequest::Method_GET, - "/testrequests.pl"); + TEST_THAT(chmod("testfiles/store/dsfdsfs98.fd", 0) == 0); + HTTPRequest request(HTTPRequest::Method_GET, "/dsfdsfs98.fd"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); + request.AddHeader("Authorization", "AWS " EXAMPLE_S3_ACCESS_KEY + ":NO9tjQuMCK83z2VZFaJOGKeDi7M="); request.SetClientKeepAliveRequested(true); - request.Send(sock, SHORT_TIMEOUT); HTTPResponse response; - response.Receive(sock, SHORT_TIMEOUT); - std::string value; - TEST_EQUAL(403, response.GetResponseCode()); - TEST_THAT(chmod("testfiles/testrequests.pl", 0755) == 0); + TEST_THAT(send_and_receive(request, response, 403)); + TEST_THAT(chmod("testfiles/store/dsfdsfs98.fd", 0755) == 0); } - #endif +#endif { - SocketStream sock; - sock.Open(Socket::TypeINET, "localhost", 1080); - - HTTPRequest request(HTTPRequest::Method_GET, - "/testrequests.pl"); + HTTPRequest request(HTTPRequest::Method_GET, "/dsfdsfs98.fd"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); + request.AddHeader("Authorization", "AWS " EXAMPLE_S3_ACCESS_KEY + ":NO9tjQuMCK83z2VZFaJOGKeDi7M="); request.SetClientKeepAliveRequested(true); - request.Send(sock, SHORT_TIMEOUT); HTTPResponse response; - response.Receive(sock, SHORT_TIMEOUT); - std::string value; - TEST_EQUAL(200, response.GetResponseCode()); - TEST_EQUAL("qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY", response.GetHeaderValue("x-amz-id-2")); + TEST_THAT(send_and_receive(request, response)); + + TEST_EQUAL("qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY", + response.GetHeaderValue("x-amz-id-2")); TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); - TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("\"dc3b8c5e57e71d31a0a9d7cbeee2e011\"", response.GetHeaderValue("ETag")); TEST_EQUAL("text/plain", response.GetContentType()); TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); + TEST_THAT(!response.IsKeepAlive()); - FileStream file("testfiles/testrequests.pl"); + FileStream file("testfiles/dsfdsfs98.fd"); TEST_THAT(file.CompareWith(response)); } @@ -458,36 +1216,441 @@ int test(int argc, const char *argv[]) SocketStream sock; sock.Open(Socket::TypeINET, "localhost", 1080); - HTTPRequest request(HTTPRequest::Method_PUT, - "/newfile"); + HTTPRequest request(HTTPRequest::Method_PUT, "/newfile"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:kfY1m6V3zTufRy2kj92FpQGKz4M="); + request.AddHeader("Authorization", "AWS " EXAMPLE_S3_ACCESS_KEY + ":kfY1m6V3zTufRy2kj92FpQGKz4M="); request.AddHeader("Content-Type", "text/plain"); - FileStream fs("testfiles/testrequests.pl"); + FileStream fs("testfiles/dsfdsfs98.fd"); HTTPResponse response; - request.SendWithStream(sock, SHORT_TIMEOUT, &fs, response); + request.SendWithStream(sock, LONG_TIMEOUT, &fs, response); std::string value; TEST_EQUAL(200, response.GetResponseCode()); TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", response.GetHeaderValue("x-amz-id-2")); TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); - TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("\"dc3b8c5e57e71d31a0a9d7cbeee2e011\"", response.GetHeaderValue("ETag")); TEST_EQUAL("", response.GetContentType()); TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); TEST_EQUAL(0, response.GetSize()); + TEST_THAT(!response.IsKeepAlive()); - FileStream f1("testfiles/testrequests.pl"); - FileStream f2("testfiles/newfile"); + FileStream f1("testfiles/dsfdsfs98.fd"); + FileStream f2("testfiles/store/newfile"); TEST_THAT(f1.CompareWith(f2)); + TEST_THAT(EMU_UNLINK("testfiles/store/newfile") == 0); + } + + // S3Client tests with S3Simulator daemon for realism + { + S3Client client("localhost", 1080, EXAMPLE_S3_ACCESS_KEY, + EXAMPLE_S3_SECRET_KEY, "johnsmith.s3.amazonaws.com"); + TEST_THAT(exercise_s3client(client)); + } + + // Test the HTTPQueryDecoder::URLEncode method. + TEST_EQUAL("AZaz09-_.~", HTTPQueryDecoder::URLEncode("AZaz09-_.~")); + TEST_EQUAL("%00%01%FF", + HTTPQueryDecoder::URLEncode(std::string("\0\x01\xff", 3))); + + // Test that we can calculate the correct signature for a known request: + // http://docs.aws.amazon.com/AWSECommerceService/latest/DG/rest-signature.html + { + HTTPRequest request(HTTPRequest::Method_GET, "/onca/xml"); + request.SetHostName("webservices.amazon.com"); + request.AddParameter("Service", "AWSECommerceService"); + request.AddParameter("AWSAccessKeyId", "AKIAIOSFODNN7EXAMPLE"); + request.AddParameter("AssociateTag", "mytag-20"); + request.AddParameter("Operation", "ItemLookup"); + request.AddParameter("ItemId", "0679722769"); + request.AddParameter("ResponseGroup", + "Images,ItemAttributes,Offers,Reviews"); + request.AddParameter("Version", "2013-08-01"); + request.AddParameter("Timestamp", "2014-08-18T12:00:00Z"); + + std::string auth_code = calculate_simpledb_signature(request, + "1234567890"); + TEST_EQUAL("j7bZM0LXZ9eXeZruTqWm2DIvDYVUU3wxPPpp+iXxzQc=", auth_code); } + // Test the S3Simulator's implementation of SimpleDB + { + std::string access_key = EXAMPLE_S3_ACCESS_KEY; + std::string secret_key = EXAMPLE_S3_SECRET_KEY; + + HTTPRequest request(HTTPRequest::Method_GET, "/"); + request.SetHostName(SIMPLEDB_SIMULATOR_HOST); + + request.AddParameter("Action", "ListDomains"); + request.AddParameter("AWSAccessKeyId", access_key); + request.AddParameter("SignatureVersion", "2"); + request.AddParameter("SignatureMethod", "HmacSHA256"); + request.AddParameter("Timestamp", "2010-01-25T15:01:28-07:00"); + request.AddParameter("Version", "2009-04-15"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + + // Send directly to in-process simulator, useful for debugging. + // CollectInBufferStream response_buffer; + // HTTPResponse response(&response_buffer); + HTTPResponse response; + + S3Simulator simulator; + simulator.Configure("testfiles/s3simulator.conf"); + simulator.Handle(request, response); + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL_LINE(200, response.GetResponseCode(), response_data); + + // Send to out-of-process simulator, useful for testing HTTP + // implementation. + TEST_THAT(send_and_receive(request, response)); + + // Check that there are no existing domains at the start + std::vector domains = simpledb_list_domains(access_key, secret_key); + std::vector expected_domains; + TEST_THAT(test_equal_lists(expected_domains, domains)); + + // Create a domain + request.SetParameter("Action", "CreateDomain"); + request.SetParameter("DomainName", "MyDomain"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + + ptree response_tree; + TEST_THAT(send_and_receive_xml(request, response_tree, + "CreateDomainResponse")); + + // List domains again, check that our new domain is present. + domains = simpledb_list_domains(access_key, secret_key); + expected_domains.push_back("MyDomain"); + TEST_THAT(test_equal_lists(expected_domains, domains)); + + // Create the same domain again. "CreateDomain is an idempotent operation; + // running it multiple times using the same domain name will not result in + // an error response." + TEST_THAT(send_and_receive_xml(request, response_tree, + "CreateDomainResponse")); + + // List domains again, check that our new domain is present only once + // (it wasn't created a second time). Therefore expected_domains is the + // same as it was above. + domains = simpledb_list_domains(access_key, secret_key); + TEST_THAT(test_equal_lists(expected_domains, domains)); + + // Create an item + request.SetParameter("Action", "PutAttributes"); + request.SetParameter("DomainName", "MyDomain"); + request.SetParameter("ItemName", "JumboFez"); + request.SetParameter("Attribute.1.Name", "Color"); + request.SetParameter("Attribute.1.Value", "Blue"); + request.SetParameter("Attribute.2.Name", "Size"); + request.SetParameter("Attribute.2.Value", "Med"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "PutAttributesResponse")); + + // Get the item back, and check that all attributes were written + // correctly. + multimap_t expected_attrs; + expected_attrs.insert(attr_t("Color", "Blue")); + expected_attrs.insert(attr_t("Size", "Med")); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Add more attributes. The Size attribute is added with the Replace + // option, so it replaces the previous value. The Color attribute is not, + // so it adds another value. + request.SetParameter("Attribute.1.Value", "Not Blue"); + request.SetParameter("Attribute.1.Replace", "true"); + request.SetParameter("Attribute.2.Value", "Large"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "PutAttributesResponse")); + + // Check that all attributes were written correctly, by getting the item + // again. + expected_attrs.erase("Color"); + expected_attrs.insert(attr_t("Color", "Not Blue")); + expected_attrs.insert(attr_t("Size", "Large")); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Conditional PutAttributes that fails (doesn't match) and therefore + // doesn't change anything. + request.SetParameter("Attribute.1.Value", "Green"); + request.SetParameter("Attribute.2.Replace", "true"); + request.SetParameter("Expected.1.Name", "Color"); + request.SetParameter("Expected.1.Value", "What?"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive(request, response, HTTPResponse::Code_Conflict)); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Conditional PutAttributes again, with the correct value for the Color + // attribute this time, so the request should succeed. + request.SetParameter("Expected.1.Value", "Not Blue"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "PutAttributesResponse")); + + // If it does, because Replace is set for the Size parameter as well, both + // Size values will be replaced by the new single value. + expected_attrs.clear(); + expected_attrs.insert(attr_t("Color", "Green")); + expected_attrs.insert(attr_t("Size", "Large")); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Test that we can delete values. We are supposed to pass some + // attribute values, but what happens if they don't match the current + // values is not specified. + request.SetParameter("Action", "DeleteAttributes"); + request.RemoveParameter("Expected.1.Name"); + request.RemoveParameter("Expected.1.Value"); + request.RemoveParameter("Attribute.1.Replace"); + request.RemoveParameter("Attribute.2.Replace"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "DeleteAttributesResponse")); + + // Since we've deleted all attributes, the server should have deleted + // the whole item, and the response to a GetAttributes request should be + // 404 not found. + TEST_THAT(simpledb_get_attributes_error(access_key, secret_key, + HTTPResponse::Code_NotFound)); + + // Create an item to use with conditional delete tests. + request.SetParameter("Action", "PutAttributes"); + request.SetParameter("DomainName", "MyDomain"); + request.SetParameter("ItemName", "JumboFez"); + request.SetParameter("Attribute.1.Name", "Color"); + request.SetParameter("Attribute.1.Value", "Blue"); + request.SetParameter("Attribute.2.Name", "Size"); + request.SetParameter("Attribute.2.Value", "Med"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "PutAttributesResponse")); + + // Conditional delete that should fail + request.SetParameter("Action", "DeleteAttributes"); + request.SetParameter("Expected.1.Name", "Color"); + request.SetParameter("Expected.1.Value", "What?"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive(request, response, HTTPResponse::Code_Conflict)); + + // Check that it did actually fail + expected_attrs.clear(); + expected_attrs.insert(attr_t("Color", "Blue")); + expected_attrs.insert(attr_t("Size", "Med")); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Conditional delete of one attribute ("Color") that should succeed + request.SetParameter("Expected.1.Value", "Blue"); + // Remove attribute 1 ("Color") from the request, so it won't be deleted. + // Attribute 2 ("Size") remains in the request, and should be deleted. + request.RemoveParameter("Attribute.1.Name"); + request.RemoveParameter("Attribute.1.Value"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "DeleteAttributesResponse")); + + // Check that the "Size" attribute is no longer present, but "Color" + // still is. + expected_attrs.erase("Size"); + TEST_THAT(simpledb_get_attributes(access_key, secret_key, expected_attrs)); + + // Conditional delete without specifying attributes, should remove all + // remaining attributes, and hence the item itself. The condition + // (expected values) set above should still be valid and match this item. + request.RemoveParameter("Attribute.2.Name"); + request.RemoveParameter("Attribute.2.Value"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "DeleteAttributesResponse")); + + // Since we've deleted all attributes, the server should have deleted + // the whole item, and the response to a GetAttributes request should be + // 404 not found. + TEST_THAT(simpledb_get_attributes_error(access_key, secret_key, + HTTPResponse::Code_NotFound)); + + // Reset for the next test + request.SetParameter("Action", "Reset"); + TEST_THAT(add_simpledb_signature(request, secret_key)); + TEST_THAT(send_and_receive_xml(request, response_tree, + "ResetResponse")); + domains = simpledb_list_domains(access_key, secret_key); + expected_domains.clear(); + TEST_THAT(test_equal_lists(expected_domains, domains)); + } + + // Test that SimpleDBClient works the same way. + { + std::string access_key = EXAMPLE_S3_ACCESS_KEY; + std::string secret_key = EXAMPLE_S3_SECRET_KEY; + SimpleDBClient client(access_key, secret_key, "localhost", 1080, + SIMPLEDB_SIMULATOR_HOST); + + // Test that date formatting produces the correct output format + // date -d "2010-01-25T15:01:28-07:00" +%s => 1264456888 + client.SetFixedTimestamp(SecondsToBoxTime(1264456888), -7 * 60); + HTTPRequest request = client.StartRequest(HTTPRequest::Method_GET, ""); + TEST_EQUAL("2010-01-25T15:01:28-07:00", + request.GetParameterString("Timestamp")); + + client.SetFixedTimestamp(SecondsToBoxTime(1264431688), 0); + request = client.StartRequest(HTTPRequest::Method_GET, ""); + TEST_EQUAL("2010-01-25T15:01:28Z", + request.GetParameterString("Timestamp")); + client.SetFixedTimestamp(0, 0); + TEST_EQUAL(20, request.GetParameterString("Timestamp").length()); + + std::vector domains = client.ListDomains(); + TEST_EQUAL(0, domains.size()); + + std::string domain = "MyDomain"; + std::string item = "JumboFez"; + client.CreateDomain(domain); + domains = client.ListDomains(); + TEST_EQUAL(1, domains.size()); + if(domains.size() > 0) + { + TEST_EQUAL(domain, domains[0]); + } + + // Create an item + SimpleDBClient::str_map_t expected_attrs; + expected_attrs["Color"] = "Blue"; + expected_attrs["Size"] = "Med"; + client.PutAttributes(domain, item, expected_attrs); + SimpleDBClient::str_map_t actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Add more attributes. SimpleDBClient always replaces existing values + // for attributes. + expected_attrs.clear(); + expected_attrs["Color"] = "Not Blue"; + expected_attrs["Size"] = "Large"; + client.PutAttributes(domain, item, expected_attrs); + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Conditional PutAttributes that fails (doesn't match) and therefore + // doesn't change anything (so we don't change expected_attrs). + SimpleDBClient::str_map_t new_attrs = expected_attrs; + new_attrs["Color"] = "Green"; + + SimpleDBClient::str_map_t conditional_attrs; + conditional_attrs["Color"] = "What?"; + + TEST_CHECK_THROWS( + client.PutAttributes(domain, item, new_attrs, conditional_attrs), + HTTPException, ConditionalRequestConflict); + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Conditional PutAttributes again, with the correct value for the Color + // attribute this time, so the request should succeed. + conditional_attrs["Color"] = "Not Blue"; + client.PutAttributes(domain, item, new_attrs, conditional_attrs); + + // If it does, because Replace is set by default (enforced) by + // SimpleDBClient, the Size value will be replaced by the new single + // value. + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(new_attrs, actual_attrs)); + + // Test that we can delete values. We are supposed to pass some + // attribute values, but what happens if they don't match the current + // values is not specified. + client.DeleteAttributes(domain, item, new_attrs); + + // Check that it has actually been removed. Since we've deleted all + // attributes, the server should have deleted the whole item, and the + // response to a GetAttributes request should be 404 not found. + TEST_CHECK_THROWS(client.GetAttributes(domain, item), + HTTPException, SimpleDBItemNotFound); + + // Create an item to use with conditional delete tests. + expected_attrs["Color"] = "Blue"; + expected_attrs["Size"] = "Med"; + client.PutAttributes(domain, item, expected_attrs); + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Conditional delete that should fail. If it succeeded, it should delete + // the whole item, because no attributes are provided. + expected_attrs["Color"] = "What?"; + SimpleDBClient::str_map_t empty_attrs; + TEST_CHECK_THROWS( + client.DeleteAttributes(domain, item, empty_attrs, // attributes + expected_attrs), // expected + HTTPException, ConditionalRequestConflict); + + // Check that the item was not actually deleted, nor any of its + // attributes, and "Color" should still be "Blue". + expected_attrs["Color"] = "Blue"; + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Conditional delete of one attribute ("Color") that should succeed + SimpleDBClient::str_map_t attrs_to_remove; + attrs_to_remove["Size"] = "Med"; + expected_attrs["Color"] = "Blue"; + client.DeleteAttributes(domain, item, attrs_to_remove, // attributes + expected_attrs); // expected + + // Check that the "Size" attribute is no longer present, but "Color" + // still is. + expected_attrs.erase("Size"); + actual_attrs = client.GetAttributes(domain, item); + TEST_THAT(test_equal_maps(expected_attrs, actual_attrs)); + + // Conditional delete without specifying attributes, should remove all + // remaining attributes, and hence the item itself. The condition + // (expected_attrs) set above should still be valid and match this item. + client.DeleteAttributes(domain, item, empty_attrs, // attributes + expected_attrs); // expected + + // Since we've deleted all attributes, the server should have deleted + // the whole item, and the response to a GetAttributes request should be + // 404 not found. + TEST_CHECK_THROWS(client.GetAttributes(domain, item), + HTTPException, SimpleDBItemNotFound); + } // Kill it TEST_THAT(StopDaemon(pid, "testfiles/s3simulator.pid", "s3simulator.memleaks", true)); - return 0; + TEST_THAT(StartSimulator()); + + // S3Client tests with s3simulator executable for even more realism + { + S3Client client("localhost", 1080, EXAMPLE_S3_ACCESS_KEY, + EXAMPLE_S3_SECRET_KEY, "johnsmith.s3.amazonaws.com"); + TEST_THAT(exercise_s3client(client)); + } + + TEST_THAT(StopSimulator()); + + TEARDOWN(); } +int test(int argc, const char *argv[]) +{ + if(argc >= 2 && ::strcmp(argv[1], "server") == 0) + { + // Run a server + TestWebServer server; + return server.Main("doesnotexist", argc - 1, argv + 1); + } + + if(argc >= 2 && ::strcmp(argv[1], "s3server") == 0) + { + // Run a server + S3Simulator server; + return server.Main("doesnotexist", argc - 1, argv + 1); + } + + TEST_THAT(test_httpserver()); + + return finish_test_suite(); +} diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp index d771f23dc..be0ad1901 100644 --- a/test/raidfile/testraidfile.cpp +++ b/test/raidfile/testraidfile.cpp @@ -498,7 +498,8 @@ bool list_matches(const std::vector &rList, const char *compareto[] int count = 0; while(compareto[count] != 0) count++; - + + TEST_EQUAL(rList.size(), count); if((int)rList.size() != count) { return false; @@ -531,12 +532,12 @@ bool list_matches(const std::vector &rList, const char *compareto[] { if(found[c] == false) { + TEST_FAIL_WITH_MESSAGE("Expected to find " << compareto[c] << " in list"); ret = false; } } delete [] found; - return ret; } @@ -813,7 +814,7 @@ int test(int argc, const char *argv[]) TEST_THAT(::rename("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf-NT", "testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf") == 0); // Delete one of the files - TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "damage.rf") == 0); // stripe 1 + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "damage.rf") == 0); // stripe 1 #ifdef TRF_CAN_INTERCEPT // Open it and read... @@ -830,7 +831,7 @@ int test(int argc, const char *argv[]) #endif //TRF_CAN_INTERCEPT // Delete another - TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf") == 0); // parity + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf") == 0); // parity TEST_CHECK_THROWS( std::auto_ptr pread2 = RaidFileRead::Open(0, "damage"), @@ -872,15 +873,15 @@ int test(int argc, const char *argv[]) TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_DirsOnly, names)); TEST_THAT(list_matches(names, dir_list1)); // Delete things - TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list1)); // Delete something else so that it's not recoverable - TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(false == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list1)); // And finally... - TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_2" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); + TEST_THAT(EMU_UNLINK("testfiles" DIRECTORY_SEPARATOR "0_2" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list2)); } diff --git a/test/s3store/testfiles/bbackupd.conf b/test/s3store/testfiles/bbackupd.conf index 77640e5e4..a6a473f59 100644 --- a/test/s3store/testfiles/bbackupd.conf +++ b/test/s3store/testfiles/bbackupd.conf @@ -10,10 +10,20 @@ DataDirectory = testfiles/bbackupd-data S3Store { HostName = localhost + S3VirtualHostName = testing.s3.amazonaws.com + + # The S3Simulator requires us to send the correct endpoint (via the Host header) to + # distinguish between S3 and SimpleDB requests, so we cannot leave it at the default, + # empty value. It must be set to exactly this value for SimpleDB requests: + SimpleDBEndpoint = sdb.localhost + SimpleDBHostName = localhost + SimpleDBPort = 22080 + Port = 22080 BasePath = /subdir/ AccessKey = 0PN5J17HBGZHT7JJ3X82 SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o + CacheDirectory = testfiles/bbackupd-cache } UpdateStoreInterval = 3 diff --git a/test/s3store/testfiles/store/subdir/dirs/create-me.txt b/test/s3store/testfiles/store/subdir/dirs/create-me.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/s3store/tests3store.cpp b/test/s3store/tests3store.cpp index 50bd2bfd2..5a4db358f 100644 --- a/test/s3store/tests3store.cpp +++ b/test/s3store/tests3store.cpp @@ -70,19 +70,35 @@ bool kill_running_daemons() TEST_THAT(kill_running_daemons()); \ TEARDOWN(); +bool check_new_account_info(); + bool test_create_account_with_account_control() { SETUP_TEST_S3SIMULATOR(); std::auto_ptr config = load_config_file(DEFAULT_BBACKUPD_CONFIG_FILE, BackupDaemonConfigVerify); - S3BackupAccountControl control(*config); - control.CreateAccount("test", 1000, 2000); + TEST_LINE_OR(config.get(), "Failed to load configuration, aborting", FAIL); + + { + S3BackupAccountControl control(*config); + control.CreateAccount("test", 1000, 2000); + TEST_THAT(check_new_account_info()); + // Exit scope to release S3BackupFileSystem now, writing the refcount db back to the + // store, before stopping the simulator daemon! + } + + TEARDOWN_TEST_S3SIMULATOR(); +} + +bool check_new_account_info() +{ + int old_failure_count_local = num_failures; FileStream fs("testfiles/store/subdir/" S3_INFO_FILE_NAME); std::auto_ptr info = BackupStoreInfo::Load(fs, fs.GetFileName(), true); // ReadOnly - TEST_EQUAL(0, info->GetAccountID()); + TEST_EQUAL(S3_FAKE_ACCOUNT_ID, info->GetAccountID()); TEST_EQUAL(1, info->GetLastObjectIDUsed()); TEST_EQUAL(1, info->GetBlocksUsed()); TEST_EQUAL(0, info->GetBlocksInCurrentFiles()); @@ -101,10 +117,37 @@ bool test_create_account_with_account_control() TEST_EQUAL(0, info->GetClientStoreMarker()); TEST_EQUAL("test", info->GetAccountName()); - FileStream root_stream("testfiles/store/subdir/dirs/0x1.dir"); + FileStream root_stream("testfiles/store/subdir/0x1.dir"); BackupStoreDirectory root_dir(root_stream); TEST_EQUAL(0, root_dir.GetNumberOfEntries()); + // Return true if no new failures. + return (old_failure_count_local == num_failures); +} + +#define BBSTOREACCOUNTS_COMMAND BBSTOREACCOUNTS " -3 -c " \ + DEFAULT_BBACKUPD_CONFIG_FILE " " + +bool test_bbstoreaccounts_commands() +{ + SETUP_TEST_S3SIMULATOR(); + + TEST_RETURN(system(BBSTOREACCOUNTS_COMMAND "create test 1000B 2000B"), 0); + TEST_THAT(check_new_account_info()); + + TEST_RETURN(system(BBSTOREACCOUNTS_COMMAND "name foo"), 0); + FileStream fs("testfiles/store/subdir/" S3_INFO_FILE_NAME); + std::auto_ptr apInfo = BackupStoreInfo::Load(fs, fs.GetFileName(), + true); // ReadOnly + TEST_EQUAL("foo", apInfo->GetAccountName()); + + TEST_RETURN(system(BBSTOREACCOUNTS_COMMAND "enabled no"), 0); + fs.Seek(0, IOStream::SeekType_Absolute); + apInfo = BackupStoreInfo::Load(fs, fs.GetFileName(), true); // ReadOnly + TEST_EQUAL(false, apInfo->IsAccountEnabled()); + + TEST_RETURN(system(BBSTOREACCOUNTS_COMMAND "info"), 0); + TEARDOWN_TEST_S3SIMULATOR(); } @@ -122,6 +165,7 @@ int test(int argc, const char *argv[]) #endif TEST_THAT(test_create_account_with_account_control()); + TEST_THAT(test_bbstoreaccounts_commands()); return finish_test_suite(); }