Move S3BackupFileSystem class into its own file.

Create BackupFileSystem, an abstract interface for reading and writing files and directories, abstracting over RaidFile, S3, FTP etc. as a parent class for S3BackupFileSystem.
boxbackup · Jul 23, 2017 · 7e7ba9b · 7e7ba9b
1 parent 5da656c
commit 7e7ba9b
Show file tree

Hide file tree

Showing 5 changed files with 211 additions and 51 deletions.
diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp
@@ -209,7 +209,8 @@ int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftL
 
 	// And an empty directory
 	BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
-	int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir);
+	mapFileSystem->PutDirectory(rootDir);
+	int64_t rootDirSize = rootDir.GetUserInfo1_SizeInBlocks();
 
 	// Update the store info to reflect the size of the root directory
 	info.ChangeBlocksUsed(rootDirSize);
@@ -234,34 +235,3 @@ int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftL
 	return 0;
 }
 
-std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID)
-{
-	std::ostringstream out;
-	out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir";
-	return out.str();
-}
-
-std::auto_ptr<HTTPResponse> S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir)
-{
-	std::string uri = GetDirectoryURI(rDir.GetObjectID());
-	HTTPResponse response = mrClient.GetObject(uri);
-	mrClient.CheckResponse(response,
-		std::string("Failed to download directory: ") + uri);
-	return std::auto_ptr<HTTPResponse>(new HTTPResponse(response));
-}
-
-int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir)
-{
-	CollectInBufferStream out;
-	rDir.WriteToStream(out);
-	out.SetForReading();
-
-	std::string uri = GetDirectoryURI(rDir.GetObjectID());
-	HTTPResponse response = mrClient.PutObject(uri, out);
-	mrClient.CheckResponse(response,
-		std::string("Failed to upload directory: ") + uri);
-
-	int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE;
-	return blocks;
-}
-
diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h
@@ -13,7 +13,8 @@
 #include <string>
 
 #include "BackupStoreAccountDatabase.h"
-#include "HTTPResponse.h"
+#include "BackupFileSystem.h"
+#include "NamedLock.h"
 #include "S3Client.h"
 
 class BackupStoreDirectory;
@@ -39,22 +40,6 @@ class BackupAccountControl
 	int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize);
 };
 
-class S3BackupFileSystem
-{
-private:
-	std::string mBasePath;
-	S3Client& mrClient;
-public:
-	S3BackupFileSystem(const Configuration& config, const std::string& BasePath,
-		S3Client& rClient)
-	: mBasePath(BasePath),
-	  mrClient(rClient)
-	{ }
-	std::string GetDirectoryURI(int64_t ObjectID);
-	std::auto_ptr<HTTPResponse> GetDirectory(BackupStoreDirectory& rDir);
-	int PutDirectory(BackupStoreDirectory& rDir);
-};
-
 class S3BackupAccountControl : public BackupAccountControl
 {
 private:

diff --git a/lib/backupstore/BackupFileSystem.cpp b/lib/backupstore/BackupFileSystem.cpp
@@ -0,0 +1,198 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupFileSystem.cpp
+//		Purpose: Generic interface for reading and writing files and
+//			 directories, abstracting over RaidFile, S3, FTP etc.
+//		Created: 2015/08/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "autogen_BackupStoreException.h"
+#include "BackupConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupFileSystem.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreRefCountDatabase.h"
+#include "BufferedStream.h"
+#include "BufferedWriteStream.h"
+#include "CollectInBufferStream.h"
+#include "Configuration.h"
+#include "BackupStoreObjectMagic.h"
+#include "HTTPResponse.h"
+#include "IOStream.h"
+#include "InvisibleTempFileStream.h"
+#include "S3Client.h"
+
+#include "MemLeakFindOn.h"
+
+int S3BackupFileSystem::GetBlockSize()
+{
+	return S3_NOTIONAL_BLOCK_SIZE;
+}
+
+std::string S3BackupFileSystem::GetObjectURL(const std::string& ObjectPath) const
+{
+	const Configuration s3config = mrConfig.GetSubConfiguration("S3Store");
+	return std::string("http://") + s3config.GetKeyValue("HostName") + ":" +
+		s3config.GetKeyValue("Port") + GetObjectURI(ObjectPath);
+}
+
+int64_t S3BackupFileSystem::GetRevisionID(const std::string& uri,
+	HTTPResponse& response) const
+{
+	std::string etag;
+
+	if(!response.GetHeader("etag", &etag))
+	{
+		THROW_EXCEPTION_MESSAGE(BackupStoreException, MissingEtagHeader,
+			"Failed to get the MD5 checksum of the file or directory "
+			"at this URL: " << GetObjectURL(uri));
+	}
+
+	if(etag[0] != '"')
+	{
+		THROW_EXCEPTION_MESSAGE(BackupStoreException, InvalidEtagHeader,
+			"Failed to get the MD5 checksum of the file or directory "
+			"at this URL: " << GetObjectURL(uri));
+	}
+
+	const char * pEnd = NULL;
+	std::string checksum = etag.substr(1, 16);
+	int64_t revID = box_strtoui64(checksum.c_str(), &pEnd, 16);
+	if(*pEnd != '\0')
+	{
+		THROW_EXCEPTION_MESSAGE(BackupStoreException, InvalidEtagHeader,
+			"Failed to get the MD5 checksum of the file or "
+			"directory at this URL: " << uri << ": invalid character '" <<
+			*pEnd << "' in '" << etag << "'");
+	}
+
+	return revID;
+}
+
+std::auto_ptr<BackupStoreInfo> S3BackupFileSystem::GetBackupStoreInfo(int32_t AccountID,
+	bool ReadOnly)
+{
+	std::string info_url = GetObjectURL(S3_INFO_FILE_NAME);
+	HTTPResponse response = GetObject(S3_INFO_FILE_NAME);
+	mrClient.CheckResponse(response, std::string("No BackupStoreInfo file exists "
+		"at this URL: ") + info_url);
+
+	std::auto_ptr<BackupStoreInfo> info =
+		BackupStoreInfo::Load(response, info_url, ReadOnly);
+
+	// Check it
+	if(info->GetAccountID() != AccountID)
+	{
+		THROW_FILE_ERROR("Found wrong account ID in store info",
+			info_url, BackupStoreException, BadStoreInfoOnLoad);
+	}
+
+	return info;
+}
+
+void S3BackupFileSystem::PutBackupStoreInfo(BackupStoreInfo& rInfo)
+{
+	CollectInBufferStream out;
+	rInfo.Save(out);
+	out.SetForReading();
+
+	HTTPResponse response = PutObject(S3_INFO_FILE_NAME, out);
+
+	std::string info_url = GetObjectURL(S3_INFO_FILE_NAME);
+	mrClient.CheckResponse(response, std::string("Failed to upload the new "
+		"BackupStoreInfo file to this URL: ") + info_url);
+}
+
+//! Returns whether an object (a file or directory) exists with this object ID, and its
+//! revision ID, which for a RaidFile is based on its timestamp and file size.
+bool S3BackupFileSystem::ObjectExists(int64_t ObjectID, int64_t *pRevisionID)
+{
+	std::string uri = GetDirectoryURI(ObjectID);
+	HTTPResponse response = mrClient.HeadObject(uri);
+
+	if(response.GetResponseCode() == HTTPResponse::Code_NotFound)
+	{
+		// A file might exist, check that too.
+		uri = GetFileURI(ObjectID);
+		response = mrClient.HeadObject(uri);
+	}
+
+	if(response.GetResponseCode() == HTTPResponse::Code_NotFound)
+	{
+		return false;
+	}
+
+	if(response.GetResponseCode() != HTTPResponse::Code_OK)
+	{
+		// Throw an appropriate exception.
+		mrClient.CheckResponse(response, std::string("Failed to check whether "
+			"a file or directory exists at this URL: ") +
+			GetObjectURL(uri));
+	}
+
+	if(pRevisionID)
+	{
+		*pRevisionID = GetRevisionID(uri, response);
+	}
+
+	return true;
+}
+
+// This URI should NOT include the BasePath, because the S3Client will add that
+// automatically when we make a request for this URI.
+std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID)
+{
+	std::ostringstream out;
+	out << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir";
+	return out.str();
+}
+
+// This URI should NOT include the BasePath, because the S3Client will add that
+// automatically when we make a request for this URI.
+std::string S3BackupFileSystem::GetFileURI(int64_t ObjectID)
+{
+	std::ostringstream out;
+	out << "files/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir";
+	return out.str();
+}
+
+//! Reads a directory with the specified ID into the supplied BackupStoreDirectory
+//! object, also initialising its revision ID and SizeInBlocks.
+void S3BackupFileSystem::GetDirectory(int64_t ObjectID, BackupStoreDirectory& rDirOut)
+{
+	std::string uri = GetDirectoryURI(ObjectID);
+	HTTPResponse response = GetObject(uri);
+	mrClient.CheckResponse(response,
+		std::string("Failed to download directory: ") + uri);
+	rDirOut.ReadFromStream(response, mrClient.GetNetworkTimeout());
+
+	rDirOut.SetRevisionID(GetRevisionID(uri, response));
+	ASSERT(false); // set the size in blocks
+	rDirOut.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(response.GetContentLength()));
+}
+
+//! Writes the supplied BackupStoreDirectory object to the store, and updates its revision
+//! ID and SizeInBlocks.
+void S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir)
+{
+	CollectInBufferStream out;
+	rDir.WriteToStream(out);
+	out.SetForReading();
+
+	std::string uri = GetDirectoryURI(rDir.GetObjectID());
+	HTTPResponse response = PutObject(uri, out);
+	mrClient.CheckResponse(response,
+		std::string("Failed to upload directory: ") + uri);
+
+	rDir.SetRevisionID(GetRevisionID(uri, response));
+	rDir.SetUserInfo1_SizeInBlocks(GetSizeInBlocks(out.GetSize()));
+}
+
diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt
@@ -72,5 +72,8 @@ UnknownObjectRefCountRequested	68	A reference count was requested for an object
 MultiplyReferencedObject	69	Attempted to modify an object with multiple references, should be uncloned first
 CorruptReferenceCountDatabase	70	The account's refcount database is corrupt and must be rebuilt by housekeeping.
 CancelledByBackgroundTask	71	The current task was cancelled on request by the background task.
-ObjectDoesNotExist		72	The specified object ID does not exist in the store.
-AccountAlreadyExists		73	Tried to create an account that already exists.
+MissingEtagHeader		72	The S3 HTTP response did not contain an ETag header as expected.
+InvalidEtagHeader		73	The S3 HTTP response contain a malformed or unexpected ETag header as expected.
+FileSystemNotInitialised	74	No BackupFileSystem has been configured for this account.
+ObjectDoesNotExist		75	The specified object ID does not exist in the store.
+AccountAlreadyExists		76	Tried to create an account that already exists.
diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h
@@ -86,6 +86,8 @@ class BackupStoreInfo
 		int64_t BlockSoftLimit, int64_t BlockHardLimit);
 
 	// Load it from the store
+	// TODO FIXME: remove this RaidFile version of Load(), let BackupFileSystem
+	// handle the RaidFile part.
 	static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0);
 
 	// Load it from a stream (file or RaidFile)
@@ -96,6 +98,8 @@ class BackupStoreInfo
 	bool IsModified() const {return mIsModified;}
 
 	// Save modified infomation back to store
+	// TODO FIXME: remove this RaidFile version of Save(), let BackupFileSystem
+	// handle the RaidFile part.
 	void Save(bool allowOverwrite = true);
 	void Save(IOStream& rOutStream);