Skip to content

Commit

Permalink
carver: Refactor carver to use the Scheduler
Browse files Browse the repository at this point in the history
  • Loading branch information
@theopolis
theopolis committed Sep 24, 2020
1 parent 49a02f7 commit cfd9e8c
Show file tree
Hide file tree
Showing 11 changed files with 416 additions and 135 deletions.
27 changes: 25 additions & 2 deletions osquery/carver/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ function(osqueryCarverMain)
endif()

generateOsqueryCarver()
generateOsqueryCarverUtils()
endfunction()

function(generateOsqueryCarver)
Expand All @@ -22,8 +23,8 @@ function(generateOsqueryCarver)
target_link_libraries(osquery_carver PUBLIC
osquery_cxx_settings
osquery_core
osquery_core_plugins
osquery_core_sql
osquery_carver_utils
osquery_dispatcher
osquery_distributed
osquery_filesystem
osquery_hashing
Expand All @@ -43,4 +44,26 @@ function(generateOsqueryCarver)
add_test(NAME osquery_carver_tests-test COMMAND osquery_carver_tests-test)
endfunction()


function(generateOsqueryCarverUtils)
add_osquery_library(osquery_carver_utils EXCLUDE_FROM_ALL
carver_utils.cpp
)

target_link_libraries(osquery_carver_utils PUBLIC
osquery_cxx_settings
osquery_core
osquery_database
osquery_utils
thirdparty_boost
thirdparty_gflags
)

set(public_header_files
carver_utils.h
)

generateIncludeNamespace(osquery_carver_utils "osquery/carver" "FILE_ONLY" ${public_header_files})
endfunction()

osqueryCarverMain()
172 changes: 80 additions & 92 deletions osquery/carver/carver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,19 @@
#include <osquery/remote/utility.h>
// clang-format on

#include <boost/algorithm/string.hpp>

#include <osquery/carver/carver.h>
#include <osquery/carver/carver_utils.h>
#include <osquery/database/database.h>
#include <osquery/distributed/distributed.h>
#include <osquery/filesystem/fileops.h>
#include <osquery/core/flags.h>
#include <osquery/hashing/hashing.h>
#include <osquery/logger/logger.h>
#include <osquery/remote/serializers/json.h>
#include <osquery/utils/conversions/split.h>
#include <osquery/core/system.h>
#include <osquery/utils/base64.h>
#include <osquery/utils/json/json.h>

#include <osquery/utils/system/system.h>
#include <osquery/utils/system/time.h>

Expand Down Expand Up @@ -55,60 +54,73 @@ CLI_FLAG(uint32,
8192,
"Size of blocks used for POSTing data back to remote endpoints");

CLI_FLAG(bool,
disable_carver,
true,
"Disable the osquery file carver (default true)");

CLI_FLAG(bool,
carver_disable_function,
FLAGS_disable_carver,
"Disable the osquery file carver function (default true)");

CLI_FLAG(bool,
carver_compression,
false,
"Compress archives using zstd prior to upload (default false)");

DECLARE_uint64(read_max);

/// Helper function to update values related to a carve
void updateCarveValue(const std::string& guid,
const std::string& key,
const std::string& value) {
std::string carve;
auto s = getDatabaseValue(kCarveDbDomain, kCarverDBPrefix + guid, carve);
if (!s.ok()) {
VLOG(1) << "Failed to update status of carve in database " << guid;
return;
}
std::atomic<bool> CarverRunnable::running_{false};

JSON tree;
s = tree.fromString(carve);
if (!s.ok()) {
VLOG(1) << "Failed to parse carve entries: " << s.what();
return;
}
void CarverRunnable::start() {
std::vector<std::string> carves;
scanDatabaseKeys(kCarves, carves, kCarverDBPrefix);

tree.add(key, value);
for (const auto& key : carves) {
std::string carve;
auto s = getDatabaseValue(kCarves, key, carve);
if (!s.ok()) {
VLOG(1) << "Failed to retrieve carve key: " << key;
deleteDatabaseValue(kCarves, key);
continue;
}

std::string out;
s = tree.toString(out);
if (!s.ok()) {
VLOG(1) << "Failed to serialize carve entries: " << s.what();
}
JSON tree;
s = tree.fromString(carve);
if (!s.ok() || !tree.doc().IsObject()) {
deleteDatabaseValue(kCarves, key);
VLOG(1) << "Failed to parse carve entries: " << s.getMessage();
continue;
}

s = setDatabaseValue(kCarveDbDomain, kCarverDBPrefix + guid, out);
if (!s.ok()) {
VLOG(1) << "Failed to update status of carve in database " << guid;
auto& doc = tree.doc();
if (!doc.HasMember("status") || !doc["status"].IsString()) {
// Malformed data.
deleteDatabaseValue(kCarves, key);
continue;
}

std::string guid(doc["carve_guid"].GetString());
std::string status(doc["status"].GetString());
if (status == kCarverStatusSuccess) {
uint64_t start_time(doc["time"].GetUint64());
auto delta = getUnixTime() - start_time;
if (delta > 86400) {
// Expire results after 1 day.
VLOG(1) << "Expiring successful carve metadata for GUID: " << guid;
deleteDatabaseValue(kCarves, key);
continue;
}
} else if (status != kCarverStatusScheduled) {
continue;
}

// Schedule the carve.
updateCarveValue(guid, "status", "STARTING");
std::set<std::string> paths;
for (const auto& path : osquery::split(doc["path"].GetString(), ",")) {
paths.insert(path);
}

auto requestId = Distributed::getCurrentRequestId();
doCarve(paths, guid, requestId);
}
}

Carver::Carver(const std::set<std::string>& paths,
const std::string& guid,
const std::string& requestId)
: InternalRunnable("Carver") {
status_ = Status(0, "Ok");
const std::string& requestId) {
for (const auto& p : paths) {
carvePaths_.insert(fs::path(p));
}
Expand All @@ -118,14 +130,15 @@ Carver::Carver(const std::set<std::string>& paths,

// Stash the work ID to be POSTed with the carve initial request
requestId_ = requestId;
}

Status Carver::createPaths() {
// TODO: Adding in a manifest file of all carved files might be nice.
carveDir_ =
fs::temp_directory_path() / fs::path(kCarvePathPrefix + carveGuid_);
auto ret = fs::create_directory(carveDir_);
if (!ret) {
status_ = Status(1, "Failed to create carve file store");
return;
return Status::failure("Failed to create carve file store");
}

// Store the path to our archive for later exfiltration
Expand All @@ -135,34 +148,36 @@ Carver::Carver(const std::set<std::string>& paths,

// Update the DB to reflect that the carve is pending.
updateCarveValue(carveGuid_, "status", "PENDING");
return Status::success();
};

Carver::~Carver() {
fs::remove_all(carveDir_);
}

void Carver::start() {
// If status_ is not Ok, the creation of our tmp FS failed
if (!status_.ok()) {
LOG(WARNING) << "Carver has not been properly constructed";
return;
Status Carver::carve() {
auto s = createPaths();
if (!s.ok()) {
updateCarveValue(carveGuid_, "status", "CREATE PATHS FAILED");
return s;
}

const auto carvedFiles = carveAll();
status_ = archive(carvedFiles, archivePath_, FLAGS_carver_block_size);
if (!status_.ok()) {
VLOG(1) << "Failed to create carve archive: " << status_.getMessage();
s = archive(carvedFiles, archivePath_, FLAGS_carver_block_size);
if (!s.ok()) {
VLOG(1) << "Failed to create carve archive: " << s.getMessage();
updateCarveValue(carveGuid_, "status", "ARCHIVE FAILED");
return;
return s;
}

fs::path uploadPath;
if (FLAGS_carver_compression) {
uploadPath = compressPath_;
status_ = compress(archivePath_, compressPath_);
if (!status_.ok()) {
VLOG(1) << "Failed to compress carve archive: " << status_.getMessage();
s = compress(archivePath_, compressPath_);
if (!s.ok()) {
VLOG(1) << "Failed to compress carve archive: " << s.getMessage();
updateCarveValue(carveGuid_, "status", "COMPRESS FAILED");
return;
return s;
}
} else {
uploadPath = archivePath_;
Expand All @@ -181,12 +196,13 @@ void Carver::start() {
}
updateCarveValue(carveGuid_, "sha256", uploadHash);

status_ = postCarve(uploadPath);
if (!status_.ok()) {
VLOG(1) << "Failed to post carve: " << status_.getMessage();
s = postCarve(uploadPath);
if (!s.ok()) {
VLOG(1) << "Failed to post carve: " << s.getMessage();
updateCarveValue(carveGuid_, "status", "DATA POST FAILED");
return;
return s;
}
return Status::success();
};

std::set<fs::path> Carver::carveAll() {
Expand Down Expand Up @@ -305,41 +321,13 @@ Status Carver::postCarve(const boost::filesystem::path& path) {
}
}

updateCarveValue(carveGuid_, "status", "SUCCESS");
updateCarveValue(carveGuid_, "status", kCarverStatusSuccess);
return Status::success();
};

Status carvePaths(const std::set<std::string>& paths) {
Status s;
auto guid = generateNewUUID();

JSON tree;
tree.add("carve_guid", guid);
tree.add("time", getUnixTime());
tree.add("status", "STARTING");
tree.add("sha256", "");
tree.add("size", -1);

if (paths.size() > 1) {
tree.add("path", boost::algorithm::join(paths, ","));
} else {
tree.add("path", *(paths.begin()));
}

std::string out;
s = tree.toString(out);
if (!s.ok()) {
VLOG(1) << "Failed to serialize carve paths: " << s.what();
return s;
}

s = setDatabaseValue(kCarveDbDomain, kCarverDBPrefix + guid, out);
if (!s.ok()) {
return s;
} else {
auto requestId = Distributed::getCurrentRequestId();
Dispatcher::addService(std::make_shared<Carver>(paths, guid, requestId));
void scheduleCarves() {
if (!CarverRunnable::running()) {
Dispatcher::addService(std::make_shared<CarverRunner<Carver>>());
}
return s;
}
} // namespace osquery
Loading

0 comments on commit cfd9e8c

Please sign in to comment.