Add new query language plugin

This commit adds a new query language plugin that allows for specifying "frontend" languages. We rewrote the Sigma rule parser as an example plugin to demonstrate the use.
tenzir · mavam · Feb 9, 2022 · Feb 9, 2022 · Feb 9, 2022 · Feb 9, 2022
commit 0206cd7b2869783df2df255a0aa80e134d52359b
diff --git a/libvast/src/system/spawn_arguments.cpp b/libvast/src/system/spawn_arguments.cpp
@@ -12,11 +12,11 @@
 #include "vast/concept/parseable/vast/expression.hpp"
 #include "vast/concept/parseable/vast/schema.hpp"
 #include "vast/detail/load_contents.hpp"
-#include "vast/detail/sigma.hpp"
 #include "vast/detail/string.hpp"
 #include "vast/error.hpp"
 #include "vast/expression.hpp"
 #include "vast/logger.hpp"
+#include "vast/plugin.hpp"
 #include "vast/schema.hpp"
 
 #include <caf/config_value.hpp>
@@ -35,19 +35,19 @@ normalized_and_validated(std::vector<std::string>::const_iterator begin,
                          std::vector<std::string>::const_iterator end) {
   if (begin == end)
     return caf::make_error(ec::syntax_error, "no query expression given");
-  auto str = detail::join(begin, end, " ");
-  // TODO: improve error handling. Right now, we silently try to parse the
-  // query expression as Sigma rule. If it fails, we only print a debug log
-  // entry and move on to parsing the input as VAST expression.
-  if (auto yaml = from_yaml(str)) {
-    if (auto e = detail::sigma::parse_rule(*yaml))
-      return normalize_and_validate(std::move(*e));
-    else
-      VAST_DEBUG("failed to parse query as Sigma rule: {}", e.error());
-  } else {
-    VAST_DEBUG("failed to parse input as YAML: {}", yaml.error());
-  }
-  if (auto e = to<expression>(str)) {
+  auto query = detail::join(begin, end, " ");
+  // We are trying all query language plugins in a non-deterministic order.
+  for (const auto& plugin : plugins::get())
+    if (const auto* query_lang = plugin.as<query_language_plugin>()) {
+      if (auto expr = query_lang->parse(query))
+        return normalize_and_validate(std::move(*expr));
+      else
+        // TODO: Demote to DEBUG when polishing the PR.
+        VAST_ERROR("failed to parse query as {} languae: {}", plugin->name(),
+                   expr.error());
+    }
+  // TODO: rewrite the VAST expression as a query language plugin.
+  if (auto e = to<expression>(query)) {
     return normalize_and_validate(std::move(*e));
   } else {
     VAST_DEBUG("failed to parse query as VAST expression: {}", e.error());

diff --git a/libvast/vast/plugin.hpp b/libvast/vast/plugin.hpp
@@ -280,6 +280,20 @@ class store_plugin : public virtual plugin {
              std::span<const std::byte> header) const = 0;
 };
 
+// -- query language plugin ---------------------------------------------------
+
+/// A query language parser to pass query in a custom language to VAST.
+/// @relates plugin
+class query_language_plugin : public virtual plugin {
+public:
+  /// Parses a query expression string into a VAST expression.
+  /// @param The string representing the custom query.
+  /// In the future, we may want to let this plugin return a substrait query
+  /// plan instead of a VAST expression.
+  [[nodiscard]] virtual caf::expected<expression>
+  parse(std::string_view query) const = 0;
+};
+
 // -- plugin_ptr ---------------------------------------------------------------
 
 /// An owned plugin and dynamically loaded plugin.

diff --git a/plugins/sigma/CHANGELOG.md b/plugins/sigma/CHANGELOG.md
@@ -0,0 +1,7 @@
+# Changelog
+
+This changelog documents all notable changes to the Sigma plugin for VAST.
+
+## v0.1.0
+
+This is the first official release.
diff --git a/plugins/sigma/CMakeLists.txt b/plugins/sigma/CMakeLists.txt
@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.18...3.22 FATAL_ERROR)
+
+project(
+  sigma
+  VERSION 0.1.0
+  DESCRIPTION "Sigma query language plugin for VAST"
+  LANGUAGES CXX)
+
+# Enable unit testing. Note that it is necessary to include CTest in the
+# top-level CMakeLists.txt file for it to create a test target, so while
+# optional for plugins built alongside VAST, it is necessary to specify this
+# line manually so plugins can be linked against an installed VAST.
+include(CTest)
+
+# FIXME: migrate unit tests
+find_package(VAST REQUIRED)
+VASTRegisterPlugin(
+  TARGET sigma
+  ENTRYPOINT sigma.cpp
+  TEST_SOURCES tests.cpp)
diff --git a/plugins/sigma/README.md b/plugins/sigma/README.md
@@ -0,0 +1,3 @@
+# Sigma Plugin for VAST
+
+to be written
diff --git a/plugins/sigma/sigma.cpp b/plugins/sigma/sigma.cpp
@@ -0,0 +1,42 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2022 The VAST Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// FIXME: migrate this code into plugin
+#include <vast/data.hpp>
+#include <vast/detail/sigma.hpp>
+#include <vast/error.hpp>
+#include <vast/plugin.hpp>
+
+#include <caf/error.hpp>
+#include <caf/expected.hpp>
+#include <fmt/format.h>
+
+namespace vast::plugins::sigma {
+
+class plugin final : public virtual query_language_plugin {
+  caf::error initialize(data) override {
+    return caf::none;
+  }
+
+  [[nodiscard]] const char* name() const override {
+    return "sigma";
+  }
+
+  [[nodiscard]] caf::expected<expression>
+  parse(std::string_view query) const override {
+    if (auto yaml = from_yaml(query))
+      return detail::sigma::parse_rule(*yaml);
+    else
+      return caf::make_error(ec::invalid_query,
+                             fmt::format("not a Sigma rule: {}", yaml.error()));
+  }
+};
+
+} // namespace vast::plugins::sigma
+
+VAST_REGISTER_PLUGIN(vast::plugins::sigma::plugin)
diff --git a/plugins/sigma/tests.cpp b/plugins/sigma/tests.cpp
@@ -0,0 +1,15 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2022 The VAST Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#define SUITE sigma
+
+namespace vast::plugins::sigma {
+
+namespace {} // namespace
+
+} // namespace vast::plugins::sigma