tenzir · mavam · Feb 11, 2022 · Feb 9, 2022 · Feb 9, 2022 · Feb 9, 2022
diff --git a/.github/workflows/vast.yaml b/.github/workflows/vast.yaml
@@ -612,6 +612,9 @@ jobs:
             path: plugins/broker
             dependencies:
               - libbroker-dev
+          - name: Sigma
+            target: sigma
+            path: plugins/sigma
     env:
       INSTALL_DIR: '${{ github.workspace }}/_install'
       BUILD_DIR: '${{ github.workspace }}/_build'

diff --git a/libvast/src/system/spawn_arguments.cpp b/libvast/src/system/spawn_arguments.cpp
@@ -12,11 +12,11 @@
 #include "vast/concept/parseable/vast/expression.hpp"
 #include "vast/concept/parseable/vast/schema.hpp"
 #include "vast/detail/load_contents.hpp"
-#include "vast/detail/sigma.hpp"
 #include "vast/detail/string.hpp"
 #include "vast/error.hpp"
 #include "vast/expression.hpp"
 #include "vast/logger.hpp"
+#include "vast/plugin.hpp"
 #include "vast/schema.hpp"
 
 #include <caf/config_value.hpp>
@@ -35,19 +35,19 @@ normalized_and_validated(std::vector<std::string>::const_iterator begin,
                          std::vector<std::string>::const_iterator end) {
   if (begin == end)
     return caf::make_error(ec::syntax_error, "no query expression given");
-  auto str = detail::join(begin, end, " ");
-  // TODO: improve error handling. Right now, we silently try to parse the
-  // query expression as Sigma rule. If it fails, we only print a debug log
-  // entry and move on to parsing the input as VAST expression.
-  if (auto yaml = from_yaml(str)) {
-    if (auto e = detail::sigma::parse_rule(*yaml))
-      return normalize_and_validate(std::move(*e));
-    else
-      VAST_DEBUG("failed to parse query as Sigma rule: {}", e.error());
-  } else {
-    VAST_DEBUG("failed to parse input as YAML: {}", yaml.error());
-  }
-  if (auto e = to<expression>(str)) {
+  auto query = detail::join(begin, end, " ");
+  // We are trying all query language plugins in a non-deterministic order.
+  for (const auto& plugin : plugins::get())
+    if (const auto* query_lang = plugin.as<query_language_plugin>()) {
+      if (auto expr = query_lang->parse(query))
+        return normalize_and_validate(std::move(*expr));
+      else
+        // TODO: Demote to DEBUG when polishing the PR.
+        VAST_ERROR("failed to parse query as {} languae: {}", plugin->name(),
+                   expr.error());
+    }
+  // TODO: rewrite the VAST expression as a query language plugin.
+  if (auto e = to<expression>(query)) {
     return normalize_and_validate(std::move(*e));
   } else {
     VAST_DEBUG("failed to parse query as VAST expression: {}", e.error());

diff --git a/libvast/vast/plugin.hpp b/libvast/vast/plugin.hpp
@@ -280,6 +280,20 @@ class store_plugin : public virtual plugin {
              std::span<const std::byte> header) const = 0;
 };
 
+// -- query language plugin ---------------------------------------------------
+
+/// A query language parser to pass query in a custom language to VAST.
+/// @relates plugin
+class query_language_plugin : public virtual plugin {
+public:
+  /// Parses a query expression string into a VAST expression.
+  /// @param The string representing the custom query.
+  /// In the future, we may want to let this plugin return a substrait query
+  /// plan instead of a VAST expression.
+  [[nodiscard]] virtual caf::expected<expression>
+  parse(std::string_view query) const = 0;
+};
+
 // -- plugin_ptr ---------------------------------------------------------------
 
 /// An owned plugin and dynamically loaded plugin.

diff --git a/plugins/sigma/CHANGELOG.md b/plugins/sigma/CHANGELOG.md
@@ -0,0 +1,7 @@
+# Changelog
+
+This changelog documents all notable changes to the Sigma plugin for VAST.
+
+## v0.1.0
+
+This is the first official release.
diff --git a/plugins/sigma/CMakeLists.txt b/plugins/sigma/CMakeLists.txt
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.18...3.22 FATAL_ERROR)
+
+project(
+  sigma
+  VERSION 1.0.0
+  DESCRIPTION "Sigma query language plugin for VAST"
+  LANGUAGES CXX)
+
+include(CTest)
+
+file(GLOB_RECURSE sigma_sources CONFIGURE_DEPENDS
+     "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp"
+     "${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp")
+
+file(GLOB_RECURSE sigma_tests CONFIGURE_DEPENDS
+     "${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cpp")
+
+find_package(VAST REQUIRED)
+VASTRegisterPlugin(
+  TARGET sigma
+  ENTRYPOINT src/plugin.cpp
+  SOURCES ${sigma_sources}
+  TEST_SOURCES ${sigma_tests}
+  INCLUDE_DIRECTORIES include)
diff --git a/plugins/sigma/README.md b/plugins/sigma/README.md
@@ -0,0 +1,3 @@
+# Sigma Plugin for VAST
+
+to be written
diff --git a/libvast/vast/detail/sigma.hpp → plugins/sigma/include/sigma/parse.hpp b/libvast/vast/detail/sigma.hpp → plugins/sigma/include/sigma/parse.hpp
@@ -15,7 +15,7 @@
 #include <caf/expected.hpp>
 
 /// Utilities to work with [Sigma](https://github.com/Neo23x0/sigma).
-namespace vast::detail::sigma {
+namespace vast::plugins::sigma {
 
 /// Parses a *rule* as VAST expression.
 /// @param yaml The rule contents.
@@ -27,4 +27,4 @@ caf::expected<expression> parse_rule(const data& yaml);
 /// @returns The VAST expression corresponding to *yaml*.
 caf::expected<expression> parse_search_id(const data& yaml);
 
-} // namespace vast::detail::sigma
+} // namespace vast::plugins::sigma
diff --git a/vast/integration/data/sigma/zeek-conn.yaml → ...gma/integration/data/sigma/zeek-conn.yaml b/vast/integration/data/sigma/zeek-conn.yaml → ...gma/integration/data/sigma/zeek-conn.yaml
diff --git a/plugins/sigma/integration/data/zeek b/plugins/sigma/integration/data/zeek
@@ -0,0 +1 @@
+../../../../vast/integration/data/zeek
diff --git a/vast/integration/reference/sigma/step_00.ref → ...a/integration/reference/sigma/step_00.ref b/vast/integration/reference/sigma/step_00.ref → ...a/integration/reference/sigma/step_00.ref
diff --git a/vast/integration/reference/sigma/step_01.ref → ...a/integration/reference/sigma/step_01.ref b/vast/integration/reference/sigma/step_01.ref → ...a/integration/reference/sigma/step_01.ref
diff --git a/plugins/sigma/integration/tests.yaml b/plugins/sigma/integration/tests.yaml
@@ -0,0 +1,26 @@
+config-file: vast.yaml
+
+fixtures:
+  ServerTester:
+    enter: | # python
+      node = Server(self.cmd,
+                    ['-e', f'127.0.0.1:{VAST_PORT}', '-i', 'node', 'start'],
+                    work_dir, name='node', port=VAST_PORT,
+                    config_file=self.config_file)
+      cmd += ['-e', f'127.0.0.1:{VAST_PORT}']
+
+    exit: | # python
+      node.stop()
+
+tests:
+  # This test only checks whether it is possible to take a Sigma rule as valid
+  # input and perform a simple query. It does not check for subtleties in the
+  # expression language itself. See the corresponding unit tests for that.
+  Sigma:
+    tags: [server, import-export, sigma, zeek]
+    fixture: ServerTester
+    steps:
+      - command: import -b zeek
+        input: data/zeek/conn.log.gz
+      - command: export json
+        input: data/sigma/zeek-conn.yaml
diff --git a/plugins/sigma/integration/vast.yaml b/plugins/sigma/integration/vast.yaml
@@ -0,0 +1,3 @@
+vast:
+  plugins:
+    - sigma
diff --git a/libvast/src/detail/sigma.cpp → plugins/sigma/src/parse.cpp b/libvast/src/detail/sigma.cpp → plugins/sigma/src/parse.cpp
@@ -6,22 +6,22 @@
 // SPDX-FileCopyrightText: (c) 2021 The VAST Contributors
 // SPDX-License-Identifier: BSD-3-Clause
 
-#include "vast/detail/sigma.hpp"
+#include "sigma/parse.hpp"
 
-#include "vast/concept/parseable/core.hpp"
-#include "vast/concept/parseable/string.hpp"
-#include "vast/detail/base64.hpp"
-#include "vast/detail/string.hpp"
-#include "vast/error.hpp"
-#include "vast/expression_visitors.hpp"
+#include <vast/concept/parseable/core.hpp>
+#include <vast/concept/parseable/string.hpp>
+#include <vast/detail/base64.hpp>
+#include <vast/detail/string.hpp>
+#include <vast/error.hpp>
+#include <vast/expression_visitors.hpp>
 
 #include <map>
 #include <regex>
 #include <string>
 #include <tuple>
 #include <vector>
 
-namespace vast::detail::sigma {
+namespace vast::plugins::sigma {
 
 // TODO: A lot of code in here is directly copied from
 // src/concept/parseable/expression.cpp. We should factor the implementation in
@@ -239,7 +239,7 @@ caf::expected<expression> parse_search_id(const data& yaml) {
   if (auto xs = caf::get_if<record>(&yaml)) {
     conjunction result;
     for (auto& [key, rhs] : *xs) {
-      auto keys = split(key, "|");
+      auto keys = detail::split(key, "|");
       auto extractor = field_extractor{std::string{keys[0]}};
       auto op = relational_operator::equal;
       auto all = false;
@@ -370,4 +370,4 @@ caf::expected<expression> parse_rule(const data& yaml) {
   return result;
 }
 
-} // namespace vast::detail::sigma
+} // namespace vast::plugins::sigma
diff --git a/plugins/sigma/src/plugin.cpp b/plugins/sigma/src/plugin.cpp
@@ -0,0 +1,42 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2022 The VAST Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include "sigma/parse.hpp"
+
+#include <vast/data.hpp>
+#include <vast/error.hpp>
+#include <vast/plugin.hpp>
+
+#include <caf/error.hpp>
+#include <caf/expected.hpp>
+#include <fmt/format.h>
+
+namespace vast::plugins::sigma {
+
+class plugin final : public virtual query_language_plugin {
+  caf::error initialize(data) override {
+    return caf::none;
+  }
+
+  [[nodiscard]] const char* name() const override {
+    return "sigma";
+  }
+
+  [[nodiscard]] caf::expected<expression>
+  parse(std::string_view query) const override {
+    if (auto yaml = from_yaml(query))
+      return parse_rule(*yaml);
+    else
+      return caf::make_error(ec::invalid_query,
+                             fmt::format("not a Sigma rule: {}", yaml.error()));
+  }
+};
+
+} // namespace vast::plugins::sigma
+
+VAST_REGISTER_PLUGIN(vast::plugins::sigma::plugin)
diff --git a/libvast/test/detail/sigma.cpp → plugins/sigma/tests/tests.cpp b/libvast/test/detail/sigma.cpp → plugins/sigma/tests/tests.cpp
@@ -4,19 +4,19 @@
 //   | |/ / __ |_\ \  / /          Across
 //   |___/_/ |_/___/ /_/       Space and Time
 //
-// SPDX-FileCopyrightText: (c) 2021 The VAST Contributors
+// SPDX-FileCopyrightText: (c) 2022 The VAST Contributors
 // SPDX-License-Identifier: BSD-3-Clause
 
 #define SUITE sigma
 
-#include "vast/detail/sigma.hpp"
+#include "sigma/parse.hpp"
 
-#include "vast/concept/parseable/to.hpp"
-#include "vast/concept/parseable/vast/expression.hpp"
-#include "vast/concept/printable/stream.hpp"
-#include "vast/concept/printable/vast/expression.hpp"
-#include "vast/expression.hpp"
-#include "vast/test/test.hpp"
+#include <vast/concept/parseable/to.hpp>
+#include <vast/concept/parseable/vast/expression.hpp>
+#include <vast/concept/printable/stream.hpp>
+#include <vast/concept/printable/vast/expression.hpp>
+#include <vast/expression.hpp>
+#include <vast/test/test.hpp>
 
 #include <caf/test/dsl.hpp>
 
@@ -27,11 +27,11 @@ using namespace vast::detail;
 namespace {
 
 expression to_search_id(std::string_view yaml) {
-  return unbox(sigma::parse_search_id(unbox(from_yaml(yaml))));
+  return unbox(plugins::sigma::parse_search_id(unbox(from_yaml(yaml))));
 }
 
 expression to_rule(std::string_view yaml) {
-  return unbox(sigma::parse_rule(unbox(from_yaml(yaml))));
+  return unbox(plugins::sigma::parse_rule(unbox(from_yaml(yaml))));
 }
 
 expression to_expr(std::string_view expr) {

diff --git a/vast/integration/vast_integration_suite.yaml b/vast/integration/vast_integration_suite.yaml
@@ -534,17 +534,6 @@ tests:
         input: data/suricata/eve.json
         transformation: sleep 4
       - command: count '#type == /zeek.*/'
-  # This test only checks whether it is possible to take a Sigma rule as valid
-  # input and perform a simple query. It does not check for subtleties in the
-  # expression language itself. See the corresponding unit tests for that.
-  Sigma:
-    tags: [server, import-export, sigma, zeek]
-    fixture: ServerTester
-    steps:
-      - command: import -b zeek
-        input: data/zeek/conn.log.gz
-      - command: export json
-        input: data/sigma/zeek-conn.yaml
   Transforms:
     tags: [server, client, import-export, transforms, suricata]
     fixture: TransformsTester
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Sigma Plugin for VAST

		to be written
mavam marked this conversation as resolved. Show resolved Hide resolved
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		../../../../vast/integration/data/zeek
dominiklohmann marked this conversation as resolved. Show resolved Hide resolved