Skip to content

Commit

Permalink
Add support for activity events
Browse files Browse the repository at this point in the history
  • Loading branch information
kpcyrd committed Jan 13, 2020
1 parent aab8a52 commit a3c5fb6
Show file tree
Hide file tree
Showing 14 changed files with 490 additions and 228 deletions.
420 changes: 208 additions & 212 deletions Cargo.lock

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions docs/keyring.rst
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
This returns a list of all keys in that namespace. Any empty list is returned
if the user doesn't have any keys in that namespace.

If you want to allow the user to select a specific script you can introduce an
option that is set by the user and then filter ``creds`` until the
``access_key`` matches.

Using access keys as source argument
------------------------------------

Expand Down
31 changes: 31 additions & 0 deletions migrations/2020-01-09-024234_activity/down.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
DROP TABLE activity;

PRAGMA foreign_keys=off;

-- ports
CREATE TABLE _ports_new (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ip_addr_id INTEGER NOT NULL,
value VARCHAR NOT NULL,
ip_addr VARCHAR NOT NULL,
port INTEGER NOT NULL,
protocol VARCHAR NOT NULL,
status VARCHAR NOT NULL,
unscoped BOOLEAN DEFAULT 0 NOT NULL,

banner VARCHAR,
service VARCHAR,
version VARCHAR,

FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
CONSTRAINT port_unique UNIQUE (value)
);

INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
FROM ports;

DROP TABLE ports;
ALTER TABLE _ports_new RENAME TO ports;

PRAGMA foreign_keys=on;
40 changes: 40 additions & 0 deletions migrations/2020-01-09-024234_activity/up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
CREATE TABLE activity (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
topic VARCHAR NOT NULL,
time DATETIME NOT NULL,
uniq VARCHAR,
latitude FLOAT,
longitude FLOAT,
content VARCHAR NOT NULL
);
CREATE UNIQUE INDEX activity_uniq ON activity(uniq);

PRAGMA foreign_keys=off;

-- ports
CREATE TABLE _ports_new (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ip_addr_id INTEGER NOT NULL,
value VARCHAR NOT NULL,
ip_addr VARCHAR NOT NULL,
port INTEGER NOT NULL,
protocol VARCHAR NOT NULL,
status VARCHAR,
unscoped BOOLEAN DEFAULT 0 NOT NULL,

banner VARCHAR,
service VARCHAR,
version VARCHAR,

FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
CONSTRAINT port_unique UNIQUE (value)
);

INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
FROM ports;

DROP TABLE ports;
ALTER TABLE _ports_new RENAME TO ports;

PRAGMA foreign_keys=on;
26 changes: 26 additions & 0 deletions modules/harness/activity-ping.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
-- Description: Log some dummy activity
-- Version: 0.1.0
-- License: GPL-3.0

function run()
local uniq = getopt('uniq')

if getopt('gps') then
lat=1.23
lon=4.56
end

while true do
db_activity({
topic='harness/activity-ping:dummy',
time=sn0int_time(),
uniq=uniq,
latitude=lat,
longitude=lon,
content={
msg='ohai',
},
})
sleep(5)
end
end
11 changes: 11 additions & 0 deletions src/db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,17 @@ impl Database {
}
}

pub fn insert_activity(&self, obj: NewActivity) -> Result<bool> {
if let Some(uniq) = &obj.uniq {
if Activity::uniq(self, uniq)?.is_some() {
// unique tag set and event already logged
return Ok(false);
}
}
obj.insert(&self)?;
Ok(true)
}

pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<Option<(DbChange, i32)>> {
if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::get_id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
Ok(Some((DbChange::None, subdomain_ipaddr_id)))
Expand Down
15 changes: 14 additions & 1 deletion src/engine/ctx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::engine::{Environment, Reporter};
use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
use crate::hlua::{self, AnyLuaValue};
use crate::keyring::KeyRingEntry;
use crate::models::{Insert, Update};
use crate::models::*;
use crate::psl::{Psl, PslReader};
use crate::lazy::Lazy;
use crate::runtime;
Expand Down Expand Up @@ -86,6 +86,18 @@ pub trait State {
reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
}

fn db_activity(&self, activity: InsertActivity) -> Result<()> {
let activity = activity.try_into_new()?;

self.send(&Event::Database(DatabaseEvent::Activity(activity)));
let reply = self.recv()?;
let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;

reply
.map(|_| ())
.map_err(|err| format_err!("Failed to add to database: {:?}", err))
}

fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
self.send(&Event::Database(DatabaseEvent::Select((family, value))));
let reply = self.recv()?;
Expand Down Expand Up @@ -432,6 +444,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
runtime::datetime(&mut lua, state.clone());
runtime::db_add(&mut lua, state.clone());
runtime::db_add_ttl(&mut lua, state.clone());
runtime::db_activity(&mut lua, state.clone());
runtime::db_select(&mut lua, state.clone());
runtime::db_update(&mut lua, state.clone());
runtime::debug(&mut lua, state.clone());
Expand Down
76 changes: 76 additions & 0 deletions src/models/activity.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
use crate::errors::*;
use crate::schema::activity;
use diesel;
use diesel::prelude::*;
use crate::models::*;
use chrono::NaiveDateTime;


#[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
#[table_name="activity"]
pub struct Activity {
pub id: i32,
pub topic: String,
pub time: NaiveDateTime,
pub uniq: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
pub content: String,
}

impl Activity {
pub fn uniq(db: &Database, my_uniq: &str) -> Result<Option<Activity>> {
use crate::schema::activity::dsl::*;
activity.filter(uniq.eq(my_uniq))
.first::<Self>(db.db())
.optional()
.map_err(|e| Error::from(e))
}
}

#[derive(Debug, Clone, Insertable, Serialize, Deserialize)]
#[table_name="activity"]
pub struct NewActivity {
pub topic: String,
pub time: NaiveDateTime,
pub uniq: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
pub content: String,
}

impl NewActivity {
pub fn insert(&self, db: &Database) -> Result<()> {
diesel::insert_into(activity::table)
.values(self)
.execute(db.db())?;
Ok(())
}
}

#[derive(Debug, Serialize, Deserialize)]
pub struct InsertActivity {
pub topic: String,
pub time: NaiveDateTime,
pub uniq: Option<String>,
pub latitude: Option<f32>,
pub longitude: Option<f32>,
pub content: serde_json::Value,
}

impl InsertToNew for InsertActivity {
type Target = NewActivity;

#[inline]
fn try_into_new(self) -> Result<NewActivity> {
let content = serde_json::to_string(&self.content)?;
Ok(NewActivity {
topic: self.topic,
time: self.time,
uniq: self.uniq,
latitude: self.latitude,
longitude: self.longitude,
content,
})
}
}
8 changes: 4 additions & 4 deletions src/models/cryptoaddr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,20 @@ impl Model for CryptoAddr {
fn get(db: &Database, query: &Self::ID) -> Result<Self> {
use crate::schema::cryptoaddrs::dsl::*;

let netblock = cryptoaddrs.filter(value.eq(query))
let cryptoaddr = cryptoaddrs.filter(value.eq(query))
.first::<Self>(db.db())?;

Ok(netblock)
Ok(cryptoaddr)
}

fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
use crate::schema::cryptoaddrs::dsl::*;

let netblock = cryptoaddrs.filter(value.eq(query))
let cryptoaddr = cryptoaddrs.filter(value.eq(query))
.first::<Self>(db.db())
.optional()?;

Ok(netblock)
Ok(cryptoaddr)
}
}

Expand Down
3 changes: 3 additions & 0 deletions src/models/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -424,3 +424,6 @@ pub use self::port::*;

mod cryptoaddr;
pub use self::cryptoaddr::*;

mod activity;
pub use self::activity::*;
21 changes: 11 additions & 10 deletions src/models/port.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub struct Port {
pub ip_addr: String,
pub port: i32,
pub protocol: String,
pub status: String,
pub status: Option<String>,
pub unscoped: bool,

pub banner: Option<String>,
Expand Down Expand Up @@ -147,7 +147,7 @@ impl Printable<PrintablePort> for Port {
pub struct DetailedPort {
id: i32,
value: String,
status: String,
status: Option<String>,
unscoped: bool,

banner: Option<String>,
Expand Down Expand Up @@ -210,7 +210,7 @@ pub struct NewPort {
pub ip_addr: String,
pub port: i32,
pub protocol: String,
pub status: String,
pub status: Option<String>,

pub banner: Option<String>,
pub service: Option<String>,
Expand Down Expand Up @@ -242,7 +242,7 @@ impl Upsertable<Port> for NewPort {
fn upsert(self, existing: &Port) -> Self::Update {
Self::Update {
id: existing.id,
status: Self::upsert_opt(Some(self.status), &Some(existing.status.clone())),
status: Self::upsert_opt(self.status, &existing.status),
banner: Self::upsert_opt(self.banner, &existing.banner),
service: Self::upsert_opt(self.service, &existing.service),
version: Self::upsert_opt(self.version, &existing.version),
Expand All @@ -264,7 +264,7 @@ pub struct InsertPort {
pub ip_addr: net::IpAddr,
pub port: i32,
pub protocol: String,
pub status: String,
pub status: Option<String>,

pub banner: Option<String>,
pub service: Option<String>,
Expand All @@ -278,10 +278,11 @@ impl InsertToNew for InsertPort {
let addr = SocketAddr::new(self.ip_addr, self.port as u16);
let value = format!("{}/{}", self.protocol, addr);

match self.status.as_str() {
"open" => (),
"closed" => (),
s => bail!("unsupported port status: {:?}", s),
match self.status.as_deref() {
Some("open") => (),
Some("closed") => (),
Some(s) => bail!("unsupported port status: {:?}", s),
None => (),
}

Ok(NewPort {
Expand Down Expand Up @@ -330,7 +331,7 @@ impl Upsert for PortUpdate {

impl Updateable<Port> for PortUpdate {
fn changeset(&mut self, existing: &Port) {
Self::clear_if_equal(&mut self.status, &Some(existing.status.clone()));
Self::clear_if_equal(&mut self.status, &existing.status);
Self::clear_if_equal(&mut self.banner, &existing.banner);
Self::clear_if_equal(&mut self.service, &existing.service);
Self::clear_if_equal(&mut self.version, &existing.version);
Expand Down
12 changes: 12 additions & 0 deletions src/runtime/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,18 @@ pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<dyn State>) {
}))
}

pub fn db_activity(lua: &mut hlua::Lua, state: Arc<dyn State>) {
lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<()> {
let v: LuaJsonValue = v.into();
let v: serde_json::Value = v.into();
let activity: InsertActivity = serde_json::from_value(v)
.map_err(|e| state.set_error(e.into()))?;

state.db_activity(activity)
.map_err(|e| state.set_error(e))
}))
}

pub fn db_select(lua: &mut hlua::Lua, state: Arc<dyn State>) {
lua.set("db_select", hlua::function2(move |family: String, value: String| -> Result<Option<i32>> {
let family = Family::from_str(&family)
Expand Down
15 changes: 14 additions & 1 deletion src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,18 @@ table! {
}
}

table! {
activity (id) {
id -> Integer,
topic -> Text,
time -> Timestamp,
uniq -> Nullable<Text>,
latitude -> Nullable<Float>,
longitude -> Nullable<Float>,
content -> Text,
}
}

table! {
autonoscope (id) {
id -> Integer,
Expand Down Expand Up @@ -184,7 +196,7 @@ table! {
ip_addr -> Text,
port -> Integer,
protocol -> Text,
status -> Text,
status -> Nullable<Text>,
unscoped -> Bool,
banner -> Nullable<Text>,
service -> Nullable<Text>,
Expand Down Expand Up @@ -246,6 +258,7 @@ joinable!(urls -> subdomains (subdomain_id));

allow_tables_to_appear_in_same_query!(
accounts,
activity,
autonoscope,
breach_emails,
breaches,
Expand Down
Loading

0 comments on commit a3c5fb6

Please sign in to comment.