Skip to content

Commit

Permalink
Fix date filters
Browse files Browse the repository at this point in the history 
Signed-off-by: Andrew Stein <steinlink@gmail.com>

# Conflicts:
#	tools/perspective-test/results.tar.gz
  • Loading branch information
@texodus
texodus committed Oct 17, 2024
1 parent 3f6e19c commit 9d312cb
Show file tree
Hide file tree
Showing 7 changed files with 117 additions and 92 deletions.
113 changes: 55 additions & 58 deletions cpp/perspective/src/cpp/server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1068,22 +1068,49 @@ coerce_to(const t_dtype dtype, const A& val) {
);
}
} else if constexpr (std::is_same_v<A, double>) {
t_tscalar scalar;
scalar.clear();
switch (dtype) {
case DTYPE_BOOL:
scalar.set(val == 1);
return scalar;
case DTYPE_UINT32:
scalar.set((std::uint32_t)val);
return scalar;
case DTYPE_UINT64:
scalar.set((std::uint64_t)val);
return scalar;
case DTYPE_INT32:
scalar.set(val);
return scalar;
case DTYPE_INT64:
scalar.set((std::int64_t)val);
return scalar;
case DTYPE_FLOAT32:
return t_tscalar(static_cast<float>(val));
case DTYPE_FLOAT64:
return t_tscalar(val);
default:
PSP_COMPLAIN_AND_ABORT("Unsupported type");
}
} else if constexpr (std::is_same_v<A, float>) {
switch (dtype) {
case DTYPE_FLOAT32:
return t_tscalar(val);
scalar.set(static_cast<float>(val));
return scalar;
case DTYPE_FLOAT64:
return t_tscalar(static_cast<double>(val));
scalar.set(val);
return scalar;
case DTYPE_DATE: {
const auto time = static_cast<time_t>(val / 1000);
std::tm* tm = std::gmtime(&time);
t_date date{
static_cast<std::int16_t>(tm->tm_year + 1900),
static_cast<std::int8_t>(tm->tm_mon),
static_cast<std::int8_t>(tm->tm_mday)
};

scalar.set(date);
return scalar;
}
case DTYPE_TIME: {
t_time time{std::chrono::milliseconds((long)val).count()};
scalar.set(time);
return scalar;
}
default:
PSP_COMPLAIN_AND_ABORT("Unsupported type");
PSP_COMPLAIN_AND_ABORT("Unsupported double type");
}
} else if constexpr (std::is_same_v<A, std::int32_t>) {
return t_tscalar(val);
Expand Down Expand Up @@ -1593,24 +1620,20 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
for (const auto& arg : f.value()) {
t_tscalar a;
a.clear();

switch (arg.scalar_case()) {
case proto::Scalar::kBool: {
a.set(arg.bool_());
args.push_back(a);
break;
}
case proto::Scalar::kFloat: {
a.set(arg.float_());
args.push_back(a);
break;
}
case proto::Scalar::kInt: {
a.set(arg.int_());
a = coerce_to(
schema->get_dtype(f.column()), arg.float_()
);

args.push_back(a);
break;
}

case proto::Scalar::kString: {
if (!schema->has_column(f.column())) {
PSP_COMPLAIN_AND_ABORT(
Expand All @@ -1623,38 +1646,6 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
args.push_back(a);
break;
}

case proto::Scalar::kDate: {
auto date_ts = arg.date();
// convert ts to date
auto tt = std::chrono::system_clock::to_time_t(
std::chrono::system_clock::time_point(
std::chrono::seconds(date_ts)
)
);

auto* date = std::localtime(&tt);

t_date d{
static_cast<std::int16_t>(date->tm_year + 1900),
static_cast<std::int8_t>(date->tm_mon),
static_cast<std::int8_t>(date->tm_mday)
};
a.set(d);
args.push_back(a);
break;
}
case proto::Scalar::kDatetime: {
auto datetime_ts = arg.datetime();
// convert ts to date
auto tt = t_time(datetime_ts);

a.set(tt);
args.push_back(a);

break;
}

case proto::Scalar::kNull:
a.set(t_none());
args.push_back(a);
Expand Down Expand Up @@ -1915,6 +1906,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
vals.push_back(scalar);
}
}

for (const auto& scalar : vals) {
auto* s = f->mutable_value()->Add();
switch (scalar.get_dtype()) {
Expand All @@ -1925,20 +1917,25 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
s->set_float_(scalar.get<double>());
break;
case DTYPE_INT64:
s->set_int_(static_cast<std::int32_t>(
scalar.get<std::int64_t>()
));
s->set_float_((double)scalar.get<std::int64_t>());
break;
case DTYPE_STR:
s->set_string(scalar.get<const char*>());
break;
case DTYPE_DATE: {
auto tm = scalar.get<t_date>().get_tm();
s->set_date(std::mktime(&tm));
auto tm = scalar.get<t_date>();
std::stringstream ss;
ss << std::setfill('0') << std::setw(4) << tm.year()
<< "-" << std::setfill('0') << std::setw(2)
<< tm.month() << "-" << std::setfill('0')
<< std::setw(2) << tm.day();
s->set_string(ss.str());
break;
}
case DTYPE_TIME:
s->set_datetime(scalar.get<t_time>().raw_value());
s->set_float_(
(double)scalar.get<t_time>().raw_value()
);
break;
case DTYPE_NONE:
s->set_null(
Expand Down
9 changes: 5 additions & 4 deletions cpp/protos/perspective.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,15 @@ message MakeTableData {
};
}

// View type scalars
// Filter type scalars - this is _not_ the same as a Columns scalar, as this
// value is used in the view config and must be JSON safe!
message Scalar {
oneof scalar {
bool bool = 1;
int64 date = 2; // TODO these are the wrong type
int64 datetime = 3;
// int64 date = 2; // TODO these are the wrong type
// int64 datetime = 3;
double float = 4;
int32 int = 5;
// int32 int = 5;
string string = 6;
google.protobuf.NullValue null = 7;
}
Expand Down
5 changes: 2 additions & 3 deletions examples/blocks/src/nypd/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,21 +109,20 @@ function set_layout_options() {
const layout_names = Object.keys(LAYOUTS);
window.layouts.innerHTML = "";
for (const layout of layout_names) {
console.log("LAYOUT: ", layout);
window.layouts.innerHTML += `<option${
layout === selected_layout ? " selected='true'" : ""
}>${layout}</option>`;
}
}

console.log("Doing?");
set_layout_options();
console.log("Done?");

window.name_input.value = layout_names[0];
window.layouts.addEventListener("change", async () => {
if (window.layouts.value.trim().length === 0) {
return;
}

window.workspace.innerHTML = "";
await window.workspace.restore(LAYOUTS[window.layouts.value]);
window.name_input.value = window.layouts.value;
Expand Down
35 changes: 28 additions & 7 deletions examples/blocks/src/nypd/layout.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@
"group_by": ["bucket(\"IncidentDate\", 'M')"],
"split_by": [],
"columns": ["ComplaintID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [],
"expressions": {
"bucket(\"IncidentDate\", 'M')": "bucket(\"IncidentDate\", 'M')"
Expand Down Expand Up @@ -130,7 +133,10 @@
"group_by": ["bucket(\"IncidentDate\", 'M')"],
"split_by": ["FADOType"],
"columns": ["AllegationID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [["AllegationID", "col asc"]],
"expressions": {
"bucket(\"IncidentDate\", 'M')": "bucket(\"IncidentDate\", 'M')"
Expand All @@ -148,7 +154,10 @@
"group_by": ["bucket(\"IncidentDate\", 'M')"],
"split_by": ["FADOType"],
"columns": ["AllegationID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [["AllegationID", "col asc"]],
"expressions": {
"bucket(\"IncidentDate\", 'M')": "bucket(\"IncidentDate\", 'M')"
Expand Down Expand Up @@ -264,7 +273,10 @@
"group_by": ["bucket(\"IncidentDate\", 'M')"],
"split_by": ["FADOType"],
"columns": ["ComplaintID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [],
"expressions": {
"bucket(\"IncidentDate\", 'M')": "bucket(\"IncidentDate\", 'M')"
Expand Down Expand Up @@ -301,7 +313,10 @@
"group_by": ["bucket(\"IncidentDate\", '3M')"],
"split_by": ["Allegation"],
"columns": ["AllegationID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [["IncidentDate", "col asc"]],
"expressions": {
"bucket(\"IncidentDate\", '3M')": "bucket(\"IncidentDate\", '3M')"
Expand Down Expand Up @@ -369,7 +384,10 @@
"group_by": ["bucket(\"IncidentDate\", 'Y')"],
"split_by": ["month_of_year(\"IncidentDate\")"],
"columns": ["ComplaintID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [],
"expressions": {
"bucket(\"IncidentDate\", 'Y')": "bucket(\"IncidentDate\", 'Y')",
Expand All @@ -388,7 +406,10 @@
"group_by": ["month_of_year(\"IncidentDate\")"],
"split_by": ["day_of_week(\"IncidentDate\")"],
"columns": ["ComplaintID"],
"filter": [["IncidentDate", ">", 473385600000]],
"filter": [
["IncidentDate", ">", "1985-01-01"],
["IncidentDate", "<", "2025-01-01"]
],
"sort": [],
"expressions": {
"bucket(\"IncidentDate\", 'Y')": "bucket(\"IncidentDate\", 'Y')",
Expand Down
17 changes: 5 additions & 12 deletions rust/perspective-client/src/rust/config/filters.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,18 @@ use ts_rs::TS;
use crate::proto;
use crate::proto::scalar;

/// This type represents the ViewConfig serializable type, which must be JSON
/// safe.
#[derive(Clone, Deserialize, Debug, PartialEq, Serialize, TS)]
#[serde(untagged)]
pub enum Scalar {
Float(f64),
String(String),
Bool(bool),
DateTime(f64),
// DateTime(i64),
// Date(String),
// Int(i32),
Null,
// // Can only have one u64 representation ...
// Date(u64)
// Int(u32)
}

impl From<&str> for Scalar {
Expand All @@ -50,7 +51,6 @@ impl Display for Scalar {
Self::Float(x) => write!(fmt, "{}", x),
Self::String(x) => write!(fmt, "{}", x),
Self::Bool(x) => write!(fmt, "{}", x),
Self::DateTime(x) => write!(fmt, "{}", x),
Self::Null => write!(fmt, ""),
}
}
Expand Down Expand Up @@ -159,10 +159,6 @@ impl From<Scalar> for proto::Scalar {
Scalar::Bool(x) => proto::Scalar {
scalar: Some(scalar::Scalar::Bool(x)),
},
// Scalar::Date(_) => todo!(),
Scalar::DateTime(x) => proto::Scalar {
scalar: Some(scalar::Scalar::Datetime(x as i64)),
},
Scalar::Null => proto::Scalar {
scalar: Some(scalar::Scalar::Null(0)),
},
Expand All @@ -175,10 +171,7 @@ impl From<proto::Scalar> for Scalar {
match value.scalar {
Some(scalar::Scalar::Bool(x)) => Scalar::Bool(x),
Some(scalar::Scalar::String(x)) => Scalar::String(x),
Some(scalar::Scalar::Int(x)) => Scalar::Float(x as f64),
Some(scalar::Scalar::Date(x)) => Scalar::DateTime(x as f64),
Some(scalar::Scalar::Float(x)) => Scalar::Float(x),
Some(scalar::Scalar::Datetime(x)) => Scalar::DateTime(x as f64),
Some(scalar::Scalar::Null(_)) => Scalar::Null,
None => Scalar::Null,
}
Expand Down
12 changes: 12 additions & 0 deletions rust/perspective-js/test/js/filters.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,18 @@ const datetime_data_local = [
table.delete();
});

test("w > date as number", async function () {
var table = await perspective.table(schema);
table.update(date_results);
var view = await table.view({
filter: [["w", ">", 1538352000000]],
});
let json = await view.to_json();
expect(json).toEqual(date_results.slice(2, 4));
view.delete();
table.delete();
});

test("w < date as string", async function () {
var table = await perspective.table(schema);
table.update(date_results);
Expand Down
Loading

0 comments on commit 9d312cb

Please sign in to comment.