Skip to content

Commit

Permalink
Merge pull request #396 from jpmorganchase/date-parser
Browse files Browse the repository at this point in the history 
C++ Date Validation
  • Loading branch information
@texodus
texodus authored Jan 23, 2019
2 parents ec82fb0 + 0c02710 commit d580705
Show file tree
Hide file tree
Showing 4 changed files with 247 additions and 110 deletions.
276 changes: 167 additions & 109 deletions packages/perspective/test/js/constructors.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,130 +626,188 @@ module.exports = perspective => {
table.delete();
});

it("Computed column of arity 0", async function() {
var table = perspective.table(data);
describe("Datetime constructors", function() {
it("Correctly parses an ISO-8601 formatted string", async function() {
let table = perspective.table({d: ["2011-10-05T14:48:00.000Z"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("datetime");
});

let table2 = table.add_computed([
{
column: "const",
type: "integer",
func: () => 1,
inputs: []
}
]);
let view = table2.view({aggregate: [{op: "count", column: "const"}]});
let result = await view.to_json();
expect(result).toEqual([{const: 1}, {const: 1}, {const: 1}, {const: 1}]);
view.delete();
table2.delete();
table.delete();
});
it("Correctly parses an ISO-8601 formatted string with timezone", async function() {
let table = perspective.table({d: ["2008-09-15T15:53:00+05:00"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("datetime");
});

it("Computed column of arity 2", async function() {
var table = perspective.table(data_3);
it("Correctly parses an RFC 2822 formatted string", async function() {
let table = perspective.table({d: ["Wed, 05 Oct 2011 22:26:12 -0400"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("datetime");
});

let table2 = table.add_computed([
{
column: "ratio",
type: "float",
func: (w, x) => w / x,
inputs: ["w", "x"]
// Not all formats covered by JS parser, test intended for C++ parser
it.skip("Correctly parses all m-d-y formatted strings", async function() {
let datestrings = ["08-15-2009", "08/15/2009", "08-15-2009", "02 28 2009", "08/15/10", "31 08 2009"];
for (let str of datestrings) {
let table = perspective.table({d: [str]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("datetime");
}
]);
let view = table2.view({aggregate: [{op: "count", column: "ratio"}]});
let result = await view.to_json();
expect(result).toEqual([{ratio: 1.5}, {ratio: 1.25}, {ratio: 1.1666666666666667}, {ratio: 1.125}]);
view.delete();
table2.delete();
table.delete();
});
});

it("Computed column of arity 2 with updates on non-dependent columns", async function() {
var meta = {
w: "float",
x: "float",
y: "string",
z: "boolean"
};
var table = perspective.table(meta, {index: "y"});
let table2 = table.add_computed([
{
column: "ratio",
type: "float",
func: (w, x) => w / x,
inputs: ["w", "x"]
}
]);
// Only implemented in the C++ date parser - skip
it.skip("Correctly parses a 'dd mm yyyy' formatted string", async function() {
let table = perspective.table({d: ["15 08 08"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("datetime");
});

table2.update(data_3);
it("Does not (for now) parse a date string in non-US formatting", async function() {
let table = perspective.table({d: ["2018/07/30"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("string");
});

let delta_upd = [{y: "a", z: false}, {y: "b", z: true}, {y: "c", z: false}, {y: "d", z: true}];
table2.update(delta_upd);
let view = table2.view({aggregate: [{op: "count", column: "y"}, {op: "count", column: "ratio"}]});
let result = await view.to_json();
let expected = [{y: "a", ratio: 1.5}, {y: "b", ratio: 1.25}, {y: "c", ratio: 1.1666666666666667}, {y: "d", ratio: 1.125}];
expect(result).toEqual(expected);
view.delete();
table2.delete();
table.delete();
it("Does not mistakenly parse a date-like string", async function() {
let table = perspective.table({d: ["Jan 14, 14"]});
let view = table.view({});
let result = await view.schema();
expect(result["d"]).toEqual("string");
});
});

it("String computed column of arity 1", async function() {
var table = perspective.table(data);

let table2 = table.add_computed([
{
column: "yes/no",
type: "string",
func: z => (z === true ? "yes" : "no"),
inputs: ["z"]
}
]);
let view = table2.view({aggregate: [{op: "count", column: "yes/no"}]});
let result = await view.to_json();
let expected = [{"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}];
expect(result).toEqual(expected);
view.delete();
table2.delete();
table.delete();
});
describe("Computed constructors", function() {
it("Computed column of arity 0", async function() {
var table = perspective.table(data);

let table2 = table.add_computed([
{
column: "const",
type: "integer",
func: () => 1,
inputs: []
}
]);
let view = table2.view({aggregate: [{op: "count", column: "const"}]});
let result = await view.to_json();
expect(result).toEqual([{const: 1}, {const: 1}, {const: 1}, {const: 1}]);
view.delete();
table2.delete();
table.delete();
});

it("Computed schema returns names and metadata", async function() {
const func = i => i + 2;
it("Computed column of arity 2", async function() {
var table = perspective.table(data_3);

let table2 = table.add_computed([
{
column: "ratio",
type: "float",
func: (w, x) => w / x,
inputs: ["w", "x"]
}
]);
let view = table2.view({aggregate: [{op: "count", column: "ratio"}]});
let result = await view.to_json();
expect(result).toEqual([{ratio: 1.5}, {ratio: 1.25}, {ratio: 1.1666666666666667}, {ratio: 1.125}]);
view.delete();
table2.delete();
table.delete();
});

const computation = {
name: "+2",
input_type: "integer",
return_type: "integer",
func: func.toString()
};
it("Computed column of arity 2 with updates on non-dependent columns", async function() {
var meta = {
w: "float",
x: "float",
y: "string",
z: "boolean"
};
var table = perspective.table(meta, {index: "y"});
let table2 = table.add_computed([
{
column: "ratio",
type: "float",
func: (w, x) => w / x,
inputs: ["w", "x"]
}
]);

table2.update(data_3);

let delta_upd = [{y: "a", z: false}, {y: "b", z: true}, {y: "c", z: false}, {y: "d", z: true}];
table2.update(delta_upd);
let view = table2.view({aggregate: [{op: "count", column: "y"}, {op: "count", column: "ratio"}]});
let result = await view.to_json();
let expected = [{y: "a", ratio: 1.5}, {y: "b", ratio: 1.25}, {y: "c", ratio: 1.1666666666666667}, {y: "d", ratio: 1.125}];
expect(result).toEqual(expected);
view.delete();
table2.delete();
table.delete();
});

const table = perspective.table(data);
it("String computed column of arity 1", async function() {
var table = perspective.table(data);

let table2 = table.add_computed([
{
column: "yes/no",
type: "string",
func: z => (z === true ? "yes" : "no"),
inputs: ["z"]
}
]);
let view = table2.view({aggregate: [{op: "count", column: "yes/no"}]});
let result = await view.to_json();
let expected = [{"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}];
expect(result).toEqual(expected);
view.delete();
table2.delete();
table.delete();
});

const table2 = table.add_computed([
{
computation: computation,
column: "plus2",
type: "integer",
inputs: ["x"],
input_type: "integer",
func: func
}
]);
it("Computed schema returns names and metadata", async function() {
const func = i => i + 2;

const result = await table2.computed_schema();
const expected = {
plus2: {
input_columns: ["x"],
const computation = {
name: "+2",
input_type: "integer",
computation: computation,
type: "integer"
}
};

expect(result).toEqual(expected);
table2.delete();
table.delete();
return_type: "integer",
func: func.toString()
};

const table = perspective.table(data);

const table2 = table.add_computed([
{
computation: computation,
column: "plus2",
type: "integer",
inputs: ["x"],
input_type: "integer",
func: func
}
]);

const result = await table2.computed_schema();
const expected = {
plus2: {
input_columns: ["x"],
input_type: "integer",
computation: computation,
type: "integer"
}
};

expect(result).toEqual(expected);
table2.delete();
table.delete();
});
});

it("Column metadata returns names and type", async function() {
Expand Down
51 changes: 51 additions & 0 deletions src/cpp/date_parser.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/******************************************************************************
*
* Copyright (c) 2019, the Perspective Authors.
*
* This file is part of the Perspective library, distributed under the terms of
* the Apache License 2.0. The full license can be found in the LICENSE file.
*
*/

#include <sstream>
#include <perspective/first.h>
#include <perspective/date_parser.h>
#include <locale>
#include <iomanip>

namespace perspective {

// Milliseconds & timezones are not currently handled
const std::string t_date_parser::VALID_FORMATS[12] = {
"%Y%m%dT%H%M%S", // ISO "%Y%m%dT%H%M%S%F%q"
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d %H:%M:%S", // ISO extended
"%A, %d %b %Y %H:%M:%S", // RFC 0822
"%Y-%m-%d\\%H:%M:%S"
"%m-%d-%Y",
"%m/%d/%Y",
"%m-%d-%Y",
"%m %d %Y",
"%m/%d/%Y",
"%m/%d/%y",
"%d %m %Y"
};

t_date_parser::t_date_parser() {}

bool
t_date_parser::is_valid(std::string const& datestring) {
for (const std::string& fmt : VALID_FORMATS) {
if (fmt != "") {
std::tm t = {};
std::stringstream ss(datestring);
ss.imbue(std::locale::classic());
ss >> std::get_time(&t, fmt.c_str());
if (!ss.fail()) {
return true;
}
}
}
return false;
}
} // end namespace perspective
2 changes: 1 addition & 1 deletion src/include/perspective/base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ void psp_abort();
ss << __FILE__ << ":" << __LINE__ << ": " << MSG << " : " \
<< perspective::get_error_str(); \
perror(ss.str().c_str()); \
PSP_ABORT(); \
psp_abort(); \
} \
}

Expand Down
28 changes: 28 additions & 0 deletions src/include/perspective/date_parser.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/******************************************************************************
*
* Copyright (c) 2019, the Perspective Authors.
*
* This file is part of the Perspective library, distributed under the terms of
* the Apache License 2.0. The full license can be found in the LICENSE file.
*
*/

#pragma once
#include <memory>
#include <vector>
#include <locale>
#include <perspective/first.h>
#include <perspective/exports.h>

namespace perspective {

class PERSPECTIVE_EXPORT t_date_parser {
public:
t_date_parser();

bool is_valid(std::string const& datestring);

private:
static const std::string VALID_FORMATS[12];
};
} // end namespace perspective

0 comments on commit d580705

Please sign in to comment.