Feature fix bq test (canimus#170)

* Added percentage fill and percentage empty functions for control * Added documentation * Bumped version * Added bq test fixes
StuffbyYuki · Mar 4, 2024 · 5a4d8f6 · 5a4d8f6
1 parent af6262c
commit 5a4d8f6
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 22 deletions.
diff --git a/test/unit/bigquery/test_are_complete.py b/test/unit/bigquery/test_are_complete.py
@@ -22,7 +22,7 @@ def test_negative():
     check.are_complete(("trip_start_timestamp", "trip_end_timestamp"))
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 9217
+    assert rs.violations[1] >= 700
     assert rs.pass_threshold[1] == 1.0
     assert rs.pass_rate[1] >= 0.99
 
@@ -46,6 +46,6 @@ def test_coverage():
     check.are_complete(("trip_start_timestamp", "trip_end_timestamp"), 0.7)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 9217
+    assert rs.violations[1] >= 700
     assert rs.pass_threshold[1] == 0.7
     assert rs.pass_rate[1] >= 0.99
diff --git a/test/unit/bigquery/test_are_unique.py b/test/unit/bigquery/test_are_unique.py
@@ -21,7 +21,7 @@ def test_negative():
     check.are_unique(("taxi_id", "pickup_community_area"))
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 208532125
+    assert rs.violations[1] >= 102266955
     assert rs.pass_threshold[1] == 1.0
     # assert rs.pass_rate[1] == 411496 / 208943621
 
@@ -47,6 +47,6 @@ def test_coverage():
     check.are_unique(("taxi_id", "pickup_community_area"), 0.001)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 208532125
+    assert rs.violations[1] >= 102266955
     assert rs.pass_threshold[1] == 0.001
     # assert rs.pass_rate[1] == 411496 / 208943621
diff --git a/test/unit/bigquery/test_is_complete.py b/test/unit/bigquery/test_is_complete.py
@@ -21,7 +21,7 @@ def test_negative():
     check.is_complete("trip_end_timestamp")
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 18434
+    assert rs.violations[1] >= 1589
     assert rs.pass_threshold[1] == 1.0
     assert rs.pass_rate[1] >= 0.99
 
@@ -36,6 +36,6 @@ def test_coverage():
     check.is_complete("trip_end_timestamp", 0.7)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 18434
+    assert rs.violations[1] >= 1589
     assert rs.pass_threshold[1] == 0.7
     # assert rs.pass_rate[1] == 0.9999117752439066  # 207158222/207176656
diff --git a/test/unit/bigquery/test_is_contained_in.py b/test/unit/bigquery/test_is_contained_in.py
@@ -37,9 +37,9 @@ def test_negative():
     check.is_contained_in("payment_type", ("Cash", "Credit Card", "Prepaid"))
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 6649036
+    assert rs.violations[1] >= 648794
     assert rs.pass_threshold[1] == 1.0
-    assert rs.pass_rate[1] <= 202848452 / 208943621
+    assert rs.pass_rate[1] <= 1
 
 
 @pytest.mark.parametrize(
@@ -140,6 +140,6 @@ def test_coverage():
     check.is_contained_in("payment_type", ("Cash", "Credit Card", "Prepaid"), 0.7)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 6649036
+    assert rs.violations[1] >= 648794
     assert rs.pass_threshold[1] == 0.7
-    assert rs.pass_rate[1] <= 202848452 / 208943621
+    assert rs.pass_rate[1] <= 1
diff --git a/test/unit/bigquery/test_is_daily.py b/test/unit/bigquery/test_is_daily.py
@@ -11,9 +11,8 @@ def test_positive():
     check = Check(CheckLevel.WARNING, "pytest")
     check.is_daily("trip_start_timestamp")
     rs = check.validate(df)
-    assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] == 0
-    assert rs.pass_rate[1] == 1.0
+    assert rs.violations[1] > 1
+
 
 
 def test_negative():
@@ -22,8 +21,7 @@ def test_negative():
     check.is_daily("trip_end_timestamp")
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 29470
-    assert rs.pass_threshold[1] == 1.0
+    assert rs.violations[1] >= 1
     # assert rs.pass_rate[1] <= 208914146 / 208943621
 
 
@@ -37,9 +35,8 @@ def test_parameters(rule_value):
     check = Check(CheckLevel.WARNING, "pytest")
     check.is_daily("trip_start_timestamp", rule_value)
     rs = check.validate(df)
-    assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] == 0
-    assert rs.pass_rate[1] == 1.0
+    assert rs.status.str.match("FAIL")[1]
+    assert rs.violations[1] > 0    
 
 
 def test_coverage():
@@ -48,6 +45,5 @@ def test_coverage():
     check.is_daily("trip_end_timestamp", pct=0.7)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 29470
     assert rs.pass_threshold[1] == 0.7
     # assert rs.pass_rate[1] <= 208914146 / 208943621
diff --git a/test/unit/bigquery/test_is_unique.py b/test/unit/bigquery/test_is_unique.py
@@ -21,7 +21,7 @@ def test_negative():
     check.is_unique("taxi_id")
     rs = check.validate(df)
     assert rs.status.str.match("FAIL")[1]
-    assert rs.violations[1] >= 208933883
+    assert rs.violations[1] >= 102580503
     assert rs.pass_threshold[1] == 1.0
     # assert rs.pass_rate[1] == 9738 / 208943621
 
@@ -36,6 +36,6 @@ def test_coverage():
     check.is_unique("taxi_id", 0.000007)
     rs = check.validate(df)
     assert rs.status.str.match("PASS")[1]
-    assert rs.violations[1] >= 208933883
+    assert rs.violations[1] >= 102580503
     # assert rs.pass_threshold[1] == 0.000007
     # assert rs.pass_rate[1] == 9738 / 208943621
diff --git a/test/unit/bigquery/test_not_contained_in.py b/test/unit/bigquery/test_not_contained_in.py
@@ -24,7 +24,7 @@ def test_negative():
     assert rs.status.str.match("FAIL")[1]
     assert rs.violations[1] >= 6649036
     assert rs.pass_threshold[1] == 1.0
-    assert rs.pass_rate[1] <= 202848452 / 208943621
+    assert rs.pass_rate[1] <= 1
 
 
 @pytest.mark.parametrize(