Allow arbitrary sql expression in option data_type.

Add a test for using sql in option data_type. Slim down image generation.
weewx · May 5, 2023 · 2bf345a · 2bf345a
1 parent 9b203d6
commit 2bf345a
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 107 deletions.
diff --git a/TODO.md b/TODO.md
@@ -1,6 +1,6 @@
 # To do
 
-Test the generation of .png files.
+Almanac values for right ascension, declination, etc. need to be returned as `ValueHelper`.
 
 When doing a skin upgrade, touch only the "core" skins.
 

diff --git a/bin/weewx/tests/test_skins/StandardTest/skin.conf b/bin/weewx/tests/test_skins/StandardTest/skin.conf
@@ -140,21 +140,7 @@
     #   THE TEST SUITES DO NOT TEST IMAGE GENERATION CURRENTLY.
     #   Nevertheless, this section is left intact in case this changes later.
     # 
-    # This section lists all the images to be generated, what SQL types are to
-    # be included in them, along with many plotting options, such as color or
-    # font. There's a default for almost everything, if not specified
-    # below. Nevertheless, I've explicitly put in values, to make it easy to see
-    # and understand the options.
-    #
-    # Fonts can be anything accepted by the Python Imaging Library (PIL), which
-    # is currently truetype (.ttf), or PIL's own font format (.pil). See
-    # http://www.pythonware.com/library/pil/handbook/imagefont.htm for more
-    # details.  Note that "font size" is only used with truetype (.ttf)
-    # fonts. For others, font size is determined by the bit-mapped size, usually
-    # encoded in the file name (e.g., courB010.pil). If a font can't be found,
-    # then a default font will be used.
-    #
-
+
     image_width = 300
     image_height = 180
     image_background_color = "#f5f5f5"
@@ -241,7 +227,11 @@
         [[[daytempchill]]]
             [[[[windchill]]]]
             [[[[heatindex]]]]
-
+
+        # Test plotting a derived type without aggregation, using the main archive table
+        [[[dayhumidex]]]
+            [[[[humidex]]]]
+
         [[[dayrain]]]
             # Make sure the y-axis increment is at least 0.02 for the rain plot:
             yscale = None, None, 0.02
@@ -257,9 +247,6 @@
                 aggregate_interval = 600
                 label = Rain (cumulative)
 
-        [[[dayrx]]]
-            [[[[rxCheckPercent]]]]
-
         [[[daywind]]]
             [[[[windSpeed]]]]
             [[[[windGust]]]]
@@ -279,59 +266,21 @@
             [[[[windvec]]]]
                 plot_type = vector
 
-    [[week_images]]
-        x_label_format = %d
-        bottom_label_format = %m/%d/%y %H:%M
-        time_length = 1w
-        aggregate_type = avg
-        aggregate_interval = 1h
-
-        [[[weekbarometer]]]
-            [[[[barometer]]]]
-
-        [[[weektempdew]]]
-            [[[[outTemp]]]]
-            [[[[dewpoint]]]]
-
-        [[[weektempchill]]]
-            [[[[windchill]]]]
-            [[[[heatindex]]]]
-
-        [[[weekrain]]]
-            yscale = None, None, 0.02
-            plot_type = bar
-            [[[[rain]]]]
-                aggregate_type = sum
-                aggregate_interval = 1d
-                label = Rain (daily avg)
-
-        [[[weekrx]]]
-            [[[[rxCheckPercent]]]]
-
-        [[[weekwind]]]
-            [[[[windSpeed]]]]
-            [[[[windGust]]]]
-                aggregate_type = max
-
-        [[[weekinside]]]
-            [[[[inTemp]]]]
-
-        [[[weekwinddir]]]
-            yscale = 0.0, 360.0, 45.0
-            line_type = None
-            marker_type = box
-            marker_size = 2
-            [[[[wind]]]]
-                aggregate_type = vecdir
-
-        [[[weekwindvec]]]
-            [[[[windvec]]]]
-                plot_type = vector
+        #Test using an arbitrary expression for the type:
+        [[[daytempdiff]]]
+            [[[[diff]]]]
+            data_type = outTemp-dewpoint
+            label = Difference
+            [[[[baseline]]]]
+                data_type = 0.0
+                label = ' '
+                color = "#ff0000"
+                width = 1
 
     [[month_images]]
         x_label_format = %d
         bottom_label_format = %m/%d/%y %H:%M
-        time_length = 2592000    # == 30 days
+        time_length = 30d
         aggregate_type = avg
         aggregate_interval = 3h
 
@@ -346,6 +295,10 @@
             [[[[windchill]]]]
             [[[[heatindex]]]]
 
+        # Test plotting a derived type with aggregation, using the main archive table
+        [[[monthhumidex]]]
+            [[[[humidex]]]]
+
         [[[monthrain]]]
             yscale = None, None, 0.02
             plot_type = bar
@@ -367,9 +320,6 @@
                 aggregate_interval = 3h
                 label = 'Wasserbilanz ges.'
 
-        [[[monthrx]]]
-            [[[[rxCheckPercent]]]]
-
         [[[monthwind]]]
             [[[[windSpeed]]]]
             [[[[windGust]]]]
@@ -422,9 +372,6 @@
                 aggregate_interval = 1w
                 label = Rain (weekly avg)
 
-        [[[yearrx]]]
-            [[[[rxCheckPercent]]]]
-
         [[[yearinside]]]
             [[[[inTemp]]]]
 
@@ -440,17 +387,6 @@
             [[[[windvec]]]]
                 plot_type = vector
 
-        # A progressive vector plot of daily gust vectors overlayed
-        # with the daily wind average would look something like this:
-#        [[[yeargustvec]]]
-#            [[[[windvec]]]]
-#                plot_type = vector
-#                aggregate_type = avg
-#            [[[[windgustvec]]]]
-#                plot_type = vector
-#                aggregate_type = max
-
-
 ############################################################################################
 
 #

diff --git a/bin/weewx/tests/test_templates.py b/bin/weewx/tests/test_templates.py
@@ -195,7 +195,7 @@ def run_engine(self, stn_info, record, testtime_ts):
                                                  % (actual_filename_abs, n, actual_line,
                                                     expected_line))
 
-                        print("Checked %d lines" % n)
+                        print(f"Checked {n:d} lines in {filename_rel}")
 
 
 class TestSqlite(Common, unittest.TestCase):

diff --git a/bin/weewx/tests/testgen.conf b/bin/weewx/tests/testgen.conf
@@ -150,9 +150,8 @@ version = test
 		driver = weedb.mysql
 
     [[alt_sqlite]]
-        # Test backwards compatibility of 'root'.
         # This also tests for an absolute path for SQLITE_ROOT.
-        root = /var/tmp/weewx_test
+        SQLITE_ROOT = /var/tmp/weewx_test/archive/
         database_name = test_alt.sdb
         driver = weedb.sqlite
 

diff --git a/bin/weewx/xtypes.py b/bin/weewx/xtypes.py
@@ -192,26 +192,31 @@ def get_series(obs_type, timespan, db_manager, aggregate_type=None, aggregate_in
         else:
 
             # No aggregation
+            sql_str = "SELECT dateTime, %s, usUnits, `interval` FROM %s " \
+                      "WHERE dateTime > ? AND dateTime <= ?" % (obs_type, db_manager.table_name)
+
             std_unit_system = None
 
-            for record in db_manager.genBatchRecords(*timespan):
+            # Hit the database. It's possible the type is not in the database, so be prepared
+            # to catch a NoColumnError:
+            try:
+                for record in db_manager.genSql(sql_str, (startstamp, stopstamp)):
 
-                if std_unit_system:
-                    if std_unit_system != record['usUnits']:
-                        raise weewx.UnsupportedFeature("Unit type cannot change within a series.")
-                else:
-                    std_unit_system = record['usUnits']
-                if obs_type in record:
-                    value = record[obs_type]
-                else:
-                    try:
-                        value = get_scalar(obs_type, record, db_manager)[0]
-                    except weewx.CannotCalculate:
-                        value = None
+                    # Unpack the record
+                    timestamp, value, unit_system, interval = record
 
-                start_vec.append(record['dateTime'] - record['interval'] * 60)
-                stop_vec.append(record['dateTime'])
-                data_vec.append(value)
+                    if std_unit_system:
+                        if std_unit_system != unit_system:
+                            raise weewx.UnsupportedFeature("Unit type cannot change "
+                                                           "within an aggregation interval.")
+                    else:
+                        std_unit_system = unit_system
+                    start_vec.append(timestamp - interval * 60)
+                    stop_vec.append(timestamp)
+                    data_vec.append(value)
+            except weedb.NoColumnError:
+                # The sql type doesn't exist. Convert to an UnknownType error
+                raise weewx.UnknownType(obs_type)
 
             unit, unit_group = weewx.units.getStandardUnitType(std_unit_system, obs_type,
                                                                aggregate_type)
@@ -810,8 +815,54 @@ def get_aggregate(obs_type, timespan, aggregate_type, db_manager, **option_dict)
 
 
 class XTypeTable(XType):
-    """Calculates for xtypes. An xtype may not necessarily be in the database, so
-    this version tries to calculate aggregates on the fly."""
+    """Calculate a series for an xtype. An xtype may not necessarily be in the database, so
+    this version calculates it on the fly. Note: this version only works if no aggregation has
+    been requested."""
+
+    @staticmethod
+    def get_series(obs_type, timespan, db_manager, aggregate_type=None, aggregate_interval=None,
+                   **option_dict):
+        """Get a series of an xtype, by using the main archive table. Works only for no
+        aggregation. """
+
+        start_vec = list()
+        stop_vec = list()
+        data_vec = list()
+
+        if aggregate_type:
+            # This version does not know how to do aggregations, although this could be
+            # added in the future.
+            raise weewx.UnknownAggregation(aggregate_type)
+
+        else:
+            # No aggregation
+
+            std_unit_system = None
+
+            # Hit the database.
+            for record in db_manager.genBatchRecords(*timespan):
+
+                if std_unit_system:
+                    if std_unit_system != record['usUnits']:
+                        raise weewx.UnsupportedFeature("Unit system cannot change "
+                                                       "within a series.")
+                else:
+                    std_unit_system = record['usUnits']
+
+                # Given a record, use the xtypes system to calculate a value:
+                try:
+                    value = get_scalar(obs_type, record, db_manager)
+                    data_vec.append(value[0])
+                except weewx.CannotCalculate:
+                    data_vec.append(None)
+                start_vec.append(record['dateTime'] - record['interval'] * 60)
+                stop_vec.append(record['dateTime'])
+
+            unit, unit_group = weewx.units.getStandardUnitType(std_unit_system, obs_type)
+
+        return (ValueTuple(start_vec, 'unix_epoch', 'group_time'),
+                ValueTuple(stop_vec, 'unix_epoch', 'group_time'),
+                ValueTuple(data_vec, unit, unit_group))
 
     @staticmethod
     def get_aggregate(obs_type, timespan, aggregate_type, db_manager, **option_dict):