Rest: Filter buildsets by sourcestamp attributes

Allow the buildsets REST endpoint to filter by sourcestamp attributes. We use the 'contains' relation to filter by a set of sourcestamp attributes: buildsets?sourcestamps__contains={"ssid":26} The list of buildsets is then only listing buildsets that have a sourcetsamp with that ssid. If the sourcestamps__contains filter is used multiple times it follows the "or" semantics like the other filters. If a single filter lists multiple attributes these follow and semantics, within the filter. The filters are implemented on SQL-level. Necessary joins are only performed if this filter is actually used.
buildbot · May 14, 2022 · c987817 · c987817
1 parent da9092c
commit c987817
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 2 deletions.
diff --git a/master/buildbot/data/buildsets.py b/master/buildbot/data/buildsets.py
@@ -65,7 +65,8 @@ def getSs(ssid):
         'complete_at': 'buildsets.complete_at',
         'results': 'buildsets.results',
         'parent_buildid': 'buildsets.parent_buildid',
-        'parent_relationship': 'buildsets.parent_relationship'
+        'parent_relationship': 'buildsets.parent_relationship',
+        'ssids': 'buildset_sourcestamps.sourcestampid',
     }
 
 

diff --git a/master/buildbot/data/sourcestamps.py b/master/buildbot/data/sourcestamps.py
@@ -15,10 +15,12 @@
 
 
 from twisted.internet import defer
+import json
 
 from buildbot.data import base
 from buildbot.data import patches
 from buildbot.data import types
+from buildbot.util import bytes2unicode
 
 
 def _db2data(ss):
@@ -89,4 +91,12 @@ class EntityType(types.Entity):
         codebase = types.String()
         patch = types.NoneOk(patches.Patch.entityType)
         created_at = types.DateTime()
+
+        def valueFromString(self, arg):
+            d = json.loads(bytes2unicode(arg))
+            d = {k:self.fields[k].valueFromString(v) for k,v in d.items()}
+            if len(d.keys()) == 0:
+                raise TypeError  # don't allow {}
+            return d
+
     entityType = EntityType(name, 'Sourcestamp')
diff --git a/master/buildbot/db/buildsets.py b/master/buildbot/db/buildsets.py
@@ -16,6 +16,8 @@
 Support for buildsets in the database
 """
 
+from functools import reduce
+from operator import and_, or_
 import json
 
 import sqlalchemy as sa
@@ -165,15 +167,32 @@ def thd(conn):
     @defer.inlineCallbacks
     def getBuildsets(self, complete=None, resultSpec=None):
         def thd(conn):
+            if resultSpec is not None:
+                sourceStampFilter = resultSpec.popFilter('sourcestamps', 'contains')
+
             bs_tbl = self.db.model.buildsets
-            q = bs_tbl.select()
+            ss_tbl = self.db.model.sourcestamps
+
+            j = bs_tbl
+            if sourceStampFilter is not None:
+                j = bs_tbl
+                j = j.join(self.db.model.buildset_sourcestamps)
+                j = j.join(ss_tbl)
+            q = sa.select(columns=[bs_tbl], from_obj=[j], distinct=True)
+
             if complete is not None:
                 if complete:
                     q = q.where(bs_tbl.c.complete != 0)
                 else:
                     q = q.where((bs_tbl.c.complete == 0) |
                                 (bs_tbl.c.complete == NULL))
+
             if resultSpec is not None:
+                if sourceStampFilter:
+                    def ssid_as_id(key):
+                        return "id" if key == "ssid" else key
+                    q = q.where(reduce(or_, [reduce(and_, [ss_tbl.c[ssid_as_id(key)] == value for key, value in filter.items()])
+                                                                                              for filter in sourceStampFilter]))
                 return resultSpec.thd_execute(conn, q, lambda x: self._thd_row2dict(conn, x))
             res = conn.execute(q)
             return [self._thd_row2dict(conn, row) for row in res.fetchall()]