Rest: Filter buildsets by sourcestamp attributes

Allow the buildsets REST endpoint to filter by sourcestamp attributes. We use the 'contains' relation to filter by a set of sourcestamp attributes: buildsets?sourcestamps__contains={"ssid":26} The list of buildsets is then only listing buildsets that have a sourcestamp with that ssid. If the sourcestamps__contains filter is used multiple times it follows the "or" semantics like the other filters. If a single filter lists multiple attributes these follow and semantics, within the filter. sourcestamps are encoded as JSON in the REST query. The filters are implemented on SQL-level. Necessary joins are only performed if this filter is actually used.
buildbot · May 17, 2022 · dc4ae9c · dc4ae9c
1 parent 145a781
commit dc4ae9c
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 1 deletion.
diff --git a/master/buildbot/data/sourcestamps.py b/master/buildbot/data/sourcestamps.py
@@ -14,11 +14,14 @@
 # Copyright Buildbot Team Members
 
 
+import json
+
 from twisted.internet import defer
 
 from buildbot.data import base
 from buildbot.data import patches
 from buildbot.data import types
+from buildbot.util import bytes2unicode
 
 
 def _db2data(ss):
@@ -89,4 +92,12 @@ class EntityType(types.Entity):
         codebase = types.String()
         patch = types.NoneOk(patches.Patch.entityType)
         created_at = types.DateTime()
+
+        def valueFromString(self, arg):
+            d = json.loads(bytes2unicode(arg))
+            d = {k: self.fields[k].valueFromString(v) for k, v in d.items()}
+            if len(d.keys()) == 0:
+                raise TypeError  # don't allow {}
+            return d
+
     entityType = EntityType(name, 'Sourcestamp')
diff --git a/master/buildbot/db/buildsets.py b/master/buildbot/db/buildsets.py
@@ -17,6 +17,9 @@
 """
 
 import json
+from functools import reduce
+from operator import and_
+from operator import or_
 
 import sqlalchemy as sa
 
@@ -165,15 +168,35 @@ def thd(conn):
     @defer.inlineCallbacks
     def getBuildsets(self, complete=None, resultSpec=None):
         def thd(conn):
+            if resultSpec is not None:
+                sourceStampFilter = resultSpec.popFilter('sourcestamps', 'contains')
+            else:
+                sourceStampFilter = None
+
             bs_tbl = self.db.model.buildsets
-            q = bs_tbl.select()
+            ss_tbl = self.db.model.sourcestamps
+
+            j = bs_tbl
+            if sourceStampFilter is not None:
+                j = bs_tbl
+                j = j.join(self.db.model.buildset_sourcestamps)
+                j = j.join(ss_tbl)
+            q = sa.select(columns=[bs_tbl], from_obj=[j], distinct=True)
+
             if complete is not None:
                 if complete:
                     q = q.where(bs_tbl.c.complete != 0)
                 else:
                     q = q.where((bs_tbl.c.complete == 0) |
                                 (bs_tbl.c.complete == NULL))
+
             if resultSpec is not None:
+                if sourceStampFilter:
+                    def ssid_as_id(key):
+                        return "id" if key == "ssid" else key
+                    q = q.where(reduce(or_, [reduce(and_, [ss_tbl.c[ssid_as_id(key)] == value
+                                for key, value in filter.items()])
+                                for filter in sourceStampFilter]))
                 return resultSpec.thd_execute(conn, q, lambda x: self._thd_row2dict(conn, x))
             res = conn.execute(q)
             return [self._thd_row2dict(conn, row) for row in res.fetchall()]