--- a/buildbot/db/build_data.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/build_data.py 2023-09-24 22:28:21.778583084 +0200 @@ -159,6 +159,15 @@ class BuildDataConnectorComponent(base.D res = yield self.db.pool.do(thd) return res + @defer.inlineCallbacks + def pruneBuildData(self, buildid): + def thd(conn): + tbl = self.db.model.build_data + q = tbl.delete() + q = q.where(tbl.c.buildid == buildid) + conn.execute(q) + yield self.db.pool.do(thd) + def _row2dict(self, conn, row): return BuildDataDict(buildid=row.buildid, name=row.name, --- a/buildbot/db/builds.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/builds.py 2023-09-24 15:31:20.598023751 +0200 @@ -206,6 +206,15 @@ class BuildsConnectorComponent(base.DBCo results=results) return self.db.pool.do(thd) + @defer.inlineCallbacks + def pruneBuild(self, buildid): + def thd(conn): + tbl = self.db.model.builds + q = tbl.delete() + q = q.where(tbl.c.id == buildid) + conn.execute(q) + yield self.db.pool.do(thd) + # returns a Deferred that returns a value def getBuildProperties(self, bid, resultSpec=None): def thd(conn): @@ -251,6 +260,15 @@ class BuildsConnectorComponent(base.DBCo {"value": value_js, "source": source}) yield self.db.pool.do(thd) + @defer.inlineCallbacks + def pruneBuildProperties(self, buildid): + def thd(conn): + bp_tbl = self.db.model.build_properties + q = bp_tbl.delete() + q = q.where(bp_tbl.c.buildid == buildid) + conn.execute(q) + yield self.db.pool.do(thd) + def _builddictFromRow(self, row): return { "id": row.id, --- a/buildbot/db/changes.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/changes.py 2023-09-24 13:53:28.166432187 +0200 @@ -333,6 +334,40 @@ class ChangesConnectorComponent(base.DBC table.delete(table.c.changeid.in_(batch))) yield self.db.pool.do(thd) + @defer.inlineCallbacks + def pruneChangesId(self, revision): + """ + Called periodically by DBConnector, this method deletes changes + than C{changeHorizon}. + """ + + def thd(conn): + changes_tbl = self.db.model.changes + ids_to_delete = [] + # First, get the list of changes to delete. This could be written + # as a subquery but then that subquery would be run for every + # table, which is very inefficient; also, MySQL's subquery support + # leaves much to be desired, and doesn't support this particular + # form. + q = changes_tbl.select() + q = q.where(changes_tbl.c.revision == revision) + res = conn.execute(q) + row = res.fetchone() + if row is not None: + ids_to_delete.append(row.changeid) + + # and delete from all relevant tables, in dependency order + for table_name in ('scheduler_changes', 'change_files', + 'change_properties', 'changes', 'change_users'): + remaining = ids_to_delete[:] + while remaining: + batch, remaining = remaining[:100], remaining[100:] + table = self.db.model.metadata.tables[table_name] + conn.execute( + table.delete(table.c.changeid.in_(batch))) + yield self.db.pool.do(thd) + + def _chdict_from_change_row_thd(self, conn, ch_row): # This method must be run in a db.pool thread, and returns a chdict # given a row from the 'changes' table --- a/buildbot/db/logs.py 2022-04-02 11:10:34.892310594 +0200 +++ b/buildbot/db/logs.py 2023-06-26 23:06:24.611959431 +0200 @@ -410,3 +410,80 @@ rv = dict(row) rv['complete'] = bool(rv['complete']) return rv + + # returns a Deferred that returns a value + def deleteLogChunks(self, buildid): + model = self.db.model + horizon_per_builder = False + + def countLogchunks(conn): + res = conn.execute(sa.select([sa.func.count(model.logchunks.c.logid)])) + count = res.fetchone()[0] + res.close() + return count + + # find the steps.id at the upper bound of steps + def getStepidMax(conn, buildid): + # N.B.: we utilize the fact that steps.id is auto-increment, thus steps.started_at + # times are effectively sorted and we only need to find the steps.id at the upper + # bound of steps to update. + + # SELECT steps.id from steps WHERE steps.buildid = buildid ORDER BY + # steps.id DESC LIMIT 1; + res = conn.execute( + sa.select([model.steps.c.id]) + .where(model.steps.c.buildid == buildid) + .order_by(model.steps.c.id.desc()) + .limit(1) + ) + res_list = res.fetchone() + stepid_max = None + if res_list: + stepid_max = res_list[0] + res.close() + return stepid_max + + # query all logs with type 'd' and delete their chunks. + def deleteLogsWithTypeD(conn): + if self.db._engine.dialect.name == 'sqlite': + # sqlite does not support delete with a join, so for this case we use a subquery, + # which is much slower + q = sa.select([model.logs.c.id]) + q = q.select_from(model.logs) + q = q.where(model.logs.c.type == 'd') + + # delete their logchunks + q = model.logchunks.delete().where(model.logchunks.c.logid.in_(q)) + else: + q = model.logchunks.delete() + q = q.where(model.logs.c.id == model.logchunks.c.logid) + q = q.where(model.logs.c.type == 'd') + + res = conn.execute(q) + res.close() + + def thddeleteLogs(conn): + count_before = countLogchunks(conn) + + # update log types that match buildid + # we do it first to avoid having UI discrepancy + + stepid_max = getStepidMax(conn, buildid) + if stepid_max: + # UPDATE logs SET logs.type = 'd' + # WHERE logs.stepid <= stepid_max AND type != 'd'; + res = conn.execute( + model.logs.update() + .where(sa.and_(model.logs.c.stepid <= stepid_max, + model.logs.c.type != 'd')) + .values(type='d') + ) + res.close() + + deleteLogsWithTypeD(conn) + + count_after = countLogchunks(conn) + count = count_before - count_after + + return count if count > 0 else 0 + return self.db.pool.do(thddeleteLogs)