Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use b4 to generate release notes from PR #1

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Use b4 to generate release notes from PR #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5b61ea8
pr: support output to stdout
amir73il Mar 12, 2022
925fab9
pr: support pr-tracker reply msgid
amir73il Mar 14, 2022
5c27ba5
mbox: try harder to find cover letter
amir73il Mar 29, 2022
1cef0cd
mbox: fix calling arguments to get_extra_series()
amir73il Mar 30, 2022
0127e9e
mbox: query latest revision of patch series by patch subject
amir73il Mar 30, 2022
fb46481
mbox: cache get_extra_series() query results
amir73il Mar 30, 2022
f596636
Introduce command "b4 rn" to produce release for pull requests
amir73il Mar 11, 2022
1bfa15f
rn: generate release notes in reStructuredText format
amir73il Apr 10, 2022
3ea3360
rn: print references to fstests in release notes
amir73il Apr 10, 2022
369aade
rn: add release notes for patches not posted
amir73il Apr 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions b4/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ def _dkim_log_filter(record):
'linkmask': LOREADDR + '/r/%s',
'trailer-order': DEFAULT_TRAILER_ORDER,
'listid-preference': '*.feeds.kernel.org,*.linux.dev,*.kernel.org,*',
'pr-tracker-email': 'pr-tracker-bot@kernel.org',
'save-maildirs': 'no',
# off: do not bother checking attestation
# check: print an attaboy when attestation is found
Expand Down Expand Up @@ -274,7 +275,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True):
for member in lser.patches:
if member is not None and member.in_reply_to is not None:
potential = self.get_by_msgid(member.in_reply_to)
if potential is not None and potential.has_diffstat and not potential.has_diff:
if potential is not None and potential.maybe_cover():
# This is *probably* the cover letter
lser.patches[0] = potential
lser.has_cover = True
Expand Down Expand Up @@ -350,7 +351,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True):

return lser

def add_message(self, msg):
def add_message(self, msg, needcover=False):
msgid = LoreMessage.get_clean_msgid(msg)
if msgid in self.msgid_map:
logger.debug('Already have a message with this msgid, skipping %s', msgid)
Expand All @@ -366,20 +367,20 @@ def add_message(self, msg):
self.followups.append(lmsg)
return

if lmsg.counter == 0 and (not lmsg.counters_inferred or lmsg.has_diffstat):
if lmsg.likely_cover():
# Cover letter
# Add it to covers -- we'll deal with them later
logger.debug(' adding as v%s cover letter', lmsg.revision)
self.covers[lmsg.revision] = lmsg
return

if lmsg.has_diff:
if lmsg.revision not in self.series:
if lmsg.revision not in self.series or needcover:
if lmsg.revision_inferred and lmsg.in_reply_to:
# We have an inferred revision here.
# Do we have an upthread cover letter that specifies a revision?
irt = self.get_by_msgid(lmsg.in_reply_to)
if irt is not None and irt.has_diffstat and not irt.has_diff:
if irt is not None and irt.likely_cover():
# Yes, this is very likely our cover letter
logger.debug(' fixed revision to v%s', irt.revision)
lmsg.revision = irt.revision
Expand Down Expand Up @@ -860,6 +861,7 @@ def __init__(self, msg):
self.pr_repo = None
self.pr_ref = None
self.pr_tip_commit = None
self.pr_merge_commit = None
self.pr_remote_tip_commit = None

# Patchwork hash
Expand Down Expand Up @@ -966,6 +968,12 @@ def __init__(self, msg):
if trailer[0].lower() not in badtrailers:
self.trailers.append(trailer)

def maybe_cover(self):
return self.has_diffstat and not self.has_diff

def likely_cover(self):
return self.counter == 0 and (not self.counters_inferred or self.mayby_cover())

def get_trailers(self, sloppy=False):
trailers = list()
mismatches = set()
Expand Down Expand Up @@ -1285,6 +1293,17 @@ def get_clean_msgid(msg, header='Message-Id'):
msgid = matches.groups()[0]
return msgid

# Get commit id from git am formatted patch
@staticmethod
def get_commit_id(msg):
commitid = None
unixhdr = msg.get_unixfrom()
if unixhdr:
matches = re.search(r'^From ([0-9a-f]+)', unixhdr)
if matches:
commitid = matches.groups()[0]
return commitid

@staticmethod
def get_preferred_duplicate(msg1, msg2):
config = get_main_config()
Expand Down Expand Up @@ -2047,7 +2066,7 @@ def get_cache_dir(appname: str = 'b4') -> str:
fullpath = os.path.join(cachedir, entry)
st = os.stat(fullpath)
if st.st_mtime < expage:
logger.debug('Cleaning up cache: %s', entry)
logger.debug('Cleaning up cache: %s mtime=%d < %d', entry, st.st_mtime, expage)
if os.path.isdir(fullpath):
shutil.rmtree(fullpath)
else:
Expand Down
16 changes: 16 additions & 0 deletions b4/command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ def cmd_diff(cmdargs):
b4.diff.main(cmdargs)


def cmd_rn(cmdargs):
import b4.rn
b4.rn.main(cmdargs)


def cmd():
# noinspection PyTypeChecker
parser = argparse.ArgumentParser(
Expand Down Expand Up @@ -191,6 +196,8 @@ def cmd():
'the identity must match a [sendemail "identity"] config section'))
sp_pr.add_argument('--dry-run', dest='dryrun', action='store_true', default=False,
help='Force a --dry-run on git-send-email invocation (use with -s)')
sp_pr.add_argument('--no-cover', dest='nocover', action='store_true', default=False,
help='Do not save the cover letter (on by default when using -o -)')
sp_pr.add_argument('msgid', nargs='?',
help='Message ID to process, or pipe a raw message')
sp_pr.set_defaults(func=cmd_pr)
Expand Down Expand Up @@ -248,6 +255,15 @@ def cmd():
help='Show all developer keys found in a thread')
sp_kr.set_defaults(func=cmd_kr)

# b4 rn
sp_rn = subparsers.add_parser('rn', help='Generate release notes from pull request')
cmd_retrieval_common_opts(sp_rn)
sp_rn.add_argument('-g', '--gitdir', default=None,
help='Operate on this git tree instead of current dir')
sp_rn.add_argument('-o', '--output-file', dest='outfile', default=None,
help='Write release notes into this file instead of outputting to stdout')
sp_rn.set_defaults(func=cmd_rn)

cmdargs = parser.parse_args()

logger.setLevel(logging.DEBUG)
Expand Down
3 changes: 2 additions & 1 deletion b4/diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ def diff_same_thread_series(cmdargs):
if not msgs:
logger.critical('Unable to retrieve thread: %s', msgid)
return
msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers, useproject=cmdargs.useproject)
msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers,
nocache=cmdargs.nocache, useproject=cmdargs.useproject)
if os.path.exists(cachedir):
shutil.rmtree(cachedir)
pathlib.Path(cachedir).mkdir(parents=True)
Expand Down
76 changes: 63 additions & 13 deletions b4/mbox.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,11 +479,15 @@ def save_as_quilt(am_msgs, q_dirname):
for patch_filename in patch_filenames:
sfh.write('%s\n' % patch_filename)


def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = None, nocache: bool = False,
useproject: Optional[str] = None) -> list:
base_msg = None
latest_revision = None
# Get older/newer revisions of a patch series from public-inbox
#
# @direction:
# 1 - look for newer revisions (default)
# -1 - look for older revisions
# 0 - look for latest revision in public-inbox (regardless of @base_msg)
def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = None, nocache: bool = False,
base_msg = None, useproject: Optional[str] = None) -> list:
latest_revision = 0
seen_msgids = set()
seen_covers = set()
obsoleted = list()
Expand Down Expand Up @@ -521,6 +525,36 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
logger.debug('Could not find cover of 1st patch in mbox')
return msgs

# For query by @base_msg, check if we have a cache of this lookup
base_msgid = b4.LoreMessage.get_clean_msgid(base_msg)
identifier = base_msgid
# Use commit id as key to cache of git am formatted @base_msg
if not identifier:
identifier = b4.LoreMessage.get_commit_id(base_msg)
if identifier is None:
logger.debug('Could not find find base msgid for series')
return msgs

cachedir = None
if identifier and len(msgs) == 0 and not wantvers:
if useproject:
identifier += '-' + useproject
if direction > 0:
identifier += '+'
elif direction < 0:
identifier += '-'
cachedir = b4.get_cache_file(identifier, suffix='extra.msgs')

if cachedir and os.path.exists(cachedir) and not nocache:
logger.debug('Using cached copy of %s at %s', identifier, cachedir)
msgs = list()
for msg in os.listdir(cachedir):
with open(os.path.join(cachedir, msg), 'rb') as fh:
msgs.append(email.message_from_binary_file(fh))
return msgs
else:
logger.debug('No cached copy for %s', identifier)

config = b4.get_main_config()
loc = urllib.parse.urlparse(config['midmask'])
if not useproject:
Expand Down Expand Up @@ -552,7 +586,7 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
else:
# Get subject info from base_msg again
lsub = b4.LoreSubject(base_msg['Subject'])
if not len(lsub.prefixes):
if direction > 0 and not len(lsub.prefixes):
logger.debug('Not checking for new revisions: no prefixes on the cover letter.')
return msgs
if direction < 0 and latest_revision <= 1:
Expand All @@ -561,18 +595,22 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
if direction < 0 and wantvers is None:
wantvers = [latest_revision - 1]

base_msgid = b4.LoreMessage.get_clean_msgid(base_msg)
fromeml = email.utils.getaddresses(base_msg.get_all('from', []))[0][1]
msgdate = email.utils.parsedate_tz(str(base_msg['Date']))
startdate = time.strftime('%Y%m%d', msgdate[:9])
if direction > 0:
q = 's:"%s" AND f:"%s" AND d:%s..' % (lsub.subject.replace('"', ''), fromeml, startdate)
queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '-1'}))
logger.critical('Checking for newer revisions on %s', listarc)
else:
elif direction < 0:
q = 's:"%s" AND f:"%s" AND d:..%s' % (lsub.subject.replace('"', ''), fromeml, startdate)
queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '1'}))
logger.critical('Checking for older revisions on %s', listarc)
else:
# Find latest revision in public-inbox to match base_msg subject
q = 's:"%s"' % (lsub.subject.replace('"', ''))
queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A'}))
logger.debug('Checking for revisions on %s', listarc)

logger.debug('Query URL: %s', queryurl)
session = b4.get_requests_session()
Expand All @@ -592,11 +630,11 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
for entry in entries:
title = entry.find('atom:title', ns).text
lsub = b4.LoreSubject(title)
if lsub.reply or lsub.counter > 1:
if lsub.reply or (direction != 0 and lsub.counter > 1):
logger.debug('Ignoring result (not interesting): %s', title)
continue
link = entry.find('atom:link', ns).get('href')
if direction > 0 and lsub.revision <= latest_revision:
if direction >= 0 and lsub.revision <= latest_revision:
logger.debug('Ignoring result (not new revision): %s', title)
continue
elif direction < 0 and lsub.revision >= latest_revision:
Expand All @@ -610,13 +648,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
continue
if lsub.revision == 1 and lsub.revision == latest_revision:
# Someone sent a separate message with an identical title but no new vX in the subject line
if direction > 0:
if direction >= 0:
# It's *probably* a new revision.
logger.debug('Likely a new revision: %s', title)
else:
# It's *probably* an older revision.
logger.debug('Likely an older revision: %s', title)
elif direction > 0 and lsub.revision > latest_revision:
elif direction >= 0 and lsub.revision > latest_revision:
logger.debug('Definitely a new revision [v%s]: %s', lsub.revision, title)
elif direction < 0 and lsub.revision < latest_revision:
logger.debug('Definitely an older revision [v%s]: %s', lsub.revision, title)
Expand All @@ -633,6 +671,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
nt_msgs += potentials
logger.info(' Added %s messages from that thread', len(potentials))

# Write results of @base_msg query to cache
if cachedir:
if os.path.exists(cachedir):
shutil.rmtree(cachedir)
pathlib.Path(cachedir).mkdir(parents=True)
at = 0

# Append all of these to the existing mailbox
for nt_msg in nt_msgs:
nt_msgid = b4.LoreMessage.get_clean_msgid(nt_msg)
Expand All @@ -643,6 +688,10 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
logger.debug('Adding: %s', nt_subject)
msgs.append(nt_msg)
seen_msgids.add(nt_msgid)
if cachedir:
with open(os.path.join(cachedir, '%04d' % at), 'wb') as fh:
fh.write(nt_msg.as_bytes(policy=b4.emlpolicy))
at += 1

return msgs

Expand Down Expand Up @@ -725,7 +774,8 @@ def main(cmdargs):
return

if len(msgs) and cmdargs.checknewer:
msgs = get_extra_series(msgs, direction=1, useproject=cmdargs.useproject)
msgs = get_extra_series(msgs, direction=1, nocache=cmdargs.nocache,
useproject=cmdargs.useproject)

if cmdargs.subcmd in ('am', 'shazam'):
make_am(msgs, cmdargs, msgid)
Expand Down
Loading