# SPDX-License-Identifier: GPL-2.0+ # # Copyright 2020 Google LLC # """Talks to the patchwork service to figure out what patches have been reviewed and commented on. Provides a way to display review tags and comments. Allows creation of a new branch based on the old but with the review tags collected from patchwork. """ import asyncio from collections import defaultdict import concurrent.futures from itertools import repeat import aiohttp import pygit2 from u_boot_pylib import terminal from u_boot_pylib import tout from patman import patchstream from patman import patchwork def to_int(vals): """Convert a list of strings into integers, using 0 if not an integer Args: vals (list): List of strings Returns: list: List of integers, one for each input string """ out = [int(val) if val.isdigit() else 0 for val in vals] return out def process_reviews(content, comment_data, base_rtags): """Process and return review data Args: content (str): Content text of the patch itself - see pwork.get_patch() comment_data (list of dict): Comments for the patch - see pwork._get_patch_comments() base_rtags (dict): base review tags (before any comments) key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string Return: tuple: dict: new review tags (noticed since the base_rtags) key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string list of patchwork.Review: reviews received on the patch """ pstrm = patchstream.PatchStream.process_text(content, True) rtags = defaultdict(set) for response, people in pstrm.commit.rtags.items(): rtags[response].update(people) reviews = [] for comment in comment_data: pstrm = patchstream.PatchStream.process_text(comment['content'], True) if pstrm.snippets: submitter = comment['submitter'] person = f"{submitter['name']} <{submitter['email']}>" reviews.append(patchwork.Review(person, pstrm.snippets)) for response, people in pstrm.commit.rtags.items(): rtags[response].update(people) # Find the tags that are not in the commit new_rtags = defaultdict(set) for tag, people in rtags.items(): for who in people: is_new = (tag not in base_rtags or who not in base_rtags[tag]) if is_new: new_rtags[tag].add(who) return new_rtags, reviews def compare_with_series(series, patches): """Compare a list of patches with a series it came from This prints any problems as warnings Args: series (Series): Series to compare against patches (:type: list of Patch): list of Patch objects to compare with Returns: tuple dict: key: Commit number (0...n-1) value: Patch object for that commit dict: key: Patch number (0...n-1) value: Commit object for that patch """ # Check the names match warnings = [] patch_for_commit = {} all_patches = set(patches) for seq, cmt in enumerate(series.commits): pmatch = [p for p in all_patches if p.subject == cmt.subject] if len(pmatch) == 1: patch_for_commit[seq] = pmatch[0] all_patches.remove(pmatch[0]) elif len(pmatch) > 1: warnings.append("Multiple patches match commit %d ('%s'):\n %s" % (seq + 1, cmt.subject, '\n '.join([p.subject for p in pmatch]))) else: warnings.append("Cannot find patch for commit %d ('%s')" % (seq + 1, cmt.subject)) # Check the names match commit_for_patch = {} all_commits = set(series.commits) for seq, patch in enumerate(patches): cmatch = [c for c in all_commits if c.subject == patch.subject] if len(cmatch) == 1: commit_for_patch[seq] = cmatch[0] all_commits.remove(cmatch[0]) elif len(cmatch) > 1: warnings.append("Multiple commits match patch %d ('%s'):\n %s" % (seq + 1, patch.subject, '\n '.join([c.subject for c in cmatch]))) else: warnings.append("Cannot find commit for patch %d ('%s')" % (seq + 1, patch.subject)) return patch_for_commit, commit_for_patch, warnings def collect_patches(series_id, pwork): """Collect patch information about a series from patchwork Uses the Patchwork REST API to collect information provided by patchwork about the status of each patch. Args: series_id (str): Patch series ID number pwork (Patchwork): Patchwork object to use for reading Returns: list of Patch: List of patches sorted by sequence number Raises: ValueError: if the URL could not be read or the web page does not follow the expected structure """ data = pwork.request('series/%s/' % series_id) # Get all the rows, which are patches patch_dict = data['patches'] count = len(patch_dict) patches = [] # Work through each row (patch) one at a time, collecting the information warn_count = 0 for pw_patch in patch_dict: patch = patchwork.Patch(pw_patch['id']) patch.parse_subject(pw_patch['name']) patches.append(patch) if warn_count > 1: tout.warning(' (total of %d warnings)' % warn_count) # Sort patches by patch number patches = sorted(patches, key=lambda x: x.seq) return patches def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, pwork): """Find new rtags collected by patchwork that we don't know about This is designed to be run in parallel, once for each commit/patch Args: new_rtag_list (list): New rtags are written to new_rtag_list[seq] list, each a dict: key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string review_list (list): New reviews are written to review_list[seq] list, each a List of reviews for the patch, each a Review seq (int): Position in new_rtag_list to update cmt (Commit): Commit object for this commit patch (Patch): Corresponding Patch object for this patch pwork (Patchwork): Patchwork object to use for reading """ if not patch: return # Get the content for the patch email itself as well as all comments data = pwork.request('patches/%s/' % patch.id) comment_data = pwork.request('patches/%s/comments/' % patch.id) new_rtags, reviews = process_reviews(data['content'], comment_data, cmt.rtags) new_rtag_list[seq] = new_rtags review_list[seq] = reviews def show_responses(col, rtags, indent, is_new): """Show rtags collected Args: col (terminal.Colour): Colour object to use rtags (dict): review tags to show key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string indent (str): Indentation string to write before each line is_new (bool): True if this output should be highlighted Returns: int: Number of review tags displayed """ count = 0 for tag in sorted(rtags.keys()): people = rtags[tag] for who in sorted(people): terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag), newline=False, colour=col.GREEN, bright=is_new, col=col) terminal.tprint(who, colour=col.WHITE, bright=is_new, col=col) count += 1 return count def create_branch(series, new_rtag_list, branch, dest_branch, overwrite, repo=None): """Create a new branch with review tags added Args: series (Series): Series object for the existing branch new_rtag_list (list): List of review tags to add, one for each commit, each a dict: key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string branch (str): Existing branch to update dest_branch (str): Name of new branch to create overwrite (bool): True to force overwriting dest_branch if it exists repo (pygit2.Repository): Repo to use (use None unless testing) Returns: int: Total number of review tags added across all commits Raises: ValueError: if the destination branch name is the same as the original branch, or it already exists and @overwrite is False """ if branch == dest_branch: raise ValueError( 'Destination branch must not be the same as the original branch') if not repo: repo = pygit2.Repository('.') count = len(series.commits) new_br = repo.branches.get(dest_branch) if new_br: if not overwrite: raise ValueError("Branch '%s' already exists (-f to overwrite)" % dest_branch) new_br.delete() if not branch: branch = 'HEAD' target = repo.revparse_single('%s~%d' % (branch, count)) repo.branches.local.create(dest_branch, target) num_added = 0 for seq in range(count): parent = repo.branches.get(dest_branch) cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1)) repo.merge_base(cherry.oid, parent.target) base_tree = cherry.parents[0].tree index = repo.merge_trees(base_tree, parent, cherry) tree_id = index.write_tree(repo) lines = [] if new_rtag_list[seq]: for tag, people in new_rtag_list[seq].items(): for who in people: lines.append('%s: %s' % (tag, who)) num_added += 1 message = patchstream.insert_tags(cherry.message.rstrip(), sorted(lines)) repo.create_commit( parent.name, cherry.author, cherry.committer, message, tree_id, [parent.target]) return num_added def _check_status(series, series_id, pwork): """Check the status of a series on Patchwork This finds review tags and comments for a series in Patchwork, displaying them to show what is new compared to the local series. Args: series (Series): Series object for the existing branch series_id (str): Patch series ID number pwork (Patchwork): Patchwork object to use for reading Return: tuple: list of Patch: List of patches sorted by sequence number dict: Patches for commit key: Commit number (0...n-1) value: Patch object for that commit list of dict: review tags: key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string list for each patch, each a: list of Review objects for the patch """ patches = collect_patches(series_id, pwork) count = len(series.commits) new_rtag_list = [None] * count review_list = [None] * count patch_for_commit, _, warnings = compare_with_series(series, patches) for warn in warnings: tout.warning(warn) patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))] with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: futures = executor.map( find_new_responses, repeat(new_rtag_list), repeat(review_list), range(count), series.commits, patch_list, repeat(pwork)) for fresponse in futures: if fresponse: raise fresponse.exception() return patches, patch_for_commit, new_rtag_list, review_list def check_patch_count(num_commits, num_patches): """Check the number of commits and patches agree Args: num_commits (int): Number of commits num_patches (int): Number of patches """ if num_patches != num_commits: tout.warning(f'Warning: Patchwork reports {num_patches} patches, ' f'series has {num_commits}') def do_show_status(patches, series, branch, show_comments, col, warnings_on_stderr=True): """Check the status of a series on Patchwork This finds review tags and comments for a series in Patchwork, displaying them to show what is new compared to the local series. Args: patches (list of Patch): Patch objects in the series series (Series): Series object for the existing branch branch (str): Existing branch to update, or None show_comments (bool): True to show the comments on each patch col (terminal.Colour): Colour object Return: tuple: int: Number of new review tags to add list: List of review tags to add, one item for each commit, each a dict: key: Response tag (e.g. 'Reviewed-by') value: Set of people who gave that response, each a name/email string list of PATCH objects """ compare = [] for pw_patch in patches: patch = patchwork.Patch(pw_patch.id) patch.parse_subject(pw_patch.series_data['name']) compare.append(patch) count = len(series.commits) new_rtag_list = [None] * count review_list = [None] * count with terminal.pager(): patch_for_commit, _, warnings = compare_with_series(series, compare) for warn in warnings: tout.do_output(tout.WARNING if warnings_on_stderr else tout.INFO, warn) for seq, pw_patch in enumerate(patches): compare[seq].patch = pw_patch for i in range(count): pat = patch_for_commit.get(i) if pat: patch_data = pat.patch.data comment_data = pat.patch.comments new_rtag_list[i], review_list[i] = process_reviews( patch_data['content'], comment_data, series.commits[i].rtags) num_to_add = _do_show_status(series, patch_for_commit, show_comments, new_rtag_list, review_list, col) return num_to_add, new_rtag_list, patches def _do_show_status(series, patch_for_commit, show_comments, new_rtag_list, review_list, col): num_to_add = 0 for seq, cmt in enumerate(series.commits): patch = patch_for_commit.get(seq) if not patch: continue terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]), colour=col.YELLOW, col=col) cmt = series.commits[seq] base_rtags = cmt.rtags new_rtags = new_rtag_list[seq] indent = ' ' * 2 show_responses(col, base_rtags, indent, False) num_to_add += show_responses(col, new_rtags, indent, True) if show_comments: for review in review_list[seq]: terminal.tprint('Review: %s' % review.meta, colour=col.RED, col=col) for snippet in review.snippets: for line in snippet: quoted = line.startswith('>') terminal.tprint( f' {line}', colour=col.MAGENTA if quoted else None, col=col) terminal.tprint() return num_to_add def show_status(series, branch, dest_branch, force, patches, show_comments, test_repo=None): """Check the status of a series on Patchwork This finds review tags and comments for a series in Patchwork, displaying them to show what is new compared to the local series. Args: series (Series): Series object for the existing branch branch (str): Existing branch to update, or None dest_branch (str): Name of new branch to create, or None force (bool): True to force overwriting dest_branch if it exists patches (list of Patch): Patches sorted by sequence number show_comments (bool): True to show the comments on each patch test_repo (pygit2.Repository): Repo to use (use None unless testing) """ col = terminal.Color() check_patch_count(len(series.commits), len(patches)) num_to_add, new_rtag_list, _ = do_show_status( patches, series, branch, show_comments, col) if not dest_branch and num_to_add: msg = ' (use -d to write them to a new branch)' else: msg = '' terminal.tprint( f"{num_to_add} new response{'s' if num_to_add != 1 else ''} " f'available in patchwork{msg}') if dest_branch: num_added = create_branch(series, new_rtag_list, branch, dest_branch, force, test_repo) terminal.tprint( f"{num_added} response{'s' if num_added != 1 else ''} added " f"from patchwork into new branch '{dest_branch}'") async def check_status(link, pwork, read_comments=False): """Set up an HTTP session and get the required state Args: link (str): Patch series ID number pwork (Patchwork): Patchwork object to use for reading read_comments (bool): True to read comments and state for each patch Return: tuple: list of Patch objects """ async with aiohttp.ClientSession() as client: patches = await pwork.series_get_state(client, link, read_comments) return patches def check_and_show_status(series, link, branch, dest_branch, force, show_comments, pwork, test_repo=None): """Read the series status from patchwork and show it to the user Args: series (Series): Series object for the existing branch link (str): Patch series ID number branch (str): Existing branch to update, or None dest_branch (str): Name of new branch to create, or None force (bool): True to force overwriting dest_branch if it exists show_comments (bool): True to show the comments on each patch pwork (Patchwork): Patchwork object to use for reading test_repo (pygit2.Repository): Repo to use (use None unless testing) """ patches = asyncio.run(check_status(link, pwork, True)) show_status(series, branch, dest_branch, force, patches, show_comments, test_repo=test_repo)