mirror of
https://source.denx.de/u-boot/u-boot.git
synced 2025-12-19 16:31:27 +01:00
24e7625085
Add a module which includes helper functions for dealing with Cseries objects. Signed-off-by: Simon Glass <sjg@chromium.org>
1525 lines
55 KiB
Python
1525 lines
55 KiB
Python
# SPDX-License-Identifier: GPL-2.0+
|
|
#
|
|
# Copyright 2025 Simon Glass <sjg@chromium.org>
|
|
#
|
|
"""Helper functions for handling the 'series' subcommand
|
|
"""
|
|
|
|
import asyncio
|
|
from collections import OrderedDict, defaultdict, namedtuple
|
|
from datetime import datetime
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
import aiohttp
|
|
import pygit2
|
|
from pygit2.enums import CheckoutStrategy
|
|
|
|
from u_boot_pylib import gitutil
|
|
from u_boot_pylib import terminal
|
|
from u_boot_pylib import tout
|
|
|
|
from patman import patchstream
|
|
from patman.database import Database, Pcommit, SerVer
|
|
from patman import patchwork
|
|
from patman.series import Series
|
|
from patman import status
|
|
|
|
|
|
# Tag to use for Change IDs
|
|
CHANGE_ID_TAG = 'Change-Id'
|
|
|
|
# Length of hash to display
|
|
HASH_LEN = 10
|
|
|
|
# Shorter version of some states, to save horizontal space
|
|
SHORTEN_STATE = {
|
|
'handled-elsewhere': 'elsewhere',
|
|
'awaiting-upstream': 'awaiting',
|
|
'not-applicable': 'n/a',
|
|
'changes-requested': 'changes',
|
|
}
|
|
|
|
# Summary info returned from Cseries.link_auto_all()
|
|
AUTOLINK = namedtuple('autolink', 'name,version,link,desc,result')
|
|
|
|
|
|
def oid(oid_val):
|
|
"""Convert a hash string into a shortened hash
|
|
|
|
The number of hex digits git uses for showing hashes depends on the size of
|
|
the repo. For the purposes of showing hashes to the user in lists, we use a
|
|
fixed value for now
|
|
|
|
Args:
|
|
str or Pygit2.oid: Hash value to shorten
|
|
|
|
Return:
|
|
str: Shortened hash
|
|
"""
|
|
return str(oid_val)[:HASH_LEN]
|
|
|
|
|
|
def split_name_version(in_name):
|
|
"""Split a branch name into its series name and its version
|
|
|
|
For example:
|
|
'series' returns ('series', 1)
|
|
'series3' returns ('series', 3)
|
|
Args:
|
|
in_name (str): Name to parse
|
|
|
|
Return:
|
|
tuple:
|
|
str: series name
|
|
int: series version, or None if there is none in in_name
|
|
"""
|
|
m_ver = re.match(r'([^0-9]*)(\d*)', in_name)
|
|
version = None
|
|
if m_ver:
|
|
name = m_ver.group(1)
|
|
if m_ver.group(2):
|
|
version = int(m_ver.group(2))
|
|
else:
|
|
name = in_name
|
|
return name, version
|
|
|
|
|
|
class CseriesHelper:
|
|
"""Helper functions for Cseries
|
|
|
|
This class handles database read/write as well as operations in a git
|
|
directory to update series information.
|
|
"""
|
|
def __init__(self, topdir=None, colour=terminal.COLOR_IF_TERMINAL):
|
|
"""Set up a new CseriesHelper
|
|
|
|
Args:
|
|
topdir (str): Top-level directory of the repo
|
|
colour (terminal.enum): Whether to enable ANSI colour or not
|
|
|
|
Properties:
|
|
gitdir (str): Git directory (typically topdir + '/.git')
|
|
db (Database): Database handler
|
|
col (terminal.Colour): Colour object
|
|
_fake_time (float): Holds the current fake time for tests, in
|
|
seconds
|
|
_fake_sleep (func): Function provided by a test; called to fake a
|
|
'time.sleep()' call and take whatever action it wants to take.
|
|
The only argument is the (Float) time to sleep for; it returns
|
|
nothing
|
|
loop (asyncio event loop): Loop used for Patchwork operations
|
|
"""
|
|
self.topdir = topdir
|
|
self.gitdir = None
|
|
self.db = None
|
|
self.col = terminal.Color(colour)
|
|
self._fake_time = None
|
|
self._fake_sleep = None
|
|
self.fake_now = None
|
|
self.loop = asyncio.get_event_loop()
|
|
|
|
def open_database(self):
|
|
"""Open the database ready for use"""
|
|
if not self.topdir:
|
|
self.topdir = gitutil.get_top_level()
|
|
if not self.topdir:
|
|
raise ValueError('No git repo detected in current directory')
|
|
self.gitdir = os.path.join(self.topdir, '.git')
|
|
fname = f'{self.topdir}/.patman.db'
|
|
|
|
# For the first instance, start it up with the expected schema
|
|
self.db, is_new = Database.get_instance(fname)
|
|
if is_new:
|
|
self.db.start()
|
|
else:
|
|
# If a previous test has already checked the schema, just open it
|
|
self.db.open_it()
|
|
|
|
def close_database(self):
|
|
"""Close the database"""
|
|
if self.db:
|
|
self.db.close()
|
|
|
|
def commit(self):
|
|
"""Commit changes to the database"""
|
|
self.db.commit()
|
|
|
|
def rollback(self):
|
|
"""Roll back changes to the database"""
|
|
self.db.rollback()
|
|
|
|
def set_fake_time(self, fake_sleep):
|
|
"""Setup the fake timer
|
|
|
|
Args:
|
|
fake_sleep (func(float)): Function to call to fake a sleep
|
|
"""
|
|
self._fake_time = 0
|
|
self._fake_sleep = fake_sleep
|
|
|
|
def inc_fake_time(self, inc_s):
|
|
"""Increment the fake time
|
|
|
|
Args:
|
|
inc_s (float): Amount to increment the fake time by
|
|
"""
|
|
self._fake_time += inc_s
|
|
|
|
def get_time(self):
|
|
"""Get the current time, fake or real
|
|
|
|
This function should always be used to read the time so that faking the
|
|
time works correctly in tests.
|
|
|
|
Return:
|
|
float: Fake time, if time is being faked, else real time
|
|
"""
|
|
if self._fake_time is not None:
|
|
return self._fake_time
|
|
return time.monotonic()
|
|
|
|
def sleep(self, time_s):
|
|
"""Sleep for a while
|
|
|
|
This function should always be used to sleep so that faking the time
|
|
works correctly in tests.
|
|
|
|
Args:
|
|
time_s (float): Amount of seconds to sleep for
|
|
"""
|
|
print(f'Sleeping for {time_s} seconds')
|
|
if self._fake_time is not None:
|
|
self._fake_sleep(time_s)
|
|
else:
|
|
time.sleep(time_s)
|
|
|
|
def get_now(self):
|
|
"""Get the time now
|
|
|
|
This function should always be used to read the datetime, so that
|
|
faking the time works correctly in tests
|
|
|
|
Return:
|
|
DateTime object
|
|
"""
|
|
if self.fake_now:
|
|
return self.fake_now
|
|
return datetime.now()
|
|
|
|
def get_ser_ver_list(self):
|
|
"""Get a list of patchwork entries from the database
|
|
|
|
Return:
|
|
list of SER_VER
|
|
"""
|
|
return self.db.ser_ver_get_list()
|
|
|
|
def get_ser_ver_dict(self):
|
|
"""Get a dict of patchwork entries from the database
|
|
|
|
Return: dict contain all records:
|
|
key (int): ser_ver id
|
|
value (SER_VER): Information about one ser_ver record
|
|
"""
|
|
svlist = self.get_ser_ver_list()
|
|
svdict = {}
|
|
for sver in svlist:
|
|
svdict[sver.idnum] = sver
|
|
return svdict
|
|
|
|
def get_upstream_dict(self):
|
|
"""Get a list of upstream entries from the database
|
|
|
|
Return:
|
|
OrderedDict:
|
|
key (str): upstream name
|
|
value (str): url
|
|
"""
|
|
return self.db.upstream_get_dict()
|
|
|
|
def get_pcommit_dict(self, find_svid=None):
|
|
"""Get a dict of pcommits entries from the database
|
|
|
|
Args:
|
|
find_svid (int): If not None, finds the records associated with a
|
|
particular series and version
|
|
|
|
Return:
|
|
OrderedDict:
|
|
key (int): record ID if find_svid is None, else seq
|
|
value (PCOMMIT): record data
|
|
"""
|
|
pcdict = OrderedDict()
|
|
for rec in self.db.pcommit_get_list(find_svid):
|
|
if find_svid is not None:
|
|
pcdict[rec.seq] = rec
|
|
else:
|
|
pcdict[rec.idnum] = rec
|
|
return pcdict
|
|
|
|
def _get_series_info(self, idnum):
|
|
"""Get information for a series from the database
|
|
|
|
Args:
|
|
idnum (int): Series ID to look up
|
|
|
|
Return: tuple:
|
|
str: Series name
|
|
str: Series description
|
|
|
|
Raises:
|
|
ValueError: Series is not found
|
|
"""
|
|
return self.db.series_get_info(idnum)
|
|
|
|
def prep_series(self, name, end=None):
|
|
"""Prepare to work with a series
|
|
|
|
Args:
|
|
name (str): Branch name with version appended, e.g. 'fix2'
|
|
end (str or None): Commit to end at, e.g. 'my_branch~16'. Only
|
|
commits up to that are processed. None to process commits up to
|
|
the upstream branch
|
|
|
|
Return: tuple:
|
|
str: Series name, e.g. 'fix'
|
|
Series: Collected series information, including name
|
|
int: Version number, e.g. 2
|
|
str: Message to show
|
|
"""
|
|
ser, version = self._parse_series_and_version(name, None)
|
|
if not name:
|
|
name = self._get_branch_name(ser.name, version)
|
|
|
|
# First check we have a branch with this name
|
|
if not gitutil.check_branch(name, git_dir=self.gitdir):
|
|
raise ValueError(f"No branch named '{name}'")
|
|
|
|
count = gitutil.count_commits_to_branch(name, self.gitdir, end)
|
|
if not count:
|
|
raise ValueError('Cannot detect branch automatically: '
|
|
'Perhaps use -U <upstream-commit> ?')
|
|
|
|
series = patchstream.get_metadata(name, 0, count, git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, ser)
|
|
msg = None
|
|
if end:
|
|
repo = pygit2.init_repository(self.gitdir)
|
|
target = repo.revparse_single(end)
|
|
first_line = target.message.splitlines()[0]
|
|
msg = f'Ending before {oid(target.id)} {first_line}'
|
|
|
|
return name, series, version, msg
|
|
|
|
def _copy_db_fields_to(self, series, in_series):
|
|
"""Copy over fields used by Cseries from one series to another
|
|
|
|
This copes desc, idnum and name
|
|
|
|
Args:
|
|
series (Series): Series to copy to
|
|
in_series (Series): Series to copy from
|
|
"""
|
|
series.desc = in_series.desc
|
|
series.idnum = in_series.idnum
|
|
series.name = in_series.name
|
|
|
|
def _handle_mark(self, branch_name, in_series, version, mark,
|
|
allow_unmarked, force_version, dry_run):
|
|
"""Handle marking a series, checking for unmarked commits, etc.
|
|
|
|
Args:
|
|
branch_name (str): Name of branch to sync, or None for current one
|
|
in_series (Series): Series object
|
|
version (int): branch version, e.g. 2 for 'mychange2'
|
|
mark (bool): True to mark each commit with a change ID
|
|
allow_unmarked (str): True to not require each commit to be marked
|
|
force_version (bool): True if ignore a Series-version tag that
|
|
doesn't match its branch name
|
|
dry_run (bool): True to do a dry run
|
|
|
|
Returns:
|
|
Series: New series object, if the series was marked;
|
|
copy_db_fields_to() is used to copy fields over
|
|
|
|
Raises:
|
|
ValueError: Series being unmarked when it should be marked, etc.
|
|
"""
|
|
series = in_series
|
|
if 'version' in series and int(series.version) != version:
|
|
msg = (f"Series name '{branch_name}' suggests version {version} "
|
|
f"but Series-version tag indicates {series.version}")
|
|
if not force_version:
|
|
raise ValueError(msg + ' (see --force-version)')
|
|
|
|
tout.warning(msg)
|
|
tout.warning(f'Updating Series-version tag to version {version}')
|
|
self.update_series(branch_name, series, int(series.version),
|
|
new_name=None, dry_run=dry_run,
|
|
add_vers=version)
|
|
|
|
# Collect the commits again, as the hashes have changed
|
|
series = patchstream.get_metadata(branch_name, 0,
|
|
len(series.commits),
|
|
git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, in_series)
|
|
|
|
if mark:
|
|
add_oid = self._mark_series(branch_name, series, dry_run=dry_run)
|
|
|
|
# Collect the commits again, as the hashes have changed
|
|
series = patchstream.get_metadata(add_oid, 0, len(series.commits),
|
|
git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, in_series)
|
|
|
|
bad_count = 0
|
|
for commit in series.commits:
|
|
if not commit.change_id:
|
|
bad_count += 1
|
|
if bad_count and not allow_unmarked:
|
|
raise ValueError(
|
|
f'{bad_count} commit(s) are unmarked; please use -m or -M')
|
|
|
|
return series
|
|
|
|
def _add_series_commits(self, series, svid):
|
|
"""Add a commits from a series into the database
|
|
|
|
Args:
|
|
series (Series): Series containing commits to add
|
|
svid (int): ser_ver-table ID to use for each commit
|
|
"""
|
|
to_add = [Pcommit(None, seq, commit.subject, None, commit.change_id,
|
|
None, None, None)
|
|
for seq, commit in enumerate(series.commits)]
|
|
|
|
self.db.pcommit_add_list(svid, to_add)
|
|
|
|
def get_series_by_name(self, name, include_archived=False):
|
|
"""Get a Series object from the database by name
|
|
|
|
Args:
|
|
name (str): Name of series to get
|
|
include_archived (bool): True to search in archives series
|
|
|
|
Return:
|
|
Series: Object containing series info, or None if none
|
|
"""
|
|
idnum = self.db.series_find_by_name(name, include_archived)
|
|
if not idnum:
|
|
return None
|
|
name, desc = self.db.series_get_info(idnum)
|
|
|
|
return Series.from_fields(idnum, name, desc)
|
|
|
|
def _get_branch_name(self, name, version):
|
|
"""Get the branch name for a particular version
|
|
|
|
Args:
|
|
name (str): Base name of branch
|
|
version (int): Version number to use
|
|
"""
|
|
return name + (f'{version}' if version > 1 else '')
|
|
|
|
def _ensure_version(self, ser, version):
|
|
"""Ensure that a version exists in a series
|
|
|
|
Args:
|
|
ser (Series): Series information, with idnum and name used here
|
|
version (int): Version to check
|
|
|
|
Returns:
|
|
list of int: List of versions
|
|
"""
|
|
versions = self._get_version_list(ser.idnum)
|
|
if version not in versions:
|
|
raise ValueError(
|
|
f"Series '{ser.name}' does not have a version {version}")
|
|
return versions
|
|
|
|
def _set_link(self, ser_id, name, version, link, update_commit,
|
|
dry_run=False):
|
|
"""Add / update a series-links link for a series
|
|
|
|
Args:
|
|
ser_id (int): Series ID number
|
|
name (str): Series name (used to find the branch)
|
|
version (int): Version number (used to update the database)
|
|
link (str): Patchwork link-string for the series
|
|
update_commit (bool): True to update the current commit with the
|
|
link
|
|
dry_run (bool): True to do a dry run
|
|
|
|
Return:
|
|
bool: True if the database was update, False if the ser_id or
|
|
version was not found
|
|
"""
|
|
if update_commit:
|
|
branch_name = self._get_branch_name(name, version)
|
|
_, ser, max_vers, _ = self.prep_series(branch_name)
|
|
self.update_series(branch_name, ser, max_vers, add_vers=version,
|
|
dry_run=dry_run, add_link=link)
|
|
if link is None:
|
|
link = ''
|
|
updated = 1 if self.db.ser_ver_set_link(ser_id, version, link) else 0
|
|
if dry_run:
|
|
self.rollback()
|
|
else:
|
|
self.commit()
|
|
|
|
return updated
|
|
|
|
def _get_autolink_dict(self, sdict, link_all_versions):
|
|
"""Get a dict of ser_vers to fetch, along with their patchwork links
|
|
|
|
Note that this returns items that already have links, as well as those
|
|
without links
|
|
|
|
Args:
|
|
sdict:
|
|
key: series ID
|
|
value: Series with idnum, name and desc filled out
|
|
link_all_versions (bool): True to sync all versions of a series,
|
|
False to sync only the latest version
|
|
|
|
Return: tuple:
|
|
dict:
|
|
key (int): svid
|
|
value (tuple):
|
|
int: series ID
|
|
str: series name
|
|
int: series version
|
|
str: patchwork link for the series, or None if none
|
|
desc: cover-letter name / series description
|
|
"""
|
|
svdict = self.get_ser_ver_dict()
|
|
to_fetch = {}
|
|
|
|
if link_all_versions:
|
|
for svinfo in self.get_ser_ver_list():
|
|
ser = sdict[svinfo.series_id]
|
|
|
|
pwc = self.get_pcommit_dict(svinfo.idnum)
|
|
count = len(pwc)
|
|
branch = self._join_name_version(ser.name, svinfo.version)
|
|
series = patchstream.get_metadata(branch, 0, count,
|
|
git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, ser)
|
|
|
|
to_fetch[svinfo.idnum] = (svinfo.series_id, series.name,
|
|
svinfo.version, svinfo.link, series)
|
|
else:
|
|
# Find the maximum version for each series
|
|
max_vers = self._series_all_max_versions()
|
|
|
|
# Get a list of links to fetch
|
|
for svid, ser_id, version in max_vers:
|
|
svinfo = svdict[svid]
|
|
ser = sdict[ser_id]
|
|
|
|
pwc = self.get_pcommit_dict(svid)
|
|
count = len(pwc)
|
|
branch = self._join_name_version(ser.name, version)
|
|
series = patchstream.get_metadata(branch, 0, count,
|
|
git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, ser)
|
|
|
|
to_fetch[svid] = (ser_id, series.name, version, svinfo.link,
|
|
series)
|
|
return to_fetch
|
|
|
|
def _get_version_list(self, idnum):
|
|
"""Get a list of the versions available for a series
|
|
|
|
Args:
|
|
idnum (int): ID of series to look up
|
|
|
|
Return:
|
|
str: List of versions
|
|
"""
|
|
if idnum is None:
|
|
raise ValueError('Unknown series idnum')
|
|
return self.db.series_get_version_list(idnum)
|
|
|
|
def _join_name_version(self, in_name, version):
|
|
"""Convert a series name plus a version into a branch name
|
|
|
|
For example:
|
|
('series', 1) returns 'series'
|
|
('series', 3) returns 'series3'
|
|
|
|
Args:
|
|
in_name (str): Series name
|
|
version (int): Version number
|
|
|
|
Return:
|
|
str: associated branch name
|
|
"""
|
|
if version == 1:
|
|
return in_name
|
|
return f'{in_name}{version}'
|
|
|
|
def _parse_series(self, name, include_archived=False):
|
|
"""Parse the name of a series, or detect it from the current branch
|
|
|
|
Args:
|
|
name (str or None): name of series
|
|
include_archived (bool): True to search in archives series
|
|
|
|
Return:
|
|
Series: New object with the name set; idnum is also set if the
|
|
series exists in the database
|
|
"""
|
|
if not name:
|
|
name = gitutil.get_branch(self.gitdir)
|
|
name, _ = split_name_version(name)
|
|
ser = self.get_series_by_name(name, include_archived)
|
|
if not ser:
|
|
ser = Series()
|
|
ser.name = name
|
|
return ser
|
|
|
|
def _parse_series_and_version(self, in_name, in_version):
|
|
"""Parse name and version of a series, or detect from current branch
|
|
|
|
Figures out the name from in_name, or if that is None, from the current
|
|
branch.
|
|
|
|
Uses the version in_version, or if that is None, uses the int at the
|
|
end of the name (e.g. 'series' is version 1, 'series4' is version 4)
|
|
|
|
Args:
|
|
in_name (str or None): name of series
|
|
in_version (str or None): version of series
|
|
|
|
Return:
|
|
tuple:
|
|
Series: New object with the name set; idnum is also set if the
|
|
series exists in the database
|
|
int: Series version-number detected from the name
|
|
(e.g. 'fred' is version 1, 'fred2' is version 2)
|
|
"""
|
|
name = in_name
|
|
if not name:
|
|
name = gitutil.get_branch(self.gitdir)
|
|
if not name:
|
|
raise ValueError('No branch detected: please use -s <series>')
|
|
name, version = split_name_version(name)
|
|
if not name:
|
|
raise ValueError(f"Series name '{in_name}' cannot be a number, "
|
|
f"use '<name><version>'")
|
|
if in_version:
|
|
if version and version != in_version:
|
|
tout.warning(
|
|
f"Version mismatch: -V has {in_version} but branch name "
|
|
f'indicates {version}')
|
|
version = in_version
|
|
if not version:
|
|
version = 1
|
|
if version > 99:
|
|
raise ValueError(f"Version {version} exceeds 99")
|
|
ser = self.get_series_by_name(name)
|
|
if not ser:
|
|
ser = Series()
|
|
ser.name = name
|
|
return ser, version
|
|
|
|
def _series_get_version_stats(self, idnum, vers):
|
|
"""Get the stats for a series
|
|
|
|
Args:
|
|
idnum (int): ID number of series to process
|
|
vers (int): Version number to process
|
|
|
|
Return:
|
|
tuple:
|
|
str: Status string, '<accepted>/<count>'
|
|
OrderedDict:
|
|
key (int): record ID if find_svid is None, else seq
|
|
value (PCOMMIT): record data
|
|
"""
|
|
svid, link = self._get_series_svid_link(idnum, vers)
|
|
pwc = self.get_pcommit_dict(svid)
|
|
count = len(pwc.values())
|
|
if link:
|
|
accepted = 0
|
|
for pcm in pwc.values():
|
|
accepted += pcm.state == 'accepted'
|
|
else:
|
|
accepted = '-'
|
|
return f'{accepted}/{count}', pwc
|
|
|
|
def get_series_svid(self, series_id, version):
|
|
"""Get the patchwork ID of a series version
|
|
|
|
Args:
|
|
series_id (int): id of the series to look up
|
|
version (int): version number to look up
|
|
|
|
Return:
|
|
str: link found
|
|
|
|
Raises:
|
|
ValueError: No matching series found
|
|
"""
|
|
return self._get_series_svid_link(series_id, version)[0]
|
|
|
|
def _get_series_svid_link(self, series_id, version):
|
|
"""Get the patchwork ID of a series version
|
|
|
|
Args:
|
|
series_id (int): series ID to look up
|
|
version (int): version number to look up
|
|
|
|
Return:
|
|
tuple:
|
|
int: record id
|
|
str: link
|
|
"""
|
|
recs = self.get_ser_ver(series_id, version)
|
|
return recs.idnum, recs.link
|
|
|
|
def get_ser_ver(self, series_id, version):
|
|
"""Get the patchwork details for a series version
|
|
|
|
Args:
|
|
series_id (int): series ID to look up
|
|
version (int): version number to look up
|
|
|
|
Return:
|
|
SER_VER: Requested information
|
|
|
|
Raises:
|
|
ValueError: There is no matching idnum/version
|
|
"""
|
|
return self.db.ser_ver_get_for_series(series_id, version)
|
|
|
|
def _prepare_process(self, name, count, new_name=None, quiet=False):
|
|
"""Get ready to process all commits in a branch
|
|
|
|
Args:
|
|
name (str): Name of the branch to process
|
|
count (int): Number of commits
|
|
new_name (str or None): New name, if a new branch is to be created
|
|
quiet (bool): True to avoid output (used for testing)
|
|
|
|
Return: tuple:
|
|
pygit2.repo: Repo to use
|
|
pygit2.oid: Upstream commit, onto which commits should be added
|
|
Pygit2.branch: Original branch, for later use
|
|
str: (Possibly new) name of branch to process
|
|
list of Commit: commits to process, in order
|
|
pygit2.Reference: Original head before processing started
|
|
"""
|
|
upstream_guess = gitutil.get_upstream(self.gitdir, name)[0]
|
|
|
|
tout.debug(f"_process_series name '{name}' new_name '{new_name}' "
|
|
f"upstream_guess '{upstream_guess}'")
|
|
dirty = gitutil.check_dirty(self.gitdir, self.topdir)
|
|
if dirty:
|
|
raise ValueError(
|
|
f"Modified files exist: use 'git status' to check: "
|
|
f'{dirty[:5]}')
|
|
repo = pygit2.init_repository(self.gitdir)
|
|
|
|
commit = None
|
|
upstream_name = None
|
|
if upstream_guess:
|
|
try:
|
|
upstream = repo.lookup_reference(upstream_guess)
|
|
upstream_name = upstream.name
|
|
commit = upstream.peel(pygit2.enums.ObjectType.COMMIT)
|
|
except KeyError:
|
|
pass
|
|
except pygit2.repository.InvalidSpecError as exc:
|
|
print(f"Error '{exc}'")
|
|
if not upstream_name:
|
|
upstream_name = f'{name}~{count}'
|
|
commit = repo.revparse_single(upstream_name)
|
|
|
|
branch = repo.lookup_branch(name)
|
|
if not quiet:
|
|
tout.info(
|
|
f'Checking out upstream commit {upstream_name}: '
|
|
f'{oid(commit.oid)}')
|
|
|
|
old_head = repo.head
|
|
if old_head.shorthand == name:
|
|
old_head = None
|
|
else:
|
|
old_head = repo.head
|
|
|
|
if new_name:
|
|
name = new_name
|
|
repo.set_head(commit.oid)
|
|
|
|
commits = []
|
|
cmt = repo.get(branch.target)
|
|
for _ in range(count):
|
|
commits.append(cmt)
|
|
cmt = cmt.parents[0]
|
|
|
|
return (repo, repo.head, branch, name, commit, list(reversed(commits)),
|
|
old_head)
|
|
|
|
def _pick_commit(self, repo, cmt):
|
|
"""Apply a commit to the source tree, without committing it
|
|
|
|
_prepare_process() must be called before starting to pick commits
|
|
|
|
This function must be called before _finish_commit()
|
|
|
|
Note that this uses a cherry-pick method, creating a new tree_id each
|
|
time, so can make source-code changes
|
|
|
|
Args:
|
|
repo (pygit2.repo): Repo to use
|
|
cmt (Commit): Commit to apply
|
|
|
|
Return: tuple:
|
|
tree_id (pygit2.oid): Oid of index with source-changes applied
|
|
commit (pygit2.oid): Old commit being cherry-picked
|
|
"""
|
|
tout.detail(f"- adding {oid(cmt.hash)} {cmt}")
|
|
repo.cherrypick(cmt.hash)
|
|
if repo.index.conflicts:
|
|
raise ValueError('Conflicts detected')
|
|
|
|
tree_id = repo.index.write_tree()
|
|
cherry = repo.get(cmt.hash)
|
|
tout.detail(f"cherry {oid(cherry.oid)}")
|
|
return tree_id, cherry
|
|
|
|
def _finish_commit(self, repo, tree_id, commit, cur, msg=None):
|
|
"""Complete a commit
|
|
|
|
This must be called after _pick_commit().
|
|
|
|
Args:
|
|
repo (pygit2.repo): Repo to use
|
|
tree_id (pygit2.oid): Oid of index with source-changes applied; if
|
|
None then the existing commit.tree_id is used
|
|
commit (pygit2.oid): Old commit being cherry-picked
|
|
cur (pygit2.reference): Reference to parent to use for the commit
|
|
msg (str): Commit subject and message; None to use commit.message
|
|
"""
|
|
if msg is None:
|
|
msg = commit.message
|
|
if not tree_id:
|
|
tree_id = commit.tree_id
|
|
repo.create_commit('HEAD', commit.author, commit.committer,
|
|
msg, tree_id, [cur.target])
|
|
return repo.head
|
|
|
|
def _finish_process(self, repo, branch, name, cur, old_head, new_name=None,
|
|
switch=False, dry_run=False, quiet=False):
|
|
"""Finish processing commits
|
|
|
|
Args:
|
|
repo (pygit2.repo): Repo to use
|
|
branch (pygit2.branch): Branch returned by _prepare_process()
|
|
name (str): Name of the branch to process
|
|
new_name (str or None): New name, if a new branch is being created
|
|
switch (bool): True to switch to the new branch after processing;
|
|
otherwise HEAD remains at the original branch, as amended
|
|
dry_run (bool): True to do a dry run, restoring the original tree
|
|
afterwards
|
|
quiet (bool): True to avoid output (used for testing)
|
|
|
|
Return:
|
|
pygit2.reference: Final commit after everything is completed
|
|
"""
|
|
repo.state_cleanup()
|
|
|
|
# Update the branch
|
|
target = repo.revparse_single('HEAD')
|
|
if not quiet:
|
|
tout.info(f'Updating branch {name} from {oid(branch.target)} to '
|
|
f'{str(target.oid)[:HASH_LEN]}')
|
|
if dry_run:
|
|
if new_name:
|
|
repo.head.set_target(branch.target)
|
|
else:
|
|
branch_oid = branch.peel(pygit2.enums.ObjectType.COMMIT).oid
|
|
repo.head.set_target(branch_oid)
|
|
repo.head.set_target(branch.target)
|
|
repo.set_head(branch.name)
|
|
else:
|
|
if new_name:
|
|
new_branch = repo.branches.create(new_name, target)
|
|
if branch.upstream:
|
|
new_branch.upstream = branch.upstream
|
|
branch = new_branch
|
|
else:
|
|
branch.set_target(cur.target)
|
|
repo.set_head(branch.name)
|
|
if old_head:
|
|
if not switch:
|
|
repo.set_head(old_head.name)
|
|
return target
|
|
|
|
def make_change_id(self, commit):
|
|
"""Make a Change ID for a commit
|
|
|
|
This is similar to the gerrit script:
|
|
git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "README"; }
|
|
| git hash-object --stdin)
|
|
|
|
Args:
|
|
commit (pygit2.commit): Commit to process
|
|
|
|
Return:
|
|
Change ID in hex format
|
|
"""
|
|
sig = commit.committer
|
|
val = hashlib.sha1()
|
|
to_hash = f'{sig.name} <{sig.email}> {sig.time} {sig.offset}'
|
|
val.update(to_hash.encode('utf-8'))
|
|
val.update(str(commit.tree_id).encode('utf-8'))
|
|
val.update(commit.message.encode('utf-8'))
|
|
return val.hexdigest()
|
|
|
|
def _filter_commits(self, name, series, seq_to_drop):
|
|
"""Filter commits to drop one
|
|
|
|
This function rebases the current branch, dropping a single commit,
|
|
thus changing the resulting code in the tree.
|
|
|
|
Args:
|
|
name (str): Name of the branch to process
|
|
series (Series): Series object
|
|
seq_to_drop (int): Commit sequence to drop; commits are numbered
|
|
from 0, which is the one after the upstream branch, to
|
|
count - 1
|
|
"""
|
|
count = len(series.commits)
|
|
(repo, cur, branch, name, commit, _, _) = self._prepare_process(
|
|
name, count, quiet=True)
|
|
repo.checkout_tree(commit, strategy=CheckoutStrategy.FORCE |
|
|
CheckoutStrategy.RECREATE_MISSING)
|
|
repo.set_head(commit.oid)
|
|
for seq, cmt in enumerate(series.commits):
|
|
if seq != seq_to_drop:
|
|
tree_id, cherry = self._pick_commit(repo, cmt)
|
|
cur = self._finish_commit(repo, tree_id, cherry, cur)
|
|
self._finish_process(repo, branch, name, cur, None, quiet=True)
|
|
|
|
def process_series(self, name, series, new_name=None, switch=False,
|
|
dry_run=False):
|
|
"""Rewrite a series commit messages, leaving code alone
|
|
|
|
This uses a 'vals' namespace to pass things to the controlling
|
|
function.
|
|
|
|
Each time _process_series() yields, it sets up:
|
|
commit (Commit): The pygit2 commit that is being processed
|
|
msg (str): Commit message, which can be modified
|
|
info (str): Initially empty; the controlling function can add a
|
|
short message here which will be shown to the user
|
|
final (bool): True if this is the last commit to apply
|
|
seq (int): Current sequence number in the commits to apply (0,,n-1)
|
|
|
|
It also sets git HEAD at the commit before this commit being
|
|
processed
|
|
|
|
The function can change msg and info, e.g. to add or remove tags from
|
|
the commit.
|
|
|
|
Args:
|
|
name (str): Name of the branch to process
|
|
series (Series): Series object
|
|
new_name (str or None): New name, if a new branch is to be created
|
|
switch (bool): True to switch to the new branch after processing;
|
|
otherwise HEAD remains at the original branch, as amended
|
|
dry_run (bool): True to do a dry run, restoring the original tree
|
|
afterwards
|
|
|
|
Return:
|
|
pygit.oid: oid of the new branch
|
|
"""
|
|
count = len(series.commits)
|
|
repo, cur, branch, name, _, commits, old_head = self._prepare_process(
|
|
name, count, new_name)
|
|
vals = SimpleNamespace()
|
|
vals.final = False
|
|
tout.info(f"Processing {count} commits from branch '{name}'")
|
|
|
|
# Record the message lines
|
|
lines = []
|
|
for seq, cmt in enumerate(series.commits):
|
|
commit = commits[seq]
|
|
vals.commit = commit
|
|
vals.msg = commit.message
|
|
vals.info = ''
|
|
vals.final = seq == len(series.commits) - 1
|
|
vals.seq = seq
|
|
yield vals
|
|
|
|
cur = self._finish_commit(repo, None, commit, cur, vals.msg)
|
|
lines.append([vals.info.strip(),
|
|
f'{oid(cmt.hash)} as {oid(cur.target)} {cmt}'])
|
|
|
|
max_len = max(len(info) for info, rest in lines) + 1
|
|
for info, rest in lines:
|
|
if info:
|
|
info += ':'
|
|
tout.info(f'- {info.ljust(max_len)} {rest}')
|
|
target = self._finish_process(repo, branch, name, cur, old_head,
|
|
new_name, switch, dry_run)
|
|
vals.oid = target.oid
|
|
|
|
def _mark_series(self, name, series, dry_run=False):
|
|
"""Mark a series with Change-Id tags
|
|
|
|
Args:
|
|
name (str): Name of the series to mark
|
|
series (Series): Series object
|
|
dry_run (bool): True to do a dry run, restoring the original tree
|
|
afterwards
|
|
|
|
Return:
|
|
pygit.oid: oid of the new branch
|
|
"""
|
|
vals = None
|
|
for vals in self.process_series(name, series, dry_run=dry_run):
|
|
if CHANGE_ID_TAG not in vals.msg:
|
|
change_id = self.make_change_id(vals.commit)
|
|
vals.msg = vals.msg + f'\n{CHANGE_ID_TAG}: {change_id}'
|
|
tout.detail(" - adding mark")
|
|
vals.info = 'marked'
|
|
else:
|
|
vals.info = 'has mark'
|
|
|
|
return vals.oid
|
|
|
|
def update_series(self, branch_name, series, max_vers, new_name=None,
|
|
dry_run=False, add_vers=None, add_link=None,
|
|
add_rtags=None, switch=False):
|
|
"""Rewrite a series to update the Series-version/Series-links lines
|
|
|
|
This updates the series in git; it does not update the database
|
|
|
|
Args:
|
|
branch_name (str): Name of the branch to process
|
|
series (Series): Series object
|
|
max_vers (int): Version number of the series being updated
|
|
new_name (str or None): New name, if a new branch is to be created
|
|
dry_run (bool): True to do a dry run, restoring the original tree
|
|
afterwards
|
|
add_vers (int or None): Version number to add to the series, if any
|
|
add_link (str or None): Link to add to the series, if any
|
|
add_rtags (list of dict): List of review tags to add, one item for
|
|
each commit, each a dict:
|
|
key: Response tag (e.g. 'Reviewed-by')
|
|
value: Set of people who gave that response, each a name/email
|
|
string
|
|
switch (bool): True to switch to the new branch after processing;
|
|
otherwise HEAD remains at the original branch, as amended
|
|
|
|
Return:
|
|
pygit.oid: oid of the new branch
|
|
"""
|
|
def _do_version():
|
|
if add_vers:
|
|
if add_vers == 1:
|
|
vals.info += f'rm v{add_vers} '
|
|
else:
|
|
vals.info += f'add v{add_vers} '
|
|
out.append(f'Series-version: {add_vers}')
|
|
|
|
def _do_links(new_links):
|
|
if add_link:
|
|
if 'add' not in vals.info:
|
|
vals.info += 'add '
|
|
vals.info += f"links '{new_links}' "
|
|
else:
|
|
vals.info += f"upd links '{new_links}' "
|
|
out.append(f'Series-links: {new_links}')
|
|
|
|
added_version = False
|
|
added_link = False
|
|
for vals in self.process_series(branch_name, series, new_name, switch,
|
|
dry_run):
|
|
out = []
|
|
for line in vals.msg.splitlines():
|
|
m_ver = re.match('Series-version:(.*)', line)
|
|
m_links = re.match('Series-links:(.*)', line)
|
|
if m_ver and add_vers:
|
|
if ('version' in series and
|
|
int(series.version) != max_vers):
|
|
tout.warning(
|
|
f'Branch {branch_name}: Series-version tag '
|
|
f'{series.version} does not match expected '
|
|
f'version {max_vers}')
|
|
_do_version()
|
|
added_version = True
|
|
elif m_links:
|
|
links = series.get_links(m_links.group(1), max_vers)
|
|
if add_link:
|
|
links[max_vers] = add_link
|
|
_do_links(series.build_links(links))
|
|
added_link = True
|
|
else:
|
|
out.append(line)
|
|
if vals.final:
|
|
if not added_version and add_vers and add_vers > 1:
|
|
_do_version()
|
|
if not added_link and add_link:
|
|
_do_links(f'{max_vers}:{add_link}')
|
|
|
|
vals.msg = '\n'.join(out) + '\n'
|
|
if add_rtags and add_rtags[vals.seq]:
|
|
lines = []
|
|
for tag, people in add_rtags[vals.seq].items():
|
|
for who in people:
|
|
lines.append(f'{tag}: {who}')
|
|
vals.msg = patchstream.insert_tags(vals.msg.rstrip(),
|
|
sorted(lines))
|
|
vals.info += (f'added {len(lines)} '
|
|
f"tag{'' if len(lines) == 1 else 's'}")
|
|
|
|
def _build_col(self, state, prefix='', base_str=None):
|
|
"""Build a patch-state string with colour
|
|
|
|
Args:
|
|
state (str): State to colourise (also indicates the colour to use)
|
|
prefix (str): Prefix string to also colourise
|
|
base_str (str or None): String to show instead of state, or None to
|
|
show state
|
|
|
|
Return:
|
|
str: String with ANSI colour characters
|
|
"""
|
|
bright = True
|
|
if state == 'accepted':
|
|
col = self.col.GREEN
|
|
elif state == 'awaiting-upstream':
|
|
bright = False
|
|
col = self.col.GREEN
|
|
elif state in ['changes-requested']:
|
|
col = self.col.CYAN
|
|
elif state in ['rejected', 'deferred', 'not-applicable', 'superseded',
|
|
'handled-elsewhere']:
|
|
col = self.col.RED
|
|
elif not state:
|
|
state = 'unknown'
|
|
col = self.col.MAGENTA
|
|
else:
|
|
# under-review, rfc, needs-review-ack
|
|
col = self.col.WHITE
|
|
out = base_str or SHORTEN_STATE.get(state, state)
|
|
pad = ' ' * (10 - len(out))
|
|
col_state = self.col.build(col, prefix + out, bright)
|
|
return col_state, pad
|
|
|
|
def _get_patches(self, series, version):
|
|
"""Get a Series object containing the patches in a series
|
|
|
|
Args:
|
|
series (str): Name of series to use, or None to use current branch
|
|
version (int): Version number, or None to detect from name
|
|
|
|
Return: tuple:
|
|
str: Name of branch, e.g. 'mary2'
|
|
Series: Series object containing the commits and idnum, desc, name
|
|
int: Version number of series, e.g. 2
|
|
OrderedDict:
|
|
key (int): record ID if find_svid is None, else seq
|
|
value (PCOMMIT): record data
|
|
str: series name (for this version)
|
|
str: patchwork link
|
|
str: cover_id
|
|
int: cover_num_comments
|
|
"""
|
|
ser, version = self._parse_series_and_version(series, version)
|
|
if not ser.idnum:
|
|
raise ValueError(f"Unknown series '{series}'")
|
|
self._ensure_version(ser, version)
|
|
svinfo = self.get_ser_ver(ser.idnum, version)
|
|
pwc = self.get_pcommit_dict(svinfo.idnum)
|
|
|
|
count = len(pwc)
|
|
branch = self._join_name_version(ser.name, version)
|
|
series = patchstream.get_metadata(branch, 0, count,
|
|
git_dir=self.gitdir)
|
|
self._copy_db_fields_to(series, ser)
|
|
|
|
return (branch, series, version, pwc, svinfo.name, svinfo.link,
|
|
svinfo.cover_id, svinfo.cover_num_comments)
|
|
|
|
def _list_patches(self, branch, pwc, series, desc, cover_id, num_comments,
|
|
show_commit, show_patch, list_patches, state_totals):
|
|
"""List patches along with optional status info
|
|
|
|
Args:
|
|
branch (str): Branch name if self.show_progress
|
|
pwc (dict): pcommit records:
|
|
key (int): seq
|
|
value (PCOMMIT): Record from database
|
|
series (Series): Series to show, or None to just use the database
|
|
desc (str): Series title
|
|
cover_id (int): Cover-letter ID
|
|
num_comments (int): The number of comments on the cover letter
|
|
show_commit (bool): True to show the commit and diffstate
|
|
show_patch (bool): True to show the patch
|
|
list_patches (bool): True to list all patches for each series,
|
|
False to just show the series summary on a single line
|
|
state_totals (dict): Holds totals for each state across all patches
|
|
key (str): state name
|
|
value (int): Number of patches in that state
|
|
|
|
Return:
|
|
bool: True if OK, False if any commit subjects don't match their
|
|
patchwork subjects
|
|
"""
|
|
lines = []
|
|
states = defaultdict(int)
|
|
count = len(pwc)
|
|
ok = True
|
|
for seq, item in enumerate(pwc.values()):
|
|
if series:
|
|
cmt = series.commits[seq]
|
|
if cmt.subject != item.subject:
|
|
ok = False
|
|
|
|
col_state, pad = self._build_col(item.state)
|
|
patch_id = item.patch_id if item.patch_id else ''
|
|
if item.num_comments:
|
|
comments = str(item.num_comments)
|
|
elif item.num_comments is None:
|
|
comments = '-'
|
|
else:
|
|
comments = ''
|
|
|
|
if show_commit or show_patch:
|
|
subject = self.col.build(self.col.BLACK, item.subject,
|
|
bright=False, back=self.col.YELLOW)
|
|
else:
|
|
subject = item.subject
|
|
|
|
line = (f'{seq:3} {col_state}{pad} {comments.rjust(3)} '
|
|
f'{patch_id:7} {oid(cmt.hash)} {subject}')
|
|
lines.append(line)
|
|
states[item.state] += 1
|
|
out = ''
|
|
for state, freq in states.items():
|
|
out += ' ' + self._build_col(state, f'{freq}:')[0]
|
|
state_totals[state] += freq
|
|
name = ''
|
|
if not list_patches:
|
|
name = desc or series.desc
|
|
name = self.col.build(self.col.YELLOW, name[:41].ljust(41))
|
|
if not ok:
|
|
out = '*' + out[1:]
|
|
print(f"{branch:16} {name} {len(pwc):5} {out}")
|
|
return ok
|
|
print(f"Branch '{branch}' (total {len(pwc)}):{out}{name}")
|
|
|
|
print(self.col.build(
|
|
self.col.MAGENTA,
|
|
f"Seq State Com PatchId {'Commit'.ljust(HASH_LEN)} Subject"))
|
|
|
|
comments = '' if num_comments is None else str(num_comments)
|
|
if desc or comments or cover_id:
|
|
cov = 'Cov' if cover_id else ''
|
|
print(self.col.build(
|
|
self.col.WHITE,
|
|
f"{cov:14} {comments.rjust(3)} {cover_id or '':7} "
|
|
f'{desc or series.desc}',
|
|
bright=False))
|
|
for seq in range(count):
|
|
line = lines[seq]
|
|
print(line)
|
|
if show_commit or show_patch:
|
|
print()
|
|
cmt = series.commits[seq] if series else ''
|
|
msg = gitutil.show_commit(
|
|
cmt.hash, show_commit, True, show_patch,
|
|
colour=self.col.enabled(), git_dir=self.gitdir)
|
|
sys.stdout.write(msg)
|
|
if seq != count - 1:
|
|
print()
|
|
print()
|
|
|
|
return ok
|
|
|
|
def _find_matched_commit(self, commits, pcm):
|
|
"""Find a commit in a list of possible matches
|
|
|
|
Args:
|
|
commits (dict of Commit): Possible matches
|
|
key (int): sequence number of patch (from 0)
|
|
value (Commit): Commit object
|
|
pcm (PCOMMIT): Patch to check
|
|
|
|
Return:
|
|
int: Sequence number of matching commit, or None if not found
|
|
"""
|
|
for seq, cmt in commits.items():
|
|
tout.debug(f"- match subject: '{cmt.subject}'")
|
|
if pcm.subject == cmt.subject:
|
|
return seq
|
|
return None
|
|
|
|
def _find_matched_patch(self, patches, cmt):
|
|
"""Find a patch in a list of possible matches
|
|
|
|
Args:
|
|
patches: dict of ossible matches
|
|
key (int): sequence number of patch
|
|
value (PCOMMIT): patch
|
|
cmt (Commit): Commit to check
|
|
|
|
Return:
|
|
int: Sequence number of matching patch, or None if not found
|
|
"""
|
|
for seq, pcm in patches.items():
|
|
tout.debug(f"- match subject: '{pcm.subject}'")
|
|
if cmt.subject == pcm.subject:
|
|
return seq
|
|
return None
|
|
|
|
def _sync_one(self, svid, series_name, version, show_comments,
|
|
show_cover_comments, gather_tags, cover, patches, dry_run):
|
|
"""Sync one series to the database
|
|
|
|
Args:
|
|
svid (int): Ser/ver ID
|
|
cover (dict or None): Cover letter from patchwork, with keys:
|
|
id (int): Cover-letter ID in patchwork
|
|
num_comments (int): Number of comments
|
|
name (str): Cover-letter name
|
|
patches (list of Patch): Patches in the series
|
|
"""
|
|
pwc = self.get_pcommit_dict(svid)
|
|
if gather_tags:
|
|
count = len(pwc)
|
|
branch = self._join_name_version(series_name, version)
|
|
series = patchstream.get_metadata(branch, 0, count,
|
|
git_dir=self.gitdir)
|
|
|
|
_, new_rtag_list = status.do_show_status(
|
|
series, cover, patches, show_comments, show_cover_comments,
|
|
self.col, warnings_on_stderr=False)
|
|
self.update_series(branch, series, version, None, dry_run,
|
|
add_rtags=new_rtag_list)
|
|
|
|
updated = 0
|
|
for seq, item in enumerate(pwc.values()):
|
|
if seq >= len(patches):
|
|
continue
|
|
patch = patches[seq]
|
|
if patch.id:
|
|
if self.db.pcommit_update(
|
|
Pcommit(item.idnum, seq, None, None, None, patch.state,
|
|
patch.id, len(patch.comments))):
|
|
updated += 1
|
|
if cover:
|
|
info = SerVer(svid, None, None, None, cover.id,
|
|
cover.num_comments, cover.name, None)
|
|
else:
|
|
info = SerVer(svid, None, None, None, None, None, patches[0].name,
|
|
None)
|
|
self.db.ser_ver_set_info(info)
|
|
|
|
return updated, 1 if cover else 0
|
|
|
|
async def _gather(self, pwork, link, show_cover_comments):
|
|
"""Sync the series status from patchwork
|
|
|
|
Creates a new client sesion and calls _sync()
|
|
|
|
Args:
|
|
pwork (Patchwork): Patchwork object to use
|
|
link (str): Patchwork link for the series
|
|
show_cover_comments (bool): True to show the comments on the cover
|
|
letter
|
|
|
|
Return: tuple:
|
|
COVER object, or None if none or not read_cover_comments
|
|
list of PATCH objects
|
|
"""
|
|
async with aiohttp.ClientSession() as client:
|
|
return await pwork.series_get_state(client, link, True,
|
|
show_cover_comments)
|
|
|
|
def _get_fetch_dict(self, sync_all_versions):
|
|
"""Get a dict of ser_vers to fetch, along with their patchwork links
|
|
|
|
Args:
|
|
sync_all_versions (bool): True to sync all versions of a series,
|
|
False to sync only the latest version
|
|
|
|
Return: tuple:
|
|
dict: things to fetch
|
|
key (int): svid
|
|
value (str): patchwork link for the series
|
|
int: number of series which are missing a link
|
|
"""
|
|
missing = 0
|
|
svdict = self.get_ser_ver_dict()
|
|
sdict = self.db.series_get_dict_by_id()
|
|
to_fetch = {}
|
|
|
|
if sync_all_versions:
|
|
for svinfo in self.get_ser_ver_list():
|
|
ser_ver = svdict[svinfo.idnum]
|
|
if svinfo.link:
|
|
to_fetch[svinfo.idnum] = patchwork.STATE_REQ(
|
|
svinfo.link, svinfo.series_id,
|
|
sdict[svinfo.series_id].name, svinfo.version, False,
|
|
False)
|
|
else:
|
|
missing += 1
|
|
else:
|
|
# Find the maximum version for each series
|
|
max_vers = self._series_all_max_versions()
|
|
|
|
# Get a list of links to fetch
|
|
for svid, series_id, version in max_vers:
|
|
ser_ver = svdict[svid]
|
|
if series_id not in sdict:
|
|
# skip archived item
|
|
continue
|
|
if ser_ver.link:
|
|
to_fetch[svid] = patchwork.STATE_REQ(
|
|
ser_ver.link, series_id, sdict[series_id].name,
|
|
version, False, False)
|
|
else:
|
|
missing += 1
|
|
|
|
# order by series name, version
|
|
ordered = OrderedDict()
|
|
for svid in sorted(
|
|
to_fetch,
|
|
key=lambda k: (to_fetch[k].series_name, to_fetch[k].version)):
|
|
sync = to_fetch[svid]
|
|
ordered[svid] = sync
|
|
|
|
return ordered, missing
|
|
|
|
async def _sync_all(self, client, pwork, to_fetch):
|
|
"""Sync all series status from patchwork
|
|
|
|
Args:
|
|
pwork (Patchwork): Patchwork object to use
|
|
sync_all_versions (bool): True to sync all versions of a series,
|
|
False to sync only the latest version
|
|
gather_tags (bool): True to gather review/test tags
|
|
|
|
Return: list of tuple:
|
|
COVER object, or None if none or not read_cover_comments
|
|
list of PATCH objects
|
|
"""
|
|
with pwork.collect_stats() as stats:
|
|
tasks = [pwork.series_get_state(client, sync.link, True, True)
|
|
for sync in to_fetch.values() if sync.link]
|
|
result = await asyncio.gather(*tasks)
|
|
return result, stats.request_count
|
|
|
|
async def _do_series_sync_all(self, pwork, to_fetch):
|
|
async with aiohttp.ClientSession() as client:
|
|
return await self._sync_all(client, pwork, to_fetch)
|
|
|
|
def _progress_one(self, ser, show_all_versions, list_patches,
|
|
state_totals):
|
|
"""Show progress information for all versions in a series
|
|
|
|
Args:
|
|
ser (Series): Series to use
|
|
show_all_versions (bool): True to show all versions of a series,
|
|
False to show only the final version
|
|
list_patches (bool): True to list all patches for each series,
|
|
False to just show the series summary on a single line
|
|
state_totals (dict): Holds totals for each state across all patches
|
|
key (str): state name
|
|
value (int): Number of patches in that state
|
|
|
|
Return: tuple
|
|
int: Number of series shown
|
|
int: Number of patches shown
|
|
int: Number of version which need a 'scan'
|
|
"""
|
|
max_vers = self._series_max_version(ser.idnum)
|
|
name, desc = self._get_series_info(ser.idnum)
|
|
coloured = self.col.build(self.col.BLACK, desc, bright=False,
|
|
back=self.col.YELLOW)
|
|
versions = self._get_version_list(ser.idnum)
|
|
vstr = list(map(str, versions))
|
|
|
|
if list_patches:
|
|
print(f"{name}: {coloured} (versions: {' '.join(vstr)})")
|
|
add_blank_line = False
|
|
total_series = 0
|
|
total_patches = 0
|
|
need_scan = 0
|
|
for ver in versions:
|
|
if not show_all_versions and ver != max_vers:
|
|
continue
|
|
if add_blank_line:
|
|
print()
|
|
_, pwc = self._series_get_version_stats(ser.idnum, ver)
|
|
count = len(pwc)
|
|
branch = self._join_name_version(ser.name, ver)
|
|
series = patchstream.get_metadata(branch, 0, count,
|
|
git_dir=self.gitdir)
|
|
svinfo = self.get_ser_ver(ser.idnum, ver)
|
|
self._copy_db_fields_to(series, ser)
|
|
|
|
ok = self._list_patches(
|
|
branch, pwc, series, svinfo.name, svinfo.cover_id,
|
|
svinfo.cover_num_comments, False, False, list_patches,
|
|
state_totals)
|
|
if not ok:
|
|
need_scan += 1
|
|
add_blank_line = list_patches
|
|
total_series += 1
|
|
total_patches += count
|
|
return total_series, total_patches, need_scan
|
|
|
|
def _summary_one(self, ser):
|
|
"""Show summary information for the latest version in a series
|
|
|
|
Args:
|
|
series (str): Name of series to use, or None to show progress for
|
|
all series
|
|
"""
|
|
max_vers = self._series_max_version(ser.idnum)
|
|
name, desc = self._get_series_info(ser.idnum)
|
|
stats, pwc = self._series_get_version_stats(ser.idnum, max_vers)
|
|
states = {x.state for x in pwc.values()}
|
|
state = 'accepted'
|
|
for val in ['awaiting-upstream', 'changes-requested', 'rejected',
|
|
'deferred', 'not-applicable', 'superseded',
|
|
'handled-elsewhere']:
|
|
if val in states:
|
|
state = val
|
|
state_str, pad = self._build_col(state, base_str=name)
|
|
print(f"{state_str}{pad} {stats.rjust(6)} {desc}")
|
|
|
|
def _series_max_version(self, idnum):
|
|
"""Find the latest version of a series
|
|
|
|
Args:
|
|
idnum (int): Series ID to look up
|
|
|
|
Return:
|
|
int: maximum version
|
|
"""
|
|
return self.db.series_get_max_version(idnum)
|
|
|
|
def _series_all_max_versions(self):
|
|
"""Find the latest version of all series
|
|
|
|
Return: list of:
|
|
int: ser_ver ID
|
|
int: series ID
|
|
int: Maximum version
|
|
"""
|
|
return self.db.series_get_all_max_versions()
|