cli/bugzilla: new module
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Anna “CyberTailor” 2024-01-10 07:53:05 +05:00
parent fbe463ae39
commit 7301e8d1b6
Signed by: CyberTaIlor
GPG Key ID: E7B76EDC50864BB1
18 changed files with 473 additions and 40 deletions

8
.reuse/dep5 Normal file
View File

@ -0,0 +1,8 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: find-work
Upstream-Contact: Anna <cyber@sysrq.in>
Source: https://find-work.sysrq.in/
Files: tests/cassettes/test_bugzilla/*
Copyright: Gentoo Authors
License: CC0-1.0

View File

@ -28,5 +28,12 @@ You can use command aliases, for example:
find-work -I rep -r gentoo_ovl_guru out
To see which installed packages have open version bump requests filed on
Bugzilla, sorted by date last updated, run:
.. prompt:: bash
find-work -I bugzilla -t outdated
All data from APIs is cached for a day, so don't hesitate running the command
again and again!

View File

@ -12,11 +12,13 @@ interested in.
The following data sources are supported:
* `Gentoo Bugzilla`
* `Repology`_
.. _Gentoo Bugzilla: https://bugs.gentoo.org/
.. _Repology: https://repology.org/
Support for other sources (like Bugzilla or Pkgcheck) is planned.
Support for other sources (like Soko or pkgcheck) is planned.
If you want to learn how to use find-work, check out the following resources:

View File

@ -8,6 +8,7 @@ from datetime import date
import click
from click_aliases import ClickAliasedGroup
import find_work.cli.bugzilla
import find_work.cli.repology
from find_work.cli import Options
from find_work.constants import VERSION
@ -36,6 +37,31 @@ def cli(ctx: click.Context, quiet: bool, installed: bool) -> None:
options.cache_key += str(today).encode() + b"\0"
@cli.group(aliases=["bug", "b"], cls=ClickAliasedGroup)
@click.option("-c", "--component",
help="Component name on Bugzilla.")
@click.option("-p", "--product",
help="Product name on Bugzilla.")
@click.option("-t", "--time", is_flag=True,
help="Sort bugs by time last modified.")
@click.pass_obj
def bugzilla(options: Options, component: str | None, product: str | None,
time: bool) -> None:
""" Use Bugzilla to find work. """
options.bugzilla.chronological_sort = time
options.cache_key += b"bugzilla" + b"\0"
options.cache_key += b"time:" + b"1" if time else b"0" + b"\0"
if product:
options.bugzilla.product = product
options.cache_key += b"product:" + product.encode() + b"\0"
if component:
options.bugzilla.component = component
options.cache_key += b"component:" + component.encode() + b"\0"
@cli.group(aliases=["rep", "r"], cls=ClickAliasedGroup)
@click.option("-r", "--repo", required=True,
help="Repository name on Repology.")
@ -49,4 +75,6 @@ def repology(options: Options, repo: str) -> None:
options.cache_key += repo.encode() + b"\0"
bugzilla.add_command(find_work.cli.bugzilla.outdated, aliases=["out", "o"])
repology.add_command(find_work.cli.repology.outdated, aliases=["out", "o"])

View File

@ -54,10 +54,24 @@ class ProgressDots:
class RepologyOptions:
""" Repology subcommand options. """
# Repository name
# Repository name.
repo: str = ""
@dataclass
class BugzillaOptions:
""" Bugzilla subcommand options. """
# Product name.
product: str = ""
# Component name.
component: str = ""
# Sort by date last modified or by ID.
chronological_sort: bool = False
@dataclass
class Options:
""" Global options. """
@ -71,11 +85,12 @@ class Options:
# Filter installed packages only
only_installed: bool = False
# String used for creating cache key
# Byte string used for creating cache key.
cache_key: bytes = b""
# Repology subcommand options
# Subcommand options.
repology: RepologyOptions = field(default_factory=RepologyOptions)
bugzilla: BugzillaOptions = field(default_factory=BugzillaOptions)
@staticmethod
def echo(*args: Any, **kwargs: Any) -> None:

168
find_work/cli/bugzilla.py Normal file
View File

@ -0,0 +1,168 @@
# SPDX-License-Identifier: WTFPL
# SPDX-FileCopyrightText: 2024 Anna <cyber@sysrq.in>
# No warranty
"""
CLI subcommands for everything Bugzilla.
This Python module also defines some regular expressions.
``pkg_re`` matches package name and version from bug summaries:
>>> ant_match = pkg_re.search(">=dev-java/ant-1.10.14: version bump - needed for jdk:21")
>>> (ant_match.group("package"), ant_match.group("version"))
('dev-java/ant', '1.10.14')
>>> libjxl_match = pkg_re.search("media-libs/libjxl: version bump")
>>> (libjxl_match.group("package"), libjxl_match.group("version"))
('media-libs/libjxl', None)
>>> tricky_match = pkg_re.search("app-foo/bar-2-baz-4.0: version bump")
>>> (tricky_match.group("package"), tricky_match.group("version"))
('app-foo/bar-2-baz', '4.0')
>>> pkg_re.search("Please bump Firefox") is None
True
``isodate_re`` matches ISO 8601 time/date strings:
>>> isodate_re.fullmatch("2024") is None
True
>>> isodate_re.fullmatch("20090916T09:04:18") is None
False
"""
import json
import re
import time
import warnings
from collections.abc import Iterable
from typing import Any
from xmlrpc.client import DateTime
import click
import gentoopm
from tabulate import tabulate
from find_work.cli import Options, ProgressDots
from find_work.constants import BUGZILLA_URL
from find_work.types import BugView
from find_work.utils import (
requests_session,
read_json_cache,
write_json_cache,
)
with warnings.catch_warnings():
# Disable annoying warning shown to LibreSSL users
warnings.simplefilter("ignore")
import bugzilla
from bugzilla.bug import Bug
# `category/package` matching according to PMS, and arbitrary version
pkg_re = re.compile(r"""(?P<package>
[\w][-+.\w]* # category
/ # single slash
[\w][+\w]* # package name before first '-'
(-[+\w]*(?=-))* # rest of package name
)
(
- # single hyphen
(?P<version>
\d+[.\w]* # arbitrary version
)
)?""",
re.VERBOSE)
isodate_re = re.compile(r"\d{4}\d{2}\d{2}T\d{2}:\d{2}:\d{2}")
class BugEncoder(json.JSONEncoder):
def default(self, o: Any) -> Any:
if isinstance(o, DateTime):
return o.value
return json.JSONEncoder.default(self, o)
def as_datetime(obj: dict) -> dict:
result: dict = {}
for key, value in obj.items():
# FIXME: every matching string will be converted to DateTime
if isinstance(value, str) and isodate_re.fullmatch(value):
result[key] = DateTime(value)
continue
result[key] = value
return result
def _bugs_from_json(data: list[dict]) -> list[Bug]:
with requests_session() as session:
bz = bugzilla.Bugzilla(BUGZILLA_URL, requests_session=session)
return [Bug(bz, dict=bug) for bug in data]
def _bugs_to_json(data: Iterable[Bug]) -> list[dict]:
return [bug.get_raw_data() for bug in data]
def _fetch_bump_requests(options: Options) -> list[Bug]:
with requests_session() as session:
bz = bugzilla.Bugzilla(BUGZILLA_URL, requests_session=session)
query = bz.build_query(
product=options.bugzilla.product or None,
component=options.bugzilla.component or None,
short_desc="version bump",
)
query["resolution"] = "---"
if options.bugzilla.chronological_sort:
query["order"] = "changeddate DESC"
else:
query["order"] = "bug_id DESC"
return bz.query(query)
def _collect_bump_requests(data: Iterable[Bug],
options: Options) -> list[BugView]:
if options.only_installed:
pm = gentoopm.get_package_manager()
result: list[BugView] = []
for bug in data:
if options.only_installed:
if (match := pkg_re.search(bug.summary)) is None:
continue
if match.group("package") not in pm.installed:
continue
date = time.strftime("%F", bug.last_change_time.timetuple())
item = BugView(bug.id, date, bug.assigned_to, bug.summary)
result.append(item)
return result
@click.command()
@click.pass_obj
def outdated(options: Options) -> None:
""" Find packages with version bump requests on Bugzilla. """
options.cache_key += b"outdated" + b"\0"
dots = ProgressDots(options.verbose)
options.vecho("Checking for cached data", nl=False, err=True)
with dots():
cached_data = read_json_cache(options.cache_key,
object_hook=as_datetime)
if cached_data is not None:
options.vecho("Loading cached data", nl=False, err=True)
with dots():
data = _bugs_from_json(cached_data)
else:
options.vecho("Fetching data from Bugzilla API", nl=False, err=True)
with dots():
data = _fetch_bump_requests(options)
options.vecho("Caching data", nl=False, err=True)
with dots():
json_data = _bugs_to_json(data)
write_json_cache(json_data, options.cache_key, cls=BugEncoder)
bumps = _collect_bump_requests(data, options)
if len(bumps) == 0:
options.secho("Congrats! You have nothing to do!", fg="green")
else:
options.echo(tabulate(bumps, tablefmt="plain")) # type: ignore

View File

@ -104,16 +104,15 @@ async def _outdated(options: Options) -> None:
write_json_cache(json_data, options.cache_key)
outdated_set = _collect_version_bumps(data.values(), options)
if len(outdated_set) == 0:
options.secho("Congrats! You have nothing to do!", fg="green")
return
for bump in outdated_set:
options.echo(bump.atom + " ", nl=False)
options.secho(bump.old_version, fg="red", nl=False)
options.echo("", nl=False)
options.secho(bump.new_version, fg="green")
if len(outdated_set) == 0:
options.secho("Congrats! You have nothing to do!", fg="green")
@click.command()
@click.pass_obj

View File

@ -13,5 +13,8 @@ VERSION = "0.2.0"
# Application homepage.
HOMEPAGE = "https://find-work.sysrq.in"
# Application's User-agent header
# Application's User-agent header.
USER_AGENT = f"Mozilla/5.0 (compatible; {PACKAGE}/{VERSION}; +{HOMEPAGE})"
# Gentoo Bugzilla location.
BUGZILLA_URL = "https://bugs.gentoo.org"

View File

@ -16,3 +16,13 @@ class VersionBump:
atom: str
old_version: str = field(compare=False)
new_version: str = field(compare=False)
@dataclass(frozen=True, order=True)
class BugView:
""" Bug listing item representation. """
bug_id: int
last_change_date: str = field(compare=False)
assigned_to: str = field(compare=False)
summary: str = field(compare=False)

View File

@ -7,6 +7,7 @@
import hashlib
import json
import tempfile
import warnings
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from pathlib import Path
@ -16,11 +17,16 @@ import aiohttp
from find_work.constants import PACKAGE, USER_AGENT
with warnings.catch_warnings():
# Disable annoying warning shown to LibreSSL users
warnings.simplefilter("ignore")
import requests
@asynccontextmanager
async def aiohttp_session() -> AsyncGenerator[aiohttp.ClientSession, None]:
"""
Construct an :py:class:`aiohttp.ClientSession` object.
Construct an :py:class:`aiohttp.ClientSession` object with out settings.
"""
headers = {"user-agent": USER_AGENT}
@ -33,18 +39,28 @@ async def aiohttp_session() -> AsyncGenerator[aiohttp.ClientSession, None]:
await session.close()
def requests_session() -> requests.Session:
"""
Construct an :py:class:`requests.Session` object with out settings.
"""
session = requests.Session()
session.headers["user-agent"] = USER_AGENT
return session
def _get_cache_path(cache_key: bytes) -> Path:
hexdigest = hashlib.sha256(cache_key).hexdigest()
file = Path(tempfile.gettempdir()) / PACKAGE / hexdigest
return file.with_suffix(".json")
def write_json_cache(data: Any, cache_key: bytes) -> None:
def write_json_cache(data: Any, cache_key: bytes, **kwargs: Any) -> None:
"""
Write a JSON cache file in a temporary directory.
Write a JSON cache file in a temporary directory. Keyword arguments are
passed to :py:function:`json.dump` as is.
:param data: data to serialize
:param cache_key: hash object to use as a key
:param cache_key: byte string to use as a key
"""
cache = _get_cache_path(cache_key)
@ -55,16 +71,17 @@ def write_json_cache(data: Any, cache_key: bytes) -> None:
with open(cache, "w") as file:
try:
json.dump(data, file)
json.dump(data, file, **kwargs)
except OSError:
pass
def read_json_cache(cache_key: bytes) -> Any | None:
def read_json_cache(cache_key: bytes, **kwargs: Any) -> Any | None:
"""
Read a JSON cache file stored in a temporary directory.
Read a JSON cache file stored in a temporary directory. Keyword arguments
are passed to :py:function:`json.load` as is.
:param cache_key: hash object to use as a key
:param cache_key: byte string to use as a key
:returns: decoded data or ``None``
"""
@ -73,4 +90,4 @@ def read_json_cache(cache_key: bytes) -> Any | None:
return None
with open(cache) as file:
return json.load(file)
return json.load(file, **kwargs)

View File

@ -40,8 +40,38 @@ The modules for
are as follows:
.Bl -tag -width repology
.It Xo
.Cm bugzilla
.Ol Fl t
.Op Fl c Ar name
.Op Fl p Ar name
.Ar command
.Xc
.Dl Pq alias: Cm bug , Cm b
.Pp
This module uses Gentoo Bugzilla to find work.
.Pp
.Ar command
can be one of the following:
.Bl -tag -width Ds
.It Ic outdated Pq alias: Ic out , Ic o
Find open version bump requests.
.El
.Pp
The options for
.Cm find-work bugzilla
are as follows:
.Bl -tag -width Ds
.It Fl c Ar name , Fl -component Ar name
Component name on Bugzilla.
.It Fl p Ar name , Fl -product Ar name
Product name on Bugzilla.
.It Fl t , Fl -time
Sort by time last modified (most recently modified first).
.El
.
.It Xo
.Cm repology
.Op Fl r Ar repo
.Fl r Ar repo
.Ar command
.Xc
.Dl Pq alias: Cm rep , Cm r
@ -60,7 +90,7 @@ The options for
are as follows:
.Bl -tag -width Ds
.It Fl r Ar repo , Fl -repo Ar repo
Repository name on repology.
Repository name on Repology.
This option is required.
Some examples for Gentoo include:
.Bl -bullet -compact -width 1n
@ -75,7 +105,7 @@ Some examples for Gentoo include:
.El
.El
.Sh ENVIRONMENT
.Bl -tag -width NOCOLOR
.Bl -tag -width NO_COLOR
.It Ev NO_COLOR
If defined, disable all color output.
.El

View File

@ -21,8 +21,11 @@ dependencies = [
"click-aliases",
"gentoopm<2",
"pydantic<3,>=2",
"python-bugzilla",
"repology-client<2,>=0.0.2",
"requests<3,>=2",
"sortedcontainers",
"tabulate",
]
classifiers = [
"Development Status :: 3 - Alpha",
@ -46,6 +49,7 @@ docs = [
test = [
"pkgcore",
"pytest",
"pytest-recording",
]
[project.scripts]
@ -70,6 +74,9 @@ include = [
"/tests",
]
[tool.pytest.ini_options]
addopts = "--doctest-modules --block-network"
[tool.mypy]
disallow_untyped_defs = true
no_implicit_optional = true
@ -84,6 +91,8 @@ check_untyped_defs = true
[[tool.mypy.overrides]]
module = [
"bugzilla",
"bugzilla.*",
"click_aliases",
"gentoopm",
"gentoopm.*",

View File

@ -0,0 +1,108 @@
interactions:
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-requests/2.31.0
method: HEAD
uri: https://bugs.gentoo.org/xmlrpc.cgi
response:
body:
string: ''
headers:
Connection:
- Keep-Alive
Content-Type:
- text/plain; charset=utf-8
Date:
- Wed, 10 Jan 2024 02:02:40 GMT
Keep-Alive:
- timeout=15, max=100
Server:
- Apache
status:
code: 200
message: OK
- request:
body: '<?xml version=''1.0''?>
<methodCall>
<methodName>Bugzilla.version</methodName>
<params>
<param>
<value><struct>
</struct></value>
</param>
</params>
</methodCall>
'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '161'
Content-Type:
- text/xml
User-Agent:
- python-bugzilla/3.2.0
method: POST
uri: https://bugs.gentoo.org/xmlrpc.cgi
response:
body:
string: <?xml version="1.0" encoding="UTF-8"?><methodResponse><params><param><value><struct><member><name>version</name><value><string>5.0.6</string></value></member></struct></value></param></params></methodResponse>
headers:
Connection:
- Keep-Alive
Content-Encoding:
- gzip
Content-Type:
- text/xml
Content-security-policy:
- frame-ancestors 'self'
Date:
- Wed, 10 Jan 2024 02:02:41 GMT
ETag:
- 7sMmJjLeC0KDNcgYXJQLEw
Keep-Alive:
- timeout=15, max=100
SOAPServer:
- SOAP::Lite/Perl/1.27
Server:
- Apache
Set-Cookie:
- Bugzilla_login_request_cookie=0jZsUHjVEQ; path=/; secure; HttpOnly
Strict-transport-security:
- max-age=15768000; includeSubDomains
Transfer-Encoding:
- chunked
Vary:
- Accept-Encoding
X-content-type-options:
- nosniff
X-frame-options:
- SAMEORIGIN
X-xss-protection:
- 1; mode=block
status:
code: 200
message: OK
version: 1

1
tests/data/bug74072.json Normal file
View File

@ -0,0 +1 @@
[{"id": 74072, "cc_detail": [{"name": "cantel", "id": 68480, "email": "cantel", "real_name": "Alex"}, {"name": "Captainsifff", "id": 32737, "email": "Captainsifff", "real_name": "Captain Sifff"}], "is_confirmed": true, "url": "", "cf_runtime_testing_required": "---", "flags": [], "is_open": false, "blocks": [], "op_sys": "Linux", "keywords": [], "component": "[OLD] Unspecified", "platform": "All", "groups": [], "depends_on": [], "qa_contact": "", "last_change_time": "20090916T09:04:18", "assigned_to": "bug-wranglers", "classification": "Unclassified", "priority": "High", "creator_detail": {"id": 17226, "name": "Augury", "real_name": "augury@vampares.org", "email": "Augury"}, "assigned_to_detail": {"name": "bug-wranglers", "id": 921, "email": "bug-wranglers", "real_name": "Gentoo Linux bug wranglers"}, "is_creator_accessible": true, "see_also": [], "cf_stabilisation_atoms": "", "alias": [], "version": "unspecified", "summary": "ld errors", "whiteboard": "", "severity": "trivial", "resolution": "WONTFIX", "is_cc_accessible": true, "creator": "Augury", "creation_time": "20041211T01:09:12", "product": "Gentoo Linux", "cc": ["cantel", "Captainsifff"], "status": "RESOLVED", "target_milestone": "---"}]

View File

@ -0,0 +1,3 @@
SPDX-FileCopyrightText: 2024 Gentoo Authors
SPDX-License-Identifier: CC0-1.0

20
tests/test_bugzilla.py Normal file
View File

@ -0,0 +1,20 @@
# SPDX-License-Identifier: WTFPL
# SPDX-FileCopyrightText: 2024 Anna <cyber@sysrq.in>
# No warranty
import json
from pathlib import Path
import pytest
from find_work.cli.bugzilla import (
_bugs_from_json,
_bugs_to_json,
)
@pytest.mark.vcr
def test_bugs_json_roundtrip():
with open(Path(__file__).parent / "data" / "bug74072.json") as file:
data: list[dict] = json.load(file)
assert data == _bugs_to_json(_bugs_from_json(data))

View File

@ -5,9 +5,9 @@
from sortedcontainers import SortedSet
from repology_client.types import Package
from find_work.types import VersionBump
from find_work.cli import Options
from find_work.cli.repology import (
VersionBump,
_collect_version_bumps,
_projects_from_json,
_projects_to_json,
@ -15,21 +15,24 @@ from find_work.cli.repology import (
def test_projects_json_roundtrip():
pkg1 = Package(
repo="gentoo",
visiblename="www-client/firefox",
version="9999",
status="test",
licenses=frozenset(["GPL-2", "LGPL-2.1", "MPL-2.0"]),
)
pkg2 = Package(
repo="gentoo",
visiblename="www-client/firefox-bin",
version="9999",
status="test",
licenses=frozenset(["GPL-2", "LGPL-2.1", "MPL-2.0"]),
)
data = {"firefox": {pkg1, pkg2}}
data = {
"firefox": {
Package(
repo="gentoo",
visiblename="www-client/firefox",
version="9999",
status="test",
licenses=frozenset(["GPL-2", "LGPL-2.1", "MPL-2.0"]),
),
Package(
repo="gentoo",
visiblename="www-client/firefox-bin",
version="9999",
status="test",
licenses=frozenset(["GPL-2", "LGPL-2.1", "MPL-2.0"]),
),
},
}
assert data == _projects_from_json(_projects_to_json(data))
@ -58,7 +61,7 @@ def test_collect_version_bumps():
version="1",
status="outdated",
),
}
},
]
expected = SortedSet([VersionBump("dev-util/examplepkg", "1", "2")])
@ -96,7 +99,7 @@ def test_collect_version_bumps_multi_versions():
version="1",
status="outdated",
),
}
},
]
expected = SortedSet([VersionBump("dev-util/examplepkg", "1", "2")])
@ -141,7 +144,7 @@ def test_collect_version_bumps_multi_names():
version="1",
status="outdated",
),
}
},
]
expected = SortedSet([

View File

@ -10,6 +10,8 @@ description = run the tests + mypy
deps =
mypy
sortedcontainers-stubs
types-requests
types-tabulate
extras =
test
commands =