#!/usr/bin/python3 # The purpose of this script is to reformat the data from the most recent MBS # build into a more concise format. It prints the basic module build # information as well as a table of the RPM tasks sorted by status. It also # checks to ensure the module builds have artifacts, and compares them with the # corresponding RHEL artifacts. import itertools import pathlib import urllib import click import git import requests import rfc3986 import texttable import yaml def nevra_to_na(nevra): nevr, arch = nevra.rsplit('.', maxsplit=1) name, _, _ = nevr.rsplit('-', maxsplit=2) return name, arch def get_centos_builds(params): response = requests.get( f'https://mbs.mbox.centos.org/module-build-service/1/module-builds/', params=params, verify=centos_ca, ) response.raise_for_status() return sorted([CentOSModuleBuild(data) for data in response.json()['items']]) def get_rhel_builds(params): response = requests.get( f'https://mbs.engineering.redhat.com/module-build-service/1/module-builds/', params=params, ) response.raise_for_status() return sorted([ RHELModuleBuild(data) for data in response.json()['items'] if '_' not in data['context'] ]) class ModuleBuild: def __init__(self, data): # basic properties self.name = data['name'] self.stream = data['stream'] self.stream_sanitized = data['stream'].replace('-', '_') self.version = data['version'] self.context = data['context'] self.koji_tag = data['koji_tag'] self.id = data['id'] self.state_name = data['state_name'] self.state_color = { 'init': 'yellow', 'build': 'yellow', 'wait': 'yellow', 'ready': 'green', 'done': 'green', 'failed': 'red', }[self.state_name] self.state_reason = data['state_reason'] self.buildrequires = [ f'{br_module_name}:{br_info["stream"]}' for br_module_name, br_info in data['buildrequires'].items() if br_module_name != 'platform' ] self.set_url() # sort components self.components = { 'pending': [], 'submitted': [], 'failed': [], 'done': [], 'reused': [], } self.failed_tasks = [] if data['tasks']: for task_name, task_data in data['tasks']['rpms'].items(): label = task_data['nvr'] or task_name if task_data['state'] is None: self.components['pending'].append(label) elif task_data['state'] == 0: self.components['submitted'].append(label) elif task_data['state'] == 1: if task_data['state_reason'] == "": self.components['done'].append(label) elif task_data['state_reason'] == "Reused component from previous module build": self.components['reused'].append(label) else: raise SystemExit(f'unrecognized task state_reason: {task_data}') elif task_data['state'] == 3: self.components['failed'].append(label) if task_data['task_id']: self.failed_tasks.append(task_data['task_id']) else: raise SystemExit(f'unrecognized task state: {task_data}') if self.done: # get artifacts self.artifacts = {} self.debug_artifacts = {} for arch in arches: response = self.get_artifact_response(arch) response.raise_for_status() artifact_data = yaml.safe_load(response.text)['data'] try: raw_artifacts = artifact_data['artifacts']['rpms'] except KeyError: self.artifacts[arch] = None self.debug_artifacts[arch] = None else: debug_artifacts = [] artifacts = [] for nevra in raw_artifacts: if '-debuginfo-' in nevra or '-debugsource-' in nevra: debug_artifacts.append(nevra_to_na(nevra)) else: artifacts.append(nevra_to_na(nevra)) self.debug_artifacts[arch] = tuple(debug_artifacts) self.artifacts[arch] = tuple(artifacts) def __lt__(self, other): return self.buildrequires < other.buildrequires def print_info(self): if self.koji_tag: click.echo(click.style('tag: ', fg='cyan') + self.koji_tag) if self.buildrequires: click.echo(click.style('build requires: ', fg='cyan') + ' '.join(self.buildrequires)) click.echo(click.style('url: ', fg='cyan') + self.url) click.echo(click.style('status: ', fg='cyan') + click.style(self.state_name, fg=self.state_color)) def print_failed(self): click.echo(click.style('reason: ', fg='cyan') + self.state_reason) click.secho('failed tasks: ', fg='cyan') for failed_task in self.failed_tasks: click.echo(f'https://koji.mbox.centos.org/koji/taskinfo?taskID={failed_task}') def print_done(self): if self.components['done']: click.secho('done components: ', fg='cyan', nl=False) click.echo(' '.join( component for component in self.components['done'] if not component.startswith('module-build-macros-') )) if self.components['reused']: click.secho('reused components: ', fg='cyan', nl=False) click.echo(' '.join(self.components['reused'])) click.secho('module builds:', fg='cyan', nl=False) click.echo(f' {self.name}-{self.stream_sanitized}-{self.version}.{self.context}', nl=False) click.echo(f' {self.name}-devel-{self.stream_sanitized}-{self.version}.{self.context}') def print_table(self): table = texttable.Texttable(max_width=0) table.add_row(['pending', 'submitted', 'failed', 'done', 'reused']) table.add_row( [ '\n'.join(self.components['pending']), '\n'.join(self.components['submitted']), '\n'.join(self.components['failed']), '\n'.join(self.components['done']), '\n'.join(self.components['reused']), ] ) click.echo(table.draw()) @property def failed(self): return self.state_name == 'failed' @property def done(self): return self.state_name in ['ready', 'done'] @property def has_components(self): return any(self.components.values()) class CentOSModuleBuild(ModuleBuild): def get_artifact_response(self, arch): return requests.get( f'https://koji.mbox.centos.org/pkgs/packages/{self.name}/{self.stream_sanitized}/' + f'{self.version}.{self.context}/files/module/modulemd.{arch}.txt', verify=centos_ca, ) def set_url(self): self.url = f'https://mbs.mbox.centos.org/module-build-service/1/module-builds/{self.id}' class RHELModuleBuild(ModuleBuild): def get_artifact_response(self, arch): return requests.get( f'http://download.eng.bos.redhat.com/brewroot/packages/{self.name}/{self.stream_sanitized}/' + f'{self.version}.{self.context}/files/module/modulemd.{arch}.txt', ) def set_url(self): self.url = f'https://mbs.engineering.redhat.com/module-build-service/1/module-builds/{self.id}' @click.command() def main(): # Set up git repo object. try: repo = git.Repo() except git.InvalidGitRepositoryError: raise click.ClickException('must be run from git checkout directory of module') # Set module name based on repo name. origin_url = rfc3986.urlparse(repo.remotes.origin.url) origin_path = pathlib.PurePath(origin_url.path) if not origin_path.match('/modules/*'): raise click.ClickException('git repo not in modules namespace') module = origin_path.stem # Set module stream by reading modulemd. try: with open(f'{module}.yaml') as f: modulemd = yaml.safe_load(f) except FileNotFoundError: raise click.ClickException(f'no {module}.yaml file found') stream = str(modulemd['data']['stream']) # Get CentOS module builds. scmurl = f'git+https://{origin_url.host}{origin_path}?#{repo.head.commit}' params = {'name': module, 'stream': stream, 'scmurl': scmurl} centos_builds = get_centos_builds(params) if not centos_builds: raise click.ClickException(f'no CentOS module builds found matching {params}') gaps = True if len(centos_builds) > 1 else False if all(centos_build.done for centos_build in centos_builds): # Get RHEL module builds. tag = repo.git.describe(tags=True, abbrev=0, exclude='*-devel-*') _, _, nsvc = urllib.parse.unquote(tag).split('/') nsv, _ = nsvc.rsplit('.', maxsplit=1) _, _, rhel_version = nsv.rsplit('-', maxsplit=2) params = {'name': module, 'stream': stream, 'version': rhel_version} rhel_builds = get_rhel_builds(params) else: rhel_builds = [] for centos_build, rhel_build in itertools.zip_longest(centos_builds, rhel_builds): if gaps: click.echo() centos_build.print_info() if centos_build.done: centos_build.print_done() click.secho('artifacts compare:', fg='cyan', nl=False) if rhel_build: for arch in arches: if centos_build.artifacts[arch] == rhel_build.artifacts[arch]: color = 'green' else: color = 'red' if centos_build.debug_artifacts[arch] == rhel_build.debug_artifacts[arch]: debug_color = 'green' else: debug_color = 'yellow' click.echo( click.style(f' {arch}', fg=color) + click.style(f'+debug', fg=debug_color), nl=False, ) click.echo() else: click.secho(' skipped due to lack of RHEL build', fg='yellow') else: if centos_build.failed: centos_build.print_failed() elif centos_build.has_components: centos_build.print_table() if gaps: click.echo() if __name__ == '__main__': centos_ca = pathlib.Path.home() / '.centos-server-ca.cert' arches = ['aarch64', 'ppc64le', 'x86_64'] main()