mirror of
https://github.com/UniXNU/UniXNU.git
synced 2026-11-18 10:01:39 +03:00
177 lines
9.3 KiB
Python
177 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Inventory every branch and differing file in a set of local fork repositories."""
|
|
import argparse
|
|
from collections import Counter
|
|
import concurrent.futures
|
|
import csv
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import urllib.parse
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.check_output([str(arg) for arg in args])
|
|
|
|
|
|
def git(repo, *args):
|
|
return command('git', '-C', repo, *args)
|
|
|
|
|
|
def tree(repo, revision):
|
|
entries = {}
|
|
for item in git(repo, 'ls-tree', '-rz', '--full-tree', revision).split(b'\0'):
|
|
if item:
|
|
metadata, path = item.split(b'\t', 1)
|
|
mode, kind, blob = metadata.decode().split()
|
|
entries[path.decode()] = (mode, kind, blob)
|
|
return entries
|
|
|
|
|
|
def classify(main, fork, bases):
|
|
if bases and all(fork == base for base in bases):
|
|
return 'upstream-only'
|
|
if bases and all(main == base for base in bases):
|
|
return 'fork-only'
|
|
if bases and any(fork == base for base in bases):
|
|
return 'multiple-bases'
|
|
return 'both-changed' if bases else 'unrelated-history'
|
|
|
|
|
|
def inspect(name, repository, owner, upstream, output, fetch):
|
|
repo = Path(repository).resolve()
|
|
main_ref = 'refs/remotes/fork-audit-upstream/main'
|
|
fork_ref = 'refs/remotes/fork-audit/'
|
|
shallow = git(repo, 'rev-parse', '--is-shallow-repository').strip() == b'true'
|
|
if shallow and not fetch:
|
|
raise RuntimeError(f'{repo}: use --fetch to retrieve complete history')
|
|
if fetch:
|
|
if shallow:
|
|
subprocess.run(['git', '-C', str(repo), 'fetch', '--quiet', '--unshallow',
|
|
'--no-tags', '--no-recurse-submodules',
|
|
f'https://github.com/{owner}/{name}.git'], check=True)
|
|
subprocess.run(['git', '-C', str(repo), 'fetch', '--quiet', '--no-tags',
|
|
'--no-recurse-submodules', f'https://github.com/{upstream}/{name}.git',
|
|
'+refs/heads/main:' + main_ref], check=True)
|
|
subprocess.run(['git', '-C', str(repo), 'fetch', '--quiet', '--prune', '--no-tags',
|
|
'--no-recurse-submodules', f'https://github.com/{owner}/{name}.git',
|
|
'+refs/heads/*:' + fork_ref + '*'], check=True)
|
|
main = git(repo, 'rev-parse', main_ref).decode().strip()
|
|
branches = []
|
|
for line in git(repo, 'for-each-ref', '--format=%(refname) %(objectname)', fork_ref).decode().splitlines():
|
|
reference, revision = line.split()
|
|
branches.append((reference[len(fork_ref):], revision))
|
|
trees = {}
|
|
|
|
def entries(revision):
|
|
if revision not in trees:
|
|
trees[revision] = tree(repo, revision)
|
|
return trees[revision]
|
|
|
|
main_tree = entries(main)
|
|
files = {}
|
|
commits = {}
|
|
branch_rows = []
|
|
for branch, revision in branches:
|
|
merge = subprocess.run(['git', '-C', str(repo), 'merge-base', '--all', main, revision], capture_output=True, check=False)
|
|
if merge.returncode not in (0, 1):
|
|
raise RuntimeError(merge.stderr.decode())
|
|
bases = merge.stdout.decode().splitlines()
|
|
base_trees = [entries(base) for base in bases]
|
|
fork_tree = entries(revision)
|
|
behind, ahead = map(int, git(repo, 'rev-list', '--left-right', '--count', main + '...' + revision).split())
|
|
counts = Counter()
|
|
for path in sorted(main_tree.keys() | fork_tree.keys()):
|
|
old, new = main_tree.get(path), fork_tree.get(path)
|
|
if old == new:
|
|
counts['identical'] += 1
|
|
continue
|
|
status = classify(old, new, [base.get(path) for base in base_trees])
|
|
counts[status] += 1
|
|
identity = hashlib.sha256(json.dumps([name, path, old, new]).encode()).hexdigest()[:16]
|
|
if identity not in files:
|
|
patch = output / 'diffs' / name / (identity + '.patch')
|
|
patch.parent.mkdir(parents=True, exist_ok=True)
|
|
patch.write_bytes(git(repo, 'diff', '--no-ext-diff', '--no-textconv', '--no-renames',
|
|
'--binary', main, revision, '--', path))
|
|
hunks = [line for line in patch.read_text(errors='replace').splitlines() if line.startswith('@@')]
|
|
files[identity] = dict(id=identity, repo=name, path=path, upstream_main=main,
|
|
upstream_entry=old, fork_entry=new, branches=[], classifications=[],
|
|
hunks=hunks, patch=str(patch.relative_to(output)),
|
|
source_url=f'https://github.com/{owner}/{name}/blob/{revision}/' + urllib.parse.quote(path),
|
|
decision='pending', reason='')
|
|
row = files[identity]
|
|
row['branches'].append(branch)
|
|
if status not in row['classifications']:
|
|
row['classifications'].append(status)
|
|
revision_commits = git(repo, 'rev-list', revision, '^' + main).decode().splitlines()
|
|
for commit in revision_commits:
|
|
if commit not in commits:
|
|
data = git(repo, 'show', '-s', '--format=%an%x00%ae%x00%aI%x00%P%x00%s', commit).decode().rstrip('\n').split('\0')
|
|
author, email, date, parents, title = data
|
|
paths = git(repo, 'diff-tree', '--root', '--no-commit-id', '--name-only', '-r', '-m', '-z', commit).split(b'\0')
|
|
commits[commit] = dict(repo=name, sha=commit, author=author, email=email, date=date,
|
|
parents=parents.split(), title=title,
|
|
paths=sorted({path.decode() for path in paths if path}), branches=[],
|
|
source_url=f'https://github.com/{owner}/{name}/commit/{commit}')
|
|
commits[commit]['branches'].append(branch)
|
|
branch_rows.append(dict(repo=name, branch=branch, sha=revision, upstream_main=main,
|
|
merge_bases=bases, ahead=ahead, behind=behind, files=dict(counts)))
|
|
return dict(repo=name, path=str(repo), upstream_main=main, branches=branch_rows,
|
|
files=list(files.values()), commits=list(commits.values()))
|
|
|
|
|
|
def write_csv(path, rows, fields):
|
|
with path.open('w', encoding='utf-8-sig', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction='ignore')
|
|
writer.writeheader()
|
|
for row in rows:
|
|
writer.writerow({key: json.dumps(value, ensure_ascii=False) if isinstance(value, (list, dict)) else value
|
|
for key, value in row.items() if key in fields})
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--repositories', type=Path, required=True, help='JSON object mapping repository names to local paths')
|
|
parser.add_argument('--owner', required=True)
|
|
parser.add_argument('--upstream', default='UniXNU')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--fetch', action='store_true', help='refresh upstream main and every fork branch')
|
|
parser.add_argument('--review', type=Path, help='JSON rows with file id, decision and reason annotations')
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
repositories = json.loads(args.repositories.read_text())
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
futures = [pool.submit(inspect, name, path, args.owner, args.upstream, output, args.fetch)
|
|
for name, path in sorted(repositories.items())]
|
|
results = [future.result() for future in futures]
|
|
branches = [row for result in results for row in result['branches']]
|
|
files = [row for result in results for row in result['files']]
|
|
commits = [row for result in results for row in result['commits']]
|
|
if args.review:
|
|
reviews = {row['id']: row for row in json.loads(args.review.read_text())}
|
|
for row in files:
|
|
if review := reviews.get(row['id']):
|
|
row.update(decision=review['decision'], reason=review['reason'])
|
|
snapshot = dict(captured_at=datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
owner=args.owner, upstream=args.upstream, repositories=results)
|
|
(output / 'audit.json').write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + '\n')
|
|
write_csv(output / 'branches.csv', branches, ['repo', 'branch', 'sha', 'upstream_main', 'ahead', 'behind', 'merge_bases', 'files'])
|
|
fields = ['id', 'repo', 'path', 'classifications', 'decision', 'reason',
|
|
'branches', 'hunks', 'upstream_entry', 'fork_entry', 'patch', 'source_url']
|
|
write_csv(output / 'files.csv', files, fields)
|
|
write_csv(output / 'fork-files.csv',
|
|
[row for row in files if any(kind != 'upstream-only' for kind in row['classifications'])], fields)
|
|
write_csv(output / 'commits.csv', commits, ['repo', 'sha', 'author', 'date', 'title', 'branches', 'paths', 'source_url'])
|
|
for result in results:
|
|
candidates = sum(any(kind != 'upstream-only' for kind in row['classifications']) for row in result['files'])
|
|
print(f"{result['repo']}: {len(result['branches'])} branches, {len(result['files'])} distinct file differences, "
|
|
f"{candidates} fork changes, {len(result['commits'])} commits")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|