mirror of
https://github.com/UniXNU/UniXNU.git
synced 2026-11-18 10:01:39 +03:00
fc0eb174b6
Add the device name beside each codename in both language indexes. Prefer the canonical registered title for each codename, with a source title as the fallback, and remove repeated codename and owner suffixes from labels. Fill in the OnePlus Ace 3 and Xiaomi Pad 6S Pro macOS labels. Identify the build-only entry as having an unspecified device. Refresh their grouped source pages together with the indexes. Validation: regenerated both language catalogs from 330 sources, compiled the generator and passed the staged whitespace check.
435 lines
21 KiB
Python
435 lines
21 KiB
Python
#!/usr/bin/env python3
|
||
"""Discover public UniXNU device repositories and refresh the documentation catalog."""
|
||
import argparse
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import tomllib
|
||
from urllib.error import HTTPError
|
||
from urllib.parse import urlencode
|
||
from urllib.request import Request, urlopen
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SNAPSHOTS = ROOT / 'docs/.vitepress/device-records'
|
||
REPOSITORY = re.compile(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+')
|
||
SLUG = re.compile(r'[a-z0-9][a-z0-9-]*')
|
||
|
||
|
||
def record_tools():
|
||
spec = importlib.util.spec_from_file_location('porting_record', ROOT / 'scripts/porting-record.py')
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
class GitHub:
|
||
def __init__(self):
|
||
self.token = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN')
|
||
if not self.token and shutil.which('gh'):
|
||
result = subprocess.run(['gh', 'auth', 'token'], capture_output=True, text=True)
|
||
if result.returncode == 0:
|
||
self.token = result.stdout.strip()
|
||
if not self.token:
|
||
raise ValueError('Set GITHUB_TOKEN or GH_TOKEN, or sign in with gh auth login')
|
||
|
||
def request(self, path, payload=None):
|
||
headers = {'Authorization': f'Bearer {self.token}',
|
||
'Accept': 'application/vnd.github+json', 'User-Agent': 'UniXNU-device-catalog'}
|
||
data = json.dumps(payload).encode() if payload is not None else None
|
||
for attempt in range(4):
|
||
try:
|
||
request = Request('https://api.github.com/' + path, data=data, headers=headers)
|
||
with urlopen(request, timeout=60) as response:
|
||
body = response.read(32 * 1024 * 1024 + 1)
|
||
if len(body) > 32 * 1024 * 1024:
|
||
raise ValueError('GitHub response exceeds 32 MiB')
|
||
return json.loads(body)
|
||
except HTTPError as error:
|
||
if error.code not in (429, 500, 502, 503, 504) or attempt == 3:
|
||
raise
|
||
time.sleep(2 ** attempt)
|
||
|
||
def search(self, query):
|
||
found = []
|
||
for page in range(1, 11):
|
||
result = self.request('search/repositories?' + urlencode({
|
||
'q': query, 'per_page': 100, 'page': page}))
|
||
# The search API enumerates at most 1000 results, so a broader query is capped there, not an error: the
|
||
# authoritative discovery is the bounded topic:unixnu and org:UniXNU queries, which a compliant device
|
||
# repository is required to satisfy (the unixnu topic), and the name/description query only supplements them.
|
||
if result.get('incomplete_results'):
|
||
raise ValueError(f'GitHub returned an incomplete search: {query}')
|
||
items = result['items']
|
||
found.extend(item['full_name'] for item in items)
|
||
if len(items) < 100:
|
||
return found
|
||
return found
|
||
|
||
def metadata(self, repositories):
|
||
def batch(names):
|
||
fields = []
|
||
for index, name in enumerate(names):
|
||
owner, repo = name.split('/')
|
||
fields.append(f'r{index}:repository(owner:{json.dumps(owner)},name:{json.dumps(repo)})' +
|
||
'{nameWithOwner databaseId isPrivate isFork parent{nameWithOwner} '
|
||
'defaultBranchRef{name target{... on Commit{oid}}}}')
|
||
result = self.request('graphql', {'query': 'query{' + ''.join(fields) + '}'})
|
||
for error in result.get('errors', []):
|
||
if error.get('type') != 'NOT_FOUND':
|
||
raise ValueError('GitHub metadata query failed: ' + error.get('message', 'unknown error'))
|
||
items = {name: result.get('data', {}).get(f'r{index}') for index, name in enumerate(names)}
|
||
fields = []
|
||
for index, name in enumerate(names):
|
||
item = items[name]
|
||
if not item or item.get('isPrivate') or not item.get('defaultBranchRef'):
|
||
continue
|
||
revision = item['defaultBranchRef']['target']['oid']
|
||
owner, repo = item['nameWithOwner'].split('/')
|
||
fields.append(f'r{index}:repository(owner:{json.dumps(owner)},name:{json.dumps(repo)})' +
|
||
'{board:object(expression:' + json.dumps(revision + ':board.toml') +
|
||
'){... on Blob{oid text}} record:object(expression:' +
|
||
json.dumps(revision + ':porting.toml') + '){... on Blob{oid text}}}')
|
||
if fields:
|
||
sources = self.request('graphql', {'query': 'query{' + ''.join(fields) + '}'})
|
||
if sources.get('errors'):
|
||
raise ValueError('Could not read the pinned device source files')
|
||
for index, name in enumerate(names):
|
||
if items[name]:
|
||
items[name].update(sources.get('data', {}).get(f'r{index}') or {})
|
||
return items
|
||
|
||
groups = [repositories[i:i + 12] for i in range(0, len(repositories), 12)]
|
||
result = {}
|
||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||
for group in pool.map(batch, groups):
|
||
result.update(group)
|
||
return result
|
||
|
||
|
||
def bootstrap_registry():
|
||
"""Register the current pages so their routes and legacy prose remain available."""
|
||
rows = {}
|
||
for line in (ROOT / 'docs/devices/index.md').read_text().splitlines():
|
||
match = re.match(r'\| \[(.*?)\]\(/devices/([^/)]+)\) \| (.*?) \| (.*?) \|', line)
|
||
if match:
|
||
title, name, soc, summary = match.groups()
|
||
rows[name] = {'title': title, 'soc': soc, 'summary': summary}
|
||
entries = []
|
||
for path in sorted((ROOT / 'docs/devices').glob('*.md')):
|
||
if path.stem == 'index':
|
||
continue
|
||
content = path.read_text()
|
||
table = re.search(r'^\| \[.*?\]\(https://github.com/([^/)]+/[^/)]+)\)', content, re.M)
|
||
other = re.search(r'https://github.com/([^/()\s]+/[^/()\s]+)', content)
|
||
match = table or other
|
||
if not match:
|
||
continue
|
||
repo = match[1].removesuffix('.git')
|
||
if not REPOSITORY.fullmatch(repo):
|
||
continue
|
||
translated = ROOT / 'docs/zh/devices' / path.name
|
||
title = content.splitlines()[0].removeprefix('# ').strip()
|
||
title_zh = translated.read_text().splitlines()[0].removeprefix('# ').strip() if translated.exists() else title
|
||
entries.append({'name': path.stem, 'repository': repo, 'title': title,
|
||
'title_zh': title_zh, 'format': 1, 'legacy_page': 'preserve',
|
||
**{key: value for key, value in rows.get(path.stem, {}).items() if key != 'title'}})
|
||
return {'schema_version': 2, 'device': entries}
|
||
|
||
|
||
def load_registry():
|
||
registry = tomllib.loads((ROOT / 'devices.toml').read_text())
|
||
if registry.get('schema_version') == 1:
|
||
return bootstrap_registry()
|
||
if registry.get('schema_version') != 2:
|
||
raise ValueError('device registry schema_version must be 2')
|
||
return registry
|
||
|
||
|
||
def write_registry(registry):
|
||
lines = ['schema_version = 2', '']
|
||
for entry in registry['device']:
|
||
lines.append('[[device]]')
|
||
for key, value in entry.items():
|
||
if value is not None:
|
||
lines.append(f'{key} = {json.dumps(value, ensure_ascii=False)}')
|
||
lines.append('')
|
||
(ROOT / 'devices.toml').write_text('\n'.join(lines))
|
||
|
||
|
||
def board_data(metadata):
|
||
value = metadata.get('board')
|
||
if not value or not value.get('text'):
|
||
return None
|
||
try:
|
||
board = tomllib.loads(value['text'])
|
||
identity = board.get('board', {})
|
||
components = board.get('components', {})
|
||
if not isinstance(identity, dict) or not isinstance(components, dict):
|
||
return None
|
||
if not isinstance(identity.get('name'), str) or not SLUG.fullmatch(identity['name']):
|
||
return None
|
||
selected = components.get('include', [])
|
||
if not isinstance(selected, list) or not all(isinstance(item, str) for item in selected):
|
||
return None
|
||
return board
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def update_entry(entry, metadata, tools):
|
||
entry['repository'] = metadata['nameWithOwner']
|
||
entry['repository_id'] = metadata.get('databaseId')
|
||
board = board_data(metadata)
|
||
if not board:
|
||
if entry.get('format') != 2:
|
||
entry['revision'] = metadata['defaultBranchRef']['target']['oid']
|
||
return
|
||
entry['revision'] = metadata['defaultBranchRef']['target']['oid']
|
||
info = board['board']
|
||
entry['board'] = info['name']
|
||
if entry.get('legacy_page') == 'link' and entry['name'] != info['name']:
|
||
suffix = f' ({entry["repository"].split("/")[0]})'
|
||
for key in ('title', 'title_zh'):
|
||
if not entry[key].endswith(suffix):
|
||
entry[key] += suffix
|
||
entry['soc'] = str(info.get('soc', ''))
|
||
entry['format'] = 2 if board.get('schema_version') == 2 else 1
|
||
if entry['format'] != 2:
|
||
for key in ('record_state', 'record_error', 'os_profile', 'components'):
|
||
entry.pop(key, None)
|
||
(SNAPSHOTS / f'{entry["name"]}.toml').unlink(missing_ok=True)
|
||
return
|
||
entry['os_profile'] = str(info.get('os_profile', ''))
|
||
entry['components'] = board.get('components', {}).get('include', [])
|
||
raw = (metadata.get('record') or {}).get('text')
|
||
entry['record_state'] = 'missing'
|
||
entry.pop('record_error', None)
|
||
snapshot = SNAPSHOTS / f'{entry["name"]}.toml'
|
||
if raw:
|
||
try:
|
||
record = tomllib.loads(raw)
|
||
tools.validate(record, board, entry['repository'], allow_custom_components=True)
|
||
except (ValueError, KeyError, TypeError, AttributeError) as error:
|
||
entry['record_state'] = 'invalid'
|
||
entry['record_error'] = str(error)
|
||
else:
|
||
snapshot.write_text(raw)
|
||
entry['record_state'] = 'available'
|
||
if entry['record_state'] != 'available':
|
||
snapshot.unlink(missing_ok=True)
|
||
|
||
|
||
def synchronize(github):
|
||
registry = load_registry()
|
||
existing = registry['device']
|
||
discovered = set()
|
||
for query in ('unixnu in:name,description fork:true is:public',
|
||
'topic:unixnu fork:true is:public', 'org:UniXNU is:public'):
|
||
discovered.update(github.search(query))
|
||
requested = sorted(discovered | {entry['repository'] for entry in existing})
|
||
metadata = github.metadata(requested)
|
||
SNAPSHOTS.mkdir(parents=True, exist_ok=True)
|
||
tools = record_tools()
|
||
by_id = {item['databaseId']: item for item in metadata.values()
|
||
if item and item.get('databaseId') and not item.get('isPrivate')}
|
||
used, signatures = set(), set()
|
||
for entry in existing:
|
||
item = metadata.get(entry['repository']) or by_id.get(entry.get('repository_id'))
|
||
if not item or item.get('isPrivate') or not item.get('defaultBranchRef'):
|
||
continue
|
||
update_entry(entry, item, tools)
|
||
used.add(item['nameWithOwner'].lower())
|
||
if item.get('board'):
|
||
signatures.add((item['board']['oid'], (item.get('record') or {}).get('oid')))
|
||
names = {entry['name'] for entry in existing}
|
||
candidates = [item for item in metadata.values() if item and not item.get('isPrivate')
|
||
and item.get('defaultBranchRef') and item['nameWithOwner'].lower() not in used]
|
||
candidates.sort(key=lambda item: (not item['nameWithOwner'].startswith('UniXNU/'),
|
||
item.get('isFork', False), item['nameWithOwner'].lower()))
|
||
added = 0
|
||
for item in candidates:
|
||
repo = item['nameWithOwner']
|
||
board = board_data(item)
|
||
if not board:
|
||
continue
|
||
name = board['board']['name']
|
||
signature = (item['board']['oid'], (item.get('record') or {}).get('oid'))
|
||
if signature in signatures or repo.lower() in used:
|
||
continue
|
||
route = name
|
||
if route in names:
|
||
route = f'{name}-{repo.split("/")[0].lower()}'
|
||
route = re.sub(r'[^a-z0-9-]', '-', route)
|
||
if route in names:
|
||
continue
|
||
title = str(board['board'].get('description') or name).replace('\n', ' ')[:160]
|
||
if route != name:
|
||
title += f' ({repo.split("/")[0]})'
|
||
entry = {'name': route, 'repository': repo, 'title': title, 'title_zh': title,
|
||
'format': 1, 'legacy_page': 'link'}
|
||
update_entry(entry, item, tools)
|
||
existing.append(entry)
|
||
names.add(route)
|
||
signatures.add(signature)
|
||
used.add(repo.lower())
|
||
added += 1
|
||
registry['device'] = sorted(existing, key=lambda entry: (entry['name'] != 'sheng', entry['name']))
|
||
write_registry(registry)
|
||
render_site(registry)
|
||
print(f'Scanned {len(discovered)} repositories; registered {len(existing)} sources; added {added}')
|
||
for entry in existing:
|
||
if entry.get('format') == 2 and entry.get('record_state') != 'available':
|
||
print(f'{entry["repository"]}: record {entry.get("record_state", "missing")} '
|
||
f'{entry.get("record_error", "")}')
|
||
return registry
|
||
|
||
|
||
def render_site(registry, local=None):
|
||
tools = record_tools()
|
||
local = local or {}
|
||
names = set()
|
||
for entry in registry['device']:
|
||
name, repo = entry['name'], entry['repository']
|
||
if not SLUG.fullmatch(name) or not REPOSITORY.fullmatch(repo) or name in names:
|
||
raise ValueError(f'Invalid or duplicate device entry: {name}')
|
||
names.add(name)
|
||
record = None
|
||
if entry.get('format') == 2:
|
||
if name in local:
|
||
record = tools.check(Path(local[name]))
|
||
elif entry.get('record_state') == 'available':
|
||
record = tomllib.loads((SNAPSHOTS / f'{name}.toml').read_text())
|
||
board = {'board': {'name': entry.get('board', name), 'soc': entry['soc'],
|
||
'os_profile': entry['os_profile']},
|
||
'components': {'include': entry.get('components', [])}}
|
||
tools.validate(record, board, name, allow_custom_components=True)
|
||
for zh in (False, True):
|
||
directory = ROOT / ('docs/zh/devices' if zh else 'docs/devices')
|
||
page = directory / f'{name}.md'
|
||
if entry.get('format') != 2:
|
||
if entry.get('legacy_page') == 'preserve' and page.exists():
|
||
continue
|
||
page.write_text(f'[{repo}](https://github.com/{repo})\n')
|
||
continue
|
||
revision = entry.get('revision', '')
|
||
if not re.fullmatch(r'[a-f0-9]{40}', revision):
|
||
raise ValueError(f'{name}: pin a full source commit')
|
||
title = tools.markdown_text(entry['title_zh' if zh else 'title'])
|
||
body = f'# {title}\n\n[{repo}](https://github.com/{repo})\n\n'
|
||
if record:
|
||
body += tools.render(record, repo, revision, zh)
|
||
else:
|
||
body += ('在设备仓库补齐并校验 [porting.toml](/zh/porting/records),随后生成各部分的适配记录。'
|
||
if zh else 'Complete and validate [porting.toml](/porting/records) in the device repository to publish its subsystem record.')
|
||
if entry.get('record_error'):
|
||
body += '\n\n' + tools.markdown_text(entry['record_error'])
|
||
page.write_text(body.rstrip() + '\n')
|
||
render_catalog(registry, tools)
|
||
|
||
|
||
def render_catalog(registry, tools):
|
||
official, groups = [], {}
|
||
entries_by_name = {entry['name']: entry for entry in registry['device']}
|
||
for entry in registry['device']:
|
||
if entry['repository'].lower() == 'unixnu/unixnu-sheng':
|
||
official.append(entry)
|
||
continue
|
||
board = entry.get('board') or entry['name']
|
||
if not SLUG.fullmatch(board):
|
||
raise ValueError(f'Invalid device codename: {board}')
|
||
groups.setdefault(board, []).append(entry)
|
||
groups = dict(sorted(groups.items()))
|
||
for entries in groups.values():
|
||
entries.sort(key=lambda entry: entry['repository'].lower())
|
||
|
||
navigation = [
|
||
{'name': entry['name'], 'title': entry['title'] + ' (official)',
|
||
'title_zh': entry['title_zh'] + '(官方维护)'}
|
||
for entry in official
|
||
]
|
||
navigation.extend(
|
||
{'name': f'models/{board}',
|
||
'title': 'sheng (community)' if board == 'sheng' else board,
|
||
'title_zh': 'sheng(社区)' if board == 'sheng' else board}
|
||
for board in groups
|
||
)
|
||
(ROOT / 'docs/.vitepress/device-catalog.json').write_text(json.dumps(navigation, ensure_ascii=False, indent=2) + '\n')
|
||
for zh in (False, True):
|
||
prefix = '/zh' if zh else ''
|
||
directory = ROOT / ('docs/zh/devices' if zh else 'docs/devices')
|
||
models = directory / 'models'
|
||
models.mkdir(parents=True, exist_ok=True)
|
||
for page in models.glob('*.md'):
|
||
if page.stem not in groups:
|
||
page.unlink()
|
||
lines = ['# ' + ('设备' if zh else 'Devices'), '',
|
||
('按机型代号查看各仓库的设备说明与适配记录。'
|
||
if zh else 'Find device documentation and porting records by codename.'), '',
|
||
('构建与验证见[构建与启动](/zh/building)、[移植步骤](/zh/porting/steps)和[日志抓取](/zh/porting/log-capture)。'
|
||
if zh else 'Follow [Build and boot](/building), [porting steps](/porting/steps) and [log capture](/porting/log-capture) for build and validation procedures.'), '']
|
||
if official:
|
||
lines += ['## ' + ('官方维护' if zh else 'Officially maintained'), '',
|
||
'| ' + ('设备 | 代号 | SoC | 仓库' if zh else 'Device | Codename | SoC | Repository') + ' |',
|
||
'| --- | --- | --- | --- |']
|
||
for entry in official:
|
||
name, repo = entry['name'], entry['repository']
|
||
title = tools.markdown_text(entry['title_zh' if zh else 'title'].replace('·', ' / ').replace('•', ' / '))
|
||
lines.append(f'| [{title}]({prefix}/devices/{name}) | sheng | '
|
||
f'{tools.markdown_text(entry.get("soc", ""))} | [{repo}](https://github.com/{repo}) |')
|
||
lines += ['', '## ' + ('社区设备' if zh else 'Community devices'), '',
|
||
'| ' + ('设备 | 机型代号 | SoC | 来源数量' if zh else 'Device | Codename | SoC | Sources') + ' |',
|
||
'| --- | --- | --- | --- |']
|
||
for board, entries in groups.items():
|
||
primary = entries_by_name.get(board, entries[0])
|
||
owner = primary['repository'].split('/')[0]
|
||
title = primary['title_zh' if zh else 'title'].removesuffix(f' ({owner})')
|
||
title = title.removesuffix(f' ({board})').removesuffix(f'({board})')
|
||
title = tools.markdown_text(title.replace('·', ' / ').replace('•', ' / '))
|
||
socs = ', '.join(sorted({entry['soc'] for entry in entries if entry.get('soc')}))
|
||
label = ('sheng(社区)' if zh else 'sheng (community)') if board == 'sheng' else board
|
||
lines.append(f'| [{title}]({prefix}/devices/models/{board}) | {label} | '
|
||
f'{tools.markdown_text(socs)} | {len(entries)} |')
|
||
page = ['# ' + label, '',
|
||
('查看各仓库的设备说明与适配记录。'
|
||
if zh else 'Browse device documentation and porting records from each repository.'), '',
|
||
'| ' + ('仓库 | 设备页面 | 文档 | SoC' if zh else 'Repository | Device page | Documentation | SoC') + ' |',
|
||
'| --- | --- | --- | --- |']
|
||
for entry in entries:
|
||
name, repo = entry['name'], entry['repository']
|
||
title = tools.markdown_text(entry['title_zh' if zh else 'title'].replace('·', ' / ').replace('•', ' / '))
|
||
state = ('适配记录' if zh else 'Porting record') if entry.get('record_state') == 'available' else (
|
||
('记录待补齐' if zh else 'Record pending') if entry.get('format') == 2 else
|
||
('设备说明' if zh else 'Device notes') if entry.get('legacy_page') == 'preserve' else
|
||
('仓库链接' if zh else 'Repository link'))
|
||
page.append(f'| [{repo}](https://github.com/{repo}) | [{title}]({prefix}/devices/{name}) | '
|
||
f'{state} | {tools.markdown_text(entry.get("soc", ""))} |')
|
||
(models / f'{board}.md').write_text('\n'.join(page) + '\n')
|
||
lines += ['', '## ' + ('维护设备记录' if zh else 'Maintain device records') + ' {#adding-a-device}', '',
|
||
('按[记录指南](/zh/porting/records#publishing-and-reuse)发布设备配置与适配记录。'
|
||
if zh else 'Publish device configuration and records using the [record guide](/porting/records#publishing-and-reuse).'), '']
|
||
(directory / 'index.md').write_text('\n'.join(lines))
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--render', action='store_true', help='render the committed catalog without scanning GitHub')
|
||
args = parser.parse_args()
|
||
try:
|
||
if args.render:
|
||
render_site(load_registry())
|
||
else:
|
||
synchronize(GitHub())
|
||
except (ValueError, OSError, KeyError) as error:
|
||
parser.exit(1, f'device catalog: {error}\n')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|