diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000..3cf21c4c52 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +paths: + .github/workflows/update-and-build.yml: + ignore: + # Remove once https://github.com/rhysd/actionlint/pull/732 is released. + - ^reusable workflow call "\$/\.github/workflows/spellcheck\.yml" at "uses" is not following + the format diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000000..09ecc9bd83 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,159 @@ +name: spellcheck + +on: + pull_request: + workflow_dispatch: + workflow_call: + inputs: + advisory: + description: Report failures without failing the calling workflow + type: boolean + default: true + version: + description: CPython documentation version + required: true + type: string + translations-ref: + description: Translation branch updated by the calling workflow + required: true + type: string + secrets: + TX_TOKEN: + description: Transifex API token with access to the Polish terminology glossary + required: true + +permissions: + contents: read + +jobs: + pospell: + runs-on: ubuntu-latest + continue-on-error: ${{ inputs.advisory || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + timeout-minutes: 15 + defaults: + run: + working-directory: cpython/Doc + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3 + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Check out workflow tools + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: spellcheck-tools + sparse-checkout: | + /spellcheck_report.py + /manage_translation.py + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Check out CPython documentation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: python/cpython + ref: ${{ inputs.version || github.base_ref || github.ref_name }} + path: cpython + persist-credentials: false + - name: Check out translations + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.translations-ref }} + path: cpython/Doc/locales/pl/LC_MESSAGES + persist-credentials: false + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl hunspell hunspell-pl unzip + make venv + # Pospell needs newer Docutils than the Sphinx used by older branches. + uv venv "$RUNNER_TEMP/spellcheck-venv" + # Pospell PR 24 provides --report; switch to a release when available. + uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" \ + sphinx-intl transifex-python six \ + 'pospell @ git+https://git.afpy.org/maciek/pospell.git@feat/json-reports' + - name: Prepare UTF-8 Polish dictionary + # The packaged dictionary uses ISO-8859-2, which cannot represent all prose. + run: | + mkdir -p "$RUNNER_TEMP/hunspell-utf8" + dictionary_encoding=$(sed -n 's/^SET //p' /usr/share/hunspell/pl_PL.aff) + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.aff \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff" + iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \ + > "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic" + - name: Download SJP Polish dictionary + run: | + curl --fail --location --retry 3 \ + https://sjp.pl/sl/ort/sjp-hunspell-pl-20261101.zip \ + --output "$RUNNER_TEMP/sjp-hunspell.zip" + echo "a29b40d80a2501ba1ad84fa17f7d177e2ebe313082f74f837a77b3135c05340c $RUNNER_TEMP/sjp-hunspell.zip" \ + | sha256sum --check --strict + # SJP wraps the dictionary files in a second ZIP archive. + unzip -p "$RUNNER_TEMP/sjp-hunspell.zip" pl_PL.zip \ + > "$RUNNER_TEMP/sjp-pl_PL.zip" + unzip -p "$RUNNER_TEMP/sjp-pl_PL.zip" pl_PL.aff \ + > "$RUNNER_TEMP/sjp-pl_PL.aff" + unzip -p "$RUNNER_TEMP/sjp-pl_PL.zip" pl_PL.dic \ + > "$RUNNER_TEMP/sjp-pl_PL.dic" + dictionary_encoding=$(sed -n 's/^SET //p' "$RUNNER_TEMP/sjp-pl_PL.aff") + iconv -f "$dictionary_encoding" -t UTF-8 "$RUNNER_TEMP/sjp-pl_PL.aff" \ + > "$RUNNER_TEMP/hunspell-utf8/pl_SJP.aff" + sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_SJP.aff" + iconv -f "$dictionary_encoding" -t UTF-8 "$RUNNER_TEMP/sjp-pl_PL.dic" \ + > "$RUNNER_TEMP/hunspell-utf8/pl_SJP.dic" + - name: Download Polish terminology glossary + working-directory: spellcheck-tools + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/python" + manage_translation.py fetch_glossary + --output "$RUNNER_TEMP/hunspell-glossary.dic" + - name: Generate POT files without literal blocks + run: | + if make -n gettext >/dev/null 2>&1; then + make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" + else + # CPython 3.10 and 3.11 do not have a dedicated gettext target. + make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0" + fi + - name: Record original translation checksums + run: > + python "$GITHUB_WORKSPACE/spellcheck-tools/spellcheck_report.py" snapshot + --translations locales/pl/LC_MESSAGES --cpython . + --output "$RUNNER_TEMP/spellcheck-source.json" + - name: Update temporary PO files + # Drop obsolete entries so removed literal blocks are not spellchecked. + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/sphinx-intl" update -p build/gettext -d locales + -l pl + --no-obsolete + - name: Check Polish spelling + id: spelling + env: + DICPATH: ${{ runner.temp }}/hunspell-utf8 + run: > + "$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL,pl_SJP + --personal-dict "$RUNNER_TEMP/hunspell-glossary.dic" + --glob 'locales/pl/LC_MESSAGES/**/*.po' + --report "$RUNNER_TEMP/pospell.json" + --report-root locales/pl/LC_MESSAGES + - name: Prepare spelling report + id: report + if: ${{ !cancelled() && (steps.spelling.outcome == 'success' || steps.spelling.outcome == 'failure') }} + env: + DOCS_VERSION: ${{ inputs.version || github.base_ref || github.ref_name }} + run: > + python "$GITHUB_WORKSPACE/spellcheck-tools/spellcheck_report.py" prepare + --input "$RUNNER_TEMP/pospell.json" + --snapshot "$RUNNER_TEMP/spellcheck-source.json" + --docs-version "$DOCS_VERSION" + --output "$RUNNER_TEMP/spellcheck.json" + - name: Upload spelling report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ !cancelled() && steps.report.outcome == 'success' }} + with: + name: spellcheck-pl + path: ${{ runner.temp }}/spellcheck.json + if-no-files-found: error + # PO updates exist only in this disposable checkout; nothing is committed. diff --git a/.github/workflows/update-and-build.yml b/.github/workflows/update-and-build.yml index fd52332169..b2f225c6a1 100644 --- a/.github/workflows/update-and-build.yml +++ b/.github/workflows/update-and-build.yml @@ -72,6 +72,20 @@ jobs: branch: ${{ matrix.version }} github_token: ${{ secrets.GITHUB_TOKEN }} + spellcheck: + needs: [update] + # These are the runs in which update publishes the fetched translations. + if: github.event_name == 'schedule' || github.ref_name == github.event.repository.default_branch + permissions: + contents: read + uses: $/.github/workflows/spellcheck.yml + with: + advisory: true + version: ${{ github.ref_name }} + translations-ref: ${{ github.ref_name }} + secrets: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + build: runs-on: ubuntu-latest strategy: diff --git a/SPELLCHECK_REPORT.md b/SPELLCHECK_REPORT.md new file mode 100644 index 0000000000..ccb79c41ed --- /dev/null +++ b/SPELLCHECK_REPORT.md @@ -0,0 +1,37 @@ +# Spellcheck reports + +The spellcheck workflow publishes a `spellcheck-pl` Actions artifact containing +`spellcheck.json`. Pospell is installed from `maciek/pospell@feat/json-reports` +until a release includes JSON reports. See the +[pospell documentation](https://git.afpy.org/maciek/pospell/src/branch/feat/json-reports/README.md) +for the report schema, finding identities, translation fingerprints, and statuses. + +Normal text diagnostics remain in the job log. The report is prepared and uploaded +even when spelling findings cause the check step to fail; the job is advisory by +default. If pospell cannot write its JSON or report preparation fails, no artifact +is published. + +## Integration fields + +`spellcheck_report.py` preserves pospell's schema-v1 findings and adds or adapts: + +- `language`: `pl`, the translation language expected by consumers. +- `dictionaries`: pospell's dictionary names, currently `pl_PL` and `pl_SJP`. +- `repository`: the GitHub repository name, such as `python/python-docs-pl`. +- `docs_version`: the documentation branch checked, such as `3.15`. +- `run_url`: the Actions run URL. +- `translations_revision` and `cpython_revision`: the checked-out Git commits. + +Catalog paths are relative to `locales/pl/LC_MESSAGES`, such as +`tutorial/guide.po`. Each catalog's `source_sha256` is replaced with the checksum +of the original PO, recorded before the temporary `sphinx-intl update`. This lets +consumers compare coverage with their repository checkout. Pospell's checksum of +the temporarily updated PO is retained as `scanned_sha256`. + +Only catalogs present both in the original checkout and in pospell's report are +published; unscanned files are not added as clean. The scan covers prose and +excludes literal blocks through the temporary gettext update. + +`potodo-docs-priority` downloads the artifact separately. It validates the source +run, selects complete reports for its requested branch, and checks local entry +identities and fingerprints. This workflow requires no dispatch to that project. diff --git a/manage_translation.py b/manage_translation.py index 7af08a87fd..c9538cdd61 100755 --- a/manage_translation.py +++ b/manage_translation.py @@ -12,9 +12,15 @@ # * recreate_tx_config: recreate configuration for all resources. # * warn_about_files_to_delete: lists files that are not available upstream # * generate_commit_msg: generates commit message with co-authors +# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary +import csv +import io import os +import re import sys +import time +import unicodedata from argparse import ArgumentParser from collections.abc import Iterable from contextlib import chdir @@ -25,12 +31,15 @@ from typing import Self from warnings import warn +import requests from polib import POFile, pofile from transifex.api import transifex_api +from transifex.api.jsonapi.exceptions import JsonApiException LANGUAGE = 'pl' PROJECT_SLUG = 'python-newest' VERSION = '3.14' +GLOSSARY_ID = 'o:python-doc:g:python_global' def fetch(): @@ -46,6 +55,59 @@ def fetch(): _call(f'msgcat --no-location -o {file} {file}') +def _download_glossary(token: str, *, max_wait: float = 300) -> str: + transifex_api.setup(auth=token) + # The SDK supports new API resource types before adding dedicated classes. + download = transifex_api.new( + type='glossaries_async_downloads', + glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID), + language=transifex_api.Language(id=f'l:{LANGUAGE}'), + ) + download.save() + deadline = time.monotonic() + max_wait + while not download.redirect: + status = download.attributes['status'] + if status == 'failed': + raise ValueError('Transifex glossary export failed.') + if status not in {'pending', 'processing'}: + raise ValueError(f'Unexpected glossary export status: {status!r}') + if time.monotonic() >= deadline: + raise ValueError('Transifex glossary export timed out.') + time.sleep(3) + download.reload() + # Download the CSV without sending the API token to the signed storage URL. + export = requests.get(download.redirect, timeout=30) + export.raise_for_status() + return export.content.decode('utf-8-sig') + + +def _glossary_words(content: str) -> list[str]: + reader = csv.DictReader(io.StringIO(content)) + translation_column = f'translation_{LANGUAGE}' + if not reader.fieldnames or translation_column not in reader.fieldnames: + raise ValueError('Transifex glossary CSV has no Polish translation column.') + words = { + word + for row in reader + for word in re.findall( + r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '') + ) + } + if not words: + raise ValueError('Transifex glossary has no Polish terms.') + return sorted(words) + + +def fetch_glossary(output: Path): + """Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary.""" + token = _get_tx_token().strip() + if not token: + raise ValueError('TX_TOKEN is required to download the Transifex glossary.') + words = _glossary_words(_download_glossary(token)) + output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8') + print(f'Added {len(words)} Polish glossary words to {output}.') + + def _call(command: str): if (return_code := call(command, shell=True)) != 0: sys.exit(return_code) @@ -224,10 +286,30 @@ def generate_commit_msg(): 'recreate_tx_config', 'warn_about_files_to_delete', 'generate_commit_msg', + 'fetch_glossary', ) parser = ArgumentParser() parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS) + parser.add_argument( + '--output', type=Path, help='Hunspell dictionary path for fetch_glossary' + ) options = parser.parse_args() - eval(options.cmd)() + if options.cmd == 'fetch_glossary': + if options.output is None: + parser.error('--output is required for fetch_glossary') + try: + fetch_glossary(options.output) + except (requests.RequestException, JsonApiException) as error: + # Exceptions may contain signed URLs; report the status without those URLs. + status = ( + error.response.status_code + if error.response is not None + else 'network error' + ) + parser.exit(1, f'Transifex glossary request failed: {status}.\n') + except (ValueError, KeyError) as error: + parser.exit(1, f'Invalid Transifex glossary export: {error}\n') + else: + eval(options.cmd)() diff --git a/spellcheck_report.py b/spellcheck_report.py new file mode 100644 index 0000000000..b53d0c31c1 --- /dev/null +++ b/spellcheck_report.py @@ -0,0 +1,99 @@ +"""Adapt pospell's JSON report for translation-priority consumers.""" + +import argparse +import hashlib +import json +import os +import subprocess +from pathlib import Path + + +def revision(root: Path) -> str: + return subprocess.check_output( + ['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True + ).strip() + + +def snapshot_catalogs(translations: Path, cpython: Path) -> dict: + """Record repository identities before sphinx-intl rewrites temporary POs.""" + return { + 'translations_revision': revision(translations), + 'cpython_revision': revision(cpython), + 'catalogs': { + path.relative_to(translations).as_posix(): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in sorted(translations.rglob('*.po')) + }, + } + + +def prepare_report( + report: dict, + snapshot: dict, + *, + docs_version: str, + repository: str, + run_url: str, +) -> dict: + """Keep pospell findings intact while describing the original checkout.""" + if report.get('schema_version') != 1: + raise ValueError('Unsupported pospell report schema') + if report.get('status') not in {'complete', 'error'}: + raise ValueError('Invalid pospell report status') + return { + **report, + 'language': 'pl', + 'dictionaries': report['language'].split(','), + 'repository': repository, + 'docs_version': docs_version, + 'run_url': run_url, + 'translations_revision': snapshot['translations_revision'], + 'cpython_revision': snapshot['cpython_revision'], + 'catalogs': { + resource: { + **catalog, + 'scanned_sha256': catalog['source_sha256'], + 'source_sha256': snapshot['catalogs'][resource], + } + for resource, catalog in report['catalogs'].items() + # Newly created catalogs have no corresponding repository file. + if resource in snapshot['catalogs'] + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + snapshot_parser = commands.add_parser('snapshot') + snapshot_parser.add_argument('--translations', type=Path, required=True) + snapshot_parser.add_argument('--cpython', type=Path, required=True) + snapshot_parser.add_argument('--output', type=Path, required=True) + report_parser = commands.add_parser('prepare') + report_parser.add_argument('--input', type=Path, required=True) + report_parser.add_argument('--snapshot', type=Path, required=True) + report_parser.add_argument('--docs-version', required=True) + report_parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.command == 'snapshot': + data = snapshot_catalogs(args.translations, args.cpython) + else: + data = prepare_report( + json.loads(args.input.read_text(encoding='utf-8')), + json.loads(args.snapshot.read_text(encoding='utf-8')), + docs_version=args.docs_version, + repository=os.environ['GITHUB_REPOSITORY'], + run_url=( + f'{os.environ["GITHUB_SERVER_URL"]}/' + f'{os.environ["GITHUB_REPOSITORY"]}/actions/runs/' + f'{os.environ["GITHUB_RUN_ID"]}' + ), + ) + args.output.write_text( + json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8' + ) + + +if __name__ == '__main__': + main() diff --git a/tests/test_spellcheck_report.py b/tests/test_spellcheck_report.py new file mode 100644 index 0000000000..6223f4124d --- /dev/null +++ b/tests/test_spellcheck_report.py @@ -0,0 +1,146 @@ +import copy +import hashlib +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from spellcheck_report import main, prepare_report, snapshot_catalogs + + +class SpellcheckReportTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.catalog = self.root / 'tutorial' / 'guide.po' + self.catalog.parent.mkdir() + self.catalog.write_text('msgid "Source"\nmsgstr "Błont"\n', encoding='utf-8') + with patch( + 'spellcheck_report.revision', side_effect=['translations', 'cpython'] + ): + self.snapshot = snapshot_catalogs(self.root, self.root) + self.finding = { + 'msgctxt': None, + 'msgid': 'Source', + 'msgid_plural': '', + 'msgstr_sha256': 'translation-digest', + 'words': ['Błont'], + } + self.raw = { + 'schema_version': 1, + 'status': 'complete', + 'created_at': '2026-11-21T00:00:00+00:00', + 'language': 'pl_PL,pl_SJP', + 'checker': {'pospell': '1.3', 'hunspell': 'test'}, + 'catalogs': { + 'tutorial/guide.po': { + 'source_sha256': 'temporary-po-digest', + 'findings': [self.finding], + }, + }, + } + + def prepare(self, raw=None): + return prepare_report( + self.raw if raw is None else raw, + self.snapshot, + docs_version='3.15', + repository='python/python-docs-pl', + run_url='https://github.com/python/python-docs-pl/actions/runs/123', + ) + + def test_original_checksum_survives_temporary_po_update(self): + original = hashlib.sha256(self.catalog.read_bytes()).hexdigest() + self.catalog.write_text('# temporary changes\n', encoding='utf-8') + result = self.prepare() + catalog = result['catalogs']['tutorial/guide.po'] + self.assertEqual(catalog['source_sha256'], original) + self.assertEqual(catalog['scanned_sha256'], 'temporary-po-digest') + self.assertEqual(catalog['findings'], [self.finding]) + self.assertEqual(result['language'], 'pl') + self.assertEqual(result['dictionaries'], ['pl_PL', 'pl_SJP']) + self.assertEqual(result['translations_revision'], 'translations') + self.assertEqual(result['cpython_revision'], 'cpython') + self.assertEqual(result['checker'], self.raw['checker']) + + def test_clean_scanned_catalogs_are_kept_but_unscanned_ones_are_not_added(self): + raw = copy.deepcopy(self.raw) + raw['catalogs']['tutorial/guide.po']['findings'] = [] + self.snapshot['catalogs']['unscanned.po'] = 'unscanned-digest' + raw['catalogs']['new.po'] = {'source_sha256': 'new', 'findings': []} + result = self.prepare(raw) + self.assertEqual(set(result['catalogs']), {'tutorial/guide.po'}) + self.assertEqual(result['catalogs']['tutorial/guide.po']['findings'], []) + + def test_error_report_is_not_promoted_to_complete(self): + self.raw.update(status='error', error='Hunspell failed') + result = self.prepare() + self.assertEqual(result['status'], 'error') + self.assertEqual(result['error'], 'Hunspell failed') + + def test_unsupported_or_invalid_reports_are_rejected(self): + for field, value in [('schema_version', 2), ('status', 'success')]: + with self.subTest(field=field), self.assertRaises(ValueError): + self.prepare({**self.raw, field: value}) + + def test_cli_adds_metadata_and_preserves_non_ascii_findings(self): + raw = self.root / 'pospell.json' + state = self.root / 'snapshot.json' + output = self.root / 'spellcheck.json' + raw.write_text(json.dumps(self.raw), encoding='utf-8') + state.write_text(json.dumps(self.snapshot), encoding='utf-8') + with ( + patch.dict( + os.environ, + { + 'GITHUB_REPOSITORY': 'python/python-docs-pl', + 'GITHUB_SERVER_URL': 'https://github.com', + 'GITHUB_RUN_ID': '123', + }, + ), + patch( + 'sys.argv', + [ + 'spellcheck_report.py', + 'prepare', + '--input', + str(raw), + '--snapshot', + str(state), + '--docs-version', + '3.15', + '--output', + str(output), + ], + ), + ): + main() + result = json.loads(output.read_text(encoding='utf-8')) + self.assertEqual(result, self.prepare()) + self.assertIn('Błont', output.read_text(encoding='utf-8')) + + def test_missing_pospell_report_cannot_become_a_clean_report(self): + output = self.root / 'spellcheck.json' + with ( + patch( + 'sys.argv', + [ + 'spellcheck_report.py', + 'prepare', + '--input', + str(self.root / 'missing.json'), + '--snapshot', + str(self.root / 'snapshot.json'), + '--docs-version', + '3.15', + '--output', + str(output), + ], + ), + self.assertRaises(FileNotFoundError), + ): + main() + self.assertFalse(output.exists())