mail/scripts/generate-spelling-names.py

#!/usr/bin/env python3
"""Write src/spelling/names_data.rs: spelling languages named in themselves, from CLDR.

The regions are those Hunspell dictionaries come in, from LibreOffice and the regional copies
Debian and Fedora install alongside them.
"""

import json
from pathlib import Path
import sys
import urllib.error
import urllib.request

ROOT = Path(__file__).resolve().parent.parent
OUTPUT = ROOT / "src/spelling/names_data.rs"
CLDR = "https://raw.githubusercontent.com/unicode-org/cldr-json/48.0.0/cldr-json"

REGIONS = {
    "af": "ZA NA", "an": "ES",
    "ar": "AE BH DZ EG IN IQ JO KW LB LY MA OM QA SA SD SY TN YE",
    "be": "BY", "bg": "BG", "bn": "BD IN", "bo": "CN IN", "br": "FR", "bs": "BA",
    "ca": "AD ES FR IT", "ckb": "IQ IR", "cs": "CZ", "cy": "GB", "da": "DK",
    "de": "AT BE CH DE LI LU", "el": "CY GR",
    "en": "AG AU BS BW BZ CA DK GB GH HK IE IN JM MW NA NG NZ PH SG TT US ZA ZM ZW",
    "eo": "", "es": "419 AR BO CL CO CR CU DO EC ES GT HN MX NI PA PE PR PY SV US UY VE",
    "et": "EE", "eu": "ES", "fa": "IR", "fi": "FI", "fo": "FO", "fr": "BE CA CH FR LU MC",
    "fy": "NL", "ga": "IE", "gd": "GB", "gl": "ES", "gn": "PY", "gu": "IN", "he": "IL",
    "hi": "IN", "hr": "HR", "hu": "HU", "hy": "AM", "ia": "", "id": "ID", "is": "IS",
    "it": "CH IT", "ka": "GE", "kk": "KZ", "km": "KH", "ko": "KR", "ku": "TR",
    "lo": "LA", "lt": "LT", "lv": "LV", "mk": "MK", "ml": "IN", "mn": "MN", "mr": "IN",
    "ms": "MY", "nb": "NO", "ne": "NP", "nl": "AW BE NL SR", "nn": "NO", "oc": "FR",
    "pa": "IN", "pl": "PL", "pt": "AO BR MZ PT", "ro": "MD RO", "ru": "RU UA", "rw": "RW",
    "si": "LK", "sk": "SK", "sl": "SI", "sq": "AL", "sr": "BA ME RS", "sv": "FI SE",
    "sw": "KE TZ", "ta": "IN", "te": "IN", "th": "TH", "tk": "TM", "tr": "TR", "uk": "UA",
    "uz": "UZ", "vi": "VN", "zu": "ZA",
}
# Dictionary names that differ from CLDR's codes: LibreOffice's Paraguayan Guarani and
# Kurmanji Kurdish.
ALIASES = {"gug": "gn", "kmr": "ku"}


def fetch(path):
    request = urllib.request.Request(
        f"{CLDR}/{path}", headers={"User-Agent": "flectar-mail-ci (+https://github.com/flectar/mail)"}
    )
    try:
        with urllib.request.urlopen(request, timeout=60) as response:
            return json.load(response)
    except urllib.error.HTTPError as error:
        if error.code == 404:
            return None
        raise


def capitalized(name):
    # CLDR capitalizes names in menus and lists for most languages; caseless scripts are
    # unchanged.
    return name[:1].upper() + name[1:]


def rust_string(value):
    return json.dumps(value, ensure_ascii=False)


def main():
    likely = fetch("cldr-core/supplemental/likelySubtags.json")["supplemental"]["likelySubtags"]
    languages = []
    regions = []
    for language, codes in sorted(REGIONS.items()):
        names = fetch(f"cldr-localenames-full/main/{language}/languages.json")
        if names is None:
            print(f"no CLDR names for {language}", file=sys.stderr)
            continue
        own = names["main"][language]["localeDisplayNames"]["languages"].get(language)
        if own is None:
            print(f"{language} has no name for itself", file=sys.stderr)
            continue
        main_region = likely.get(language, "").split("-")[-1]
        if len(main_region) != 2 or main_region not in codes.split():
            main_region = None
        languages.append((language, capitalized(own), main_region))
        territories = fetch(f"cldr-localenames-full/main/{language}/territories.json")
        territories = (
            territories["main"][language]["localeDisplayNames"]["territories"] if territories else {}
        )
        for region in codes.split():
            if region in territories:
                regions.append((language, region, territories[region]))
    lines = [
        "// Spelling languages named in themselves, from CLDR 48.",
        "// Generated by scripts/generate-spelling-names.py; change the script and run it again.",
        "",
        "/// Each language's name in itself, and the region a dictionary for it means by default.",
        "pub(super) const LANGUAGES: &[(&str, &str, Option<&str>)] = &[",
    ]
    for language, name, main_region in languages:
        region = f'Some("{main_region}")' if main_region else "None"
        lines.append(f'    ("{language}", {rust_string(name)}, {region}),')
    lines += [
        "];",
        "",
        "/// Regions named in the language a dictionary for them is in.",
        "pub(super) const REGIONS: &[(&str, &str, &str)] = &[",
    ]
    for language, region, name in regions:
        lines.append(f'    ("{language}", "{region}", {rust_string(name)}),')
    lines += [
        "];",
        "",
        "/// Dictionary language codes that CLDR names with another code.",
        "pub(super) const ALIASES: &[(&str, &str)] = &[",
    ]
    for dictionary, cldr in sorted(ALIASES.items()):
        lines.append(f'    ("{dictionary}", "{cldr}"),')
    lines.append("];")
    OUTPUT.write_text("\n".join(lines) + "\n", encoding="utf-8")


if __name__ == "__main__":
    main()