generate.py

#!/usr/bin/env python3

"""
# generate.py

Parse `codepoint-character-sequence.txt`, generating `sequence-characters.txt`.

Licensed under "MIT No Attribution" (MIT-0),
see <https://spdx.org/licenses/MIT-0>.
"""


import itertools
import re
from collections import defaultdict
from string import Template


CODEPOINT_CHARACTER_SEQUENCE_FILE_NAME = 'codepoint-character-sequence.txt'
SEQUENCE_CHARACTERS_FILE_NAME = 'sequence-characters.txt'
CHARACTERS_TRADITIONAL_FILE_NAME = 'characters-traditional.txt'
CHARACTERS_SIMPLIFIED_FILE_NAME = 'characters-simplified.txt'

STROKE_DATA_NOTICE = '''\
# Part of 'Conway Stroke Data',
# see <https://github.com/stroke-input/stroke-input-data>.\
'''

CREATIVE_COMMONS_NOTICE = '''\
# Copyright 2021--2024 Conway.
# Licensed under Creative Commons Attribution 4.0 International (CC-BY-4.0),
# see <https://creativecommons.org/licenses/by/4.0/>.\
'''

AUTOMATIC_GENERATION_NOTICE = f'''\
# This file is automatically generated by running `generate.py`
# in <https://github.com/stroke-input/stroke-input-data>.
# It should NOT be edited manually.
# Manual edits should be made to `{CODEPOINT_CHARACTER_SEQUENCE_FILE_NAME}`.\
'''

PUBLIC_DOMAIN_NOTICE = f'''\
# Released into the public domain,
# see <https://creativecommons.org/publicdomain/zero/1.0/>.\
'''

SEQUENCE_CHARACTERS_TEMPLATE = Template(
    f'# # {SEQUENCE_CHARACTERS_FILE_NAME}\n\n'
    f'{STROKE_DATA_NOTICE}\n\n'
    f'{CREATIVE_COMMONS_NOTICE}\n\n'
    f'# Contains tab-separated (stroke sequence, characters) pairs.\n\n'
    f'{AUTOMATIC_GENERATION_NOTICE}\n\n'
    f'$body\n'
)

CHARACTERS_TRADITIONAL_TEMPLATE = Template(
    f'# # {CHARACTERS_TRADITIONAL_FILE_NAME}\n\n'
    f'{STROKE_DATA_NOTICE}\n\n'
    f'{PUBLIC_DOMAIN_NOTICE}\n\n'
    f'# Contains traditional characters.\n\n'
    f'{AUTOMATIC_GENERATION_NOTICE}\n\n'
    f'$body\n'
)

CHARACTERS_SIMPLIFIED_TEMPLATE = Template(
    f'# # {CHARACTERS_SIMPLIFIED_FILE_NAME}\n\n'
    f'{STROKE_DATA_NOTICE}\n\n'
    f'{PUBLIC_DOMAIN_NOTICE}\n\n'
    f'# Contains simplified characters.\n\n'
    f'{AUTOMATIC_GENERATION_NOTICE}\n\n'
    f'$body\n'
)


def supplant_capture_group(match, alternatives_storage):
    """
    Replace a capture group match object with a back reference, and store the alternatives.
    """
    alternatives = set(match.group('alternatives').split('|'))
    alternatives_storage.append(alternatives)
    group_number = len(alternatives_storage)
    back_reference = fr'\{group_number}'

    return back_reference


def realise_back_reference(match, alternatives_combo):
    """
    Replace a back reference with the appropriate alternative.
    """
    group_number = int(match.group('group_number'))
    alternative = alternatives_combo[group_number - 1]

    return alternative


def to_sequence_set(sequence_regex):
    """
    Convert stroke sequence regex to a stroke sequence set.
    Assumes capture groups:
        1. number at most 9,
        2. are not nested,
        3. contain pure digits separated by pipes, and
        4. are referred to by a backslash followed by a positive decimal digit.
    """
    alternatives_storage = []
    back_reference_template = re.sub(
        r'\( (?P<alternatives>[1-5|]*) \)',
        lambda match: supplant_capture_group(match, alternatives_storage),
        sequence_regex,
        flags=re.VERBOSE,
    )

    sequences = {
        re.sub(
            r'\\ (?P<group_number>[1-9])',
            lambda match: realise_back_reference(match, alternatives_combo),
            back_reference_template,
            flags=re.VERBOSE,
        )
        for alternatives_combo in itertools.product(*alternatives_storage)
    }

    return sequences


def main():
    traditional_characters = set()
    simplified_characters = set()
    dual_characters = set()

    characters_from_sequence = defaultdict(list)

    with open(CODEPOINT_CHARACTER_SEQUENCE_FILE_NAME, 'r', encoding='utf-8') as codepoint_character_sequence_file:
        lines = codepoint_character_sequence_file.readlines()

    for line in lines:
        compliant_match = re.fullmatch(
            r'''
                U[+] (?P<codepoint_hex> [0-9A-F]{4,5} )
                    \t
                (?P<character> \S ) (?P<character_type> [\^*]? )
                    \t
                (?P<sequence_regex> [1-5|()\\]+ )
                    \n
            ''',
            line,
            flags=re.VERBOSE,
        )

        if not compliant_match:
            continue

        codepoint_hex = compliant_match.group('codepoint_hex')
        character = compliant_match.group('character')
        character_type = compliant_match.group('character_type')
        sequence_regex = compliant_match.group('sequence_regex')

        if int(codepoint_hex, 16) != ord(character):
            continue

        if character_type == '^':
            traditional_characters.add(character)
        elif character_type == '*':
            simplified_characters.add(character)
        else:
            dual_characters.add(character)

        for sequence in to_sequence_set(sequence_regex):
            characters_from_sequence[sequence].append(character)

    character_count = len({*traditional_characters, *simplified_characters, *dual_characters})
    print(f'Finished parsing `{CODEPOINT_CHARACTER_SEQUENCE_FILE_NAME}` ({character_count} characters).')

    sequence_count = len(characters_from_sequence)
    sequence_characters_lines = '\n'.join(
        f'{sequence}\t{"".join(characters_from_sequence[sequence])}'
        for sequence in sorted(characters_from_sequence.keys())
    )
    with open(SEQUENCE_CHARACTERS_FILE_NAME, 'w', encoding='utf-8') as sequence_characters_file:
        sequence_characters_file.write(SEQUENCE_CHARACTERS_TEMPLATE.substitute(body=sequence_characters_lines))
    print(f'Finished generating `{SEQUENCE_CHARACTERS_FILE_NAME}` ({sequence_count} stroke sequences).')

    traditional_character_count = len(traditional_characters)
    traditional_character_lines = '\n'.join(sorted(traditional_characters))
    with open(CHARACTERS_TRADITIONAL_FILE_NAME, 'w', encoding='utf-8') as characters_traditional_file:
        characters_traditional_file.write(CHARACTERS_TRADITIONAL_TEMPLATE.substitute(body=traditional_character_lines))
    print(f'Finished generating `{CHARACTERS_TRADITIONAL_FILE_NAME}` ({traditional_character_count} characters).')

    simplified_character_count = len(simplified_characters)
    simplified_character_lines = '\n'.join(sorted(simplified_characters))
    with open(CHARACTERS_SIMPLIFIED_FILE_NAME, 'w', encoding='utf-8') as characters_simplified_file:
        characters_simplified_file.write(CHARACTERS_SIMPLIFIED_TEMPLATE.substitute(body=simplified_character_lines))
    print(f'Finished generating `{CHARACTERS_SIMPLIFIED_FILE_NAME}` ({simplified_character_count} characters).')

    sequences_per_character = sequence_count / character_count
    print(f'On average, there are {sequences_per_character:.4f} stroke sequences per character.')

    traditional_fraction = traditional_character_count / character_count
    print(f'{traditional_fraction:6.2%} of characters are traditional-only.')

    simplified_fraction = simplified_character_count / character_count
    print(f'{simplified_fraction:6.2%} of characters are simplified-only.')

    dual_fraction = 1 - traditional_fraction - simplified_fraction
    print(f'{dual_fraction:6.2%} of characters are dual.')


if __name__ == '__main__':
    main()