build.py

# coding: utf-8
'''
Parse extensions/*.toml files, output a static site with following structure:
public/
  |-my-extension-1/
  |  |-index.json  <- extension info
  |  |-index.html  <- extension entrance (component)
  |  |-dist  <- extension resources
  |  |-...  <- other files
  |
  |-index.json  <- repo info, contain all extensions' info
'''
import glob
import json
import os
import shutil
from subprocess import run

import toml

def onerror(func, path, exc_info):
    """
    Error handler for ``shutil.rmtree``.

    If the error is due to an access error (read only file)
    it attempts to add write permission and then retries.

    If the error is for another reason it re-raises the error.

    Usage : ``shutil.rmtree(path, onerror=onerror)``
    """
    import stat
    if not os.access(path, os.W_OK):
        # Is the error an access error ?
        os.chmod(path, stat.S_IWUSR)
        func(path)
    else:
        raise

def main(base_url):
    while base_url.endswith('/'):
        base_url = base_url[:-1]

    print('Fetching extensions...')
    base_dir = os.path.dirname(os.path.abspath(__file__))
    extension_dir = os.path.join(base_dir, 'extensions')
    public_dir = os.path.join(base_dir, 'public')

    if os.path.exists(public_dir):
        shutil.rmtree(public_dir, ignore_errors=False, onerror=onerror)

    os.makedirs(public_dir)
    os.chdir(public_dir)

    extensions = []

    # Read and parse all extension info
    for fname in os.listdir(extension_dir):
        if not fname.endswith('.toml'):
            continue

        with open(os.path.join(extension_dir, fname)) as rf:
            ext = toml.load(rf)

        # Build extension info
        repo_name = ext['github'].split('/')[-1]
        # https://example.com/my-extension/index.html
        extension_url = '/'.join([base_url, repo_name, ext['main']])
        # https://example.com/my-extension/index.json
        extension_info_url = '/'.join([base_url, repo_name, 'index.json'])
        extension = dict(
            identifier=ext['id'],
            name=ext['name'],
            content_type=ext['content_type'],
            area=ext.get('area', None),
            version=ext['version'],
            description=ext.get('description', None),
            marketing_url=ext.get('marketing_url', None),
            thumbnail_url=ext.get('thumbnail_url', None),
            valid_until='2030-05-16T18:35:33.000Z',
            url=extension_url,
            download_url='https://github.com/{github}/archive/{version}.zip'.format(**ext),
            latest_url=extension_info_url,
            flags=ext.get('flags', []),
            dock_icon=ext.get('dock_icon', {}),
            layerable=ext.get('layerable', None),
            deletion_warning=ext.get('deletion_warning', None),
        )

        # mfa-link (2FA manager) has no expiration
        if ext.get('no_expire', False):
            extension.pop('valid_until', None)

        # Strip empty values
        extension = {k: v for k, v in extension.items() if v}

        # That is very strange, StandardNotes does not upload some npm packages
        # when extensions get updated. We'll have to handle them by git.
        # git clone --branch {version} --depth 1 {github_url}
        run([
            'git', '-c', 'advice.detachedHead=false',
            'clone',
            '--branch', ext['version'],
            '--depth', '1',
            'https://github.com/{github}.git'.format(**ext),
        ])
        shutil.rmtree(os.path.join(public_dir, repo_name, '.git'), ignore_errors=False, onerror=onerror)

        # Generate JSON file for each extension
        with open(os.path.join(public_dir, repo_name, 'index.json'), 'w') as wf:
            json.dump(extension, wf)

        extensions.append(extension)
        print('Loaded extension: {}'.format(ext['name']))

    os.chdir('..')

    # Generate the index JSON file
    with open(os.path.join(public_dir, 'index.json'), 'w') as wf:
        json.dump(
            dict(
                content_type='SN|Repo',
                valid_until='2030-05-16T18:35:33.000Z',
                packages=extensions,
            ),
            wf,
        )

    # Cleanup unnecessary files
    print('Cleaning up...')
    for filename in ['README.md', 'Gruntfile.js', '.babelrc', 'package.json', '*.map', 'src', 'vendor']:
        for matched_fpath in glob.glob(f'public/*/{filename}'):
            if os.path.isdir(matched_fpath):
                shutil.rmtree(matched_fpath, ignore_errors=False, onerror=onerror)
            elif os.path.isfile(matched_fpath):
                os.remove(matched_fpath)
            else:
                continue

            print(f'> Removed {matched_fpath}')

    # Distribute header rules for Netlify
    shutil.copyfile('_headers', os.path.join(public_dir, '_headers'))

    print('Build finished')


if __name__ == '__main__':
    main(os.getenv('URL', 'https://standardnotes-extensions.netlify.app/'))