update.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

This script updates and rebuilds wiki sources from Github and from
parameters on the test server.

It is intended to be run on the main wiki server or
locally within the project's Vagrant environment.

Build notes:

* First step is always a fetch and pull from git (master).
  * Default is just a normal fetch and pull from master
  * If the --clean option is "True" then git will reset to head

* Common topics are copied from /common/source/docs.
  * Topics are copied based on information in the copywiki shortcode.
    For example a topic marked as below would only be copied to copter
     and plane wikis:
    [copywiki destination="copter,plane"]
  * Topics that don't have a [copywiki] will be copied to wikis
    the DEFAULT_COPY_WIKIS list
  * Copied topics are stripped of the 'copywiki' shortcode in the destination.
  * Copied topics are stripped of any content not marked for the target wiki
    using the "site" shortcode:
    [site wiki="plane,rover"]conditional content[/site]

Parameters files are fetched from autotest using a Wget

"""
from __future__ import print_function, unicode_literals

import argparse
import distutils
import errno
import filecmp
import glob
import hashlib
import multiprocessing
import os
import platform
import re
import shutil
import subprocess
import sys
import time

from codecs import open
from datetime import datetime
# while flake8 says this is unused, distutils.dir_util.mkpath fails
# without the following import on old versions of Python:
from distutils import dir_util  # noqa
 
DEFAULT_COPY_WIKIS = ['copter', 'plane', 'rover']
ALL_WIKIS = [
    'copter',
    'plane',
    'rover',
    'antennatracker',
    'dev',
    'planner',
    'planner2',
    'ardupilot',
    'mavproxy',
    'frontend',
    'blimp',
]
COMMON_DIR = 'common'

# GIT_REPO = ''

PARAMETER_SITE = {
    'rover': 'APMrover2',
    'copter': 'ArduCopter',
    'plane': 'ArduPlane',
    'antennatracker': 'AntennaTracker',
    'AP_Periph': 'AP_Periph',
    'blimp': 'Blimp',
}
LOGMESSAGE_SITE = {
    'rover': 'Rover',
    'copter': 'Copter',
    'plane': 'Plane',
    'antennatracker': 'Tracker',
    'blimp': 'Blimp',
}
error_count = 0
N_BACKUPS_RETAIN = 10

VERBOSE = False


def debug(str_to_print):
    """Debug output if verbose is set."""
    if VERBOSE:
        print("[update.py] " + str_to_print)


def error(str_to_print):
    """Show and count the errors."""
    global error_count
    error_count += 1
    print("[update.py][error]: " + str(str_to_print))


def remove_if_exists(filepath):
    try:
        os.remove(filepath)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise e


def fetchparameters(site=None, cache=None):
    """Fetches the parameters for all the sites from the test server and
    copies them to the correct location.

    This is always run as part of a build (i.e. no checking to see if
    parameters have changed.)

    """
    # remove any parameters files in root
    remove_if_exists('Parameters.rst')

    for key, value in PARAMETER_SITE.items():
        fetchurl = 'https://autotest.ardupilot.org/Parameters/%s/Parameters.rst' % value  # noqa
        targetfile = './%s/source/docs/parameters.rst' % key
        if key == 'AP_Periph':
            targetfile = './dev/source/docs/AP_Periph-Parameters.rst'
        if cache:
            if not os.path.exists(targetfile):
                raise Exception("Asked to use cached parameter files, but (%s) does not exist" % (targetfile,))  # noqa
            continue
        if site == key or site is None:
            if platform.system() == "Windows":
                subprocess.check_call(["powershell.exe", "Start-BitsTransfer", "-Source", fetchurl])
            else:
                subprocess.check_call(["wget", fetchurl])

            # move in new file
            if os.path.exists(targetfile):
                os.unlink(targetfile)
            os.rename('Parameters.rst', targetfile)


def fetchlogmessages(site=None, cache=None):
    """
    Fetches the parameters for all the sites from the autotest server and
    copies them to the correct location.

    This is always run as part of a build (i.e. no checking to see if
    logmessages have changed.)
    """
    for key, value in LOGMESSAGE_SITE.items():
        fetchurl = 'https://autotest.ardupilot.org/LogMessages/%s/LogMessages.rst' % value  # noqa
        targetfile = './%s/source/docs/logmessages.rst' % key
        if cache:
            if not os.path.exists(targetfile):
                raise(Exception("Asked to use cached parameter files, but (%s) does not exist" % (targetfile,)))  # noqa
            continue
        if site == key or site is None:
            if platform.system() == "Windows":
                subprocess.check_call(["powershell.exe", "Start-BitsTransfer", "-Source", fetchurl])
            else:
                subprocess.check_call(["wget", fetchurl])
            # move in new file
            if os.path.exists(targetfile):
                os.unlink(targetfile)
            os.rename('LogMessages.rst', targetfile)


def build_one(wiki):
    '''build one wiki'''
    print('make and clean: %s' % wiki)
    if platform.system() == "Windows":
        # This will fail if there's no folder to clean, so no check_call here
        subprocess.run(["make.bat", "clean"], cwd=wiki, shell=True)
        subprocess.check_call(["make.bat", "html"], cwd=wiki, shell=True)
    else:
        subprocess.check_call(["nice", "make", "clean"], cwd=wiki)
        subprocess.check_call(["nice", "make", "html"], cwd=wiki)


def sphinx_make(site, parallel):
    """
    Calls 'make html' to build each site
    """
    done = set()
    wikis = set(ALL_WIKIS[:])
    procs = []

    while len(done) != len(wikis):
        wiki = list(wikis.difference(done))[0]
        done.add(wiki)
        if site == 'common' or site == 'frontend':
            continue
        if wiki == 'frontend'    :
            continue
        if site is not None and not site == wiki:
            continue
        p = multiprocessing.Process(target=build_one, args=(wiki,))
        p.start()
        procs.append(p)
        while parallel != -1 and len(procs) >= parallel:
            for p in procs:
                if p.exitcode is not None:
                    p.join()
                    procs.remove(p)
                    if p.exitcode != 0:
                        error('Error making sphinx(1)')
            time.sleep(0.1)
    while len(procs) > 0:
        for p in procs[:]:
            if p.exitcode is not None:
                p.join()
                procs.remove(p)
                if p.exitcode != 0:
                    error('Error making sphinx(2)')
        time.sleep(0.1)


def copy_build(site, destdir):
    """
    Copies each site into the target location
    """
    for wiki in ALL_WIKIS:
        if site == 'common':
            continue
        if site is not None and not site == wiki:
            continue
        if wiki == 'frontend':
            continue            
        print('copy: %s' % wiki)
        targetdir = os.path.join(destdir, wiki)
        print("DEBUG: Creating backup")
        olddir = os.path.join(destdir, 'old')
        print('DEBUG: recreating %s' % olddir)
        if os.path.exists(olddir):
            shutil.rmtree(olddir)
        os.makedirs(olddir)
        if os.path.exists(targetdir):
            print('DEBUG: moving %s into %s' % (targetdir, olddir))
            shutil.move(targetdir, olddir)
        # copy new dir to targetdir
        # print("DEBUG: targetdir: %s" % targetdir)
        # sourcedir='./%s/build/html/*' % wiki
        sourcedir = './%s/build/html/' % wiki
        # print("DEBUG: sourcedir: %s" % sourcedir)
        # print('DEBUG: mv %s %s' % (sourcedir, destdir) )

        html_moved_dir = os.path.join(destdir, 'html')
        try:
            shutil.move(sourcedir, html_moved_dir)
            # Rename move! (single move to html/* failed)
            shutil.move(html_moved_dir, targetdir)
            print("DEBUG: Moved to %s" % targetdir)
        except Exception:  # FIXME: narrow exception type
            print("DEBUG: FAIL moving output to %s" % targetdir)
            error('Error moving output')

        # delete the old directory
        print('DEBUG: removing %s' % olddir)
        shutil.rmtree(olddir)


def copy_and_keep_build(site, destdir, backupdestdir):
    """
    Copies each site into target location and keep last "n" builds as backups
    """
    for wiki in ALL_WIKIS:
        if site == 'common':
            continue
        if site is not None and site != wiki:
            continue
        if wiki == 'frontend':
            continue             
        debug('copying: %s' % wiki)
        targetdir = os.path.join(destdir, wiki)
        distutils.dir_util.mkpath(targetdir)

        if os.path.exists(targetdir):
            olddir = os.path.join(
                backupdestdir,
                str(building_time + '-wiki-bkp'),
                str(wiki))
            debug('Checking %s' % olddir)
            distutils.dir_util.mkpath(olddir)
            debug('Moving %s into %s' % (targetdir, olddir))
            shutil.move(targetdir, olddir)

            sourcedir = os.path.join(
                os.path.abspath(os.getcwd()),
                str(wiki),
                'build',
                'html')
            html_moved_dir = os.path.join(destdir, 'html')
            try:
                shutil.move(sourcedir, html_moved_dir)
                # Rename move! (single move to html/* failed)
                shutil.move(html_moved_dir, targetdir)
                debug("Moved to %s" % targetdir)
            except Exception:  # FIXME: narrow exception type
                error("FAIL moving output to %s" % targetdir)
            finally:
                debug("Creating a backup in %s" % olddir)
                # subprocess.check_call(['cp', '-r', targetdir ,olddir])
                distutils.dir_util.copy_tree(targetdir,
                                             olddir,
                                             preserve_symlinks=0)
        else:
            error("FAIL when looking for folder %s" % targetdir)


def delete_old_wiki_backups(folder, n_to_keep):
    try:
        debug('Checking number of number of backups in folder %s' % folder)
        backup_folders = glob.glob(folder + "/*-wiki-bkp/")
        backup_folders.sort()
        if len(backup_folders) > n_to_keep:
            for i in range(0, len(backup_folders) - n_to_keep):
                if '-wiki-bkp' in str(backup_folders[i]):
                    debug('Deleting folder %s' % str(backup_folders[i]))
                    shutil.rmtree(str(backup_folders[i]))
                else:
                    debug('Ignoring folder %s because it does not look like a auto generated wiki backup folder' %  # noqa
                          str(backup_folders[i]))
        else:
            debug('No old backups to delete in %s' % folder)
    except Exception as e:
        error('Error on deleting some previous wiki backup folders: %s' % e)


def generate_copy_dict(start_dir=COMMON_DIR):
    """
    This creates a dict which indexes copy targets for all common docs.
    """

    # Clean existing common topics (easiest way to guarantee old ones
    # are removed)
    # Cost is that these will have to be rebuilt even if not changed
    import glob
    for wiki in ALL_WIKIS:
        files = glob.glob('%s/source/docs/common-*.rst' % wiki)
        for f in files:
            print('remove: %s' % f)
            os.remove(f)

    # Create destination folders that might be needed (if don't exist)
    for wiki in ALL_WIKIS:
        try:
            os.mkdir(wiki)
        except Exception:  # FIXME: narrow exception type
            pass

        try:
            os.mkdir('%s/source' % wiki)
        except Exception:  # FIXME: narrow exception type
            pass

        try:
            os.mkdir('%s/source/docs' % wiki)
        except Exception:  # FIXME: narrow exception type
            pass

        try:
            os.mkdir('%s/source/_static' % wiki)
        except Exception:  # FIXME: narrow exception type
            pass

    for root, dirs, files in os.walk(start_dir):
        for file in files:
            if file.endswith(".rst"):
                print("FILE: %s" % file)
                source_file_path = os.path.join(root, file)
                source_file = open(source_file_path, 'r', 'utf-8')
                source_content = source_file.read()
                source_file.close()
                targets = get_copy_targets(source_content)
                # print(targets)
                for wiki in targets:
                    # print("CopyTarget: %s" % wiki)
                    content = strip_content(source_content, wiki)
                    targetfile = '%s/source/docs/%s' % (wiki, file)
                    print(targetfile)
                    destination_file = open(targetfile, 'w', 'utf-8')
                    destination_file.write(content)
                    destination_file.close()
            elif file.endswith(".css"):
                for wiki in ALL_WIKIS:
                    shutil.copy2(os.path.join(root, file),
                                 '%s/source/_static/' % wiki)
            elif file.endswith(".js"):
                source_file_path = os.path.join(root, file)
                source_file = open(source_file_path, 'r', 'utf-8')
                source_content = source_file.read()
                source_file.close()
                targets = get_copy_targets(source_content)
                # print("JS: " + str(targets))
                for wiki in targets:
                    content = strip_content(source_content, wiki)
                    targetfile = '%s/source/_static/%s' % (wiki, file)
                    print(targetfile)
                    destination_file = open(targetfile, 'w', 'utf-8')
                    destination_file.write(content)
                    destination_file.close()


def get_copy_targets(content):
    p = re.compile(r'\[copywiki.*?destination\=\"(.*?)\".*?\]', flags=re.S)
    m = p.search(content)
    targetset = set()
    if m:
        targets = m.group(1).split(',')
        for item in targets:
            targetset.add(item.strip())
    else:
        targetset = set(DEFAULT_COPY_WIKIS)
    return targetset


def strip_content(content, site):
    """
    Strips the copywiki shortcode. Removes content for other sites and
    the [site] shortcode itself.
    """

    def fix_copywiki_shortcode(matchobj):
        """
        Strip the copywiki shortcode if found (just return "nothing" to
        result of re)
        """
        # logmatch_code(matchobj, 'STRIP')
        # print("STRIPPED")
        return ''

    # Remove the copywiki from content
    newText = re.sub(r'\[copywiki.*?\]',
                     fix_copywiki_shortcode,
                     content,
                     flags=re.M)

    def fix_site_shortcode(matchobj):
        # logmatch_code(matchobj, 'SITESC_')
        sitelist = matchobj.group(1)
        # print("SITES_BLOCK: %s" % sitelist)
        if site not in sitelist:
            # print("NOT")
            return ''
        else:
            # print("YES")
            return matchobj.group(2)
    # Remove the site shortcode from content
    newText = re.sub(r'\[site\s.*?wiki\=\"(.*?)\".*?\](.*?)\[\/site\]',
                     fix_site_shortcode,
                     newText,
                     flags=re.S)

    return newText


def logmatch_code(matchobj, prefix):

    try:
        print("%s m0: %s" % (prefix, matchobj.group(0)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m0" % prefix)

    try:
        print("%s m1: %s" % (prefix, matchobj.group(1)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m1" % prefix)

    try:
        print("%s m2: %s" % (prefix, matchobj.group(2)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m1" % prefix)

    try:
        print("%s m3: %s" % (prefix, matchobj.group(3)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m3" % prefix)

    try:
        print("%s m4: %s" % (prefix, matchobj.group(4)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m4" % prefix)
    try:
        print("%s m5: %s" % (prefix, matchobj.group(5)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m5" % prefix)
    try:
        print("%s m6: %s" % (prefix, matchobj.group(6)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m6" % prefix)
    try:
        print("%s m7: %s" % (prefix, matchobj.group(7)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except 7" % prefix)
    try:
        print("%s m8: %s" % (prefix, matchobj.group(8)))
    except Exception:  # FIXME: narrow exception type
        print("%s: except m8" % prefix)


def is_the_same_file(file1, file2):
    """ Compare two files using their SHA256 hashes"""
    digests = []
    for filename in [file1, file2]:
        hasher = hashlib.sha256()
        with open(filename, 'rb') as f:
            buf = f.read()
            hasher.update(buf)
            a = hasher.hexdigest()
            digests.append(a)

    return(digests[0] == digests[1])


def fetch_versioned_parameters(site=None):
    """
    It relies on "build_parameters.py" be executed before the "update.py"

    Once the generated files are on ../new_params_mversion it tut all
    parameters and JSON files in their destinations.
    """

    for key, value in PARAMETER_SITE.items():

        if key == 'AP_Periph': # workaround until create a versioning for AP_Periph in firmware server
            fetchurl = 'https://autotest.ardupilot.org/Parameters/%s/Parameters.rst' % value  # noqa
            subprocess.check_call(["wget", fetchurl])
            targetfile = './dev/source/docs/AP_Periph-Parameters.rst'
            os.rename('Parameters.rst', targetfile)

        else: #regular versining 

            if site == key or site is None:
                # Remove old param single file
                single_param_file = './%s/source/docs/parameters.rst' % key
                debug("Erasing " + single_param_file)
                remove_if_exists(single_param_file)

                # Remove old versioned param files
                if 'antennatracker' in key.lower():  # To main the original script approach instead of the build_parameters.py approach.  # noqa
                    old_parameters_mask = (os.getcwd() +
                                        '/%s/source/docs/parameters-%s-' %
                                        ("AntennaTracker", "AntennaTracker"))
                else:
                    old_parameters_mask = (os.getcwd() +
                                        '/%s/source/docs/parameters-%s-' %
                                        (key, key.title()))
                try:
                    old_parameters_files = [
                        f for f in glob.glob(old_parameters_mask + "*.rst")]
                    for filename in old_parameters_files:
                        debug("Erasing rst " + filename)
                        os.remove(filename)
                except Exception as e:
                    error(e)
                    pass

                # Remove old json file
                if 'antennatracker' in key.lower():  # To main the original script approach instead of the build_parameters.py approach.  # noqa
                    target_json_file = ('./%s/source/_static/parameters-%s.json' %
                                        ("AntennaTracker", "AntennaTracker"))
                else:
                    target_json_file = ('./%s/source/_static/parameters-%s.json' %
                                        (value, key.title()))
                debug("Erasing json " + target_json_file)
                remove_if_exists(target_json_file)

                # Moves the updated JSON file
                if 'antennatracker' in key.lower():  # To main the original script approach instead of the build_parameters.py approach.  # noqa
                    vehicle_json_file = os.getcwd() + '/../new_params_mversion/%s/parameters-%s.json' % ("AntennaTracker", "AntennaTracker")  # noqa
                else:
                    vehicle_json_file = os.getcwd() + '/../new_params_mversion/%s/parameters-%s.json' % (value,key.title())  # noqa
                new_file = (
                    key +
                    "/source/_static/" +
                    vehicle_json_file[str(vehicle_json_file).rfind("/")+1:])
                try:
                    debug("Moving " + vehicle_json_file)
                    # os.rename(vehicle_json_file, new_file)
                    shutil.copy2(vehicle_json_file, new_file)
                except Exception as e:
                    error(e)
                    pass

                # Copy all parameter files to vehicle folder IFF it is new
                try:
                    new_parameters_folder = (os.getcwd() +
                                            '/../new_params_mversion/%s/' % value)
                    new_parameters_files = [
                        f for f in glob.glob(new_parameters_folder + "*.rst")
                    ]
                except Exception as e:
                    error(e)
                    pass
                for filename in new_parameters_files:
                    # Check possible cached version
                    try:
                        new_file = (key +
                                    "/source/docs/" +
                                    filename[str(filename).rfind("/")+1:])
                        if not os.path.isfile(new_file):
                            debug("Copying %s to %s (target file does not exist)" %
                                (filename, new_file))
                            shutil.copy2(filename, new_file)
                        elif os.path.isfile(filename.replace("new_params_mversion","old_params_mversion")): # The cached file exists?  # noqa

                            # Temporary debug messages to help with cache tasks.
                            debug("Check cache: %s against %s" % (filename, filename.replace("new_params_mversion","old_params_mversion")))  # noqa
                            debug("Check cache with filecmp.cmp: %s" % filecmp.cmp(filename, filename.replace("new_params_mversion","old_params_mversion")))  # noqa
                            debug("Check cache with sha256: %s" % is_the_same_file(filename, filename.replace("new_params_mversion","old_params_mversion")))  # noqa

                            if ("parameters.rst" in filename) or (not filecmp.cmp(filename, filename.replace("new_params_mversion","old_params_mversion"))):    # It is different?  OR is this one the latest. | Latest file must be built everytime in order to enable Sphinx create the correct references across the wiki.  # noqa
                                debug("Overwriting %s to %s" %
                                    (filename, new_file))
                                shutil.copy2(filename, new_file)
                            else:
                                debug("It will reuse the last build of " +
                                    new_file)
                        else:   # If not cached, copy it anyway.
                            debug("Copying %s to %s" %
                                (filename, new_file))
                            shutil.copy2(filename, new_file)

                    except Exception as e:
                        error(e)
                        pass


def create_latest_parameter_redirect(default_param_file, vehicle):
    """
    For a given vehicle create a file called parameters.rst that
    redirects to the latest parameters file.(Create to maintaim retro
    compatibility.)
    """
    out_line = "======================\nParameters List (Full)(\n======================\n"  # noqa
    out_line += "\n.. raw:: html\n\n"
    out_line += "   <script>location.replace(\"" + default_param_file[:-3] + "html" + "\")</script>"  # noqa
    out_line += "\n\n"

    filename = vehicle + "/source/docs/parameters.rst"
    with open(filename, "w") as text_file:
        text_file.write(out_line)

    debug("Created html automatic redirection from parameters.html to %shtml" %
          default_param_file[:-3])


def cache_parameters_files(site=None):
    """
    For each vechile: put new_params_mversion/ content in
    old_params_mversion/ folders and .html built files as well.
    """
    for key, value in PARAMETER_SITE.items():
        if (site == key or site is None) and (key != 'AP_Periph'):  # and (key != 'AP_Periph') workaround until create a versioning for AP_Periph in firmware server
            try:
                old_parameters_folder = (os.getcwd() +
                                         '/../old_params_mversion/%s/' % value)
                old_parameters_files = [
                    f for f in glob.glob(old_parameters_folder + "*.*")
                ]
                for file in old_parameters_files:
                    debug("Removing %s" % file)
                    os.remove(file)

                new_parameters_folder = (os.getcwd() +
                                         '/../new_params_mversion/%s/' % value)
                new_parameters_files = [
                    f for f in glob.glob(new_parameters_folder +
                                         "parameters-*.rst")
                ]
                for filename in new_parameters_files:
                    debug("Copying %s to %s" %
                          (filename, old_parameters_folder))
                    shutil.copy2(filename, old_parameters_folder)

                built_folder = os.getcwd() + "/" + key + "/build/html/docs/"
                built_parameters_files = [
                    f for f in glob.glob(built_folder + "parameters-*.html")
                ]
                for built in built_parameters_files:
                    debug("Copying %s to %s" %
                          (built, old_parameters_folder))
                    shutil.copy2(built, old_parameters_folder)

            except Exception as e:
                error(e)
                pass


def put_cached_parameters_files_in_sites(site=None):
    """
    For each vechile: put built .html files in site folder

    """
    for key, value in PARAMETER_SITE.items():
        if (site == key or site is None) and (key != 'AP_Periph'): # and (key != 'AP_Periph') workaround until create a versioning for AP_Periph in firmware server
            try:
                built_folder = (os.getcwd() +
                                '/../old_params_mversion/%s/' % value)
                built_parameters_files = [
                    f for f in glob.glob(built_folder + "parameters-*.html")
                ]
                vehicle_folder = os.getcwd() + "/" + key + "/build/html/docs/"
                debug("Site %s getting previously built files from %s" %
                      (site, built_folder))
                for built in built_parameters_files:
                    if ("latest" not in built):  # latest parameters files must be built every time  # noqa
                        debug("Reusing built %s in %s " %
                              (built, vehicle_folder))
                        shutil.copy(built, vehicle_folder)
            except Exception as e:
                error(e)
                pass

def update_frotend_json():
    """
    Frontend get posts from Forum server and insert it into JSON
    """
    debug('Running script to get last posts from forum server.')
    try:
        if platform.system() == "Windows":
            subprocess.check_call(["python", "./frontend/scripts/get_discourse_posts.py"])
        else:
            subprocess.check_call(["python3", "./frontend/scripts/get_discourse_posts.py"])
    except Exception as e:
        error(e)
        pass

def copy_static_html_sites(site, destdir):
    """
    Copy pure HMTL folder the same way that Sphinx builds it
    """
    debug('Copying static sites (only frontend so far).')

    if (site == 'frontend') or (site is None):
        update_frotend_json()
        folder = 'frontend'
        try:    
            site_folder = os.getcwd() + "/" + folder
            targetdir = os.path.join(destdir, folder)
            shutil.rmtree(targetdir, ignore_errors=True)
            shutil.copytree(site_folder, targetdir)
        except Exception as e:
            error(e)
            pass


#######################################################################

if __name__ == "__main__":
    if platform.system() == "Windows":
        multiprocessing.freeze_support()

    # Set up option parsing to get connection string
    parser = argparse.ArgumentParser(
        description='Copy Common Files as needed, stripping out non-relevant wiki content',   # noqa
    )
    parser.add_argument(
        '--site',
        help="If you just want to copy to one site, you can do this. Otherwise will be copied.",  # noqa
    )
    parser.add_argument(
        '--clean',
        action='store_true',
        help="Does a very clean build - resets git to master head (and TBD cleans up any duplicates in the output).",  # noqa
    )
    parser.add_argument(
        '--cached-parameter-files',
        action='store_true',
        help="Do not re-download parameter files",
    )
    parser.add_argument(
        '--parallel',
        type=int,
        help="limit parallel builds, -1 for unlimited",
        default=1,
    )
    parser.add_argument(
        '--destdir',
        default="/var/sites/wiki/web" if platform.system() != "Windows" else os.path.join(os.path.dirname(__file__), "..", "wiki"),
        help="Destination directory for compiled docs",
    )
    parser.add_argument(
        '--enablebackups',
        action='store_true',
        default=False,
        help="Enable several backups up to const N_BACKUPS_RETAIN in --backupdestdir folder",  # noqa
    )
    parser.add_argument(
        '--backupdestdir',
        default="/var/sites/wiki-backup/web",
        help="Destination directory for compiled docs",
    )
    parser.add_argument(
        '--paramversioning',
        action='store_true',
        default=False,
        help="Build multiple parameters pages for each vehicle based on its firmware repo.",  # noqa
    )
    parser.add_argument(
        '--verbose',
        dest='verbose',
        action='store_false',
        default=True,
        help="show debugging output",
    )

    args = parser.parse_args()
    # print(args.site)
    # print(args.clean)
    
    VERBOSE = args.verbose

    now = datetime.now()
    building_time = now.strftime("%Y-%m-%d-%H-%M-%S")

    if args.paramversioning:
        # Parameters for all versions availble on firmware.ardupilot.org:
        fetch_versioned_parameters(args.site)
    else:
        # Single parameters file. Just present the latest parameters:
        fetchparameters(args.site, args.cached_parameter_files)

    # Fetch most recent LogMessage metadata from autotest:
    fetchlogmessages(args.site, args.cached_parameter_files)


    copy_static_html_sites(args.site, args.destdir)
    generate_copy_dict()
    sphinx_make(args.site, args.parallel)

    if args.paramversioning:
        put_cached_parameters_files_in_sites(args.site)
        cache_parameters_files(args.site)


    if args.enablebackups:
        copy_and_keep_build(args.site, args.destdir, args.backupdestdir)
        delete_old_wiki_backups(args.backupdestdir, N_BACKUPS_RETAIN)
    elif args.destdir:
        copy_build(args.site, args.destdir)

    # To navigate locally and view versioning script for parameters
    # working is necessary run Chrome as "chrome
    # --allow-file-access-from-files". Otherwise it will appear empty
    # locally and working once is on the server.

    if error_count > 0:
        print("%u errors during Wiki build" % (error_count,))
        sys.exit(1)

    sys.exit(0)