build-restler.py

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import argparse
import os
import shutil
import sys
import subprocess
import contextlib
from pathlib import Path

class Dirs:
    """ Global directories """
    def __init__(self, dest_dir, repository_root_dir, python_path):
        self.dest_dir = Path(dest_dir)
        self.engine_dest_dir = self.dest_dir.joinpath('engine')
        self.engine_build_dir = self.dest_dir.joinpath('build')
        self.repository_root_dir = Path(repository_root_dir)
        self.python_path = python_path

@contextlib.contextmanager
def usedir(dir):
    """ Helper for 'with' statements that changes the current directory to
    @dir and then changes the directory back to its original once the 'with' ends.

    Can be thought of like pushd with an auto popd after the 'with' scope ends
    """
    curr = os.getcwd()
    os.chdir(dir)
    try:
        yield
    finally:
        os.chdir(curr)

def _copy_py_files(src_root_dir, root_sub_dir, dest_dir):
    """ Helper function that copies all .py files from one directory to
    another while maintaining the directory structure.

    @param src_root_dir: The root portion of the source directory. This portion
            will be filtered out of the destination directory when
            creating the new directory tree.
    @type  src_root_dir: Path or str
    @param root_sub_dir: The subdirectory of the root where we will begin copying from.
    @type  root_sub_dir: Path or str
    @param dest_dir: The destination directory
    @type  dest_dir: Path or str

    Example:
      src_root_dir = /home/rest.fuzzing/restler/; root_sub_dir = engine; dest_dir = /home/drop/
      Recursively copies all .py files from /home/rest.fuzzing/restler/engine/*
      to /home/drop/engine/...

    """
    restler_dir = Path(f'{src_root_dir}/{root_sub_dir}')
    for dirpath, dirs, files in os.walk(restler_dir):
        for file in files:
            if file.endswith('.py'):
                # Combines the current file's directory with the destination directory after
                # removing the @src_root_dir portion of the current file's directory.
                # Example:
                #   src_root_dir=/home/rest.fuzzing/restler/; dirpath=/home/rest.fuzzing/restler/engine/core/;
                #   dest_dir=/home/drop/; dest_path=/home/drop/engine/core
                dest_path = Path(dest_dir).joinpath(*Path(dirpath).parts[len(src_root_dir.parts):])
                if not os.path.exists(dest_path):
                    os.makedirs(dest_path)
                shutil.copy(Path(f'{dirpath}/{file}'), dest_path.joinpath(file))

def copy_python_files(repo_root, dest_dir):
    """ Copies python files from repository to destination directory

    @param repo_root: The repository root
    @type  repo_root: Path or str
    @param dest_dir: The destination directory
    @type  dest_dir: Path or str

    """
    print("Copying all python files...")
    if not os.path.exists(dest_dir):
        os.makedirs(dest_dir)

    restler = Path(f'{repo_root}/restler')
    for file in restler.glob('*.py'):
       shutil.copy(file, dest_dir)

    _copy_py_files(restler, 'engine', dest_dir)
    _copy_py_files(restler, 'checkers', dest_dir)
    _copy_py_files(restler, 'utils', dest_dir)

def get_compilation_errors(stdout):
    """ Helper that extracts compilation errors from the build's stdout """
    Error_Start = '***'
    Error_End = '\\r\\n\\r\\n'
    stdout_index = stdout.find(Error_Start)
    errors = []
    while stdout_index >= 0:
        # Partition stdout to extract the error from between Error_Start and Error_End
        parts = stdout[stdout_index + len(Error_Start):].partition(Error_End)
        # Add this error to the error list
        errors.append(parts[0])
        # Search for more errors in the rest of stdout that is beyond the previous error
        parts_index = parts[2].find(Error_Start)
        if parts_index > 0:
            # Increment the index in stdout by adding the error partitions that were already used
            stdout_index += parts_index+len(parts[0]) + len(parts[1])
        else:
            break
    return errors

def publish_engine_py(dirs):
    """ Publish the Python RESTler engine as .py files.

    Will also do a quick compilation of the files to verify that no exception occurs

    """
    def print_compilation_errors(text):
        errors = get_compilation_errors(text)
        if errors:
            print("Compilation errors found.")
            for err in errors:
                print("\nError found!\n")
                print(err.replace('\\r\\n', '\r\n'))

    # Copy files to a build directory to test for basic compilation failure
    print("Testing compilation of Python files...")
    try:
        copy_python_files(dirs.repository_root_dir, dirs.engine_build_dir)
        try:
            completed_process = subprocess.run(f'\"{dirs.python_path}\" -m compileall -q \"{dirs.engine_build_dir}\"', shell=True, capture_output=True, check=True)
        except subprocess.CalledProcessError as e:
            print("Build failed!")
            print(f"Exit code: {e.returncode}")
            if len(e.stderr) > 0:
                print(e.stderr)
            print_compilation_errors(f"{e.stdout}")
            sys.exit(-1)
        output=f"{completed_process.stdout}"
        errors = get_compilation_errors(output)
        if (len(errors) > 0):
            print_compilation_errors(output)
            sys.exit(1)
    finally:
        print("Removing engine compilation build directory...")
        shutil.rmtree(dirs.engine_build_dir)

    # Copy files to drop
    copy_python_files(dirs.repository_root_dir, dirs.engine_dest_dir)

def publish_dotnet_apps(dirs, configuration, dotnet_package_source):
    """ Publishes the dotnet components (compiler, driver, and results analyzer)

    @param dirs: The global directories
    @type  dirs: Dirs
    @param configuration: The build configuration
    @type  configuration: Str

    """
    print("Publishing dotnet core apps...")
    dotnetcore_projects = {
        "compiler": os.path.join(f'{dirs.repository_root_dir}','src','compiler','Restler.CompilerExe','Restler.CompilerExe.fsproj'),
        "resultsAnalyzer": os.path.join(f'{dirs.repository_root_dir}','src','ResultsAnalyzer','ResultsAnalyzer.fsproj'),
        "restler": os.path.join(f'{dirs.repository_root_dir}','src','driver','Restler.Driver.fsproj')
    }

    for key in dotnetcore_projects.keys():
        target_dir_name = key
        proj_output_dir = os.path.join(f'{dirs.dest_dir}', f'{target_dir_name}')
        proj_file_path = dotnetcore_projects[target_dir_name]
        print(f"Publishing project {proj_file_path} to output dir {proj_output_dir}")

        restore_args = f"dotnet restore \"{proj_file_path}\" --use-lock-file --locked-mode --force"
        if dotnet_package_source is not None:
            restore_args = f"{restore_args} -s {dotnet_package_source}"
        try:
            subprocess.run(restore_args, shell=True, stderr=subprocess.PIPE, check=True)
        except subprocess.CalledProcessError as e:
            print("Build failed!")
            print(f"Exit code: {e.returncode}")
            if len(e.stderr) > 0:
                print(e.stderr)

            sys.exit(-1)
        try:
            subprocess.run(f"dotnet publish \"{proj_file_path}\" --no-restore -o \"{proj_output_dir}\" -c {configuration} -f net6.0", shell=True, stderr=subprocess.PIPE, check=True)
        except subprocess.CalledProcessError as e:
            print("Build failed!")
            print(f"Exit code: {e.returncode}")
            if len(e.stderr) > 0:
                print(e.stderr)
            sys.exit(-1)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--repository_root_dir',
                        help='The root of the rest.fuzzing repository.'
                        'If not specified, the root will be inferred from the location of this script in this repository.',
                        type=str, default=os.path.dirname(os.path.realpath(__file__)))
    parser.add_argument('--dest_dir',
                        help='The destination directory for the drop.',
                        type=str, required=True)
    parser.add_argument('--configuration',
                        help='The build configuration',
                        type=str, default='release', required=False)
    parser.add_argument('--python_path',
                        help='The path or python command to use for compilation. (Default: python command that initiated this script)',
                        type=str, default=sys.executable, required=False)
    parser.add_argument('--compile_type',
                        help='all: driver/compiler & engine as python files\n'
                        'engine: engine only, as python files\n'
                        'compiler: compiler only\n'
                        '(Default: all)',
                        type=str, default='all', required=False)
    parser.add_argument('--dotnet_package_source',
                        help='Overrides the dotnet package source. (Default: none)',
                        type=str, default=None, required=False)

    args = parser.parse_args()

    if not os.path.exists(args.dest_dir):
        os.makedirs(args.dest_dir)

    dirs = Dirs(args.dest_dir, args.repository_root_dir, args.python_path)

    print("Generating a new RESTler binary drop...")
    if args.compile_type == 'all':
        publish_dotnet_apps(dirs, args.configuration, args.dotnet_package_source)
        publish_engine_py(dirs)
    elif args.compile_type == 'compiler':
        publish_dotnet_apps(dirs, args.configuration, args.dotnet_package_source)
    elif args.compile_type == 'engine':
        publish_engine_py(dirs)
    else:
        print(f"Invalid compileType specified: {args.compile_type!s}")