From 964d8204d1f021330f46858d060e91be100fb13a Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 25 Aug 2023 15:27:46 +0200 Subject: [PATCH 01/11] Initial refactoring of some of the repo handling code to move it out from SubGit class into it's own contained class SubGitRepo that enables simpler consumption of data/values Refactor "subgit status" command to use this new repo handling code --- subgit/constants.py | 9 +++- subgit/core.py | 122 ++++++++++++++++++++------------------------ subgit/repo.py | 84 ++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 69 deletions(-) create mode 100644 subgit/repo.py diff --git a/subgit/constants.py b/subgit/constants.py index 01450aa..120103e 100644 --- a/subgit/constants.py +++ b/subgit/constants.py @@ -2,9 +2,16 @@ DEFAULT_REPO_DICT = {"repos": []} +REVISION_BRANCH = "branch" +REVISION_COMMIT = "commit" +REVISION_TAG = "tag" + WORKER_COUNT = 8 __all__ = [ "DEFAULT_REPO_DICT", - "WORKER_COUNT" + "REVISION_BRANCH", + "REVISION_COMMIT", + "REVISION_TAG", + "WORKER_COUNT", ] diff --git a/subgit/core.py b/subgit/core.py index e8c79dc..29ddf51 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -22,6 +22,8 @@ from subgit.constants import * from subgit.enums import * from subgit.exceptions import * +from subgit.repo import SubGitRepo + log = logging.getLogger(__name__) @@ -50,7 +52,7 @@ def __init__(self, config_file_path=None, answer_yes=False): self.subgit_config_file_name = ".subgit.yml" self.subgit_config_file_path = Path.cwd() / ".subgit.yml" - def _get_recursive_config_path(self): + def _resolve_recursive_config_path(self): """ Looks for either .sgit.yml or .subgit.yml recursively """ @@ -96,7 +98,6 @@ def init_repo(self, repo_name=None, repo_url=None): .subgit.yml config file as the first repo in your config. If these values is anything else the initial config vill we written as empty. """ - if self.subgit_config_file_path.exists(): log.error(f"File '{self.subgit_config_file_name}' already exists on disk") return 1 @@ -140,36 +141,36 @@ def _dump_config_file(self, config_data): default_flow_style=False, ) - def repo_status(self): - self._get_recursive_config_path() - config = self._get_config_file() - repos = config.get("repos", {}) + def build_repo_objects(self, repos_config): + repos = [] - if not repos: - print(" No repos found") - return 1 + for repo_data in repos_config["repos"]: + repo = SubGitRepo(repo_data) + repos.append(repo) - for repo_data in repos: - repo_name = repo_data["name"] + return repos - print(f"{repo_name}") - print(f" Url: {repo_data.get('url', 'NOT SET')}") + def repo_status(self): + self._resolve_recursive_config_path() + config = self._get_config_file() - repo_disk_path = Path().cwd() / repo_name - print(f" Disk path: {repo_disk_path}") + repos = self.build_repo_objects(config) - try: - repo = Repo(repo_disk_path) - cloned_to_disk = True - except git.exc.NoSuchPathError: - cloned_to_disk = False + if len(repos) == 0: + print(" No data for repositories found in config file") + return 1 - print(f" Cloned: {'Yes' if cloned_to_disk else 'No'}") + for repo in repos: + print("") + print(f"{repo.name}") + print(f" {repo.url}") + print(f" {repo.repo_root().resolve()}") + print(f" Is cloned to disk? {repo.is_cloned_to_disk_str()}") - if cloned_to_disk: - file_cwd = Path().cwd() / repo_name / ".git/FETCH_HEAD" + if repo.is_cloned_to_disk: + fetch_file_path = repo.git_fetch_head_file_path - if file_cwd.exists(): + if fetch_file_path.exists(): output, stderr = run_cmd(f"stat -c %y {file_cwd}") parsed_output = str(output).replace('\\n', '') @@ -179,61 +180,48 @@ def repo_status(self): else: print(" Last pull/fetch: UNKNOWN repo not cloned to disk") - if cloned_to_disk: - repo = Repo(repo_disk_path) - print(f" Repo is dirty? {'Yes' if repo.is_dirty() else 'No'}") - else: - print(" Repo is dirty? ---") - - branch = repo_data['revision'].get('branch', '---') - commit = repo_data['revision'].get('commit', '---') - tag = repo_data['revision'].get('tag', '---') + print(f" Repo is dirty? {repo.is_git_repo_dirty_str()}") + print(f" Revision:") - print(" Revision:") + if repo.revision_type == REVISION_BRANCH: + print(f" branch: {repo.revision_value}") - print(f" branch: {branch}") - if branch != "---": - if cloned_to_disk: + if repo.is_cloned_to_disk: if branch in repo.heads: - commit_hash = str(repo.heads[branch].commit) - commit_message = str(repo.heads[branch].commit.summary) - has_new = repo.remotes.origin.refs["master"].commit != repo.heads[branch].commit + git_commit = repo.git_repo.heads[self.revision_value].commit + + commit_hash = str(git_commit) + commit_message = str(git_commit.summary) + has_newer_commit = repo.git_repo.remotes.origin.refs["master"].commit != git_commit is_in_origin = f"{branch in repo.remotes.origin.refs}" else: commit_hash = "Local branch not found" commit_message = "Local branch not found" - has_new = "---" + has_newer_commit = "---" is_in_origin = "Local branch not found" else: commit_hash = "Repo not cloned to disk" commit_message = "Repo not cloned to disk" - has_new = "Repo not cloned to disk" + has_newer_commit = "Repo not cloned to disk" is_in_origin = "Repo not cloned to disk" print(f" commit hash: {commit_hash}") print(f" commit message: '{commit_message}'") print(f" branch exists in origin? {is_in_origin}") - print(f" has newer commit in origin? {has_new}") - - print(f" commit: {commit}") - if commit != "---": - print("FIXME: Not implemented yet") - - # Extract tag from inner value if that is set - # {revision: {tag: {select: {value: foo}}}} - if isinstance(tag, dict): - tag = tag["select"] + print(f" has newer commit in origin? {has_newer_commit}") - if isinstance(tag, dict): - tag = tag["value"] + if repo.revision_type == REVISION_COMMIT: + print(" FIXME: Not implemented yet") + print(f" commit: {repo.revision_value}") - print(f" tag: {tag}") + if repo.revision_type == REVISION_TAG: + print(f" tag: {repo.revision_value}") - if tag != "---": - if cloned_to_disk: - if tag in repo.tags: - commit_hash = str(repo.tags[tag].commit) - commit_summary = str(repo.tags[tag].commit.summary) + if repo.is_cloned_to_disk: + if repo.revision_value in repo.git_repo.tags: + tags_commit = repo.git_repo.tags[repo.revision_value].commit + commit_hash = str(tags_commit) + commit_summary = str(tags_commit.summary) else: commit_hash = "Tag not found" commit_summary = "---" @@ -242,9 +230,7 @@ def repo_status(self): commit_summary = "Repo not cloned to disk" print(f" commit hash: {commit_hash}") - print(f" commit message: '{commit_summary}'") - - print("") + print(f" commit message: {commit_summary}") def yes_no(self, question): print(question) @@ -267,7 +253,7 @@ def fetch(self, repos): A empty list of items will not fetch any repo. """ - self._get_recursive_config_path() + self._resolve_recursive_config_path() log.debug(f"repo fetch input - {repos}") @@ -345,7 +331,7 @@ def pull(self, names): To pull a subset of repos send in a list of strings names=["repo1", "repo2"] """ - self._get_recursive_config_path() + self._resolve_recursive_config_path() log.debug(f"Repo pull - {names}") @@ -635,7 +621,7 @@ def delete(self, repo_names=None): more of them creates a conflict. (e.g repo(s) is not in the config file, path(s) is not to a valid git repo or repo(s) is dirty) """ - self._get_recursive_config_path() + self._resolve_recursive_config_path() has_dirty_repos = False good_repos = [] repos_dict = self._get_repos(repo_names) @@ -669,7 +655,7 @@ def reset(self, repo_names=None, hard_flag=None): Will take a list of repos and find any diffs and reset them back to the same state they were when they were first pulled. """ - self._get_recursive_config_path() + self._resolve_recursive_config_path() dirty_repos = [] repos_dict = self._get_repos(repo_names) repos = repos_dict["repos"] @@ -709,7 +695,7 @@ def clean(self, repo_names=None, recurse_into_dir=None, force=None, dry_run=None """ Method to look through a list of repos and remove untracked files """ - self._get_recursive_config_path() + self._resolve_recursive_config_path() dirty_repos = [] recursive_flag = "d" if recurse_into_dir else "" force_flag = "f" if force else "" diff --git a/subgit/repo.py b/subgit/repo.py new file mode 100644 index 0000000..d21b85e --- /dev/null +++ b/subgit/repo.py @@ -0,0 +1,84 @@ +import git + +from git import Git, Repo +from pathlib import Path +from subgit.constants import * + + +class SubGitRepo(object): + + def __init__(self, repo_config, *args, **kwargs): + self.raw_config = repo_config + + self.repo_cwd = Path(".") + self.name = self.raw_config.get("name", None) + self.clone_point = Path(".") # Clone point + + self.url = repo_config.get("url", "NOT SET") # repo url:en som git fattar + self.revision_type = "" # branch, tag, commit + self.revision_value = "" # namnet på saken + + try: + self.git_repo = Repo(self.repo_root()) + self.is_cloned_to_disk = True + except git.exc.NoSuchPathError: + self.is_cloned_to_disk = False + + # TODO: Simplify this to not be as repetetive as now + repo_branch = repo_config["revision"].get("branch", None) + repo_commit = repo_config["revision"].get("commit", None) + repo_tag = repo_config["revision"].get("tag", None) + + if repo_branch: + self.revision_type = REVISION_BRANCH + self.revision_value = repo_branch + if repo_commit: + self.revision_type = REVISION_COMMIT + self.revision_value = repo_commit + if repo_tag: + self.revision_type = REVISION_TAG + + if isinstance(repo_tag, str): + self.revision_value = repo_tag + elif isinstance(repo_tag, dict): + self.revision_value = repo_tag["select"] + + # If value is nested inside a dict explicit + if isinstance(self.revision_value, dict): + self.revision_value = repo_tag["select"]["value"] + + @property + def git_fetch_head_file_path(self): + """ + Dynamically calculate this repo path each time + """ + return self.repo_root() / ".git/FETCH_HEAD" + + def is_git_repo_dirty(self): + return self.git_repo.is_dirty() + + def is_git_repo_dirty_str(self): + return "Yes" if self.is_git_repo_dirty else "No" + + def is_cloned_to_disk_str(self): + return "Yes" if self.is_cloned_to_disk else "No" + + def repo_root(self): + """ + Combines repo_cwd + clone_point into the root folder that + the repo should be cloned to + """ + return self.repo_cwd / self.name / self.clone_point + + def print_status(self): + pass + + def to_dict(self): + """ + Compiles and returns the object as a dict. This is not the same as the raw config + values that was set from repo_config but all the calculated values after initialization + """ + return vars(self) + + def __repr__(self): + return f"SubGitRepo - {self.name} - {self.repo_root().resolve()} - {self.url} - {self.revision_type}/{self.revision_value}" From e3ef755bf469aba6c24d0555d92c7fba0dc130c0 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 28 Aug 2023 13:13:10 +0200 Subject: [PATCH 02/11] Fix typo in variable name --- subgit/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subgit/core.py b/subgit/core.py index 29ddf51..a48cfc2 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -171,7 +171,7 @@ def repo_status(self): fetch_file_path = repo.git_fetch_head_file_path if fetch_file_path.exists(): - output, stderr = run_cmd(f"stat -c %y {file_cwd}") + output, stderr = run_cmd(f"stat -c %y {fetch_file_path}") parsed_output = str(output).replace('\\n', '') print(f" Last pull/fetch: {parsed_output}") From bf37ea7fbb947883649bbc71e0598ff85e7dbc38 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 28 Aug 2023 13:31:25 +0200 Subject: [PATCH 03/11] Rewrite fetch command to use new SubGitRepo class --- subgit/core.py | 65 ++++++++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/subgit/core.py b/subgit/core.py index a48cfc2..e97d064 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -153,7 +153,6 @@ def build_repo_objects(self, repos_config): def repo_status(self): self._resolve_recursive_config_path() config = self._get_config_file() - repos = self.build_repo_objects(config) if len(repos) == 0: @@ -243,70 +242,52 @@ def yes_no(self, question): return answer.lower().startswith("y") - def fetch(self, repos): + def fetch(self, repos_to_fetch): """ - Runs "git fetch" on one or more git repos. + Runs "git fetch" on one or more git repos - To fetch all enabled repos send in None as value. + To fetch all enabled repos send in None as value - To fetch a subset of repo names, send in them as a list of strings. + To fetch a subset of repo names, send in them as a list of strings - A empty list of items will not fetch any repo. + A empty list of items will not fetch any repo """ + log.debug(f"Repo names fetch input - {repos_to_fetch}") + self._resolve_recursive_config_path() - - log.debug(f"repo fetch input - {repos}") - config = self._get_config_file() + repos = self.build_repo_objects(config) - repos_to_fetch = [] - - if repos is None: - for repo_data in config["repos"]: - repo_name = repo_data["name"] - repos_to_fetch.append(repo_name) - - if isinstance(repos, list): - for repo_data in repos: - repo_name = repo_data["name"] - - if repo_name in config["repos"]: - repos_to_fetch.append(repo_name) - else: - log.warning(f"repo '{repo_name}' not found in configuration") + if isinstance(repos_to_fetch, list): + # If we provide a list of repos we need to filter them out from all repo objects + repos = list(filter(lambda obj: obj.name in repos_to_fetch, repos)) - log.info(f"repos to fetch: {repos_to_fetch}") + log.info(f"Git repos to fetch: {repos}") - if len(repos_to_fetch) == 0: - log.error("No repos to fetch found") + if len(repos) == 0: + log.critical("No repos found to filter out. Check config file and cli arguments") return 1 missing_any_repo = False - for repo_name in repos_to_fetch: - try: - repo_path = Path().cwd() / repo_name - Repo(repo_path) - except git.exc.NoSuchPathError: - log.error(f"Repo {repo_name} not found on disk. You must pull to do a initial clone before fetching can be done") + for repo in repos: + if not repo.is_cloned_to_disk: + log.error(f"Repo {repo.name} not found on disk. You must pull to do a initial clone before fetching can be done") missing_any_repo = True if missing_any_repo: return 1 with Pool(WORKER_COUNT) as pool: - pool.map(self.fetch_repo, repos_to_fetch) + pool.map(self.fetch_repo, repos) - log.info("Fetching for all repos completed") + log.info("Fetch command run on all git repos completed") return 0 - def fetch_repo(self, repo_name): - repo_path = Path().cwd() / repo_name - git_repo = Repo(repo_path) - - log.info(f"Fetching git repo '{repo_name}'") - fetch_results = git_repo.remotes.origin.fetch() - log.info(f"Fetching completed for repo '{repo_name}'") + def fetch_repo(self, repo): + log.info(f"Fetching git repo '{repo.name}'") + fetch_results = repo.git_repo.remotes.origin.fetch() + log.info(f"Fetching completed for repo '{repo.name}'") for fetch_result in fetch_results: log.info(f" - Fetch result: {fetch_result.name}") From 9b738d76a85c937c04bdb3afcbbc8b30ad1c9690 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 29 Aug 2023 15:04:14 +0200 Subject: [PATCH 04/11] Rewrite pull command to use the new SubGitRepo class --- subgit/core.py | 198 +++++++++++++++++++------------------------------ subgit/repo.py | 51 +++++++++++-- 2 files changed, 120 insertions(+), 129 deletions(-) diff --git a/subgit/core.py b/subgit/core.py index e97d064..2f11f36 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -186,13 +186,13 @@ def repo_status(self): print(f" branch: {repo.revision_value}") if repo.is_cloned_to_disk: - if branch in repo.heads: - git_commit = repo.git_repo.heads[self.revision_value].commit + if repo.revision_value in repo.git_repo.heads: + git_commit = repo.git_repo.heads[repo.revision_value].commit commit_hash = str(git_commit) commit_message = str(git_commit.summary) has_newer_commit = repo.git_repo.remotes.origin.refs["master"].commit != git_commit - is_in_origin = f"{branch in repo.remotes.origin.refs}" + is_in_origin = f"{repo.revision_value in repo.git_repo.remotes.origin.refs}" else: commit_hash = "Local branch not found" commit_message = "Local branch not found" @@ -265,6 +265,7 @@ def fetch(self, repos_to_fetch): log.info(f"Git repos to fetch: {repos}") if len(repos) == 0: + # TODO: Convert to an exception log.critical("No repos found to filter out. Check config file and cli arguments") return 1 @@ -276,6 +277,7 @@ def fetch(self, repos_to_fetch): missing_any_repo = True if missing_any_repo: + # TODO: Convert to an exception return 1 with Pool(WORKER_COUNT) as pool: @@ -286,68 +288,50 @@ def fetch(self, repos_to_fetch): def fetch_repo(self, repo): log.info(f"Fetching git repo '{repo.name}'") - fetch_results = repo.git_repo.remotes.origin.fetch() + fetch_results = repo.fetch_repo() log.info(f"Fetching completed for repo '{repo.name}'") for fetch_result in fetch_results: log.info(f" - Fetch result: {fetch_result.name}") - def _get_active_repos(self, config): + def pull(self, repos_to_pull): """ - Helper method that will return only the repos that is enabled and active for usage - """ - active_repos = [] - - for repo_data in config.get("repos", []): - repo_name = repo_data["name"] - - if repo_data.get("enable", True): - active_repos.append(repo_name) + To pull all repos defined in the configuration send in repos_to_pull=None - return active_repos - - def pull(self, names): + To pull a subset of repos send in a list of strings repos_to_pull=["repo1", "repo2"] """ - To pull all repos defined in the configuration send in names=None + log.debug(f"Repo pull - {repos_to_pull}") - To pull a subset of repos send in a list of strings names=["repo1", "repo2"] - """ self._resolve_recursive_config_path() - - log.debug(f"Repo pull - {names}") - config = self._get_config_file() + repos = self.build_repo_objects(config) - active_repos = self._get_active_repos(config) - - repos = [] + # Filter out any repo object that is not enabled + repos = list(filter(lambda obj: obj.is_enabled is True, repos)) - if len(active_repos) == 0: + if len(repos) == 0: + # TODO: Convert to an exception log.error("There is no repos defined or enabled in the config") return 1 - if names is None: - repos = config.get("repos", []) - repo_choices = ", ".join(active_repos) - + if repos_to_pull is None: + repo_choices = ", ".join([repo.name for repo in repos]) answer = self.yes_no(f"Are you sure you want to 'git pull' the following repos '{repo_choices}'") - if not answer: + if answer is False: + # TODO: Convert to an exception log.warning("User aborted pull step") return 1 - elif isinstance(names, list): - # Validate that all provided repo names exists in the config - for name in names: - if name not in active_repos: - choices = ", ".join(active_repos) - log.error(f'Repo with name "{name}" not found in config file. Choices are "{choices}"') - return 1 - - for repo_data in config["repos"]: - if repo_data["name"] == name: - repos.append(repo_data) + elif isinstance(repos_to_pull, list): + # Validate that all names we have sent in really exists in the config + is_all_repos_valid = all([repo.name in repos_to_pull for repo in repos]) + print(is_all_repos_valid) + + # If we send in a list of a subset of repos we want to pull, filter out the repos + # that do not match these names + repos = list(filter(lambda obj: obj.name in repos_to_pull, repos)) else: - log.debug(f"Names {names}") + log.debug(f"Names {repos_to_pull}") raise SubGitConfigException("Unsuported value type for argument names") if not repos: @@ -360,20 +344,19 @@ def pull(self, names): has_dirty = False - for repo_data in repos: - name = repo_data["name"] - repo_path = Path().cwd() / name - + for repo in repos: # If the path do not exists then the repo can't be dirty - if not repo_path.exists(): + # if not repo_path.exists(): + if not repo.repo_root().exists(): continue - repo = Repo(repo_path) - # A dirty repo means there is uncommited changes in the tree - if repo.is_dirty(): - log.error(f'The repo "{name}" is dirty and has uncommited changes in the following files') - dirty_files = [item.a_path for item in repo.index.diff(None)] + if repo.git_repo.is_dirty(): + log.error(f'The repo "{repo.name}" is dirty and has uncommited changes in the following files') + dirty_files = [ + item.a_path + for item in repo.git_repo.index.diff(None) + ] for file in dirty_files: log.info(f" - {file}") @@ -381,76 +364,45 @@ def pull(self, names): has_dirty = True if has_dirty: - log.error("\nFound one or more dirty repos. Resolve it before continue...") - return 1 - - bad_repo_configs = [] - - for repo_data in repos: - name = repo_data["name"] - revision = repo_data["revision"] - - if "branch" in revision: - if not revision["branch"]: - bad_repo_configs.append(name) - - if bad_repo_configs: - bad_repos_string = ", ".join(repo for repo in bad_repo_configs) - log.error(f"One or more repos in congif file has an invalid branch option... {bad_repos_string}") + # TODO: Convert to an exception + log.error("Found one or more dirty repos. Resolve it before continue...") return 1 # Repos looks good to be pulled. Run the pull logic for each repo in sequence - - for repo_data in repos: - name = repo_data["name"] + for repo in repos: log.info("") - repo_path = Path().cwd() / name - revision = repo_data["revision"] - # Boolean value wether repo is newly cloned. cloned = False - if not repo_path.exists(): - clone_url = repo_data.get("url", None) + # if not repo_path.exists(): + if not repo.is_cloned_to_disk: cloned = True - if not clone_url: - raise SubGitConfigException(f"Missing required key 'url' on repo '{name}'") - try: # Cloning a repo w/o a specific commit/branch/tag it will clone out whatever default # branch or where the origin HEAD is pointing to. After we clone we can then move our # repo to the correct revision we want. - repo = Repo.clone_from( - repo_data["url"], - repo_path, - ) - log.info(f'Successfully cloned repo "{name}" from remote server') + repo.clone_repo() + log.info(f'Successfully cloned repo "{repo.name}" from remote server') except Exception as e: - raise SubGitException(f'Clone "{name}" failed, exception: {e}') from e + raise SubGitException(f'Clone "{repo.name}" failed') from e log.debug("TODO: Parse for any changes...") # TODO: Check that origin remote exists - p = Path().cwd() / name - repo = Repo(p) - g = Git(p) - # Fetch all changes from upstream git repo - repo.remotes.origin.fetch() + repo.fetch_repo() + revision = repo.revision_value # How to handle the repo when a branch is specified - if "branch" in revision: + if repo.revision_type == REVISION_BRANCH: log.debug("Handling branch pull case") - # Extract the sub tag data - branch_revision = revision["branch"] - if not cloned: try: - latest_remote_sha = str(repo.rev_parse(f"origin/{branch_revision}")) - latest_local_sha = str(repo.head.commit.hexsha) + latest_remote_sha = str(repo.git_repo.rev_parse(f"origin/{revision}")) + latest_local_sha = str(repo.git_repo.head.commit.hexsha) if latest_remote_sha != latest_local_sha: repo.remotes.origin.pull() @@ -458,20 +410,21 @@ def pull(self, names): log.error(er) # Ensure the local version of the branch exists and points to the origin ref for that branch - repo.create_head(f"{branch_revision}", f"origin/{branch_revision}") + repo.git_repo.create_head(f"{revision}", f"origin/{revision}") # Checkout the selected revision # TODO: This only support branches for now - repo.heads[branch_revision].checkout() + repo.git_repo.heads[revision].checkout() - log.info(f'Successfully pull repo "{name}" to latest commit on branch "{branch_revision}"') - log.info(f"Current git hash on HEAD: {str(repo.head.commit)}") - elif "tag" in revision: + log.info(f'Successfully pull repo "{repo.name}" to latest commit on branch "{repo.revision_value}"') + log.info(f"Current git hash on HEAD: {str(repo.git_repo.head.commit)}") + return 1 + elif repo.revision_type == REVISION_TAG: # # Parse and extract out all relevant config options and determine if they are nested # dicts or single values. The values will later be used as input into each operation. # - tag_config = revision["tag"] + tag_config = repo.revision_value if isinstance(tag_config, str): # All options should be set to default' @@ -492,6 +445,7 @@ def pull(self, names): raise SubGitConfigException("filter option must be a list of items or a single string") order_config = tag_config.get("order", None) + if order_config is None: order_algorithm = OrderAlgorithms.SEMVER else: @@ -502,6 +456,7 @@ def pull(self, names): select_config = tag_config.get("select", None) select_method = None + if select_config is None: raise SubGitConfigException("select key is required in all tag revisions") @@ -519,9 +474,10 @@ def pull(self, names): if select_method is None: raise SubGitConfigException(f"Unsupported select method chosen: {select_method_value.upper()}") else: + # By default we should treat the select value as a semver string select_method = SelectionMethods.SEMVER else: - raise SubGitConfigException(f"Key revision.tag for repo {name} must be a string or dict object") + raise SubGitConfigException(f"Key revision.tag for repo {repo.name} must be a string or dict object") log.debug(f"{filter_config}") log.debug(f"{order_config}") @@ -531,7 +487,7 @@ def pull(self, names): # Main tag parsing logic - git_repo_tags = list(repo.tags) + git_repo_tags = list(repo.git_repo.tags) log.debug(f"Raw git tags from git repo {git_repo_tags}") filter_output = self._filter(git_repo_tags, filter_config) @@ -548,36 +504,34 @@ def pull(self, names): if not select_output: raise SubGitRepoException("No git tag could be parsed out with the current repo configuration") - log.info(f"Attempting to checkout tag '{select_output}' for repo '{name}'") + log.info(f"Attempting to checkout tag '{select_output}' for repo '{repo.name}'") # Otherwise atempt to checkout whatever we found. If our selection is still not something valid # inside the git repo, we will get sub exceptions raised by git module. - g.checkout(select_output) + repo.git.checkout(select_output) - log.info(f"Checked out tag '{select_output}' for repo '{name}'") - log.info(f"Current git hash on HEAD: {str(repo.head.commit)}") - log.info(f"Current commit summary on HEAD in git repo '{name}': ") - log.info(f" {str(repo.head.commit.summary)}") + log.info(f"Checked out tag '{select_output}' for repo '{repo.name}'") + log.info(f"Current git hash on HEAD: {str(repo.git_repo.head.commit)}") + log.info(f"Current commit summary on HEAD in git repo '{repo.name}': ") + log.info(f" {str(repo.git_repo.head.commit.summary)}") # Handle sparse checkout by configure the repo - sparse_repo_config = repo_data.get("sparse", None) - - if sparse_repo_config: - log.info(f"Enable sparse checkout on repo {name}") + if repo.sparse_checkout_enabled: + log.info(f"Enable sparse checkout on repo {repo.name}") # Always ensure that sparse is enabled - g.sparse_checkout("init") + repo.git.sparse_checkout("init") repos = [ str(path) - for path in sparse_repo_config["paths"] + for path in repo.sparse_checkout_config["paths"] ] # Set what paths we defined to be checked out - g.sparse_checkout("set", *repos) + repo.git.sparse_checkout("set", *repos) # List all items (files and directories) in the cwd - items = list(p.iterdir()) + items = list(repo.repo_root().iterdir()) # Filter out the '.git' folder visible_items = [ @@ -593,8 +547,8 @@ def pull(self, names): else: # By always setting disable as a default, this will automatically revert any repo # that used to have sparse enabled but no longer is ensabled - log.debug(f"Disabling sparse checkout on repo {name}") - g.sparse_checkout("disable") + log.debug(f"Disabling sparse checkout on repo {repo.name}") + repo.git.sparse_checkout("disable") def delete(self, repo_names=None): """ diff --git a/subgit/repo.py b/subgit/repo.py index d21b85e..120547e 100644 --- a/subgit/repo.py +++ b/subgit/repo.py @@ -3,6 +3,7 @@ from git import Git, Repo from pathlib import Path from subgit.constants import * +from subgit.exceptions import * class SubGitRepo(object): @@ -11,18 +12,22 @@ def __init__(self, repo_config, *args, **kwargs): self.raw_config = repo_config self.repo_cwd = Path(".") - self.name = self.raw_config.get("name", None) + self.name = repo_config.get("name", None) self.clone_point = Path(".") # Clone point - + self.is_enabled = repo_config.get("enable", True) self.url = repo_config.get("url", "NOT SET") # repo url:en som git fattar self.revision_type = "" # branch, tag, commit self.revision_value = "" # namnet på saken + self.is_cloned_to_disk = None + self.sparse_checkout_config = repo_config.get("sparse", None) + self.sparse_checkout_enabled = True if self.sparse_checkout_config else False - try: - self.git_repo = Repo(self.repo_root()) - self.is_cloned_to_disk = True - except git.exc.NoSuchPathError: - self.is_cloned_to_disk = False + # try: + self.refresh_git_repo() + # self.git_repo = Repo(self.repo_root()) + # self.is_cloned_to_disk = True + # except git.exc.NoSuchPathError: + # self.is_cloned_to_disk = False # TODO: Simplify this to not be as repetetive as now repo_branch = repo_config["revision"].get("branch", None) @@ -47,6 +52,38 @@ def __init__(self, repo_config, *args, **kwargs): if isinstance(self.revision_value, dict): self.revision_value = repo_tag["select"]["value"] + def clone_repo(self): + """ + Helper action that will clone the configured git repo based on the url + and the root folder path + """ + if not self.url: + raise SubGitConfigException(f"Missing required key 'url' on repo '{self.name}'") + + Repo.clone_from( + self.url, + self.repo_root(), + ) + + self.refresh_git_repo() + + def fetch_repo(self): + return self.git_repo.remotes.origin.fetch() + + def refresh_git_repo(self): + """ + Helper method that is supposed to recreate the Repo and Git objects + """ + try: + self.git_repo = Repo(self.repo_root()) + self.git = Git(self.repo_root()) + + # If we do not get any exception, then we assume that folder and git + # repo exists locally on our machine + self.is_cloned_to_disk = True + except git.exc.NoSuchPathError: + self.is_cloned_to_disk = False + @property def git_fetch_head_file_path(self): """ From 9a5eff57a8402624e3232a818c1cb9530d9b22e7 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Thu, 31 Aug 2023 12:12:53 +0200 Subject: [PATCH 05/11] Update delete method with new SubGitRepo class --- README.md | 1 + subgit/core.py | 45 +++++++++++++++++++++++++++------------------ subgit/repo.py | 6 ------ 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2ab6444..4ae2f9d 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ You can reset repos in which changes have been made such as new files or new com ```bash # Reset all repos in sequence if they are dirty + subgit reset # Reset one specified repo if it's dirty diff --git a/subgit/core.py b/subgit/core.py index 2f11f36..1b46c25 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -550,6 +550,9 @@ def pull(self, repos_to_pull): log.debug(f"Disabling sparse checkout on repo {repo.name}") repo.git.sparse_checkout("disable") + def _filter_for_selected_repos(self, repos, filter_list): + return list(filter(lambda repo: repo.name in filter_list, repos)) + def delete(self, repo_names=None): """ Helper method that recieves a list of repos. Deletes them as long as not one or @@ -557,33 +560,38 @@ def delete(self, repo_names=None): path(s) is not to a valid git repo or repo(s) is dirty) """ self._resolve_recursive_config_path() + config = self._get_config_file() + repos = self.build_repo_objects(config) + repos = self._filter_for_selected_repos(repos, repo_names) + has_dirty_repos = False good_repos = [] - repos_dict = self._get_repos(repo_names) - repos = repos_dict["repos"] - active_repos = repos_dict["active_repos"] - repo_choices = repos_dict["repo_choices"] - repo_paths = self._get_working_repos(repos, active_repos) - if not repo_paths: + if not repos: + log.critical(f"No git repos to delete selected") return 1 + repo_choices = [ + repo.name + for repo in repos + ] + answer = self.yes_no(f"Are you sure you want to delete the following repos '{repo_choices}'?") if answer: - for path in repo_paths: - current_repo = Repo(path) - - if self._check_remote(current_repo): - has_dirty_repos = True - log.critical(f"'{path.name}' has some diff(s) in the local repo or the remote that needs be taken care of before deletion.") + for repo in repos: + if not repo.is_cloned_to_disk: + # This repo is already not present or cloned on disk, continue to next repo + log.info(f"Git repo '{repo.name}' is already deleted on disk") + continue + + if self._check_remote(repo.git_repo): + # If repo is dirty, or uncommited changes, the folder can't be removed until that is fixed + log.critical(f"'{repo.name}' has some diff(s) in the local repo or the remote that needs be taken care of before deletion") else: - good_repos.append(path) - - if not has_dirty_repos: - for repo in good_repos: - shutil.rmtree(repo) - log.info(f"Successfully removed repo: {path.name}") + # If repo is clean then we try to delete it + shutil.rmtree(repo.repo_root().resolve()) + log.info(f"Successfully removed folder: {repo.repo_root()} for repo '{repo.name}'") def reset(self, repo_names=None, hard_flag=None): """ @@ -706,6 +714,7 @@ def _check_remote(self, repo): differences in remote and local commits, and/or has any untracked files. """ has_remote_difference = False + for remote in repo.remotes: for branch in repo.branches: try: diff --git a/subgit/repo.py b/subgit/repo.py index 120547e..d26b0fc 100644 --- a/subgit/repo.py +++ b/subgit/repo.py @@ -10,7 +10,6 @@ class SubGitRepo(object): def __init__(self, repo_config, *args, **kwargs): self.raw_config = repo_config - self.repo_cwd = Path(".") self.name = repo_config.get("name", None) self.clone_point = Path(".") # Clone point @@ -22,12 +21,7 @@ def __init__(self, repo_config, *args, **kwargs): self.sparse_checkout_config = repo_config.get("sparse", None) self.sparse_checkout_enabled = True if self.sparse_checkout_config else False - # try: self.refresh_git_repo() - # self.git_repo = Repo(self.repo_root()) - # self.is_cloned_to_disk = True - # except git.exc.NoSuchPathError: - # self.is_cloned_to_disk = False # TODO: Simplify this to not be as repetetive as now repo_branch = repo_config["revision"].get("branch", None) From 93f6a265d3ae50ce2e9fac63ddc70f182d8f6b18 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 1 Sep 2023 13:01:36 +0200 Subject: [PATCH 06/11] Update clean command to new SubGitRepo class Fixed a logic bug with clone-point Minor other minor fixes Removed some unused functions that will be converted later --- subgit/core.py | 200 +++++++++++++++---------------------------------- subgit/repo.py | 4 +- 2 files changed, 64 insertions(+), 140 deletions(-) diff --git a/subgit/core.py b/subgit/core.py index 1b46c25..e4bf8ab 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -141,7 +141,7 @@ def _dump_config_file(self, config_data): default_flow_style=False, ) - def build_repo_objects(self, repos_config): + def _build_repo_objects(self, repos_config): repos = [] for repo_data in repos_config["repos"]: @@ -153,7 +153,7 @@ def build_repo_objects(self, repos_config): def repo_status(self): self._resolve_recursive_config_path() config = self._get_config_file() - repos = self.build_repo_objects(config) + repos = self._build_repo_objects(config) if len(repos) == 0: print(" No data for repositories found in config file") @@ -256,7 +256,7 @@ def fetch(self, repos_to_fetch): self._resolve_recursive_config_path() config = self._get_config_file() - repos = self.build_repo_objects(config) + repos = self._build_repo_objects(config) if isinstance(repos_to_fetch, list): # If we provide a list of repos we need to filter them out from all repo objects @@ -281,12 +281,12 @@ def fetch(self, repos_to_fetch): return 1 with Pool(WORKER_COUNT) as pool: - pool.map(self.fetch_repo, repos) + pool.map(self._fetch_repo, repos) log.info("Fetch command run on all git repos completed") return 0 - def fetch_repo(self, repo): + def _fetch_repo(self, repo): log.info(f"Fetching git repo '{repo.name}'") fetch_results = repo.fetch_repo() log.info(f"Fetching completed for repo '{repo.name}'") @@ -304,7 +304,7 @@ def pull(self, repos_to_pull): self._resolve_recursive_config_path() config = self._get_config_file() - repos = self.build_repo_objects(config) + repos = self._build_repo_objects(config) # Filter out any repo object that is not enabled repos = list(filter(lambda obj: obj.is_enabled is True, repos)) @@ -375,7 +375,6 @@ def pull(self, repos_to_pull): # Boolean value wether repo is newly cloned. cloned = False - # if not repo_path.exists(): if not repo.is_cloned_to_disk: cloned = True @@ -418,7 +417,6 @@ def pull(self, repos_to_pull): log.info(f'Successfully pull repo "{repo.name}" to latest commit on branch "{repo.revision_value}"') log.info(f"Current git hash on HEAD: {str(repo.git_repo.head.commit)}") - return 1 elif repo.revision_type == REVISION_TAG: # # Parse and extract out all relevant config options and determine if they are nested @@ -551,6 +549,9 @@ def pull(self, repos_to_pull): repo.git.sparse_checkout("disable") def _filter_for_selected_repos(self, repos, filter_list): + if not filter_list: + return repos + return list(filter(lambda repo: repo.name in filter_list, repos)) def delete(self, repo_names=None): @@ -561,7 +562,7 @@ def delete(self, repo_names=None): """ self._resolve_recursive_config_path() config = self._get_config_file() - repos = self.build_repo_objects(config) + repos = self._build_repo_objects(config) repos = self._filter_for_selected_repos(repos, repo_names) has_dirty_repos = False @@ -571,10 +572,10 @@ def delete(self, repo_names=None): log.critical(f"No git repos to delete selected") return 1 - repo_choices = [ + repo_choices = ", ".join([ repo.name for repo in repos - ] + ]) answer = self.yes_no(f"Are you sure you want to delete the following repos '{repo_choices}'?") @@ -599,26 +600,35 @@ def reset(self, repo_names=None, hard_flag=None): to the same state they were when they were first pulled. """ self._resolve_recursive_config_path() + config = self._get_config_file() + repos = self._build_repo_objects(config) + repos = self._filter_for_selected_repos(repos, repo_names) + # TODO: Add back filtering out of inactive repos + dirty_repos = [] - repos_dict = self._get_repos(repo_names) - repos = repos_dict["repos"] - active_repos = repos_dict["active_repos"] - repo_choices = repos_dict["repo_choices"] - repo_paths = self._get_working_repos(repos, active_repos) - if not repo_paths: - return 1 + if not repos: + log.critical(f"No git repos to delete selected") + return + + repo_choices = ", ".join([ + repo.name + for repo in repos + ]) answer = self.yes_no(f"Are you sure you want to reset the following repos '{repo_choices}'?") if answer: - for path in repo_paths: - current_repo = Repo(path) + for repo in repos: + if not repo.is_cloned_to_disk: + # This repo is already not present or cloned on disk, continue to next repo + log.info(f" - Unable to reset git repo '{repo.name}', it is not cloned to disk. Pull the repo first") + continue - if self._check_remote(current_repo): - dirty_repos.append(path) + if self._check_remote(repo.git_repo): + dirty_repos.append(repo) else: - log.info(f"{repo.name} is clean") + log.info(f" - {repo.name} is clean, nothing to reset") if not dirty_repos: log.error("No repos found to reset. Exiting...") @@ -627,86 +637,59 @@ def reset(self, repo_names=None, hard_flag=None): # Resets repo back to the latest remote commit. # If the repo has untracked files, it will not be removed unless '--hard' flag is specified. for repo in dirty_repos: - repo_to_reset = Repo(repo) flag = "--hard" if hard_flag else None - repo_to_reset.git.reset(flag) - log.info(f"Successfully reset {repo.name}") - - return 0 + repo.git.reset(flag) + log.info(f" - Successfully reset {repo.name}") def clean(self, repo_names=None, recurse_into_dir=None, force=None, dry_run=None): """ Method to look through a list of repos and remove untracked files """ self._resolve_recursive_config_path() + config = self._get_config_file() + repos = self._build_repo_objects(config) + repos = self._filter_for_selected_repos(repos, repo_names) + # TODO: Add back filtering out of inactive repos + dirty_repos = [] recursive_flag = "d" if recurse_into_dir else "" force_flag = "f" if force else "" dry_run_flag = "n" if dry_run else "" flags = f"-{recursive_flag}{force_flag}{dry_run_flag}" - repos_dict = self._get_repos(repo_names) - repos = repos_dict["repos"] - active_repos = repos_dict["active_repos"] - repo_paths = self._get_working_repos(repos, active_repos) - if not repo_paths: - return 1 + if not repos: + log.error(f"No git repos to clean selected") + return - for path in repo_paths: - current_repo = Repo(path) + for repo in repos: + if not repo.is_cloned_to_disk: + log.warning(f" - Git repo '{repo.name}' is not cloned to disk. Skipping") + continue - if self._check_remote(current_repo): - dirty_repos.append(path) + if self._check_remote(repo.git_repo): + log.info(f" - Dirty Git repo '{repo.name}' found, adding to clean list") + dirty_repos.append(repo) + else: + log.info(f" - Git repo '{repo.name}' is not dirty, nothing to clean") if not dirty_repos: - log.error("No repos found to reset. Exiting...") - return 1 + log.error(" - No dirty git repos found to clean") + return - for repo in repo_paths: - current_repo = Repo(repo) + for dirty_repo in dirty_repos: try: - clean_return = current_repo.git.clean(flags) + clean_return = dirty_repo.git.clean(flags) - if clean_return: - log.info(f"Repo: {repo}") - log.info(clean_return) + log.info(f" Clean Repo output: {repo.name}") + log.info(f" '{clean_return}'") except git.exc.GitCommandError as er: - print(er) - return 1 + log.critical(er) + return if not dry_run: log.info("Successfully cleaned repo(s)") - - return 0 - - def _get_repos(self, repos=None): - """ - Method takes zero or one repo name as argument. - - Checks if the repo(s) are valid git repo(s) and if there are any untracked changes. - """ - config = self._get_config_file() - active_repos = self._get_active_repos(config) - active_repos_dict = config.get("repos", []) - - if not repos: - repos = active_repos - - # If no names are specified in the command - if repos is None: - repo_choices = ", ".join(active_repos_dict) - - # If at least one name was specified - if isinstance(repos, list): - repo_choices = ", ".join(repos) - - working_repos = { - "active_repos": active_repos, - "repos": repos, - "repo_choices": repo_choices - } - - return working_repos + else: + log.info("Command was a dry run. No changes has been saved") def _check_remote(self, repo): """ @@ -728,65 +711,6 @@ def _check_remote(self, repo): return has_remote_difference - def _is_valid_repo(self, path): - """ - Method that checks if the given path is a valid git repo. Returns false if not. - """ - try: - Repo(path) - except git.exc.InvalidGitRepositoryError: - return False - - return True - - def _get_working_repos(self, repos, active_repos): - """ - Method to return two lists of repos. - One that contains the names of the repos specified in subgit command. - One that contains all repos listed in the working subgit config file. - - Example purpose is to easier check if the repo(s) specified in the subgit command, - exists in the subgit config file. etc. - """ - repo_paths = [] - in_conf_file = True - bad_path = [] - valid_repos = True - - for repo in repos: - repo_paths.append(Path().cwd() / repo) - - if repo not in active_repos: - log.critical(f"'{repo}' does not exist in {self.subgit_config_file_name} config file.") - in_conf_file = False - - if not in_conf_file: - return False - - for path in repo_paths: - repo_name = path.name - - if not path.exists(): - log.warning(f"Path to repo does not exist: {repo_name} | Skipping {repo_name}") - bad_path.append(path) - - for path in bad_path: - repo_paths.remove(path) - - if not repo_paths: - log.error("It appears no repos have been cloned. Exiting...") - return False - - for path in repo_paths: - if not self._is_valid_repo(path): - log.critical(f"'{path.name}' is not a git repo") - valid_repos = False - - if not valid_repos: - return False - - return repo_paths - def _filter(self, sequence, regex_list): """ Given a sequence of git objects, clean them against all regex items in the provided regex_list. diff --git a/subgit/repo.py b/subgit/repo.py index d26b0fc..110428f 100644 --- a/subgit/repo.py +++ b/subgit/repo.py @@ -12,7 +12,7 @@ def __init__(self, repo_config, *args, **kwargs): self.raw_config = repo_config self.repo_cwd = Path(".") self.name = repo_config.get("name", None) - self.clone_point = Path(".") # Clone point + self.clone_point = Path(repo_config.get("clone_point", ".")) # Clone point self.is_enabled = repo_config.get("enable", True) self.url = repo_config.get("url", "NOT SET") # repo url:en som git fattar self.revision_type = "" # branch, tag, commit @@ -99,7 +99,7 @@ def repo_root(self): Combines repo_cwd + clone_point into the root folder that the repo should be cloned to """ - return self.repo_cwd / self.name / self.clone_point + return self.repo_cwd / self.clone_point / self.name def print_status(self): pass From a45c1b0daeeb96c0aa82a3d1eba6b0e4f982d551 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 1 Sep 2023 14:20:46 +0200 Subject: [PATCH 07/11] Add in invoke task runner and make a super simple basic smoke-testcase --- .gitignore | 3 ++ setup.py | 1 + subgit/core.py | 1 - tasks.py | 108 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tasks.py diff --git a/.gitignore b/.gitignore index 3e65374..af3b7a2 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,6 @@ bare/ .idea/* code/ _manifest/ + + +smoke_tests/ diff --git a/setup.py b/setup.py index f18e6e4..d10b801 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ extras_require={ "test": ["pytest", "pytest-mock", "mock", "tox", "tox-gh-actions"], "dev": [ + "invoke>=2.2.0", "ruff>=0.0.285", "ptpython", ], diff --git a/subgit/core.py b/subgit/core.py index e4bf8ab..9307318 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -325,7 +325,6 @@ def pull(self, repos_to_pull): elif isinstance(repos_to_pull, list): # Validate that all names we have sent in really exists in the config is_all_repos_valid = all([repo.name in repos_to_pull for repo in repos]) - print(is_all_repos_valid) # If we send in a list of a subset of repos we want to pull, filter out the repos # that do not match these names diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..1f472d0 --- /dev/null +++ b/tasks.py @@ -0,0 +1,108 @@ +import os +import shutil +import subgit +import uuid + +from invoke import task +from pathlib import Path +from ruamel import yaml + +# TODO: Base all files from where the tasks.py file is located + +test_config = { + "repos": [ + { + "name": "pyk", + "url": "git@github.com:Grokzen/pykwalify.git", + "sparse": { + "paths": [ + "docs", + "tests", + ], + }, + "revision": { + "branch": "master", + }, + }, + { + "name": "pykwalify", + "url": "git@github.com:Grokzen/pykwalify.git", + "sparse": { + "paths": [ + "docs", + "tests", + ], + }, + "clone_point": "pyk/docs", + "revision": { + "filter": "[0-9].[0-9].[0-9]", + "order": "semver", + "select": "last", + }, + }, + ] +} + + +@task +def smoke_test(c): + subgit.init_logging(5 if "DEBUG" in os.environ else 4) + + print(f"Running smoke tests") + + run_folder = str(uuid.uuid4()) + + tests_folder = Path("./smoke_tests") + + # Check if the directory exists + if not tests_folder.exists(): + # If it doesn't exist, create it + tests_folder.mkdir(parents=True) + + run_tests_folder = tests_folder / run_folder + + if not run_tests_folder.exists(): + run_tests_folder.mkdir(parents=True) + + yaml_file = run_tests_folder / ".subgit.yml" + + if yaml_file.exists() is False: + yaml_file.touch() + + with open(yaml_file, "w") as stream: + yaml.dump( + test_config, + stream, + indent=2, + default_flow_style=False, + ) + + # Move into this run:s folder + os.chdir(run_tests_folder) + + from subgit.core import SubGit + + core = SubGit(".subgit.yml") + core.repo_status() + core.pull(None) + core.pull(["pyk"]) + core.pull(["pykwalify"]) + core.repo_status() + core.fetch(None) + core.fetch(["pyk"]) + core.fetch(["pykwalify"]) + core.clean(repo_names=None, recurse_into_dir=True, dry_run=True) + core.clean(repo_names=["pyk"], recurse_into_dir=True, dry_run=True) + core.clean(repo_names=["pykwalify"], recurse_into_dir=True, dry_run=True) + + +@task +def clean_smoke_tests(c): + folder = Path("smoke_tests") + + if folder.exists() is False: + print("Smoke test folder do not exists on disk") + return + + print("Removing smoketest folder") + shutil.rmtree(folder) From 1c061d2e17a1ab678a2fdd8843b1bf2391d822a0 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Fri, 1 Sep 2023 14:28:31 +0200 Subject: [PATCH 08/11] Update default repo dict with a better example that points back to subgit.git repo itself --- subgit/constants.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/subgit/constants.py b/subgit/constants.py index 120103e..9303590 100644 --- a/subgit/constants.py +++ b/subgit/constants.py @@ -1,6 +1,16 @@ # -*- coding: utf-8 -*- -DEFAULT_REPO_DICT = {"repos": []} +DEFAULT_REPO_DICT = { + "repos": [ + { + "name": "subgit", + "url": "git@github.com:dynamist/subgit.git", + "revision": { + "branch": "master", + }, + } + ] +} REVISION_BRANCH = "branch" REVISION_COMMIT = "commit" From d5e04080b6209d7b5b71023df538c562c52f7ca7 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 4 Sep 2023 14:14:55 +0200 Subject: [PATCH 09/11] Fix broken unittests Change out some prints to proper log messages --- subgit/core.py | 45 ++++++++++++++++++++++----------------------- tests/conftest.py | 14 +++++++++++++- tests/test_core.py | 4 +++- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/subgit/core.py b/subgit/core.py index 9307318..1676d3f 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -156,15 +156,15 @@ def repo_status(self): repos = self._build_repo_objects(config) if len(repos) == 0: - print(" No data for repositories found in config file") + log.critical(" No data for repositories found in config file") return 1 for repo in repos: - print("") - print(f"{repo.name}") - print(f" {repo.url}") - print(f" {repo.repo_root().resolve()}") - print(f" Is cloned to disk? {repo.is_cloned_to_disk_str()}") + log.info("") + log.info(f"{repo.name}") + log.info(f" {repo.url}") + log.info(f" {repo.repo_root().resolve()}") + log.info(f" Is cloned to disk? {repo.is_cloned_to_disk_str()}") if repo.is_cloned_to_disk: fetch_file_path = repo.git_fetch_head_file_path @@ -173,17 +173,17 @@ def repo_status(self): output, stderr = run_cmd(f"stat -c %y {fetch_file_path}") parsed_output = str(output).replace('\\n', '') - print(f" Last pull/fetch: {parsed_output}") + log.info(f" Last pull/fetch: {parsed_output}") else: - print(" Last pull/fetch: Repo has not been pulled or fetch since initial clone") + log.info(" Last pull/fetch: Repo has not been pulled or fetch since initial clone") else: - print(" Last pull/fetch: UNKNOWN repo not cloned to disk") + log.info(" Last pull/fetch: UNKNOWN repo not cloned to disk") - print(f" Repo is dirty? {repo.is_git_repo_dirty_str()}") - print(f" Revision:") + log.info(f" Repo is dirty? {repo.is_git_repo_dirty_str()}") + log.info(f" Revision:") if repo.revision_type == REVISION_BRANCH: - print(f" branch: {repo.revision_value}") + log.info(f" branch: {repo.revision_value}") if repo.is_cloned_to_disk: if repo.revision_value in repo.git_repo.heads: @@ -204,17 +204,17 @@ def repo_status(self): has_newer_commit = "Repo not cloned to disk" is_in_origin = "Repo not cloned to disk" - print(f" commit hash: {commit_hash}") - print(f" commit message: '{commit_message}'") - print(f" branch exists in origin? {is_in_origin}") - print(f" has newer commit in origin? {has_newer_commit}") + log.info(f" commit hash: {commit_hash}") + log.info(f" commit message: '{commit_message}'") + log.info(f" branch exists in origin? {is_in_origin}") + log.info(f" has newer commit in origin? {has_newer_commit}") if repo.revision_type == REVISION_COMMIT: - print(" FIXME: Not implemented yet") - print(f" commit: {repo.revision_value}") + log.info(" FIXME: Not implemented yet") + log.info(f" commit: {repo.revision_value}") if repo.revision_type == REVISION_TAG: - print(f" tag: {repo.revision_value}") + log.info(f" tag: {repo.revision_value}") if repo.is_cloned_to_disk: if repo.revision_value in repo.git_repo.tags: @@ -228,11 +228,11 @@ def repo_status(self): commit_hash = "Repo not cloned to disk" commit_summary = "Repo not cloned to disk" - print(f" commit hash: {commit_hash}") - print(f" commit message: {commit_summary}") + log.info(f" commit hash: {commit_hash}") + log.info(f" commit message: {commit_summary}") def yes_no(self, question): - print(question) + log.info(question) if self.answer_yes: log.info("--yes flag set, automatically answer yes to question") @@ -345,7 +345,6 @@ def pull(self, repos_to_pull): for repo in repos: # If the path do not exists then the repo can't be dirty - # if not repo_path.exists(): if not repo.repo_root().exists(): continue diff --git a/tests/conftest.py b/tests/conftest.py index 4ce86b0..057eba9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- # python std lib +import mock -# rediscluster imports +# subgit imports from subgit.core import SubGit # 3rd party imports @@ -15,3 +16,14 @@ def subgit(tmpdir, *args, **kwargs): conf_file = tmpdir.join(".subgit.yml") return SubGit(config_file_path=conf_file, *args, **kwargs) + + +@pytest.fixture(scope='function', autouse=True) +def universal_fixture(tmpdir): + """ + Currently all tests require that we init an empty repos config + """ + PATCHED_DEFAULT_REPO_DICT = {"repos": []} + + with mock.patch('subgit.core.DEFAULT_REPO_DICT', PATCHED_DEFAULT_REPO_DICT): + yield diff --git a/tests/test_core.py b/tests/test_core.py index 6baf95a..57ae24a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ # python std lib import json +import mock import os # subgit imports @@ -79,7 +80,8 @@ def test_init_repo(subgit): with open(subgit.subgit_config_file_path, "r") as stream: content = yaml.load(stream) - assert content == DEFAULT_REPO_DICT + # We patch the DEFAULT_REPO_DICT in conftest to another value + assert content == {"repos": []} def test_init_repo_file_exists(subgit): From c7d267e0494a7a2e595c441588dcea5f5784b702 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Mon, 4 Sep 2023 15:40:22 +0200 Subject: [PATCH 10/11] Change out the code version of smoke-tests to run through the entire cli with invoke run method --- tasks.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tasks.py b/tasks.py index 1f472d0..e2f8dd2 100644 --- a/tasks.py +++ b/tasks.py @@ -80,20 +80,17 @@ def smoke_test(c): # Move into this run:s folder os.chdir(run_tests_folder) - from subgit.core import SubGit - - core = SubGit(".subgit.yml") - core.repo_status() - core.pull(None) - core.pull(["pyk"]) - core.pull(["pykwalify"]) - core.repo_status() - core.fetch(None) - core.fetch(["pyk"]) - core.fetch(["pykwalify"]) - core.clean(repo_names=None, recurse_into_dir=True, dry_run=True) - core.clean(repo_names=["pyk"], recurse_into_dir=True, dry_run=True) - core.clean(repo_names=["pykwalify"], recurse_into_dir=True, dry_run=True) + c.run("subgit status") + c.run("subgit pull") + c.run("subgit pull pyk") + c.run("subgit pull pykwalify") + c.run("subgit status") + c.run("subgit fetch") + c.run("subgit fetch pyk") + c.run("subgit fetch pykwalify") + c.run("subgit clean") + c.run("subgit clean pyk -d -n") + c.run("subgit clean pykwalify -d -n") @task From 306da5c514aa8121cab138abf9901b775975d130 Mon Sep 17 00:00:00 2001 From: Johan Andersson Date: Tue, 5 Sep 2023 12:02:52 +0200 Subject: [PATCH 11/11] Change option "clone_point" to "path" Refactor out the Yes/No string convert to a generic helper function --- subgit/core.py | 5 +++-- subgit/repo.py | 12 +++--------- subgit/utils.py | 6 ++++++ 3 files changed, 12 insertions(+), 11 deletions(-) create mode 100644 subgit/utils.py diff --git a/subgit/core.py b/subgit/core.py index 1676d3f..0dc25f4 100644 --- a/subgit/core.py +++ b/subgit/core.py @@ -23,6 +23,7 @@ from subgit.enums import * from subgit.exceptions import * from subgit.repo import SubGitRepo +from subgit.utils import bool_to_str log = logging.getLogger(__name__) @@ -164,7 +165,7 @@ def repo_status(self): log.info(f"{repo.name}") log.info(f" {repo.url}") log.info(f" {repo.repo_root().resolve()}") - log.info(f" Is cloned to disk? {repo.is_cloned_to_disk_str()}") + log.info(f" Is cloned to disk? {bool_to_str(repo.is_cloned_to_disk)}") if repo.is_cloned_to_disk: fetch_file_path = repo.git_fetch_head_file_path @@ -179,7 +180,7 @@ def repo_status(self): else: log.info(" Last pull/fetch: UNKNOWN repo not cloned to disk") - log.info(f" Repo is dirty? {repo.is_git_repo_dirty_str()}") + log.info(f" Repo is dirty? {bool_to_str(repo.is_git_repo_dirty)}") log.info(f" Revision:") if repo.revision_type == REVISION_BRANCH: diff --git a/subgit/repo.py b/subgit/repo.py index 110428f..db19830 100644 --- a/subgit/repo.py +++ b/subgit/repo.py @@ -12,7 +12,7 @@ def __init__(self, repo_config, *args, **kwargs): self.raw_config = repo_config self.repo_cwd = Path(".") self.name = repo_config.get("name", None) - self.clone_point = Path(repo_config.get("clone_point", ".")) # Clone point + self.path = Path(repo_config.get("path", ".")) # Clone point self.is_enabled = repo_config.get("enable", True) self.url = repo_config.get("url", "NOT SET") # repo url:en som git fattar self.revision_type = "" # branch, tag, commit @@ -88,18 +88,12 @@ def git_fetch_head_file_path(self): def is_git_repo_dirty(self): return self.git_repo.is_dirty() - def is_git_repo_dirty_str(self): - return "Yes" if self.is_git_repo_dirty else "No" - - def is_cloned_to_disk_str(self): - return "Yes" if self.is_cloned_to_disk else "No" - def repo_root(self): """ - Combines repo_cwd + clone_point into the root folder that + Combines repo_cwd + path into the root folder that the repo should be cloned to """ - return self.repo_cwd / self.clone_point / self.name + return self.repo_cwd / self.path / self.name def print_status(self): pass diff --git a/subgit/utils.py b/subgit/utils.py new file mode 100644 index 0000000..c6d893f --- /dev/null +++ b/subgit/utils.py @@ -0,0 +1,6 @@ + +def bool_to_str(input_bool): + """ + Helper function that converts a boolean to Yes or No as a string + """ + return "Yes" if input_bool else "No"