From 6aafd6639b1158f060ea79cb996c95b446d2e737 Mon Sep 17 00:00:00 2001 From: Andrew Block Date: Mon, 27 Jul 2020 22:39:32 -0500 Subject: [PATCH 1/3] Added Quay statistics script --- README.md | 35 +++++++++++++++ quay-stats.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100755 quay-stats.py diff --git a/README.md b/README.md index e6fc42e..0cf5527 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,7 @@ export GITLAB_API_TOKEN='' Execute the script: +``` $ ./gitlab-stats.py === Statistics for GitLab Group 'redhat-cop' ==== @@ -215,6 +216,7 @@ $ ./hangouts-chat.py" - containers - 55 Members - openshift-development - 311 Members +``` ## Mailman @@ -224,9 +226,42 @@ A script called [mailman-subscribers.py](mailman-subscribers.py) is available to Execute the script: +``` $ ./mailman-subscribers [ options ] hostname listname password ashore@redhat.com csmart@redhat.com mmurray@redhat.com ``` + +## Quay Registry Statistics + +### Script + +A script called [quay-stats.py](quay-stats.py) is available to query statistics from [Quay](https://quay.io). + +The Quay API can be queried using an Application OAuth Access token. Refer to the [Quay documentation](https://access.redhat.com/documentation/en-us/red_hat_quay/3/html/red_hat_quay_api_guide/using_the_red_hat_quay_api) on the steps necessary to generate a token. + +Configure the required environment variable: + +``` +export QUAY_API_TOKEN=`` +``` +Execute the script: + +``` +$ ./quay-stats.py -r group-sync-operator +=== Statistics for Quay Organization 'redhat-cop' ==== + +- Repository 'group-sync-operator' - + +Earliest Record: Mon, 27 Jul 2020 05:15:22 -0000 +Most Recent Record: Mon, 27 Jul 2020 20:30:07 -0000 + +Country Counts: +DE - 3 +ES - 2 +US - 1 +FR - 2 +AU - 4 +``` diff --git a/quay-stats.py b/quay-stats.py new file mode 100755 index 0000000..6a9a8b5 --- /dev/null +++ b/quay-stats.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +import os +import json +import requests +import sys +import argparse +from datetime import datetime + +QUAY_API_TOKEN_NAME = 'QUAY_API_TOKEN' +QUAY_ORG_DEFAULT = 'redhat-cop' +QUAY_HOSTNAME_DEFAULT = 'quay.io' + + +def valid_date(s): + try: + return datetime.strptime(s, "%Y-%m-%d") + except ValueError: + msg = "Not a valid date: '{0}'.".format(s) + raise argparse.ArgumentTypeError(msg) + + +def generate_default_date(): + today_date = datetime.now() + return datetime.strptime("{0}-{1}-{02}".format( + today_date.year, today_date.month, today_date.day), "%Y-%m-%d") + + +def get_logs(session, url, params, next_page): + + if next_page is not None: + params["next_page"] = next_page + + log_request = session.get(url, params=params) + log_request.raise_for_status() + + log_request_json = log_request.json() + + if 'next_page' in log_request_json: + return log_request.json()['logs'] + get_logs(session, url, params, log_request_json['next_page']) + else: + return log_request.json()['logs'] + + +def print_repository_statistics(organization, repository, logs): + print( + "\n- Repository '{1}' -\n".format(organization, repository)) + + if len(logs) > 0: + country_count = {} + + for log in logs: + if 'resolved_ip' in log['metadata']: + + if log['metadata']['resolved_ip']['country_iso_code'] not in country_count: + country_count[log['metadata'] + ['resolved_ip']['country_iso_code']] = 1 + else: + country_count[log['metadata']['resolved_ip']['country_iso_code'] + ] = int(country_count[log['metadata']['resolved_ip']['country_iso_code']]) + 1 + + print("Earliest Record: {}".format(logs[-1]['datetime'])) + print("Most Recent Record: {}".format(logs[0]['datetime'])) + + print( + "\nCountry Counts:") + + for country_key in country_count.keys(): + print("{} - {}".format(country_key, country_count[country_key])) + else: + print("No Records Found") + + +parser = argparse.ArgumentParser(description='Gather GitHub Statistics.') +parser.add_argument("-s", "--start-date", + help="The start date to query from", type=valid_date) +parser.add_argument("-o", "--organization", + help="Organization name", default=QUAY_ORG_DEFAULT) +parser.add_argument("-r", "--repositories", + help="Repositories", nargs='+', default=[]) +parser.add_argument("-q", "--quay", + help="Quay hostname", default=QUAY_HOSTNAME_DEFAULT) +args = parser.parse_args() + +repositories = args.repositories +start_date = args.start_date +organization = args.organization +quay_host = args.quay + +if start_date is None: + start_date = generate_default_date() + +quay_api_token = os.environ.get(QUAY_API_TOKEN_NAME) + +if not quay_api_token: + print("Error: Quay API Key is Required!") + sys.exit(1) + + +session = requests.Session() +session.headers = { + 'Authorization': 'Bearer {0}'.format(quay_api_token), + 'Accept': "application/json" +} + +if len(repositories) > 0: + print( + "=== Statistics for Quay Organization '{0}' ====\n".format(organization)) + +for repository in repositories: + + params = { + "starttime": start_date.strftime("%m/%d/%Y") + } + + url = "https://{}/api/v1/repository/{}/{}/logs".format( + quay_host, organization, repository) + + logs = get_logs(session, url, params, None) + + print_repository_statistics(organization, repository, logs) From 462f1298ac7cbf91048ab8a421431af87ebe8989 Mon Sep 17 00:00:00 2001 From: Andrew Block Date: Wed, 5 Aug 2020 21:54:54 -0500 Subject: [PATCH 2/3] Additional logic to handle multiple dates --- quay-stats.py | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/quay-stats.py b/quay-stats.py index 6a9a8b5..89d4edf 100755 --- a/quay-stats.py +++ b/quay-stats.py @@ -5,7 +5,8 @@ import requests import sys import argparse -from datetime import datetime +from datetime import datetime, timedelta +import logging QUAY_API_TOKEN_NAME = 'QUAY_API_TOKEN' QUAY_ORG_DEFAULT = 'redhat-cop' @@ -14,7 +15,7 @@ def valid_date(s): try: - return datetime.strptime(s, "%Y-%m-%d") + return datetime.strptime("{} UTC".format(s), "%Y-%m-%d %Z") except ValueError: msg = "Not a valid date: '{0}'.".format(s) raise argparse.ArgumentTypeError(msg) @@ -50,14 +51,15 @@ def print_repository_statistics(organization, repository, logs): country_count = {} for log in logs: - if 'resolved_ip' in log['metadata']: + if log['kind'] == 'pull_repo': + if 'resolved_ip' in log['metadata']: - if log['metadata']['resolved_ip']['country_iso_code'] not in country_count: - country_count[log['metadata'] - ['resolved_ip']['country_iso_code']] = 1 - else: - country_count[log['metadata']['resolved_ip']['country_iso_code'] - ] = int(country_count[log['metadata']['resolved_ip']['country_iso_code']]) + 1 + if log['metadata']['resolved_ip']['country_iso_code'] not in country_count: + country_count[log['metadata'] + ['resolved_ip']['country_iso_code']] = 1 + else: + country_count[log['metadata']['resolved_ip']['country_iso_code'] + ] = int(country_count[log['metadata']['resolved_ip']['country_iso_code']]) + 1 print("Earliest Record: {}".format(logs[-1]['datetime'])) print("Most Recent Record: {}".format(logs[0]['datetime'])) @@ -87,9 +89,15 @@ def print_repository_statistics(organization, repository, logs): organization = args.organization quay_host = args.quay +current_date = datetime.utcnow() + if start_date is None: start_date = generate_default_date() +if start_date > current_date: + print("Start date cannot be greater than current date") + + quay_api_token = os.environ.get(QUAY_API_TOKEN_NAME) if not quay_api_token: @@ -109,13 +117,21 @@ def print_repository_statistics(organization, repository, logs): for repository in repositories: - params = { - "starttime": start_date.strftime("%m/%d/%Y") - } + logs = [] + while True: + params = { + "starttime": start_date.strftime("%m/%d/%Y"), + "endtime": start_date.strftime("%m/%d/%Y") + } + + url = "https://{}/api/v1/repository/{}/{}/logs".format( + quay_host, organization, repository) + + logs.extend(get_logs(session, url, params, None)) - url = "https://{}/api/v1/repository/{}/{}/logs".format( - quay_host, organization, repository) + start_date = start_date + timedelta(days=1) - logs = get_logs(session, url, params, None) + if start_date > current_date: + break print_repository_statistics(organization, repository, logs) From 9c1247a3f2c08fc7fe7b1b7e3310d1b0af598e9f Mon Sep 17 00:00:00 2001 From: Andrew Block Date: Thu, 6 Aug 2020 08:14:11 -0500 Subject: [PATCH 3/3] Fixed issue with start date --- quay-stats.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/quay-stats.py b/quay-stats.py index 89d4edf..72c0852 100755 --- a/quay-stats.py +++ b/quay-stats.py @@ -118,10 +118,11 @@ def print_repository_statistics(organization, repository, logs): for repository in repositories: logs = [] + current_repository_start_date = start_date while True: params = { - "starttime": start_date.strftime("%m/%d/%Y"), - "endtime": start_date.strftime("%m/%d/%Y") + "starttime": current_repository_start_date.strftime("%m/%d/%Y"), + "endtime": current_repository_start_date.strftime("%m/%d/%Y") } url = "https://{}/api/v1/repository/{}/{}/logs".format( @@ -129,9 +130,10 @@ def print_repository_statistics(organization, repository, logs): logs.extend(get_logs(session, url, params, None)) - start_date = start_date + timedelta(days=1) + current_repository_start_date = current_repository_start_date + \ + timedelta(days=1) - if start_date > current_date: + if current_repository_start_date > current_date: break print_repository_statistics(organization, repository, logs)