From f3fea9c18c5b323f1034fbcaf56064a01c1a0c8c Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Mon, 11 Mar 2024 16:05:32 -0700 Subject: [PATCH 01/13] create file to read dash outputs --- resspect/read_dash.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 resspect/read_dash.py diff --git a/resspect/read_dash.py b/resspect/read_dash.py new file mode 100644 index 00000000..ffb6dd2f --- /dev/null +++ b/resspect/read_dash.py @@ -0,0 +1,49 @@ +# Copyright 2020 resspect software +# Author: Amanda Wasserman +# +# created on 11 March 2024 +# +# Licensed GNU General Public License v3.0; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#import logging +#import numpy as np +#import pandas as pd +import csv + +__all__ = ["get_id_type"] + + + + +def get_id_type(file='/Users/arw/Desktop/spec_sims_binned_spaced/DASH_matches.txt'): + #read text file + data = open(file, "r") + + #retrieve lines with id and type + sns =[] + for line in data: + if '.txt' in line: + sns.append(line) + + ids = [id[0:7] for id in sns] + + type = [] + #parse file for type + for obj in sns: + indxb = obj.find("('") + indxe = obj.find(") ") + temp = obj[indxb+2:indxe] + temp = temp.split(',')[0] + type.append(temp[0:-1]) + return ids, type + From 4dc41ced3244064b5bf31ea30e15bceec221736c Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Wed, 13 Mar 2024 15:28:39 -0700 Subject: [PATCH 02/13] implement fitting TOM inputs --- resspect/fit_lightcurves.py | 64 ++++++++++++++++++++++++- resspect/lightcurves_utils.py | 31 ++++++++++++ resspect/read_dash.py | 8 +--- resspect/request_webservice_elasticc.py | 59 +++++++++++++++++++++++ 4 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 resspect/request_webservice_elasticc.py diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index d40a1f25..071e6363 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -1,5 +1,5 @@ # Copyright 2020 resspect software -# Author: Rupesh Durgesh and Emille Ishida +# Author: Rupesh Durgesh, Emille Ishida, and Amanda Wasserman # # created on 14 April 2022 # @@ -32,12 +32,14 @@ from resspect.lightcurves_utils import get_resspect_header_data from resspect.lightcurves_utils import read_plasticc_full_photometry_data from resspect.lightcurves_utils import SNPCC_FEATURES_HEADER +from resspect.lightcurves_utils import TOM_FEATURES_HEADER +from resspect.lightcurves_utils import TOM_MALANCHEV_FEATURES_HEADER from resspect.lightcurves_utils import SNPCC_MALANCHEV_FEATURES_HEADER from resspect.lightcurves_utils import find_available_key_name_in_header from resspect.lightcurves_utils import PLASTICC_TARGET_TYPES from resspect.lightcurves_utils import PLASTICC_RESSPECT_FEATURES_HEADER -__all__ = ["fit_snpcc", "fit_plasticc"] +__all__ = ["fit_snpcc", "fit_plasticc", "fit_TOM"] FEATURE_EXTRACTOR_MAPPING = { @@ -238,6 +240,64 @@ def fit_plasticc(path_photo_file: str, path_header_file: str, light_curve_data, plasticc_features_file) logging.info("Features have been saved to: %s", output_file) +def _TOM_sample_fit( + id: str, dic: dict, feature_extractor: str): + """ + Reads SNPCC file and performs fit. + + Parameters + ---------- + id + SNID + feature_extractor + Function used for feature extraction. + Options are 'bazin', 'bump', or 'malanchev'. + """ + light_curve_data = FEATURE_EXTRACTOR_MAPPING[feature_extractor]() + light_curve_data.photometry = pd.DataFrame(dic[id]['photometry']) + light_curve_data.dataset_name = 'TOM' + light_curve_data.filters = ['u', 'g', 'r', 'i', 'z', 'Y'] + + light_curve_data.fit_all() + + return light_curve_data + +def fit_TOM(data_dic: dict, features_file: str, + number_of_processors: int = 1, + feature_extractor: str = 'bazin'): + """ + Perform fit to all objects from the TOM data. + + Parameters + ---------- + data_dic: str + Dictionary containing the photometry for all light curves. + features_file: str + Path to output file where results should be stored. + number_of_processors: int, default 1 + Number of cpu processes to use. + feature_extractor: str, default bazin + Function used for feature extraction. + """ + if feature_extractor == 'bazin': + header = TOM_FEATURES_HEADER + elif feature_extractor == 'malanchev': + header = TOM_MALANCHEV_FEATURES_HEADER + + multi_process = multiprocessing.Pool(number_of_processors) + logging.info("Starting TOM " + feature_extractor + " fit...") + with open(features_file, 'w') as snpcc_features_file: + snpcc_features_file.write(','.join(header) + '\n') + + for light_curve_data in multi_process.starmap( + _TOM_sample_fit, zip( + data_dic, repeat(data_dic), repeat(feature_extractor))): + if 'None' not in light_curve_data.features: + write_features_to_output_file( + light_curve_data, snpcc_features_file) + logging.info("Features have been saved to: %s", features_file) + + def main(): return None diff --git a/resspect/lightcurves_utils.py b/resspect/lightcurves_utils.py index 3439a91b..b529dd02 100644 --- a/resspect/lightcurves_utils.py +++ b/resspect/lightcurves_utils.py @@ -79,6 +79,37 @@ 'zotsu_lower_to_all_ratio', 'zlinear_fit_slope', 'zlinear_fit_slope_sigma','zlinear_fit_reduced_chi2' ] +TOM_FEATURES_HEADER = [ + 'id', 'redshift', 'type', 'code', 'orig_sample', + 'uA', 'uB', 'ut0', 'utfall', 'utrise', + 'gA', 'gB', 'gt0', 'gtfall', 'gtrise', 'rA', 'rB', + 'rt0', 'rtfall', 'rtrise', 'iA', 'iB', 'it0', 'itfall', + 'itrise', 'zA', 'zB', 'zt0', 'ztfall', 'ztrise', + 'YA', 'YB', 'Yt0', 'Ytfall', 'Ytrise' +] + +TOM_MALANCHEV_FEATURES_HEADER = [ + 'id', 'redshift', 'type', 'code', 'orig_sample', + 'uanderson_darling_normal','uinter_percentile_range_5', + 'uchi2','ustetson_K','uweighted_mean','uduration', 'uotsu_mean_diff','uotsu_std_lower', 'uotsu_std_upper', + 'uotsu_lower_to_all_ratio', 'ulinear_fit_slope', 'ulinear_fit_slope_sigma','ulinear_fit_reduced_chi2', + 'ganderson_darling_normal','ginter_percentile_range_5', + 'gchi2','gstetson_K','gweighted_mean','gduration', 'gotsu_mean_diff','gotsu_std_lower', 'gotsu_std_upper', + 'gotsu_lower_to_all_ratio', 'glinear_fit_slope', 'glinear_fit_slope_sigma','glinear_fit_reduced_chi2', + 'randerson_darling_normal', 'rinter_percentile_range_5', + 'rchi2', 'rstetson_K', 'rweighted_mean','rduration', 'rotsu_mean_diff','rotsu_std_lower', 'rotsu_std_upper', + 'rotsu_lower_to_all_ratio', 'rlinear_fit_slope', 'rlinear_fit_slope_sigma','rlinear_fit_reduced_chi2', + 'ianderson_darling_normal','iinter_percentile_range_5', + 'ichi2', 'istetson_K', 'iweighted_mean','iduration', 'iotsu_mean_diff','iotsu_std_lower', 'iotsu_std_upper', + 'iotsu_lower_to_all_ratio', 'ilinear_fit_slope', 'ilinear_fit_slope_sigma','ilinear_fit_reduced_chi2', + 'zanderson_darling_normal','zinter_percentile_range_5', + 'zchi2', 'zstetson_K', 'zweighted_mean','zduration', 'zotsu_mean_diff','zotsu_std_lower', 'zotsu_std_upper', + 'zotsu_lower_to_all_ratio', 'zlinear_fit_slope', 'zlinear_fit_slope_sigma','zlinear_fit_reduced_chi2', + 'Yanderson_darling_normal','Yinter_percentile_range_5', + 'Ychi2','Ystetson_K','Yweighted_mean','Yduration', 'Yotsu_mean_diff','Yotsu_std_lower', 'Yotsu_std_upper', + 'Yotsu_lower_to_all_ratio', 'Ylinear_fit_slope', 'Ylinear_fit_slope_sigma','Ylinear_fit_reduced_chi2' +] + PLASTICC_RESSPECT_FEATURES_HEADER = [ 'id', 'redshift', 'type', 'code', 'orig_sample', 'uA', 'uB', 'ut0', 'utfall', 'utrise', 'gA', 'gB', 'gt0', 'gtfall','gtrise', 'rA', 'rB', diff --git a/resspect/read_dash.py b/resspect/read_dash.py index ffb6dd2f..a3809514 100644 --- a/resspect/read_dash.py +++ b/resspect/read_dash.py @@ -15,17 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -#import logging -#import numpy as np -#import pandas as pd import csv __all__ = ["get_id_type"] - - - -def get_id_type(file='/Users/arw/Desktop/spec_sims_binned_spaced/DASH_matches.txt'): +def get_id_type(file: str): #read text file data = open(file, "r") diff --git a/resspect/request_webservice_elasticc.py b/resspect/request_webservice_elasticc.py new file mode 100644 index 00000000..f5729f8b --- /dev/null +++ b/resspect/request_webservice_elasticc.py @@ -0,0 +1,59 @@ +# Copyright 2020 resspect software +# Author: Amanda Wasserman +# +# created on 12 March 2024 +# +# Licensed GNU General Public License v3.0; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +class TomClient: + def __init__(self, url='https://desc-tom.lbl.gov', username=None, + password=None, passwordfile = None, connect=True): + self._url = url + self._username = username + self._password = password + self._rqs=None + if self._password is None: + if passwordfile is None: + raise RuntimeError('No password or passwordfile provided') + with open(passwordfile) as ifp: + self._password = ifp.readline().strip() + + if connect: + self.connect() + + def connect(self): + self._rqs = requests.session() #should this be capitalized S? + res = self._rqs.get(f'{self._url}/acounts/login/') + if res.status_code != 200: + raise RuntimeError(f'Failed to connect to {self._url}') + res = self._rqs.post(f'{self._url}/accounts/login/', + data={'username':self._username, + 'password':self._password, + 'csrfmiddlewaretoken': self._rqs.cookies['csrftoken']}) + if res.status_code != 200: + raise RuntimeError(f'Failed to login.') + if 'Please enter a correct' in res.text: + raise RuntimeError("failed to log in.") + self._rqs.headers.update({'X-CSRFToken': self._rqs.cookies['csrftoken']}) + + def request(self, method="GET", page=None, **kwargs): + + return self._rqs.request(method=method, url=f"{self._url}/{page}", **kwargs) + +#def request(website: str): +# r = requests.get(website) +# status = r.status_code +# text = r.text +# return text \ No newline at end of file From 1eac524f2d904414a53a443ddc690f59cd0f8142 Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Wed, 13 Mar 2024 15:50:19 -0700 Subject: [PATCH 03/13] add tom_client class to access TOM --- resspect/fit_lightcurves.py | 12 ++- resspect/request_webservice_elasticc.py | 59 --------------- resspect/tom_client.py | 97 +++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 61 deletions(-) delete mode 100644 resspect/request_webservice_elasticc.py create mode 100644 resspect/tom_client.py diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 071e6363..9e8f627b 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -38,8 +38,9 @@ from resspect.lightcurves_utils import find_available_key_name_in_header from resspect.lightcurves_utils import PLASTICC_TARGET_TYPES from resspect.lightcurves_utils import PLASTICC_RESSPECT_FEATURES_HEADER +from resspect.tom_client import TomClient -__all__ = ["fit_snpcc", "fit_plasticc", "fit_TOM"] +__all__ = ["fit_snpcc", "fit_plasticc", "fit_TOM", "request_TOM_data"] FEATURE_EXTRACTOR_MAPPING = { @@ -296,7 +297,14 @@ def fit_TOM(data_dic: dict, features_file: str, write_features_to_output_file( light_curve_data, snpcc_features_file) logging.info("Features have been saved to: %s", features_file) - + +def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = None, + passwordfile: str = None, password: str = None): + tom = TomClient(url = url, username = username, passwordfile = passwordfile, + password = password) + res = tom.request( 'POST', 'elasticc2/gethotsne/10/' ) + data_dic = res.json() + return data_dic def main(): diff --git a/resspect/request_webservice_elasticc.py b/resspect/request_webservice_elasticc.py deleted file mode 100644 index f5729f8b..00000000 --- a/resspect/request_webservice_elasticc.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2020 resspect software -# Author: Amanda Wasserman -# -# created on 12 March 2024 -# -# Licensed GNU General Public License v3.0; -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.gnu.org/licenses/gpl-3.0.en.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests - -class TomClient: - def __init__(self, url='https://desc-tom.lbl.gov', username=None, - password=None, passwordfile = None, connect=True): - self._url = url - self._username = username - self._password = password - self._rqs=None - if self._password is None: - if passwordfile is None: - raise RuntimeError('No password or passwordfile provided') - with open(passwordfile) as ifp: - self._password = ifp.readline().strip() - - if connect: - self.connect() - - def connect(self): - self._rqs = requests.session() #should this be capitalized S? - res = self._rqs.get(f'{self._url}/acounts/login/') - if res.status_code != 200: - raise RuntimeError(f'Failed to connect to {self._url}') - res = self._rqs.post(f'{self._url}/accounts/login/', - data={'username':self._username, - 'password':self._password, - 'csrfmiddlewaretoken': self._rqs.cookies['csrftoken']}) - if res.status_code != 200: - raise RuntimeError(f'Failed to login.') - if 'Please enter a correct' in res.text: - raise RuntimeError("failed to log in.") - self._rqs.headers.update({'X-CSRFToken': self._rqs.cookies['csrftoken']}) - - def request(self, method="GET", page=None, **kwargs): - - return self._rqs.request(method=method, url=f"{self._url}/{page}", **kwargs) - -#def request(website: str): -# r = requests.get(website) -# status = r.status_code -# text = r.text -# return text \ No newline at end of file diff --git a/resspect/tom_client.py b/resspect/tom_client.py new file mode 100644 index 00000000..24e3f3c8 --- /dev/null +++ b/resspect/tom_client.py @@ -0,0 +1,97 @@ +# Copyright 2020 resspect software +# Author: Rob Knop +# +# created on 12 March 2024 +# +# Licensed GNU General Public License v3.0; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +class TomClient: + """A thin class that supports sending requests via "requests" to the DESC tom. + + Usage: initialize one of these, giving it the url, your TOM + username, and either your TOM password, or a file that has your TOM + password in it: + + tc = TomClient( username='rknop', passwordfile='/home/raknop/secrets/tom_rknop_passwd' ) + + (You can give it a url with url=; it defaults to https://desc-tom.lbl.gov.) + + Thereafter, just do something like + + res = tc.request( "POST", "elasticc2/ppdbdiaobject/55772173" ) + + and res will come back with a string that you can load into JSON + that will have all the fun data about PPDBDiaObject number 55772173. + + tc.request is just a thin front-end to python requests.request. The + only reason to use this client rather than the python requests + module directly is that this class takes care of the stupid fiddly + bits of getting some headers that django demands set up right in the + request object when you log in. + + """ + + def __init__( self, url="https://desc-tom.lbl.gov", username=None, password=None, passwordfile=None, connect=True ): + self._url = url + self._username = username + self._password = password + self._rqs = None + if self._password is None: + if passwordfile is None: + raise RuntimeError( "Must give either password or passwordfile. " ) + with open( passwordfile ) as ifp: + self._password = ifp.readline().strip() + + if connect: + self.connect() + + def connect( self ): + self._rqs = requests.session() + res = self._rqs.get( f'{self._url}/accounts/login/' ) + if res.status_code != 200: + raise RuntimeError( f"Got status {res.status_code} from first attempt to connect to {self._url}" ) + res = self._rqs.post( f'{self._url}/accounts/login/', + data={ 'username': self._username, + 'password': self._password, + 'csrfmiddlewaretoken': self._rqs.cookies['csrftoken'] } ) + if res.status_code != 200: + raise RuntimeError( f"Failed to log in; http status: {res.status_code}" ) + if 'Please enter a correct' in res.text: + # This is a very cheesy attempt at checking if the login failed. + # I haven't found clean documentation on how to log into a django site + # from an app like this using standard authentication stuff. So, for + # now, I'm counting on the HTML that happened to come back when + # I ran it with a failed login one time. One of these days I'll actually + # figure out how Django auth works and make a version of /accounts/login/ + # designed for use in API scripts like this one, rather than desgined + # for interactive users. + raise RuntimeError( "Failed to log in. I think. Put in a debug break and look at res.text" ) + self._rqs.headers.update( { 'X-CSRFToken': self._rqs.cookies['csrftoken'] } ) + + def request( self, method="GET", page=None, **kwargs ): + """Send a request to the TOM + + method : a string with the HTTP method ("GET", "POST", etc.) + + page : the page to get; this is the URL but with the url you + passed to the constructor removed. So, if you wanted to get + https://desc-tom.lbl.gov/elasticc, you'd pass just "elasticc" + here. + + **kwargs : additional keyword arguments are passed on to + requests.request + + """ + return self._rqs.request( method=method, url=f"{self._url}/{page}", **kwargs ) From aa9910bff7a935add0ac792268cb8b11c44f4df2 Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Wed, 13 Mar 2024 16:05:46 -0700 Subject: [PATCH 04/13] fill in metadata --- resspect/fit_lightcurves.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 9e8f627b..99b942d1 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -258,6 +258,11 @@ def _TOM_sample_fit( light_curve_data.photometry = pd.DataFrame(dic[id]['photometry']) light_curve_data.dataset_name = 'TOM' light_curve_data.filters = ['u', 'g', 'r', 'i', 'z', 'Y'] + light_curve_data.id = id + light_curve_data.redshift = -99 + light_curve_data.sntype = 'unknown' + light_curve_data.sncode = -99 + light_curve_data.sample = 'N/A' light_curve_data.fit_all() From 3783cad27652cf587674eebca2558d46f08d0767 Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Wed, 13 Mar 2024 16:12:14 -0700 Subject: [PATCH 05/13] refer to dic for redshift/sncode --- resspect/fit_lightcurves.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 99b942d1..6b84919a 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -259,9 +259,9 @@ def _TOM_sample_fit( light_curve_data.dataset_name = 'TOM' light_curve_data.filters = ['u', 'g', 'r', 'i', 'z', 'Y'] light_curve_data.id = id - light_curve_data.redshift = -99 + light_curve_data.redshift = dic[id]['redshift'] light_curve_data.sntype = 'unknown' - light_curve_data.sncode = -99 + light_curve_data.sncode = dic[id]['sncode'] light_curve_data.sample = 'N/A' light_curve_data.fit_all() From 4048019072f860c6eb75fcc5bf14da35106fa407 Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Fri, 5 Apr 2024 12:08:05 -0700 Subject: [PATCH 06/13] add more TOM capabilities and feature storage --- resspect/fit_lightcurves.py | 21 +++++- resspect/time_domain_loop.py | 13 ++++ resspect/tom_client.py | 13 ++++ resspect/update_stashes.py | 133 +++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 resspect/update_stashes.py diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 6b84919a..48e77191 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -304,13 +304,30 @@ def fit_TOM(data_dic: dict, features_file: str, logging.info("Features have been saved to: %s", features_file) def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = None, - passwordfile: str = None, password: str = None): + passwordfile: str = None, password: str = None, detected_since_mjd: float = None, + detected_in_last_days: float = None,): tom = TomClient(url = url, username = username, passwordfile = passwordfile, password = password) - res = tom.request( 'POST', 'elasticc2/gethotsne/10/' ) + dic = {} + if detected_since_mjd is not None: + dic['detected_since_mjd'] = detected_since_mjd + if detected_in_last_days is not None: + dic['detected_in_last_days'] = detected_in_last_days + res = tom.post('elasticc2/gethotsne', dic) data_dic = res.json() return data_dic +def submit_queries_to_TOM(objectids: list, priorities: list, requester: str='resspect'): + req = { 'requester': requester, + 'objectids': objectids, + 'priorities': priorities} + res = TomClient.request( 'POST', 'elasticc2/askforspectrum', json=req ) + dic = res.json() + if res.satus_code != 200: + raise ValueError('Request failed, ' + res.text + ". Status code: " + str(res.status_code)) + + if dic['status'] == 'error': + raise ValueError('Request failed, ' + dic.json()['error']) def main(): return None diff --git a/resspect/time_domain_loop.py b/resspect/time_domain_loop.py index ddd88031..67f2a67d 100644 --- a/resspect/time_domain_loop.py +++ b/resspect/time_domain_loop.py @@ -799,6 +799,19 @@ def process_next_day_loop( return light_curve_data +def submit_queries_to_TOM(objectids: list, priorities: list, requester: str='resspect'): + req = { 'requester': requester, + 'objectids': objectids, + 'priorities': priorities} + res = TomClient.request( 'POST', 'elasticc2/askforspectrum', json=req ) + dic = res.json() + if res.satus_code != 200: + raise ValueError('Request failed, ' + res.text + ". Status code: " + str(res.status_code)) + + if dic['status'] == 'error': + raise ValueError('Request failed, ' + dic.json()['error']) + + # TODO: Too many arguments. Refactor and update docs def run_time_domain_active_learning_loop( light_curve_data: DataBase, learning_days: list, diff --git a/resspect/tom_client.py b/resspect/tom_client.py index 24e3f3c8..0873e567 100644 --- a/resspect/tom_client.py +++ b/resspect/tom_client.py @@ -95,3 +95,16 @@ def request( self, method="GET", page=None, **kwargs ): """ return self._rqs.request( method=method, url=f"{self._url}/{page}", **kwargs ) + + def post( self, page=None, **kwargs ): + """Shortand for TomClient.request( "POST", ... )""" + return self.request( "POST", page, **kwargs ) + + def get( self, page=None, **kwargs ): + """Shortand for TomClient.request( "GET", ... )""" + return self.request( "GET", page, **kwargs ) + + def put( self, page=None, **kwargs ): + """Shortand for TomClient.request( "PUT", ... )""" + return self.request( "PUT", page, **kwargs ) + diff --git a/resspect/update_stashes.py b/resspect/update_stashes.py new file mode 100644 index 00000000..16554842 --- /dev/null +++ b/resspect/update_stashes.py @@ -0,0 +1,133 @@ +# Copyright 2020 resspect software +# Author: Amanda Wasserman +# +# created on 18 March 2024 +# +# Licensed GNU General Public License v3.0; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ["update_pool_stash", "update_training_stash"] + +### Need to remove old objs at some point (once they are no longer hot) +# Need to remove objects that have been moved from pool to training +def update_pool_stash(current_stash_path: str, new_night_path: str): + #should we store old features somewhere? makes it easier to add training objs + #would want to add current MJD, maybe first MJD, and peak MJD + + #read in current stash as list of strings + with open(current_stash_path, 'r') as f: + current_stash = f.readlines() + + #read in new night as list of strings + with open(new_night_path, 'r') as f: + new_night = f.readlines() + + #if id is already in stash, replace it. else append it + for obj in new_night: + replaced = 0 + id = obj.split(',')[0] + for old_obj in current_stash: + if id in old_obj: + current_stash[current_stash.index(old_obj)] = obj + replaced = 1 + break + if replaced == 0: + current_stash.append(obj) + + #write new stash + with open(current_stash_path, 'w') as f: + for obj in current_stash: + f.write(obj) + +def update_training_stash_with_new_classification(current_training_stash_path: str, new_obj_ids: list, + new_obj_classes: list, new_obj_redshifts: list, current_stash_path: str): + #add new obj id and class for each point on the training obj light curve going forward + #(how do we want to do this? add A,B,C, etc to the end of the id?) + with open(current_stash_path, 'r') as f: + current_stash = f.readlines() + + with open(current_training_stash_path, 'r') as f: + training_stash = f.readlines() + + #find obj in current stash + for idx, obj in new_obj_ids: + for old_obj in current_stash: + if idx in old_obj: + line = old_obj.split(',') + break + line[1] = new_obj_redshifts[idx] + line[2] = new_obj_classes[idx] + train_to_append = ','.join(line) + training_stash.append(train_to_append) + + #write new training stash + with open(current_training_stash_path, 'w') as f: + for obj in training_stash: + f.write(obj) + + +def update_training_stash_with_new_features(new_night_path: str, current_training_stash_path: str): + #add new obj id and features for each point on the training obj light curve going forward + #(how do we want to do this? add A,B,C, etc to the end of the id?) + with open(new_night_path, 'r') as f: + new_night = f.readlines() + + with open(current_training_stash_path, 'r') as f: + training_stash = f.readlines() + + #append training set with new features + for obj in new_night: + id = obj.split(',')[0] + for old_obj in training_stash: + if id in old_obj: + #append new features + old_split = old_obj.split(',') + split = [id+mjd, old_split[1], old_split[2], 'N/A', 'N/A'] ###we need mjd (or something) to differentiate the ids + split.extend(obj.split(',')[5:]) + training_stash.append(','.join(split)) + + #write new stash + with open(current_training_stash_path, 'w') as f: + for obj in training_stash: + f.write(obj) + +def remove_training_from_pool(current_stash_path: str, new_obj_ids: list): + #remove objects that have been moved from pool to training + with open(current_stash_path, 'r') as f: + current_stash = f.readlines() + + for id in new_obj_ids: + for obj in current_stash: + if id in obj: + current_stash.remove(obj) + + #write new stash + with open(current_stash_path, 'w') as f: + for obj in current_stash: + f.write(obj) + +def remove_not_hot_objects(current_stash_path: str): + #remove objects that are no longer hot + #need MJDs + #make sure we're not getting old objects from rob or we'll be wasting computing power adding/removing + with open(current_stash_path, 'r') as f: + current_stash = f.readlines() + + for obj in current_stash: + if obj[current_mjd-discovery_mjd] >5 0: + current_stash.remove(obj) + + #write new stash + with open(current_stash_path, 'w') as f: + for obj in current_stash: + f.write(obj) + From 8d9b7a3f5200d5a0d85240cba2a9420fa6a2dd4f Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Wed, 29 May 2024 12:17:35 -0500 Subject: [PATCH 07/13] TOM updates --- resspect/database.py | 45 ++++++++++++++++++++++++++++++------ resspect/fit_lightcurves.py | 43 ++++++++++++++-------------------- resspect/time_domain_loop.py | 8 ++++--- 3 files changed, 60 insertions(+), 36 deletions(-) diff --git a/resspect/database.py b/resspect/database.py index 7246d7c1..0da3f926 100644 --- a/resspect/database.py +++ b/resspect/database.py @@ -276,7 +276,6 @@ def load_features_from_file(self, path_to_features_file: str, screen=False, 'rp1', 'rp2', 'rp3', 'rmax_flux', 'ip1', 'ip2', 'ip3', 'imax_flux', 'zp1', 'zp2', 'zp3', 'zmax_flux'] - elif feature_extractor == 'malanchev': self.features_names = ['ganderson_darling_normal','ginter_percentile_range_5', 'gchi2','gstetson_K','gweighted_mean','gduration', @@ -310,12 +309,44 @@ def load_features_from_file(self, path_to_features_file: str, screen=False, self.metadata_names = self.metadata_names + ['cost_' + name] elif survey == 'LSST': - self.features_names = ['uA', 'uB', 'ut0', 'utfall', 'utrise', - 'gA', 'gB', 'gt0', 'gtfall', 'gtrise', - 'rA', 'rB', 'rt0', 'rtfall', 'rtrise', - 'iA', 'iB', 'it0', 'itfall', 'itrise', - 'zA', 'zB', 'zt0', 'ztfall', 'ztrise', - 'YA', 'YB', 'Yt0', 'Ytfall', 'Ytrise'] + if feature_extractor == "bazin": + self.features_names = ['uA', 'uB', 'ut0', 'utfall', 'utrise', + 'gA', 'gB', 'gt0', 'gtfall', 'gtrise', + 'rA', 'rB', 'rt0', 'rtfall', 'rtrise', + 'iA', 'iB', 'it0', 'itfall', 'itrise', + 'zA', 'zB', 'zt0', 'ztfall', 'ztrise', + 'YA', 'YB', 'Yt0', 'Ytfall', 'Ytrise'] + elif feature_extractor == "malanchev": + self.features_names = ['uanderson_darling_normal','uinter_percentile_range_5', + 'uchi2','ustetson_K','uweighted_mean','uduration', + 'uotsu_mean_diff','uotsu_std_lower', 'uotsu_std_upper', + 'uotsu_lower_to_all_ratio', 'ulinear_fit_slope', + 'ulinear_fit_slope_sigma','ulinear_fit_reduced_chi2', + 'ganderson_darling_normal','ginter_percentile_range_5', + 'gchi2','gstetson_K','gweighted_mean','gduration', + 'gotsu_mean_diff','gotsu_std_lower', 'gotsu_std_upper', + 'gotsu_lower_to_all_ratio', 'glinear_fit_slope', + 'glinear_fit_slope_sigma','glinear_fit_reduced_chi2', + 'randerson_darling_normal', 'rinter_percentile_range_5', + 'rchi2', 'rstetson_K', 'rweighted_mean','rduration', + 'rotsu_mean_diff','rotsu_std_lower', 'rotsu_std_upper', + 'rotsu_lower_to_all_ratio', 'rlinear_fit_slope', + 'rlinear_fit_slope_sigma','rlinear_fit_reduced_chi2', + 'ianderson_darling_normal','iinter_percentile_range_5', + 'ichi2', 'istetson_K', 'iweighted_mean','iduration', + 'iotsu_mean_diff','iotsu_std_lower', 'iotsu_std_upper', + 'iotsu_lower_to_all_ratio', 'ilinear_fit_slope', + 'ilinear_fit_slope_sigma','ilinear_fit_reduced_chi2', + 'zanderson_darling_normal','zinter_percentile_range_5', + 'zchi2', 'zstetson_K', 'zweighted_mean','zduration', + 'zotsu_mean_diff','zotsu_std_lower', 'zotsu_std_upper', + 'zotsu_lower_to_all_ratio', 'zlinear_fit_slope', + 'zlinear_fit_slope_sigma','zlinear_fit_reduced_chi2', + 'Yanderson_darling_normal','Yinter_percentile_range_5', + 'Ychi2', 'Ystetson_K', 'Yweighted_mean','Yduration', + 'Yotsu_mean_diff','Yotsu_std_lower', 'Yotsu_std_upper', + 'Yotsu_lower_to_all_ratio', 'Ylinear_fit_slope', + 'Ylinear_fit_slope_sigma','Ylinear_fit_reduced_chi2'] if 'objid' in data.keys(): self.metadata_names = ['objid', 'redshift', 'type', 'code', diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 48e77191..3d24a86a 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -242,7 +242,7 @@ def fit_plasticc(path_photo_file: str, path_header_file: str, logging.info("Features have been saved to: %s", output_file) def _TOM_sample_fit( - id: str, dic: dict, feature_extractor: str): + obj_dic: dict, feature_extractor: str): """ Reads SNPCC file and performs fit. @@ -255,20 +255,20 @@ def _TOM_sample_fit( Options are 'bazin', 'bump', or 'malanchev'. """ light_curve_data = FEATURE_EXTRACTOR_MAPPING[feature_extractor]() - light_curve_data.photometry = pd.DataFrame(dic[id]['photometry']) + light_curve_data.photometry = pd.DataFrame(obj_dic['photometry']) light_curve_data.dataset_name = 'TOM' light_curve_data.filters = ['u', 'g', 'r', 'i', 'z', 'Y'] - light_curve_data.id = id - light_curve_data.redshift = dic[id]['redshift'] + light_curve_data.id = obj_dic['objectid'] + light_curve_data.redshift = obj_dic['redshift'] light_curve_data.sntype = 'unknown' - light_curve_data.sncode = dic[id]['sncode'] + light_curve_data.sncode = obj_dic['sncode'] light_curve_data.sample = 'N/A' light_curve_data.fit_all() return light_curve_data -def fit_TOM(data_dic: dict, features_file: str, +def fit_TOM(data_dic: dict, output_features_file: str, number_of_processors: int = 1, feature_extractor: str = 'bazin'): """ @@ -278,7 +278,7 @@ def fit_TOM(data_dic: dict, features_file: str, ---------- data_dic: str Dictionary containing the photometry for all light curves. - features_file: str + output_features_file: str Path to output file where results should be stored. number_of_processors: int, default 1 Number of cpu processes to use. @@ -289,23 +289,23 @@ def fit_TOM(data_dic: dict, features_file: str, header = TOM_FEATURES_HEADER elif feature_extractor == 'malanchev': header = TOM_MALANCHEV_FEATURES_HEADER - + multi_process = multiprocessing.Pool(number_of_processors) logging.info("Starting TOM " + feature_extractor + " fit...") - with open(features_file, 'w') as snpcc_features_file: - snpcc_features_file.write(','.join(header) + '\n') + with open(output_features_file, 'w') as TOM_features_file: + TOM_features_file.write(','.join(header) + '\n') for light_curve_data in multi_process.starmap( _TOM_sample_fit, zip( - data_dic, repeat(data_dic), repeat(feature_extractor))): + data_dic, repeat(feature_extractor))): if 'None' not in light_curve_data.features: write_features_to_output_file( - light_curve_data, snpcc_features_file) - logging.info("Features have been saved to: %s", features_file) + light_curve_data, TOM_features_file) + logging.info("Features have been saved to: %s", output_features_file) def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = None, passwordfile: str = None, password: str = None, detected_since_mjd: float = None, - detected_in_last_days: float = None,): + detected_in_last_days: float = None, mjdnow: float = None): tom = TomClient(url = url, username = username, passwordfile = passwordfile, password = password) dic = {} @@ -313,21 +313,12 @@ def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = No dic['detected_since_mjd'] = detected_since_mjd if detected_in_last_days is not None: dic['detected_in_last_days'] = detected_in_last_days - res = tom.post('elasticc2/gethotsne', dic) + if mjdnow is not None: + dic['mjd_now'] = mjdnow + res = tom.post('elasticc2/gethottransients', json = dic) data_dic = res.json() return data_dic -def submit_queries_to_TOM(objectids: list, priorities: list, requester: str='resspect'): - req = { 'requester': requester, - 'objectids': objectids, - 'priorities': priorities} - res = TomClient.request( 'POST', 'elasticc2/askforspectrum', json=req ) - dic = res.json() - if res.satus_code != 200: - raise ValueError('Request failed, ' + res.text + ". Status code: " + str(res.status_code)) - - if dic['status'] == 'error': - raise ValueError('Request failed, ' + dic.json()['error']) def main(): return None diff --git a/resspect/time_domain_loop.py b/resspect/time_domain_loop.py index 67f2a67d..bc044d37 100644 --- a/resspect/time_domain_loop.py +++ b/resspect/time_domain_loop.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['time_domain_loop', 'load_dataset'] +__all__ = ['time_domain_loop', 'load_dataset', 'submit_queries_to_TOM'] import os from typing import Union, Tuple @@ -24,6 +24,7 @@ import progressbar from resspect import DataBase +from resspect.tom_client import TomClient def load_dataset(file_names_dict: dict, survey_name: str = 'DES', @@ -799,11 +800,12 @@ def process_next_day_loop( return light_curve_data -def submit_queries_to_TOM(objectids: list, priorities: list, requester: str='resspect'): +def submit_queries_to_TOM(username, passwordfile, objectids: list, priorities: list, requester: str='resspect'): + tom = TomClient(url = "https://desc-tom-2.lbl.gov", username = username, passwordfile = passwordfile) req = { 'requester': requester, 'objectids': objectids, 'priorities': priorities} - res = TomClient.request( 'POST', 'elasticc2/askforspectrum', json=req ) + res = tom.request( 'POST', 'elasticc2/askforspectrum', json=req ) dic = res.json() if res.satus_code != 200: raise ValueError('Request failed, ' + res.text + ". Status code: " + str(res.status_code)) From e84a3bce07aed738a4a24b4c61115b89fedcbb00 Mon Sep 17 00:00:00 2001 From: AmandaWasserman Date: Thu, 19 Sep 2024 14:35:13 -0500 Subject: [PATCH 08/13] add cheat_gentype functionality --- resspect/fit_lightcurves.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resspect/fit_lightcurves.py b/resspect/fit_lightcurves.py index 3d24a86a..2af77824 100644 --- a/resspect/fit_lightcurves.py +++ b/resspect/fit_lightcurves.py @@ -305,7 +305,7 @@ def fit_TOM(data_dic: dict, output_features_file: str, def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = None, passwordfile: str = None, password: str = None, detected_since_mjd: float = None, - detected_in_last_days: float = None, mjdnow: float = None): + detected_in_last_days: float = None, mjdnow: float = None, cheat_gentypes: list = None): tom = TomClient(url = url, username = username, passwordfile = passwordfile, password = password) dic = {} @@ -315,6 +315,8 @@ def request_TOM_data(url: str = "https://desc-tom-2.lbl.gov", username: str = No dic['detected_in_last_days'] = detected_in_last_days if mjdnow is not None: dic['mjd_now'] = mjdnow + if cheat_gentypes is not None: + dic['cheat_gentypes'] = cheat_gentypes res = tom.post('elasticc2/gethottransients', json = dic) data_dic = res.json() return data_dic From 0c7b2d985c96367a5eca726f4b5f3fcf49db6ba4 Mon Sep 17 00:00:00 2001 From: Max West Date: Thu, 26 Sep 2024 14:57:21 -0700 Subject: [PATCH 09/13] copier/ppt application --- .copier-answers.yml | 20 +++ .git_archival.txt | 4 + .gitattributes | 24 +++ .github/ISSUE_TEMPLATE/0-general_issue.md | 8 + .github/ISSUE_TEMPLATE/1-bug_report.md | 17 ++ .github/ISSUE_TEMPLATE/2-feature_request.md | 18 ++ .github/dependabot.yml | 14 +- .github/pull_request_template.md | 63 +++++++ .github/workflows/asv-main.yml | 101 +++++++++++ .github/workflows/asv-nightly.yml | 93 ++++++++++ .github/workflows/asv-pr.yml | 86 +++++++++ .github/workflows/pre-commit-ci.yml | 35 ++++ .github/workflows/publish-benchmarks-pr.yml | 53 ++++++ .github/workflows/publish-to-pypi.yml | 37 ++++ .github/workflows/smoke-test.yml | 42 +++++ .github/workflows/testing-and-coverage.yml | 38 ++++ .gitignore | 165 +++++++++++++++--- .pre-commit-config.yaml | 59 +++++++ .setup_dev.sh | 42 +++++ LICENSE | 21 +++ .../__init__.py | 0 benchmarks/asv.conf.json | 80 +++++++++ benchmarks/benchmarks.py | 16 ++ pyproject.toml | 105 ++++++++--- resspect/snanapipe/snana_hook.pyc | Bin 1075 -> 0 bytes {resspect => src/resspect}/__init__.py | 3 +- {resspect => src/resspect}/batch_functions.py | 0 {resspect => src/resspect}/bazin.py | 0 .../resspect}/build_plasticc_canonical.py | 0 .../resspect}/build_plasticc_metadata.py | 0 .../resspect}/build_snpcc_canonical.py | 0 {resspect => src/resspect}/bump.py | 0 {resspect => src/resspect}/classifiers.py | 0 .../resspect}/cosmo_metric_utils.py | 0 {resspect => src/resspect}/database.py | 0 src/resspect/example_benchmarks.py | 14 ++ .../resspect}/exposure_time_calculator.py | 0 .../resspect/feature_extractors}/__init__.py | 0 .../resspect}/feature_extractors/bazin.py | 0 .../resspect}/feature_extractors/bump.py | 0 .../feature_extractors/light_curve.py | 0 .../resspect}/feature_extractors/malanchev.py | 0 {resspect => src/resspect}/fit_lightcurves.py | 0 {resspect => src/resspect}/learn_loop.py | 0 .../resspect}/lightcurves_utils.py | 0 {resspect => src/resspect}/metrics.py | 0 {resspect => src/resspect}/plot_results.py | 0 .../resspect}/query_budget_strategies.py | 0 .../resspect}/query_strategies.py | 0 {resspect => src/resspect}/read_dash.py | 0 {resspect => src/resspect}/readme.rst | 0 {resspect => src/resspect}/salt3_utils.py | 0 {resspect => src/resspect}/samples_utils.py | 0 .../resspect}/scripts/__init__.py | 0 .../resspect}/scripts/build_canonical.py | 0 .../scripts/build_time_domain_plasticc.py | 0 .../scripts/build_time_domain_snpcc.py | 0 .../scripts/calculate_cosmology_metric.py | 0 .../resspect}/scripts/fit_dataset.py | 0 .../resspect}/scripts/make_metrics_plots.py | 0 .../resspect}/scripts/run_loop.py | 0 .../resspect}/scripts/run_time_domain.py | 0 .../resspect}/snana_fits_to_pd.py | 0 src/resspect/snanapipe/__init__.py | 0 .../resspect}/snanapipe/samplerun.py | 0 .../resspect}/snanapipe/snana_hook.py | 0 .../resspect}/time_domain_loop.py | 0 .../resspect}/time_domain_plasticc.py | 0 .../resspect}/time_domain_snpcc.py | 0 {resspect => src/resspect}/tom_client.py | 0 {resspect => src/resspect}/update_stashes.py | 0 tests/resspect/__init__.py | 0 tests/{ => resspect}/conftest.py | 2 +- tests/{ => resspect}/run-snpcc-e2e.sh | 0 tests/{ => resspect}/test_batch_functions.py | 0 tests/{ => resspect}/test_bazin.py | 0 tests/{ => resspect}/test_bump.py | 0 .../{ => resspect}/test_cosmo_metric_utils.py | 0 tests/{ => resspect}/test_database.py | 0 tests/{ => resspect}/test_example.py | 0 tests/{ => resspect}/test_fit_lightcurves.py | 0 tests/{ => resspect}/test_learn_loop.py | 0 tests/{ => resspect}/test_metrics.py | 0 83 files changed, 1110 insertions(+), 50 deletions(-) create mode 100644 .copier-answers.yml create mode 100644 .git_archival.txt create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/0-general_issue.md create mode 100644 .github/ISSUE_TEMPLATE/1-bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/2-feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/asv-main.yml create mode 100644 .github/workflows/asv-nightly.yml create mode 100644 .github/workflows/asv-pr.yml create mode 100644 .github/workflows/pre-commit-ci.yml create mode 100644 .github/workflows/publish-benchmarks-pr.yml create mode 100644 .github/workflows/publish-to-pypi.yml create mode 100644 .github/workflows/smoke-test.yml create mode 100644 .github/workflows/testing-and-coverage.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .setup_dev.sh create mode 100644 LICENSE rename {resspect/feature_extractors => benchmarks}/__init__.py (100%) create mode 100644 benchmarks/asv.conf.json create mode 100644 benchmarks/benchmarks.py delete mode 100644 resspect/snanapipe/snana_hook.pyc rename {resspect => src/resspect}/__init__.py (99%) rename {resspect => src/resspect}/batch_functions.py (100%) rename {resspect => src/resspect}/bazin.py (100%) rename {resspect => src/resspect}/build_plasticc_canonical.py (100%) rename {resspect => src/resspect}/build_plasticc_metadata.py (100%) rename {resspect => src/resspect}/build_snpcc_canonical.py (100%) rename {resspect => src/resspect}/bump.py (100%) rename {resspect => src/resspect}/classifiers.py (100%) rename {resspect => src/resspect}/cosmo_metric_utils.py (100%) rename {resspect => src/resspect}/database.py (100%) create mode 100644 src/resspect/example_benchmarks.py rename {resspect => src/resspect}/exposure_time_calculator.py (100%) rename {resspect/snanapipe => src/resspect/feature_extractors}/__init__.py (100%) rename {resspect => src/resspect}/feature_extractors/bazin.py (100%) rename {resspect => src/resspect}/feature_extractors/bump.py (100%) rename {resspect => src/resspect}/feature_extractors/light_curve.py (100%) rename {resspect => src/resspect}/feature_extractors/malanchev.py (100%) rename {resspect => src/resspect}/fit_lightcurves.py (100%) rename {resspect => src/resspect}/learn_loop.py (100%) rename {resspect => src/resspect}/lightcurves_utils.py (100%) rename {resspect => src/resspect}/metrics.py (100%) rename {resspect => src/resspect}/plot_results.py (100%) rename {resspect => src/resspect}/query_budget_strategies.py (100%) rename {resspect => src/resspect}/query_strategies.py (100%) rename {resspect => src/resspect}/read_dash.py (100%) rename {resspect => src/resspect}/readme.rst (100%) rename {resspect => src/resspect}/salt3_utils.py (100%) rename {resspect => src/resspect}/samples_utils.py (100%) rename {resspect => src/resspect}/scripts/__init__.py (100%) rename {resspect => src/resspect}/scripts/build_canonical.py (100%) rename {resspect => src/resspect}/scripts/build_time_domain_plasticc.py (100%) rename {resspect => src/resspect}/scripts/build_time_domain_snpcc.py (100%) rename {resspect => src/resspect}/scripts/calculate_cosmology_metric.py (100%) rename {resspect => src/resspect}/scripts/fit_dataset.py (100%) rename {resspect => src/resspect}/scripts/make_metrics_plots.py (100%) rename {resspect => src/resspect}/scripts/run_loop.py (100%) rename {resspect => src/resspect}/scripts/run_time_domain.py (100%) rename {resspect => src/resspect}/snana_fits_to_pd.py (100%) create mode 100644 src/resspect/snanapipe/__init__.py rename {resspect => src/resspect}/snanapipe/samplerun.py (100%) rename {resspect => src/resspect}/snanapipe/snana_hook.py (100%) rename {resspect => src/resspect}/time_domain_loop.py (100%) rename {resspect => src/resspect}/time_domain_plasticc.py (100%) rename {resspect => src/resspect}/time_domain_snpcc.py (100%) rename {resspect => src/resspect}/tom_client.py (100%) rename {resspect => src/resspect}/update_stashes.py (100%) create mode 100644 tests/resspect/__init__.py rename tests/{ => resspect}/conftest.py (99%) rename tests/{ => resspect}/run-snpcc-e2e.sh (100%) rename tests/{ => resspect}/test_batch_functions.py (100%) rename tests/{ => resspect}/test_bazin.py (100%) rename tests/{ => resspect}/test_bump.py (100%) rename tests/{ => resspect}/test_cosmo_metric_utils.py (100%) rename tests/{ => resspect}/test_database.py (100%) rename tests/{ => resspect}/test_example.py (100%) rename tests/{ => resspect}/test_fit_lightcurves.py (100%) rename tests/{ => resspect}/test_learn_loop.py (100%) rename tests/{ => resspect}/test_metrics.py (100%) diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 00000000..8130fcdb --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,20 @@ +# Changes here will be overwritten by Copier +_commit: v2.0.2 +_src_path: gh:lincc-frameworks/python-project-template +author_email: contact@cosmostatistics-initiative.org +author_name: The RESSPECT team +create_example_module: false +custom_install: true +enforce_style: [] +failure_notification: [] +include_benchmarks: true +include_docs: false +mypy_type_checking: none +package_name: resspect +project_license: MIT +project_name: resspect +project_organization: LSSTDESC +python_versions: +- '3.9' +- '3.10' +- '3.11' diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..b1a286bb --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..343a7554 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,24 @@ +# For explanation of this file and uses see +# https://git-scm.com/docs/gitattributes +# https://developer.lsst.io/git/git-lfs.html#using-git-lfs-enabled-repositories +# https://lincc-ppt.readthedocs.io/en/latest/practices/git-lfs.html +# +# Used by https://github.com/lsst/afwdata.git +# *.boost filter=lfs diff=lfs merge=lfs -text +# *.dat filter=lfs diff=lfs merge=lfs -text +# *.fits filter=lfs diff=lfs merge=lfs -text +# *.gz filter=lfs diff=lfs merge=lfs -text +# +# apache parquet files +# *.parq filter=lfs diff=lfs merge=lfs -text +# +# sqlite files +# *.sqlite3 filter=lfs diff=lfs merge=lfs -text +# +# gzip files +# *.gz filter=lfs diff=lfs merge=lfs -text +# +# png image files +# *.png filter=lfs diff=lfs merge=lfs -text + +.git_archival.txt export-subst \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/0-general_issue.md b/.github/ISSUE_TEMPLATE/0-general_issue.md new file mode 100644 index 00000000..84bb0d72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/0-general_issue.md @@ -0,0 +1,8 @@ +--- +name: General issue +about: Quickly create a general issue +title: '' +labels: '' +assignees: '' + +--- \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/1-bug_report.md b/.github/ISSUE_TEMPLATE/1-bug_report.md new file mode 100644 index 00000000..16b6b711 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-bug_report.md @@ -0,0 +1,17 @@ +--- +name: Bug report +about: Tell us about a problem to fix +title: 'Short description' +labels: 'bug' +assignees: '' + +--- +**Bug report** + + +**Before submitting** +Please check the following: + +- [ ] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem. +- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a description of what I expected instead. +- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list. diff --git a/.github/ISSUE_TEMPLATE/2-feature_request.md b/.github/ISSUE_TEMPLATE/2-feature_request.md new file mode 100644 index 00000000..908ff720 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: 'Short description' +labels: 'enhancement' +assignees: '' + +--- + +**Feature request** + + +**Before submitting** +Please check the following: + +- [ ] I have described the purpose of the suggested change, specifying what I need the enhancement to accomplish, i.e. what problem it solves. +- [ ] I have included any relevant links, screenshots, environment information, and data relevant to implementing the requested feature, as well as pseudocode for how I want to access the new functionality. +- [ ] If I have ideas for how the new feature could be implemented, I have provided explanations and/or pseudocode and/or task lists for the steps. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fd49e1c2..3b5ca194 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,10 @@ version: 2 updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: daily - time: "09:00" - open-pull-requests-limit: 10 \ No newline at end of file + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..76e043ca --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,63 @@ + + +## Change Description + +- [ ] My PR includes a link to the issue that I am addressing + + + +## Solution Description + + + + +## Code Quality +- [ ] I have read the Contribution Guide +- [ ] My code follows the code style of this project +- [ ] My code builds (or compiles) cleanly without any errors or warnings +- [ ] My code contains relevant comments and necessary documentation + +## Project-Specific Pull Request Checklists + + +### Bug Fix Checklist +- [ ] My fix includes a new test that breaks as a result of the bug (if possible) +- [ ] My change includes a breaking change + - [ ] My change includes backwards compatibility and deprecation warnings (if possible) + +### New Feature Checklist +- [ ] I have added or updated the docstrings associated with my feature using the [NumPy docstring format](https://numpydoc.readthedocs.io/en/latest/format.html) +- [ ] I have updated the tutorial to highlight my new feature (if appropriate) +- [ ] I have added unit/End-to-End (E2E) test cases to cover my new feature +- [ ] My change includes a breaking change + - [ ] My change includes backwards compatibility and deprecation warnings (if possible) + +### Documentation Change Checklist +- [ ] Any updated docstrings use the [NumPy docstring format](https://numpydoc.readthedocs.io/en/latest/format.html) + +### Build/CI Change Checklist +- [ ] If required or optional dependencies have changed (including version numbers), I have updated the README to reflect this +- [ ] If this is a new CI setup, I have added the associated badge to the README + + + +### Other Change Checklist +- [ ] Any new or updated docstrings use the [NumPy docstring format](https://numpydoc.readthedocs.io/en/latest/format.html). +- [ ] I have updated the tutorial to highlight my new feature (if appropriate) +- [ ] I have added unit/End-to-End (E2E) test cases to cover any changes +- [ ] My change includes a breaking change + - [ ] My change includes backwards compatibility and deprecation warnings (if possible) diff --git a/.github/workflows/asv-main.yml b/.github/workflows/asv-main.yml new file mode 100644 index 00000000..f6a6f297 --- /dev/null +++ b/.github/workflows/asv-main.yml @@ -0,0 +1,101 @@ +# This workflow will run benchmarks with airspeed velocity (asv), +# store the new results in the "benchmarks" branch and publish them +# to a dashboard on GH Pages. + +name: Run ASV benchmarks for main + +on: + push: + branches: [ main ] + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + setup-python: + runs-on: ubuntu-latest + + steps: + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + asv-main: + runs-on: ubuntu-latest + needs: setup-python + + permissions: + contents: write + + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + + steps: + - name: Checkout main branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv tabulate + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + + - name: Fetch previous results from the "benchmarks" branch + run: | + if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then + git merge origin/benchmarks \ + --allow-unrelated-histories \ + --no-commit + mv ../_results . + fi + + - name: Run ASV for the main branch + run: asv run ALL --skip-existing --verbose || true + + - name: Submit new results to the "benchmarks" branch + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: benchmarks + folder: ${{ env.WORKING_DIR }}/_results + target-folder: _results + + - name: Generate dashboard HTML + run: | + asv show + asv publish + + - name: Deploy to Github pages + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: gh-pages + folder: ${{ env.WORKING_DIR }}/_html \ No newline at end of file diff --git a/.github/workflows/asv-nightly.yml b/.github/workflows/asv-nightly.yml new file mode 100644 index 00000000..80a2d785 --- /dev/null +++ b/.github/workflows/asv-nightly.yml @@ -0,0 +1,93 @@ +# This workflow will run daily at 06:45. +# It will run benchmarks with airspeed velocity (asv) +# and compare performance with the previous nightly build. + +name: Run benchmarks nightly job + +on: + schedule: + - cron: 45 6 * * * + workflow_dispatch: + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + NIGHTLY_HASH_FILE: nightly-hash + +jobs: + + asv-nightly: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + + steps: + - name: Checkout main branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + + - name: Fetch previous results from the "benchmarks" branch + run: | + if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then + git merge origin/benchmarks \ + --allow-unrelated-histories \ + --no-commit + mv ../_results . + fi + + - name: Get nightly dates under comparison + id: nightly-dates + run: | + echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT + echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Use last nightly commit hash from cache + uses: actions/cache@v4 + with: + path: ${{ env.WORKING_DIR }} + key: nightly-results-${{ steps.nightly-dates.outputs.yesterday }} + + - name: Run comparison of main against last nightly build + run: | + HASH_FILE=${{ env.NIGHTLY_HASH_FILE }} + CURRENT_HASH=${{ github.sha }} + if [ -f $HASH_FILE ]; then + PREV_HASH=$(cat $HASH_FILE) + asv continuous $PREV_HASH $CURRENT_HASH --verbose || true + asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose + fi + echo $CURRENT_HASH > $HASH_FILE + + - name: Update last nightly hash in cache + uses: actions/cache@v4 + with: + path: ${{ env.WORKING_DIR }} + key: nightly-results-${{ steps.nightly-dates.outputs.today }} \ No newline at end of file diff --git a/.github/workflows/asv-pr.yml b/.github/workflows/asv-pr.yml new file mode 100644 index 00000000..bf5aed6d --- /dev/null +++ b/.github/workflows/asv-pr.yml @@ -0,0 +1,86 @@ +# This workflow will run benchmarks with airspeed velocity (asv) for pull requests. +# It will compare the performance of the main branch with the performance of the merge +# with the new changes. It then publishes a comment with this assessment by triggering +# the publish-benchmarks-pr workflow. +# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. +name: Run benchmarks for PR + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.10" + WORKING_DIR: ${{ github.workspace }}/benchmarks + ARTIFACTS_DIR: ${{ github.workspace }}/artifacts + +jobs: + setup-python: + runs-on: ubuntu-latest + steps: + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + asv-pr: + runs-on: ubuntu-latest + needs: setup-python + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + steps: + - name: Checkout PR branch of the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Display Workflow Run Information + run: | + echo "Workflow Run ID: ${{ github.run_id }}" + - name: Cache Python ${{ env.PYTHON_VERSION }} + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: python-${{ env.PYTHON_VERSION }} + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install asv==0.6.1 virtualenv tabulate lf-asv-formatter + - name: Make artifacts directory + run: mkdir -p ${{ env.ARTIFACTS_DIR }} + - name: Save pull request number + run: echo ${{ github.event.pull_request.number }} > ${{ env.ARTIFACTS_DIR }}/pr + - name: Get current job logs URL + uses: Tiryoh/gha-jobid-action@v1 + id: jobs + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: ${{ github.job }} + - name: Create ASV machine config file + run: asv machine --machine gh-runner --yes + - name: Save comparison of PR against main branch + run: | + git remote add upstream https://github.com/${{ github.repository }}.git + git fetch upstream + asv continuous upstream/main HEAD --verbose || true + asv compare upstream/main HEAD --sort ratio --verbose | tee output + python -m lf_asv_formatter --asv_version "$(echo asv --version)" + printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output + mv output ${{ env.ARTIFACTS_DIR }} + env: + STEP_URL: "${{ steps.jobs.outputs.html_url }}#step:11:1" + - name: Upload artifacts (PR number and benchmarks output) + uses: actions/upload-artifact@v4 + with: + name: benchmark-artifacts + path: ${{ env.ARTIFACTS_DIR }} \ No newline at end of file diff --git a/.github/workflows/pre-commit-ci.yml b/.github/workflows/pre-commit-ci.yml new file mode 100644 index 00000000..a57e2219 --- /dev/null +++ b/.github/workflows/pre-commit-ci.yml @@ -0,0 +1,35 @@ +# This workflow runs pre-commit hooks on pushes and pull requests to main +# to enforce coding style. To ensure correct configuration, please refer to: +# https://lincc-ppt.readthedocs.io/en/latest/practices/ci_precommit.html +name: Run pre-commit hooks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit-ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install .[dev] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files --verbose + env: + SKIP: "check-lincc-frameworks-template-version,no-commit-to-branch,check-added-large-files,validate-pyproject,sphinx-build,pytest-check" + - uses: pre-commit-ci/lite-action@v1.0.2 + if: failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false \ No newline at end of file diff --git a/.github/workflows/publish-benchmarks-pr.yml b/.github/workflows/publish-benchmarks-pr.yml new file mode 100644 index 00000000..45ed9280 --- /dev/null +++ b/.github/workflows/publish-benchmarks-pr.yml @@ -0,0 +1,53 @@ +# This workflow publishes a benchmarks comment on a pull request. It is triggered after the +# benchmarks are computed in the asv-pr workflow. This separation of concerns allows us limit +# access to the target repository private tokens and secrets, increasing the level of security. +# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. +name: Publish benchmarks comment to PR + +on: + workflow_run: + workflows: ["Run benchmarks for PR"] + types: [completed] + +jobs: + upload-pr-comment: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + issues: write + pull-requests: write + steps: + - name: Display Workflow Run Information + run: | + echo "Workflow Run ID: ${{ github.event.workflow_run.id }}" + echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" + echo "Head Branch: ${{ github.event.workflow_run.head_branch }}" + echo "Conclusion: ${{ github.event.workflow_run.conclusion }}" + echo "Event: ${{ github.event.workflow_run.event }}" + - name: Download artifact + uses: dawidd6/action-download-artifact@v3 + with: + name: benchmark-artifacts + run_id: ${{ github.event.workflow_run.id }} + - name: Extract artifacts information + id: pr-info + run: | + printf "PR number: $(cat pr)\n" + printf "Output:\n$(cat output)" + printf "pr=$(cat pr)" >> $GITHUB_OUTPUT + - name: Find benchmarks comment + uses: peter-evans/find-comment@v3 + id: find-comment + with: + issue-number: ${{ steps.pr-info.outputs.pr }} + comment-author: 'github-actions[bot]' + body-includes: view all benchmarks + - name: Create or update benchmarks comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ steps.pr-info.outputs.pr }} + body-path: output + edit-mode: replace \ No newline at end of file diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 00000000..f7cecc2e --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,37 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://github.com/pypa/gh-action-pypi-publish#trusted-publishing + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml new file mode 100644 index 00000000..31075605 --- /dev/null +++ b/.github/workflows/smoke-test.yml @@ -0,0 +1,42 @@ +# This workflow will run daily at 06:45. +# It will install Python dependencies and run tests with a variety of Python versions. +# See documentation for help debugging smoke test issues: +# https://lincc-ppt.readthedocs.io/en/latest/practices/ci_testing.html#version-culprit + +name: Unit test smoke test + +on: + + # Runs this workflow automatically + schedule: + - cron: 45 6 * * * + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install -e .[dev] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: List dependencies + run: | + pip list + - name: Run unit tests with pytest + run: | + python -m pytest \ No newline at end of file diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml new file mode 100644 index 00000000..9d8522fd --- /dev/null +++ b/.github/workflows/testing-and-coverage.yml @@ -0,0 +1,38 @@ +# This workflow will install Python dependencies, run tests and report code coverage with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Unit test and code coverage + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + python -m pip install --upgrade pip + pip install -e .[dev] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run unit tests with pytest + run: | + python -m pytest --cov=resspect --cov-report=xml + - name: Upload coverage report to codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index b1fc6b3f..50990fec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,25 +1,150 @@ -resspect.egg-info/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python build/ -build/lib/ +develop-eggs/ dist/ -work/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +_version.py + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +_readthedocs/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ venv/ -resspect/__init__.pyc -resspect/__pycache__/ -resspect/scripts/__pycache__/ -resspect/actConfig/ -resspect/analysis_functions/ -resspect/snanapipe/ -data/SIMGEN_PUBLIC_DES/ -env.yml -examples/ -venv2/ -resspect/.ipynb_checkpoints/ -.ipynb_checkpoints/ -__pycache__/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# vscode +.vscode/ + +# dask +dask-worker-space/ + +# tmp directory +tmp/ + +# Mac OS +.DS_Store -# PyCharm Files -.idea +# Airspeed Velocity performance results +_results/ +_html/ -auxiliary_files/cosmo -auxiliary_files/cosmo.hpp +# Project initialization script +.initialize_new_project.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..7811ea40 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,59 @@ +repos: + # Compare the local template version to the latest remote template version + # This hook should always pass. It will print a message if the local version + # is out of date. + - repo: https://github.com/lincc-frameworks/pre-commit-hooks + rev: v0.1.2 + hooks: + - id: check-lincc-frameworks-template-version + name: Check template version + description: Compare current template version against latest + verbose: true + # Clear output from jupyter notebooks so that only the input cells are committed. + - repo: local + hooks: + - id: jupyter-nb-clear-output + name: Clear output from Jupyter notebooks + description: Clear output from Jupyter notebooks. + files: \.ipynb$ + stages: [commit] + language: system + entry: jupyter nbconvert --clear-output + # Prevents committing directly branches named 'main' and 'master'. + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: no-commit-to-branch + name: Prevent main branch commits + description: Prevent the user from committing directly to the primary branch. + - id: check-added-large-files + name: Check for large files + description: Prevent the user from committing very large files. + args: ['--maxkb=500'] + # Verify that pyproject.toml is well formed + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.12.1 + hooks: + - id: validate-pyproject + name: Validate pyproject.toml + description: Verify that pyproject.toml adheres to the established schema. + # Verify that GitHub workflows are well formed + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.28.0 + hooks: + - id: check-github-workflows + args: ["--verbose"] + # Run unit tests, verify that they pass. Note that coverage is run against + # the ./src directory here because that is what will be committed. In the + # github workflow script, the coverage is run against the installed package + # and uploaded to Codecov by calling pytest like so: + # `python -m pytest --cov= --cov-report=xml` + - repo: local + hooks: + - id: pytest-check + name: Run unit tests + description: Run unit tests with pytest. + entry: bash -c "if python -m pytest --co -qq; then python -m pytest --cov=./src --cov-report=html; fi" + language: system + pass_filenames: false + always_run: true diff --git a/.setup_dev.sh b/.setup_dev.sh new file mode 100644 index 00000000..d8cd955c --- /dev/null +++ b/.setup_dev.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# This script should be run by new developers to install this package in +# editable mode and configure their local environment + +echo "Checking virtual environment" +if [ -z "${VIRTUAL_ENV}" ] && [ -z "${CONDA_PREFIX}" ]; then + echo 'No virtual environment detected: none of $VIRTUAL_ENV or $CONDA_PREFIX is set.' + echo + echo "=== This script is going to install the project in the system python environment ===" + echo "Proceed? [y/N]" + read -r RESPONCE + if [ "${RESPONCE}" != "y" ]; then + echo "See https://lincc-ppt.readthedocs.io/ for details." + echo "Exiting." + exit 1 + fi + +fi + +echo "Checking pip version" +MINIMUM_PIP_VERSION=22 +pipversion=( $(python -m pip --version | awk '{print $2}' | sed 's/\./ /g') ) +if let "${pipversion[0]}<${MINIMUM_PIP_VERSION}"; then + echo "Insufficient version of pip found. Requires at least version ${MINIMUM_PIP_VERSION}." + echo "See https://lincc-ppt.readthedocs.io/ for details." + exit 1 +fi + +echo "Installing package and runtime dependencies in local environment" +python -m pip install -e . > /dev/null + +echo "Installing developer dependencies in local environment" +python -m pip install -e .'[dev]' > /dev/null +if [ -f docs/requirements.txt ]; then python -m pip install -r docs/requirements.txt; fi + +echo "Installing pre-commit" +pre-commit install > /dev/null + +####################################################### +# Include any additional configurations below this line +####################################################### diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..435c3ed1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 The RESSPECT team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/resspect/feature_extractors/__init__.py b/benchmarks/__init__.py similarity index 100% rename from resspect/feature_extractors/__init__.py rename to benchmarks/__init__.py diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 00000000..08dea6a3 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,80 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + // The name of the project being benchmarked. + "project": "resspect", + // The project's homepage. + "project_url": "https://github.com/LSSTDESC/resspect", + // The URL or local path of the source code repository for the + // project being benchmarked. + "repo": "..", + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "tip" (for mercurial). + "branches": [ + "HEAD" + ], + "install_command": [ + "python -m pip install {wheel_file}" + ], + "build_command": [ + "python -m build --wheel -o {build_cache_dir} {build_dir}" + ], + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "virtualenv", + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/LSSTDESC/resspect/commit/", + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": [ + "3.10" + ], + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "Cython": [], + "build": [], + "packaging": [] + }, + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks". + "benchmark_dir": ".", + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env". + "env_dir": "env", + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "_results", + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "_html", + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + "build_cache_size": 8 + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } +} \ No newline at end of file diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py new file mode 100644 index 00000000..79ca5a2d --- /dev/null +++ b/benchmarks/benchmarks.py @@ -0,0 +1,16 @@ +"""Two sample benchmarks to compute runtime and memory usage. + +For more information on writing benchmarks: +https://asv.readthedocs.io/en/stable/writing_benchmarks.html.""" + +from resspect import example_benchmarks + + +def time_computation(): + """Time computations are prefixed with 'time'.""" + example_benchmarks.runtime_computation() + + +def mem_list(): + """Memory computations are prefixed with 'mem' or 'peakmem'.""" + return example_benchmarks.memory_computation() diff --git a/pyproject.toml b/pyproject.toml index 8ecf9cdd..1e219e38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,20 @@ -[build-system] -requires = ["setuptools>=65.5.0", "wheel"] - [project] name = "resspect" -version = "0.1.0" -description = "resspect - Recommendation System for Spectroscopic Follow-up" +license = {file = "LICENSE"} readme = "README.md" authors = [ { name = "The RESSPECT team", email = "contact@cosmostatistics-initiative.org" } ] -license = { text = "GPL-3.0-or-later" } +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", +] +dynamic = ["version"] +requires-python = ">=3.9" dependencies = [ "flask==2.3.2", "astropy>=5.2.1", @@ -42,23 +47,81 @@ run_time_domain = "resspect.scripts.run_time_domain:main" [project.optional-dependencies] dev = [ - "pytest", + "asv==0.6.1", # Used to compute performance benchmarks + "jupyter", # Clears output from Jupyter notebooks + "pre-commit", # Used to run checks before finalizing a git commit + "pytest", + "pytest-cov", # Used to report total code coverage ] -[tool.setuptools.packages.find] -include = ["resspect*"] +[build-system] +requires = [ + "setuptools>=62", # Used to build and package the Python project + "setuptools_scm>=6.2", # Gets release version from git. Makes it available programmatically +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/resspect/_version.py" -[tool.pytest] -addopts = ["--pyargs"] +[tool.pytest.ini_options] +testpaths = [ + "tests", +] -[tool.tox] -legacy_tox_ini = """ -[tox] -envlist = py38 +[tool.black] +line-length = 110 +target-version = ["py39"] + +[tool.isort] +profile = "black" +line_length = 110 + +[tool.ruff] +line-length = 110 +target-version = "py39" + +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + "W", + # Pyflakes + "F", + # pep8-naming + "N", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", + # docstrings + "D101", + "D102", + "D103", + "D106", + "D206", + "D207", + "D208", + "D300", + "D417", + "D419", + # Numpy v2.0 compatibility + "NPY201", +] + +ignore = [ + "UP006", # Allow non standard library generics in type hints + "UP007", # Allow Union in type hints + "SIM114", # Allow if with same arms + "B028", # Allow default warning level + "SIM117", # Allow nested with + "UP015", # Allow redundant open parameters + "UP028", # Allow yield in for loop +] -[testenv] -extras = dev -isolated_build = true -commands = - pytest -s -""" +[tool.coverage.run] +omit=["src/resspect/_version.py"] diff --git a/resspect/snanapipe/snana_hook.pyc b/resspect/snanapipe/snana_hook.pyc deleted file mode 100644 index 525f268c6119d96016c222d3eb24c1254c3b58fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1075 zcmaiyO>fgc5Qb-MCnXUTMO@(m7mGu4=oKLXQiTLW$l*c`SSH>iZv7Eu*Fr(*Df};f z6h8pock)qjsA^>=v%531&wPY`PW!*(m*1wc|3dt~!E%=nIetb;k%@>ExiWGUNFK=6 zh~Zr*Whhl7lL-4rY)58B@{arx*@}Ffbm=~hy9ibM#W!+Kp$-;JWd~I;sMeLui)=6+ ze*AJVa9QauVgm!;gV4iev9!D0XgC_aZ<^~PC=K6Yxfc+Co8V>P%M&0PRYJ1zB$8wt zGqY&`aaix@eG0KLvJpu&-?0*mZaIih)XUb#^z#v&Y&r8xYPP$8CNYIZb9Gi{=>n|h z%e8-i<1{Vm!lx-+9HWSt68}GtXE^X+(K4gJH-OT5Z()wi4q!|}AOE!e475b4c(`h7 zMNEzs)tc~1b_ZT^0!LQcF)uNXc3_M^@DErnfdCk43bw77n?N=JX930as5Pn_vFSOw zeIj#AhlGX}A%{RU+#fG7I=CFij6X{FtkyKoe5Oj(Mo4Q;hL*Id3 zPG^NzZEk(l#ylgX=-KzSD09^=v(oKjHTA5Rw=1hp%*)p5IP9KhTdsd(tGUzLduv8} zj*aU>ux8Iq-#iHvW8d#Ot8AKj8l)*QYD;Vr Date: Thu, 26 Sep 2024 15:42:56 -0700 Subject: [PATCH 10/13] fix test_data_path --- tests/resspect/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/resspect/conftest.py b/tests/resspect/conftest.py index 212550b2..650dfece 100644 --- a/tests/resspect/conftest.py +++ b/tests/resspect/conftest.py @@ -10,11 +10,11 @@ @pytest.fixture def test_data_path(): - return Path(__file__).parent.parent / "data" / "tests" + return Path(__file__).parent.parent.parent / "data" / "tests" @pytest.fixture def test_des_data_path(): - return Path(__file__).parent.parent / "data" / "tests" / "DES_data" + return Path(__file__).parent.parent.parent / "data" / "tests" / "DES_data" @pytest.fixture(scope="session") From 6008ef72f9abc1d5f3be1c2a9afe971041eb3cbd Mon Sep 17 00:00:00 2001 From: Max West Date: Fri, 27 Sep 2024 10:56:00 -0700 Subject: [PATCH 11/13] update github actions to run on python=3.9 --- .github/workflows/e2e-tests.yml | 4 ++-- .github/workflows/unit-tests.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ba1eaf78..bb64b8d3 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -17,11 +17,11 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: Install package inside virtual environment run: | - python3.8 -m venv venv + python3.9 -m venv venv source venv/bin/activate python -m pip install -U pip setuptools python -m pip install . diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 24df78bd..8c7e9dbb 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: Run Tox run: pipx run tox -e py38 -v From 6338666a56d248262a62428bf811fd83dab888bd Mon Sep 17 00:00:00 2001 From: Max West Date: Fri, 27 Sep 2024 11:00:18 -0700 Subject: [PATCH 12/13] py39 again --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8c7e9dbb..292af7a4 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -20,4 +20,4 @@ jobs: python-version: 3.9 - name: Run Tox - run: pipx run tox -e py38 -v + run: pipx run tox -e py39 -v From f0b67aca43aec0258ce801e779ee02d1a432c247 Mon Sep 17 00:00:00 2001 From: Max West Date: Fri, 27 Sep 2024 11:04:04 -0700 Subject: [PATCH 13/13] fix e2e paths --- .github/workflows/e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index bb64b8d3..4a40babb 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -29,4 +29,4 @@ jobs: - name: Run pipeline for SNPCC run: | source venv/bin/activate - ./tests/run-snpcc-e2e.sh + ./tests/resspect/run-snpcc-e2e.sh