diff --git a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/__init__.py b/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/__init__.py deleted file mode 100644 index 3770aa06cf..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .scaling import ( - apply_scalings, - compute_individual_scalings, - estimate_positive_score_shift, - estimate_score_deviation, -) diff --git a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling.py b/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling.py deleted file mode 100644 index 94cf12f3bd..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling.py +++ /dev/null @@ -1,121 +0,0 @@ -import logging - -import pandas as pd - -from solidago.pipeline import PipelineInput -from solidago.primitives import qr_quantile, qr_standard_deviation -from .scaling_step import compute_scaling - - -def compute_individual_scalings( - individual_scores: pd.DataFrame, - tournesol_input: PipelineInput, - W: float, -) -> pd.DataFrame: - """ - Inputs: - - individual_scores: DataFrame with columns - * user_id - * entity_id - * raw_score - * raw_uncertainty - - - tournesol_input: PipelineInput used to fetch user details (used in calibration step) - - Returns: - - scalings: DataFrame with index `user_id` and columns: - * `s`: scaling factor - * `tau`: translation value - * `delta_s`: uncertainty on `s` - * `delta_tau`: uncertainty on `tau` - """ - if len(individual_scores) == 0: - return pd.DataFrame( - columns=["s", "tau", "delta_s", "delta_tau"] - ) - - ratings = tournesol_input.ratings_properties.set_index(["user_id", "entity_id"]) - - # Calibration - calibration_ratings = ratings[ratings.is_scaling_calibration_user] - calibration_users_scores = individual_scores.join(calibration_ratings, on=["user_id", "entity_id"], how="inner") - ## Init columns "score" and "uncertainty" read by `compute_scaling()` - calibration_users_scores["score"] = calibration_users_scores["raw_score"] - calibration_users_scores["uncertainty"] = calibration_users_scores["raw_uncertainty"] - calibration_users_scalings = compute_scaling( - calibration_users_scores, - tournesol_input=tournesol_input, - W=W, - calibration=True, - ) - - non_calibration_user_ids = ratings[~ratings.is_scaling_calibration_user].index.unique("user_id") - calibration_user_ids = ratings[ratings.is_scaling_calibration_user].index.unique("user_id") - all_users_scores = individual_scores.join(ratings, on=["user_id", "entity_id"], how="left") - all_users_scores["is_public"].fillna(False, inplace=True) - all_users_scores["is_scaling_calibration_user"].fillna(False, inplace=True) - - # Apply scaling for calibration users - all_users_scores = apply_scalings(all_users_scores, scalings=calibration_users_scalings) - - # Apply scaling for non-calibration users - logging.info( - "Computing scaling for %s non-calibration users, based on %s calibration users", - len(non_calibration_user_ids), - len(calibration_user_ids), - ) - non_calibration_users_scalings = compute_scaling( - all_users_scores, - tournesol_input=tournesol_input, - users_to_compute=non_calibration_user_ids, - reference_users=calibration_user_ids, - calibration=False, - W=W, - ) - all_users_scalings = pd.concat( - df - for df in [calibration_users_scalings, non_calibration_users_scalings] - if not df.empty - ) - return all_users_scalings - - -def apply_scalings(individual_scores: pd.DataFrame, scalings: pd.DataFrame) -> pd.DataFrame: - df = individual_scores.join(scalings, on="user_id") - df["s"].fillna(1.0, inplace=True) - df["tau"].fillna(0.0, inplace=True) - df["delta_s"].fillna(0.0, inplace=True) - df["delta_tau"].fillna(0.0, inplace=True) - df["score"] = df["s"] * df["raw_score"] + df["tau"] - df["uncertainty"] = ( - df["s"] * df["raw_uncertainty"] + df["delta_s"] * df["raw_score"].abs() + df["delta_tau"] - ) - return df.drop(columns=["s", "tau", "delta_s", "delta_tau"]) - - -def estimate_positive_score_shift( - scaled_individual_scores: pd.DataFrame, W: float, quantile: float = 0.05 -) -> float: - w = 1 / scaled_individual_scores.groupby("user_id")["score"].transform("size") - x = scaled_individual_scores.score - delta = scaled_individual_scores.uncertainty - return qr_quantile( - lipschitz=1/W, - quantile=quantile, - values=x.to_numpy(), - voting_rights=w.to_numpy(), - left_uncertainties=delta.to_numpy(), - ) - - -def estimate_score_deviation(scaled_individual_scores, W, quantile=0.5): - w = 1 / scaled_individual_scores.groupby("user_id")["score"].transform("size") - x = scaled_individual_scores.score - delta = scaled_individual_scores.uncertainty - return qr_standard_deviation( - lipschitz=1/W, - values=x.to_numpy(), - quantile_dev=quantile, - voting_rights=w.to_numpy(), - left_uncertainties=delta.to_numpy(), - ) diff --git a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling_step.py b/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling_step.py deleted file mode 100644 index e16c60b052..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/collaborative_scaling/scaling_step.py +++ /dev/null @@ -1,292 +0,0 @@ -import numpy as np -import pandas as pd -from numba import njit - -from solidago.pipeline import PipelineInput -from solidago.primitives import ( - lipschitz_resilient_mean, - qr_median, - qr_uncertainty, -) - - -# This limit allows to index pairs of entity_id into a usual Index with dtype 'uint64'. -# We originally used a MultiIndex that consumed significantly more memory, due to how -# pandas may cache MultiIndex values as an array of Python tuples. -ENTITY_ID_MAX = 2**32 - 1 - - -@njit -def get_significantly_different_pairs_indices(scores, uncertainties): - indices = [] - n_alternatives = len(scores) - for idx_a in range(0, n_alternatives): - for idx_b in range(idx_a + 1, n_alternatives): - if abs(scores[idx_a] - scores[idx_b]) >= 2 * ( - uncertainties[idx_a] + uncertainties[idx_b] - ): - indices.append((idx_a, idx_b)) - if len(indices) == 0: - return np.empty((0, 2), dtype=np.int64) - return np.array(indices) - - - -def get_user_scaling_weights(input: PipelineInput, W: float): - ratings_properties = input.ratings_properties[ - ["user_id", "trust_score", "is_scaling_calibration_user"] - ].copy() - df = ratings_properties.groupby("user_id").first() - df["scaling_weight"] = df["trust_score"] - df["scaling_weight"].mask( - df.is_scaling_calibration_user, - W, - inplace=True, - ) - return df["scaling_weight"].to_dict() - - -def get_significantly_different_pairs(scores: pd.DataFrame): - """ - Find the set of pairs of alternatives - that are significantly different, according to the contributor scores. - (Used for collaborative preference scaling) - """ - scores = scores[["entity_id", "score", "uncertainty"]] - indices = get_significantly_different_pairs_indices( - scores["score"].to_numpy(), scores["uncertainty"].to_numpy() - ) - scores_a = scores.iloc[indices[:, 0]] - scores_b = scores.iloc[indices[:, 1]] - - entity_pairs_index = pd.Index( - # As a memory optimization, a pair of entity_id is represented as a single uint64 - scores_a["entity_id"].to_numpy() * (ENTITY_ID_MAX + 1) + scores_b["entity_id"].to_numpy(), - dtype="uint64", - ) - return pd.DataFrame( - { - "score_a": scores_a["score"].to_numpy(), - "score_b": scores_b["score"].to_numpy(), - "uncertainty_a": scores_a["uncertainty"].to_numpy(), - "uncertainty_b": scores_b["uncertainty"].to_numpy(), - }, - index=entity_pairs_index, - ) - - -def compute_scaling( - df: pd.DataFrame, - tournesol_input: PipelineInput, - W: float, - users_to_compute=None, - reference_users=None, - calibration=False, -): - scaling_weights = get_user_scaling_weights(tournesol_input, W=W) - - if users_to_compute is None: - users_to_compute = set(df.user_id.unique()) - else: - users_to_compute = set(users_to_compute) - - if reference_users is None: - reference_users = set(df.user_id.unique()) - else: - reference_users = set(reference_users) - - s_dict = {} - delta_s_dict = {} - - if len(df) > 0 and df["entity_id"].max() > ENTITY_ID_MAX: - raise AssertionError("Values of entity_id are too large.") - - ref_user_scores_pairs = {} - ref_user_scores_by_entity_id = {} - for (ref_user_id, ref_user_scores) in df[df["user_id"].isin(reference_users)].groupby( - "user_id" - ): - ref_user_scores_pairs[ref_user_id] = get_significantly_different_pairs(ref_user_scores) - ref_user_scores_by_entity_id[ref_user_id] = ref_user_scores.set_index("entity_id") - - for (user_n, user_n_scores) in df[df["user_id"].isin(users_to_compute)].groupby("user_id"): - s_nqm = [] - delta_s_nqm = [] - s_weights = [] - - if calibration: - ABn_all = ref_user_scores_pairs[user_n] - else: - ABn_all = get_significantly_different_pairs(user_n_scores) - - if len(ABn_all) > 0: - ABn_all_index_set = set(ABn_all.index) - for user_m in reference_users - {user_n}: - try: - ABm = ref_user_scores_pairs[user_m] - except KeyError: - # the reference user may not have contributed on the current criterion - continue - - if all((idx not in ABn_all_index_set) for idx in ABm.index): - # Quick path: the intersection is empty, no need to call expensive inner join. - continue - - ABnm = ABn_all.join(ABm, how="inner", lsuffix="_n", rsuffix="_m") - s_nqmab = ( - (ABnm.score_a_m - ABnm.score_b_m).abs() - / (ABnm.score_a_n - ABnm.score_b_n).abs() - ).to_numpy() - delta_s_nqmab = (( - ( - (ABnm.score_a_m - ABnm.score_b_m).abs() - + ABnm.uncertainty_a_m - + ABnm.uncertainty_b_m - ) - / ( - (ABnm.score_a_n - ABnm.score_b_n).abs() - - ABnm.uncertainty_a_n - - ABnm.uncertainty_b_n - ) - ) - s_nqmab).to_numpy() - - s = qr_median( - lipschitz=1.0, - voting_rights=1.0, - values=s_nqmab - 1, - left_uncertainties=delta_s_nqmab, - ) - s_nqm.append(s + 1) - delta_s_nqm.append(qr_uncertainty( - lipschitz=1.0, - default_dev=1.0, - voting_rights=1.0, - values=s_nqmab - 1, - left_uncertainties=delta_s_nqmab, - median=s, - )) - s_weights.append(scaling_weights[user_m]) - - s_weights = np.array(s_weights) - theta_inf = np.max(user_n_scores.score.abs()) - s_nqm = np.array(s_nqm) - delta_s_nqm = np.array(delta_s_nqm) - lipshitz = 1/(8*W*theta_inf) if theta_inf > 0.0 else 1e18 - if calibration: - s_dict[user_n] = lipschitz_resilient_mean( - lipschitz=lipshitz, - values=s_nqm, - voting_rights=s_weights, - left_uncertainties=delta_s_nqm, - default_value=1.0, - ) - delta_s_dict[user_n] = qr_uncertainty( - lipschitz=lipshitz, - default_dev=1.0, - values=s_nqm, - voting_rights=s_weights, - left_uncertainties=delta_s_nqm, - ) - else: - qr_med = qr_median( - lipschitz=lipshitz, - voting_rights=s_weights, - values=s_nqm - 1, - left_uncertainties=delta_s_nqm, - ) - s_dict[user_n] = 1 + qr_med - delta_s_dict[user_n] = qr_uncertainty( - lipschitz=lipshitz, - default_dev=1.0, - voting_rights=s_weights, - values=s_nqm-1, - left_uncertainties=delta_s_nqm, - median=qr_med, - ) - - tau_dict = {} - delta_tau_dict = {} - for (user_n, user_n_scores) in df[df.user_id.isin(users_to_compute)].groupby("user_id"): - tau_nqm = [] - delta_tau_nqm = [] - s_weights = [] - user_n_scores = user_n_scores.set_index("entity_id") - user_n_scores_entity_ids = set(user_n_scores.index) - - for user_m in reference_users - {user_n}: - try: - user_m_scores = ref_user_scores_by_entity_id[user_m] - except KeyError: - # the reference user may not have contributed on the current criterion - continue - common_entity_ids = list(user_n_scores_entity_ids.intersection(user_m_scores.index)) - if len(common_entity_ids) == 0: - continue - - m_scores = user_m_scores.loc[common_entity_ids] - n_scores = user_n_scores.loc[common_entity_ids] - - s_m = s_dict.get(user_m, 1) - s_n = s_dict[user_n] - tau_nqmab = (s_m * m_scores.score - s_n * n_scores.score).to_numpy() - delta_tau_nqmab = (s_n * n_scores.uncertainty + s_m * m_scores.uncertainty).to_numpy() - - tau = qr_median( - lipschitz=1.0, - voting_rights=1.0, - values=tau_nqmab, - left_uncertainties=delta_tau_nqmab, - ) - tau_nqm.append(tau) - delta_tau_nqm.append(qr_uncertainty( - lipschitz=1.0, - default_dev=1.0, - voting_rights=1.0, - values=tau_nqmab, - left_uncertainties=delta_tau_nqmab, - median=tau, - )) - s_weights.append(scaling_weights[user_m]) - - s_weights = np.array(s_weights) - tau_nqm = np.array(tau_nqm) - delta_tau_nqm = np.array(delta_tau_nqm) - if calibration: - tau_dict[user_n] = lipschitz_resilient_mean( - lipschitz=1/(8*W), - voting_rights=s_weights, - values=tau_nqm, - left_uncertainties=delta_tau_nqm, - ) - delta_tau_dict[user_n] = qr_uncertainty( - lipschitz=1/(8*W), - default_dev=1.0, - voting_rights=s_weights, - values=tau_nqm, - left_uncertainties=delta_tau_nqm, - ) - else: - qr_med = qr_median( - lipschitz=1/(8*W), - voting_rights=s_weights, - values=tau_nqm, - left_uncertainties=delta_tau_nqm, - ) - tau_dict[user_n] = qr_med - delta_tau_dict[user_n] = qr_uncertainty( - lipschitz=1/(8*W), - default_dev=1.0, - voting_rights=s_weights, - values=tau_nqm, - left_uncertainties=delta_tau_nqm, - median=qr_med, - ) - - return pd.DataFrame( - { - "s": s_dict, - "tau": tau_dict, - "delta_s": delta_s_dict, - "delta_tau": delta_tau_dict, - } - ) diff --git a/solidago/src/solidago/pipeline/legacy2023/criterion_pipeline.py b/solidago/src/solidago/pipeline/legacy2023/criterion_pipeline.py deleted file mode 100644 index b70c739840..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/criterion_pipeline.py +++ /dev/null @@ -1,118 +0,0 @@ -import logging -from typing import Callable, Optional - -from .global_scores import ( - add_voting_rights, - aggregate_scores, - get_squash_function, - ALL_SCORE_MODES, -) -from solidago.pipeline import PipelineInput, PipelineOutput -from . import collaborative_scaling -from .individual_scores import get_individual_scores -from .parameters import PipelineParameters - - -logger = logging.getLogger(__name__) - - -def run_pipeline_for_criterion( - criterion: str, - input: PipelineInput, - parameters: PipelineParameters, - output: PipelineOutput, - done_callback: Optional[Callable] = None, -): - """ - Run Pipepeline for a single criterion - """ - logger.info("Starting Solidago pipeline for criterion '%s'", criterion) - - logger.info("Computing individual scores for criterion '%s'...", criterion) - indiv_scores = get_individual_scores(input, criteria=criterion, parameters=parameters) - logger.info("Computing individual scores for criterion '%s' DONE", criterion) - - logger.info("Computing individual scalings for criterion '%s'...", criterion) - scalings = collaborative_scaling.compute_individual_scalings( - individual_scores=indiv_scores, - tournesol_input=input, - W=parameters.W, - ) - scaled_scores = collaborative_scaling.apply_scalings( - individual_scores=indiv_scores, scalings=scalings - ) - - if len(scaled_scores) > 0: - score_shift = collaborative_scaling.estimate_positive_score_shift( - scaled_scores, - W=parameters.score_shift_W, - quantile=parameters.score_shift_quantile, - ) - score_std = collaborative_scaling.estimate_score_deviation( - scaled_scores, - W=parameters.score_shift_W, - quantile=parameters.score_deviation_quantile, - ) - scaled_scores.score -= score_shift - scaled_scores.score /= score_std - scaled_scores.uncertainty /= score_std - - output.save_individual_scalings(scalings) - logger.info("Computing individual scalings for criterion '%s' DONE", criterion) - - logger.info("Computing aggregated scores for criterion '%s'...", criterion) - # Join ratings columns ("is_public", "trust_score", etc.) - ratings = input.ratings_properties.set_index(["user_id", "entity_id"]) - scaled_scores = scaled_scores.join( - ratings, - on=["user_id", "entity_id"], - ) - - logger.info(" computing voting rights scores for criterion '%s'...", criterion) - scaled_scores_with_voting_rights_per_score_mode = { - mode: add_voting_rights(scaled_scores, params=parameters, score_mode=mode) - for mode in ALL_SCORE_MODES - } - logger.info(" computing voting rights scores for criterion '%s' DONE", criterion) - for mode in ALL_SCORE_MODES: - scaled_scores_with_voting_rights = scaled_scores_with_voting_rights_per_score_mode[mode] - global_scores = aggregate_scores(scaled_scores_with_voting_rights, W=parameters.W) - global_scores["criteria"] = criterion - - # Apply squashing - squash_function = get_squash_function(parameters) - global_scores["uncertainty"] = 0.5 * ( - squash_function(global_scores["score"] + global_scores["uncertainty"]) - - squash_function(global_scores["score"] - global_scores["uncertainty"]) - ) - global_scores["score"] = squash_function(global_scores["score"]) - - logger.info( - "Mehestan: scores computed for crit '%s' and mode '%s'", - criterion, - mode, - ) - output.save_entity_scores(global_scores, score_mode=mode) - logger.info("Computing aggregated scores for criterion '%s' DONE", criterion) - - logger.info("Computing squashed individual scores for criterion '%s'...", criterion) - squash_function = get_squash_function(parameters) - scaled_scores = scaled_scores_with_voting_rights_per_score_mode["default"] - scaled_scores["uncertainty"] = 0.5 * ( - squash_function(scaled_scores["score"] + scaled_scores["uncertainty"]) - - squash_function(scaled_scores["score"] - scaled_scores["uncertainty"]) - ) - scaled_scores["score"] = squash_function(scaled_scores["score"]) - scaled_scores["criteria"] = criterion - output.save_individual_scores(scaled_scores) - logger.info("Computing squashed individual scores for criterion '%s' DONE", criterion) - - logger.info( - "Solidago pipeline for criterion '%s' DONE.", - criterion, - ) - - if done_callback is not None: - done_callback() - - return output diff --git a/solidago/src/solidago/pipeline/legacy2023/global_scores.py b/solidago/src/solidago/pipeline/legacy2023/global_scores.py deleted file mode 100644 index 2215197c97..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/global_scores.py +++ /dev/null @@ -1,99 +0,0 @@ -from typing import Literal, List, get_args - -import pandas as pd -import numpy as np - -from solidago.primitives import qr_uncertainty, qr_median -from solidago.voting_rights import compute_voting_rights -from .parameters import PipelineParameters - - -ScoreMode = Literal["default", "trusted_only", "all_equal"] -ALL_SCORE_MODES: List[ScoreMode] = list(get_args(ScoreMode)) - - -def add_voting_rights( - ratings_properties_df: pd.DataFrame, - params: PipelineParameters, - score_mode: ScoreMode = "default", -): - """ - Add a "voting_right" column to the ratings_df DataFrame - - Parameters - ---------- - - ratings_properties_df: - DataFrame of ratings. All ratings must be on the same criteria and the df must have - columns: user_id, entity_id, trust_score, is_public - """ - ratings_df = ratings_properties_df.copy(deep=True) - ratings_df["voting_right"] = 0.0 - ratings_df["privacy_penalty"] = ratings_df.is_public.map( - { - True: params.vote_weight_public_ratings, - False: params.vote_weight_private_ratings, - } - ) - if score_mode == "trusted_only": - ratings_df = ratings_df[ratings_df["trust_score"] >= 0.8] - ratings_df["voting_right"] = 1 - elif score_mode == "all_equal": - ratings_df["voting_right"] = 1 - elif score_mode == "default": - # trust score would be possibly None (NULL) when new users are created and when - # computation of trust scores fail for any reason (e.g. no user pre-trusted) - ratings_df.trust_score.fillna(0.0, inplace=True) - for (_, ratings_group) in ratings_df.groupby("entity_id"): - ratings_df.loc[ratings_group.index, "voting_right"] = compute_voting_rights( - ratings_group.trust_score.to_numpy(), - ratings_group.privacy_penalty.to_numpy(), - over_trust_bias=params.over_trust_bias, - over_trust_scale=params.over_trust_scale, - ) - else: - raise ValueError(f"Unknown score mode '{score_mode}'") - - return ratings_df - - -def get_squash_function(params: PipelineParameters): - def squash(x): - return params.max_squashed_score * x / ( - np.sqrt(1 + x * x) - ) - return squash - - -def aggregate_scores(scaled_scores: pd.DataFrame, W: float): - if len(scaled_scores) == 0: - return pd.DataFrame( - columns=["entity_id", "score", "uncertainty", "deviation", "n_contributors"] - ) - - global_scores = {} - for (entity_id, scores) in scaled_scores.groupby("entity_id"): - w = scores.voting_right.to_numpy() - theta = scores.score.to_numpy() - delta = scores.uncertainty.to_numpy() - rho = qr_median( - lipschitz=1/W, - values=theta, - voting_rights=w, - left_uncertainties=delta, - ) - rho_uncertainty = qr_uncertainty( - lipschitz=1/W, - values=theta, - voting_rights=w, - left_uncertainties=delta, - median=rho, - ) - global_scores[entity_id] = { - "score": rho, - "uncertainty": rho_uncertainty, - "n_contributors": len(scores), - } - - result = pd.DataFrame.from_dict(global_scores, orient="index") - result.index.name = "entity_id" - return result.reset_index() diff --git a/solidago/src/solidago/pipeline/legacy2023/individual_scores.py b/solidago/src/solidago/pipeline/legacy2023/individual_scores.py deleted file mode 100644 index a8d13bf029..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/individual_scores.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import Optional - -import pandas as pd - -from solidago.pipeline import PipelineInput -from .parameters import PipelineParameters - - -def get_individual_scores( - input: PipelineInput, - criteria: str, - parameters: PipelineParameters, - single_user_id: Optional[int] = None, -) -> pd.DataFrame: - comparisons_df = input.get_comparisons(criterion=criteria, user_id=single_user_id) - # Legacy pipeline assumes all comparisons use the same 'score_max' - score_max_series = comparisons_df.pop("score_max") - if score_max_series.nunique() > 1: - raise RuntimeError( - "Legacy pipeline does not support multiple 'score_max' in comparisons. " - f"Found {dict(score_max_series.value_counts())}" - ) - - individual_scores = input.get_individual_scores(criterion=criteria, user_id=single_user_id) - if individual_scores is not None and "raw_score" in individual_scores: - initial_contributor_scores = individual_scores.groupby("user_id") - else: - initial_contributor_scores = None - - individual_scores = [] - for user_id, user_comparisons in comparisons_df.groupby("user_id"): - if initial_contributor_scores is None: - initial_entity_scores = None - else: - try: - contributor_score_df = initial_contributor_scores.get_group(user_id) - initial_entity_scores = pd.Series( - data=contributor_score_df.raw_score, index=contributor_score_df.entity_id - ) - except KeyError: - initial_entity_scores = None - scores = parameters.indiv_algo.compute_individual_scores( - user_comparisons, initial_entity_scores=initial_entity_scores - ) - if scores is None: - continue - scores["user_id"] = user_id - individual_scores.append(scores.reset_index()) - - if len(individual_scores) == 0: - return pd.DataFrame(columns=["user_id", "entity_id", "raw_score", "raw_uncertainty"]) - - result = pd.concat(individual_scores, ignore_index=True, copy=False) - return result.reindex(columns=["user_id", "entity_id", "raw_score", "raw_uncertainty"]) diff --git a/solidago/src/solidago/pipeline/legacy2023/parameters.py b/solidago/src/solidago/pipeline/legacy2023/parameters.py deleted file mode 100644 index edff57f880..0000000000 --- a/solidago/src/solidago/pipeline/legacy2023/parameters.py +++ /dev/null @@ -1,28 +0,0 @@ -from dataclasses import dataclass -from functools import cached_property - -from solidago.comparisons_to_scores import ContinuousBradleyTerry - - -@dataclass -class PipelineParameters: - # comparisons to scores - alpha: float = 0.02 - r_max: float = 10.0 - - # collaborative scaling and aggregation - max_squashed_score: float = 100.0 - score_shift_W: float = 1. - score_shift_quantile: float = 0.15 - score_deviation_quantile: float = 0.9 - W: float = 10.0 - - # voting rights - over_trust_bias: float = 2 - over_trust_scale: float = 0.1 - vote_weight_public_ratings: float = 1.0 - vote_weight_private_ratings: float = 0.5 - - @cached_property - def indiv_algo(self): - return ContinuousBradleyTerry(r_max=self.r_max, alpha=self.alpha) diff --git a/solidago/tests/test_scaling.py b/solidago/tests/test_scaling.py deleted file mode 100644 index df8d91c6c7..0000000000 --- a/solidago/tests/test_scaling.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest -import pandas as pd - -from solidago.pipeline.legacy2023.collaborative_scaling import estimate_positive_score_shift - - -def test_score_shift_when_all_scores_are_equal(): - scaled_individual_scores = pd.DataFrame(dict( - user_id=[f"user_{i}" for i in range(1000)], - entity_id=[f"entity_{i}" for i in range(1000)], - score=12.12, - uncertainty=1e-12, - )) - tau = estimate_positive_score_shift( - scaled_individual_scores, - W=1.0, - quantile=0.05 - ) - assert tau == pytest.approx(12.12) - -def test_all_users_equal_voting_right_for_score_shift(): - scaled_individual_scores = pd.DataFrame(dict( - user_id=["louis"] * 998 + ["adrien", "aidan"], - entity_id=["entity_{i}" for i in range(1000)], - score=[-10] * 998 + [10, 10], - uncertainty=1e-12, - )) - tau = estimate_positive_score_shift( - scaled_individual_scores, - W=1.0, - quantile=1/3 - ) - assert tau == pytest.approx(0, abs=1e-4) - -def test_5_percent_percentile_score_shift_has_expected_value(): - scaled_individual_scores = pd.DataFrame(dict( - user_id=["user_{i}" for i in range(1000)], - entity_id=["entity_{i}" for i in range(1000)], - score=[i for i in range(1, 1001)], - uncertainty=1e-12, - )) - tau = estimate_positive_score_shift( - scaled_individual_scores, - W=1e-12, - quantile=0.05 - ) - assert tau == pytest.approx(50, abs=1e-4) - -def test_large_W_pulls_score_shift_towards_zero(): - scaled_individual_scores = pd.DataFrame(dict( - user_id=["user_{i}" for i in range(1000)], - entity_id=["entity_{i}" for i in range(1000)], - score=[i for i in range(1, 1001)], - uncertainty=1e-12, - )) - tau = estimate_positive_score_shift( - scaled_individual_scores, - W=100, - quantile=0.05 - ) - assert tau < 50