Merge pull request #21 from Oxid15/develop

v0.3.0
Oxid15 · May 7, 2023 · 4a55a77 · 4a55a77
2 parents dcf2cea + eb97561
commit 4a55a77
Show file tree

Hide file tree

Showing 84 changed files with 366 additions and 239 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -5,53 +5,3 @@ on:
     branches: [ "master" ]
   pull_request:
     branches: [ "master", "develop" ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.9", "3.10"]
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r explainers_requirements.txt
-        pip install -r https://raw.githubusercontent.com/oxid15/cascade/main/utils_requirements.txt
-        pip install .
-
-    - name: Feature importance
-      run: |
-        cd xaib/evaluation/feature_importance/
-        ./evaluate.sh
-        zip -r repo .
-
-    - name: Example selection
-      run: |
-        pwd
-        ls
-        cd xaib/evaluation/example_selection
-        ./evaluate.sh
-        zip -r repo .
-
-    - name: Archive feature importance results
-      uses: actions/upload-artifact@v3
-      with:
-        name: Feature importance results
-        path: xaib/evaluation/feature_importance/repo.zip
-
-    - name: Archive example selection results
-      uses: actions/upload-artifact@v3
-      with:
-        name: Example selection results
-        path: xaib/evaluation/example_selection/repo.zip
diff --git a/docs/_modules/index.html b/docs/_modules/index.html
diff --git a/docs/_modules/xaib/base/base.html b/docs/_modules/xaib/base/base.html
diff --git a/docs/_modules/xaib/cases/example_selection/continuity_case.html b/docs/_modules/xaib/cases/example_selection/continuity_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/coherence_case.html b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/compactness_case.html b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/continuity_case.html b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/correctness_case.html b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
diff --git a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
diff --git a/docs/_modules/xaib/datasets/synthetic_dataset.html b/docs/_modules/xaib/datasets/synthetic_dataset.html
diff --git a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
diff --git a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
diff --git a/docs/_modules/xaib/explainers/example_selection/random_explainer.html b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
diff --git a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
diff --git a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
diff --git a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
diff --git a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
diff --git a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
diff --git a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
diff --git a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
diff --git a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
diff --git a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/label_difference.html b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
diff --git a/docs/_modules/xaib/metrics/feature_importance/sparsity.html b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
diff --git a/docs/_modules/xaib/utils/utils.html b/docs/_modules/xaib/utils/utils.html
diff --git a/docs/cases.html b/docs/cases.html
diff --git a/docs/contribute.html b/docs/contribute.html
diff --git a/docs/contribute/add_dataset.html b/docs/contribute/add_dataset.html
diff --git a/docs/contribute/add_explainer.html b/docs/contribute/add_explainer.html
diff --git a/docs/contribute/add_metric.html b/docs/contribute/add_metric.html
diff --git a/docs/contribute/add_model.html b/docs/contribute/add_model.html
diff --git a/docs/datasets.html b/docs/datasets.html
diff --git a/docs/genindex.html b/docs/genindex.html
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/installation.html b/docs/installation.html
diff --git a/docs/metrics.html b/docs/metrics.html
diff --git a/docs/models.html b/docs/models.html
diff --git a/docs/modules.html b/docs/modules.html
diff --git a/docs/objects.inv b/docs/objects.inv
diff --git a/docs/results.html b/docs/results.html
diff --git a/docs/results/example_selection.html b/docs/results/example_selection.html
diff --git a/docs/results/feature_importance.html b/docs/results/feature_importance.html
diff --git a/docs/search.html b/docs/search.html
diff --git a/docs/searchindex.js b/docs/searchindex.js
diff --git a/docs/use_cases.html b/docs/use_cases.html
diff --git a/docs/use_cases/evaluate_method.html b/docs/use_cases/evaluate_method.html
diff --git a/docs/use_cases/try_method.html b/docs/use_cases/try_method.html
diff --git a/docs/xaib/base.html b/docs/xaib/base.html
diff --git a/docs/xaib/cases.example_selection.html b/docs/xaib/cases.example_selection.html
diff --git a/docs/xaib/cases.feature_importance.html b/docs/xaib/cases.feature_importance.html
diff --git a/docs/xaib/cases.html b/docs/xaib/cases.html
diff --git a/docs/xaib/datasets.html b/docs/xaib/datasets.html
diff --git a/docs/xaib/explainers.example_selection.html b/docs/xaib/explainers.example_selection.html
diff --git a/docs/xaib/explainers.feature_importance.html b/docs/xaib/explainers.feature_importance.html
diff --git a/docs/xaib/explainers.html b/docs/xaib/explainers.html
diff --git a/docs/xaib/metrics.example_selection.html b/docs/xaib/metrics.example_selection.html
diff --git a/docs/xaib/metrics.feature_importance.html b/docs/xaib/metrics.feature_importance.html
diff --git a/docs/xaib/metrics.html b/docs/xaib/metrics.html
diff --git a/docs/xaib/models.html b/docs/xaib/models.html
diff --git a/docs/xaib/utils.html b/docs/xaib/utils.html
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-cascade-ml<0.11.0
-scikit-learn
+cascade-ml>=0.11.0
 plotly
-kaleido
+kaleido
+numpy==1.23.5
diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="xai-benchmark",
-    version="0.2.0",
+    version="0.3.0",
     author="Ilia Moiseev",
     author_email="ilia.moiseev.5@yandex.ru",
     license="MIT",
@@ -20,5 +20,5 @@
     package_dir={"xaib": "./xaib"},
     packages=setuptools.find_packages(),
     python_requires=">=3.8",
-    install_requires=["cascade-ml>=0.7.2", "scikit-learn", "plotly", "kaleido"],
+    install_requires=["cascade-ml", "scikit-learn", "plotly", "kaleido"],
 )
diff --git a/xaib/__init__.py b/xaib/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 __author__ = "Ilia Moiseev"
 __author_email__ = "ilia.moiseev.5@yandex.ru"
 

diff --git a/xaib/base/base.py b/xaib/base/base.py
@@ -1,3 +1,4 @@
+from .. import __version__ as version
 from typing import Union, List, Dict, Any, Callable
 from cascade import data as cdd
 from cascade import models as cdm
@@ -72,7 +73,7 @@ def evaluate(
         expl: Explainer,
         batch_size: int = 1,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -82,6 +83,8 @@ def evaluate(
         self.params["direction"] = self.direction
         self.params["dataset"] = self._ds.name
         self.params["model"] = self._model.name
+        self.params["model_params"] = self._model.params
+        self.params["model_metrics"] = self._model.metrics
         self.metrics[self.name] = value
 
 
@@ -95,6 +98,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
         self._metric_objs = dict()
+        self._meta_prefix.update({"xaib_version": version})
 
     def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric
@@ -104,7 +108,7 @@ def evaluate(
         name: str,
         expl: Explainer,
         metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
@@ -146,9 +150,12 @@ def _get(self, name: str) -> Any:
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == "all":
-            return self._get_all()
-        return self._get(name)
+        try:
+            if name == "all":
+                return self._get_all()
+            return self._get(name)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create object {name} in {self}") from e
 
     def add(
         self,
@@ -158,3 +165,6 @@ def add(
     ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs
+
+    def get_names(self):
+        return list(self._constructors.keys())
diff --git a/xaib/datasets/sk_dataset.py b/xaib/datasets/sk_dataset.py
@@ -1,20 +1,30 @@
 from typing import Any, Dict
+from cascade.base import PipeMeta
 
 import numpy as np
 from cascade import data as cdd
-from sklearn.datasets import load_iris, load_digits
+from sklearn.datasets import (
+    load_iris,
+    load_digits,
+    load_wine,
+    load_breast_cancer,
+)
 from sklearn.model_selection import train_test_split
 
 
 class SkDataset(cdd.SizedDataset):
+    # TODO: this is classification dataset - should be renamed
     def __init__(self, name, split, frac=0.8, *args, **kwargs) -> None:
         super().__init__(**kwargs)
 
         self.name = name
+        self.split = split
 
         constructors = {
             "iris": load_iris(return_X_y=True),
+            "wine": load_wine(return_X_y=True),
             "digits": load_digits(return_X_y=True),
+            "breast_cancer": load_breast_cancer(return_X_y=True),
         }
 
         if name not in constructors:
@@ -37,3 +47,10 @@ def __len__(self) -> int:
 
     def __getitem__(self, index: int) -> Dict[str, Any]:
         return {"item": self.x[index], "label": self.y[index]}
+
+    def get_meta(self):
+        meta = super().get_meta()
+        meta[0]["dataset_name"] = self.name
+        meta[0]["split"] = self.split
+        meta[0]["labels"] = self.labels
+        return meta
diff --git a/xaib/datasets/synthetic_dataset.py b/xaib/datasets/synthetic_dataset.py
@@ -31,3 +31,8 @@ def __len__(self) -> int:
 
     def __getitem__(self, index: int) -> Dict[str, Any]:
         return {"item": self.x[index], "label": self.y[index]}
+
+    def get_meta(self):
+        meta = super().get_meta()
+        meta[0]["dataset_name"] = self.name
+        return meta
diff --git a/xaib/docs/source/conf.py b/xaib/docs/source/conf.py
@@ -78,7 +78,7 @@
     "html_minify": True,
     "css_minify": True,
     "nav_title": "XAIB - Open and extensible benchmark for XAI methods",
-    "logo_icon": "<ruby>細部<rt>さいぶ</rt></ruby>",  # "&#x8cea",  #'&#x7d30&#x90e8'
+    "logo_icon": "細部",  # "&#x8cea",  #'&#x7d30&#x90e8'
 }
 
 html_sidebars = {

diff --git a/xaib/evaluation/dataset_factory.py b/xaib/evaluation/dataset_factory.py
@@ -16,7 +16,7 @@ def __init__(self) -> None:
         super().__init__()
         self._constructors["synthetic"] = lambda: generate_dataset(
             SyntheticDataset,
-            n_samples=100,
+            n_samples=1000,
             n_features=14,
             random_state=0,
             n_informative=14,
@@ -28,7 +28,7 @@ def __init__(self) -> None:
         self._constructors["synthetic_noisy"] = lambda: generate_dataset(
             SyntheticDataset,
             name="synthetic_noisy",
-            n_samples=100,
+            n_samples=1000,
             n_features=14,
             random_state=0,
             n_informative=7,
@@ -40,6 +40,12 @@ def __init__(self) -> None:
         self._constructors["iris"] = lambda: generate_dataset(
             SkDataset, "iris", frac=0.8
         )
+        self._constructors["wine"] = lambda: generate_dataset(
+            SkDataset, "wine", frac=0.8
+        )
         self._constructors["digits"] = lambda: generate_dataset(
             SkDataset, "digits", frac=0.8
         )
+        self._constructors["breast_cancer"] = lambda: generate_dataset(
+            SkDataset, "breast_cancer", frac=0.8
+        )
diff --git a/xaib/evaluation/example_selection/case_factory.py b/xaib/evaluation/example_selection/case_factory.py
@@ -20,7 +20,7 @@ def correctness(test_ds, model):
 
 
 class CaseFactory(Factory):
-    def __init__(self, test_ds: Dataset, model: Model) -> None:
+    def __init__(self, test_ds: Dataset = None, model: Model = None) -> None:
         super().__init__()
         self._constructors["continuity"] = lambda: continuity(test_ds, model)
         self._constructors["contrastivity"] = lambda: ContrastivityCase(test_ds, model)

diff --git a/xaib/evaluation/example_selection/example_selection.py b/xaib/evaluation/example_selection/example_selection.py
@@ -1,11 +1,14 @@
 import os
 import sys
 
-from cascade import data as cdd
 from cascade.models import ModelRepo
 from cascade.utils.sk_model import SkModel
 from xaib.evaluation import DatasetFactory, ModelFactory
-from xaib.evaluation.example_selection import ExperimentFactory, ExplainerFactory
+from xaib.evaluation.example_selection import (
+    ExperimentFactory,
+    ExplainerFactory,
+    CaseFactory,
+)
 
 SCRIPT_DIR = os.path.dirname(__file__)
 # xaib/results/...
@@ -14,35 +17,37 @@
 )
 
 sys.path.append(os.path.abspath(os.path.dirname(SCRIPT_DIR)))
-from utils import WrapperModel, visualize_results
-
-
-class SkWrapper(SkModel):
-    def __init__(self, *args, blocks=None, name=None, **kwargs) -> None:
-        super().__init__(*args, blocks=blocks, **kwargs)
-        self.name = name
+from utils import Setup, visualize_results
 
 
-BS = 5
+BS = 100
 
 # Overwrite previous run
 ModelRepo(REPO_PATH, overwrite=True)
 
-for dataset in ["synthetic_noisy", "synthetic"]:
-    for model in ["knn"]:
-        train_ds, test_ds = DatasetFactory().get(dataset)
-        print(train_ds.get_meta())
+factories = (DatasetFactory(), ModelFactory(), ExplainerFactory(), CaseFactory())
+setups = [Setup(*factories, models=["knn"])]
+
+for setup in setups:
+    for dataset in setup.datasets:
+        for model in setup.models:
+            train_ds, test_ds = DatasetFactory().get(dataset)
+            print(train_ds.get_meta())
+
+            model = ModelFactory(train_ds, test_ds).get(model)
+            print(model.get_meta())
 
-        model = ModelFactory(train_ds, test_ds).get(model)
-        print(model.get_meta())
+            explainers = {
+                explainer: ExplainerFactory(train_ds, model).get(explainer)
+                for explainer in setup.explainers
+            }
 
-        explainers = ExplainerFactory(train_ds, model).get("all")
-        experiment_factory = ExperimentFactory(
-            REPO_PATH, explainers, test_ds, model, BS
-        )
+            experiment_factory = ExperimentFactory(
+                REPO_PATH, explainers, test_ds, model, BS
+            )
 
-        experiments = experiment_factory.get("all")
-        for name in experiments:
-            experiments[name]()
+            for case in setup.cases:
+                experiment = experiment_factory.get(case)
+                experiment()
 
 visualize_results(REPO_PATH, REPO_PATH)
diff --git a/xaib/evaluation/example_selection/experiment_factory.py b/xaib/evaluation/example_selection/experiment_factory.py
@@ -5,7 +5,9 @@
 
 
 class ExperimentFactory(Factory):
-    def __init__(self, repo_path, explainers, test_ds, model, batch_size) -> None:
+    def __init__(
+        self, repo_path=None, explainers=None, test_ds=None, model=None, batch_size=None
+    ) -> None:
         super().__init__()
 
         case_factory = CaseFactory(test_ds, model)

diff --git a/xaib/evaluation/example_selection/explainer_factory.py b/xaib/evaluation/example_selection/explainer_factory.py
@@ -5,7 +5,7 @@
 
 
 class ExplainerFactory(Factory):
-    def __init__(self, train_ds: Dataset, model: Model) -> None:
+    def __init__(self, train_ds: Dataset = None, model: Model = None) -> None:
         super().__init__()
         self._constructors["const"] = lambda: ConstantExplainer(
             train_ds, train_ds[0]["item"]

diff --git a/xaib/evaluation/feature_importance/case_factory.py b/xaib/evaluation/feature_importance/case_factory.py
@@ -21,7 +21,9 @@ def continuity(test_ds, model):
 
 
 class CaseFactory(Factory):
-    def __init__(self, test_ds: Dataset, model: Model, labels) -> None:
+    def __init__(
+        self, test_ds: Dataset = None, model: Model = None, labels=None
+    ) -> None:
         super().__init__()
         self._constructors["correctness"] = lambda: correctness(
             test_ds, model, labels=labels

diff --git a/xaib/evaluation/feature_importance/experiment_factory.py b/xaib/evaluation/feature_importance/experiment_factory.py
@@ -6,7 +6,13 @@
 
 class ExperimentFactory(Factory):
     def __init__(
-        self, repo_path, explainers, test_ds, model, labels, batch_size
+        self,
+        repo_path=None,
+        explainers=None,
+        test_ds=None,
+        model=None,
+        labels=None,
+        batch_size=None,
     ) -> None:
         super().__init__()
 

diff --git a/xaib/evaluation/feature_importance/explainer_factory.py b/xaib/evaluation/feature_importance/explainer_factory.py
@@ -6,7 +6,9 @@
 
 
 class ExplainerFactory(Factory):
-    def __init__(self, train_ds: Dataset, model: Model, labels=[0, 1]) -> None:
+    def __init__(
+        self, train_ds: Dataset = None, model: Model = None, labels=None
+    ) -> None:
         super().__init__()
         self._constructors["const"] = lambda: ConstantExplainer(constant=1)
         self._constructors["random"] = lambda: RandomExplainer(shift=-15, magnitude=10)