diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d75efe4..7c7f36b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,53 +5,3 @@ on: branches: [ "master" ] pull_request: branches: [ "master", "develop" ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r explainers_requirements.txt - pip install -r https://raw.githubusercontent.com/oxid15/cascade/main/utils_requirements.txt - pip install . - - - name: Feature importance - run: | - cd xaib/evaluation/feature_importance/ - ./evaluate.sh - zip -r repo . - - - name: Example selection - run: | - pwd - ls - cd xaib/evaluation/example_selection - ./evaluate.sh - zip -r repo . - - - name: Archive feature importance results - uses: actions/upload-artifact@v3 - with: - name: Feature importance results - path: xaib/evaluation/feature_importance/repo.zip - - - name: Archive example selection results - uses: actions/upload-artifact@v3 - with: - name: Example selection results - path: xaib/evaluation/example_selection/repo.zip diff --git a/docs/_modules/index.html b/docs/_modules/index.html index 682c572..81320f4 100644 --- a/docs/_modules/index.html +++ b/docs/_modules/index.html @@ -1 +1 @@ - Overview: module code — xai-benchmark 0.2.0 documentation Skip to content

All modules for which code is available

\ No newline at end of file + Overview: module code — xai-benchmark 0.2.0 documentation Skip to content

All modules for which code is available

\ No newline at end of file diff --git a/docs/_modules/xaib/base/base.html b/docs/_modules/xaib/base/base.html index 74275cf..f9df119 100644 --- a/docs/_modules/xaib/base/base.html +++ b/docs/_modules/xaib/base/base.html @@ -1,5 +1,6 @@ - xaib.base.base — xai-benchmark 0.2.0 documentation Skip to content

Source code for xaib.base.base

-from typing import Union, List, Dict, Any, Callable
+                        xaib.base.base — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.base.base
+from .. import __version__ as version
+from typing import Union, List, Dict, Any, Callable
 from cascade import data as cdd
 from cascade import models as cdm
 
@@ -73,7 +74,7 @@
         expl: Explainer,
         batch_size: int = 1,
         expl_kwargs: Union[Dict[Any, Any], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if expl_kwargs is None:
             expl_kwargs = {}
@@ -83,6 +84,8 @@
         self.params["direction"] = self.direction
         self.params["dataset"] = self._ds.name
         self.params["model"] = self._model.name
+        self.params["model_params"] = self._model.params
+        self.params["model_metrics"] = self._model.metrics
         self.metrics[self.name] = value
 
 
@@ -95,7 +98,8 @@
 
[docs]    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.name = None
-        self._metric_objs = dict()

+        self._metric_objs = dict()
+        self._meta_prefix.update({"xaib_version": version})

 
 
[docs]    def add_metric(self, name: str, metric: Metric) -> None:
         self._metric_objs[name] = metric

@@ -105,7 +109,7 @@
         name: str,
         expl: Explainer,
         metrics_kwargs: Union[Dict[str, Dict[Any, Any]], None] = None,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         if metrics_kwargs is None:
             metrics_kwargs = {name: {} for _ in self._metric_objs}
@@ -147,9 +151,12 @@
         return constructor(**kwargs)
 
     def get(self, name: str) -> Union[Dict[str, Any], Any]:
-        if name == "all":
-            return self._get_all()
-        return self._get(name)
+        try:
+            if name == "all":
+                return self._get_all()
+            return self._get(name)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create object {name} in {self}") from e
 
     def add(
         self,
@@ -159,4 +166,7 @@
     ) -> None:
         self._constructors[name] = constructor
         self._constructors_kwargs[name] = constr_kwargs
+
+    def get_names(self):
+        return list(self._constructors.keys())
\ No newline at end of file diff --git a/docs/_modules/xaib/cases/example_selection/continuity_case.html b/docs/_modules/xaib/cases/example_selection/continuity_case.html index 15d56c0..ec4edf0 100644 --- a/docs/_modules/xaib/cases/example_selection/continuity_case.html +++ b/docs/_modules/xaib/cases/example_selection/continuity_case.html @@ -1,4 +1,4 @@ - xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0 documentation Skip to content

Source code for xaib.cases.example_selection.continuity_case

+                        xaib.cases.example_selection.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.example_selection.continuity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/coherence_case.html b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
index bfbf9db..e4b0c65 100644
--- a/docs/_modules/xaib/cases/feature_importance/coherence_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/coherence_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
+                        xaib.cases.feature_importance.coherence_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.coherence_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/compactness_case.html b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
index 37d74b6..6a4cd0f 100644
--- a/docs/_modules/xaib/cases/feature_importance/compactness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/compactness_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
+                        xaib.cases.feature_importance.compactness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.compactness_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/continuity_case.html b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
index 232345c..808d6c3 100644
--- a/docs/_modules/xaib/cases/feature_importance/continuity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/continuity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
+                        xaib.cases.feature_importance.continuity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.continuity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
index 8f82247..3a4e0e2 100644
--- a/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/contrastivity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
+                        xaib.cases.feature_importance.contrastivity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.contrastivity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/correctness_case.html b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
index eba1378..df84884 100644
--- a/docs/_modules/xaib/cases/feature_importance/correctness_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/correctness_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
+                        xaib.cases.feature_importance.correctness_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.correctness_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
index 052e9d8..cde6c99 100644
--- a/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
+++ b/docs/_modules/xaib/cases/feature_importance/covariate_complexity_case.html
@@ -1,4 +1,4 @@
-                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
+                        xaib.cases.feature_importance.covariate_complexity_case — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.cases.feature_importance.covariate_complexity_case
 from typing import Any
 
 from ...base import Case, Dataset, Model
diff --git a/docs/_modules/xaib/datasets/synthetic_dataset.html b/docs/_modules/xaib/datasets/synthetic_dataset.html
index fa51e37..aff3a56 100644
--- a/docs/_modules/xaib/datasets/synthetic_dataset.html
+++ b/docs/_modules/xaib/datasets/synthetic_dataset.html
@@ -1,4 +1,4 @@
-                        xaib.datasets.synthetic_dataset — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.datasets.synthetic_dataset
+                        xaib.datasets.synthetic_dataset — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.datasets.synthetic_dataset
 from typing import Any, Dict
 
 from cascade import data as cdd
@@ -31,5 +31,10 @@
         return len(self.x)
 
     def __getitem__(self, index: int) -> Dict[str, Any]:
-        return {"item": self.x[index], "label": self.y[index]}

+        return {"item": self.x[index], "label": self.y[index]}
+
+
[docs]    def get_meta(self):
+        meta = super().get_meta()
+        meta[0]["dataset_name"] = self.name
+        return meta

 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
index fab636a..1ed9cb6 100644
--- a/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/constant_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
+                        xaib.explainers.example_selection.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.constant_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
index f056b85..5dba701 100644
--- a/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/knn_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
+                        xaib.explainers.example_selection.knn_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.knn_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/example_selection/random_explainer.html b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
index 4e78602..a934bc5 100644
--- a/docs/_modules/xaib/explainers/example_selection/random_explainer.html
+++ b/docs/_modules/xaib/explainers/example_selection/random_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
+                        xaib.explainers.example_selection.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.example_selection.random_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
index 4f9cb4a..5014968 100644
--- a/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/constant_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
+                        xaib.explainers.feature_importance.constant_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.constant_explainer
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
index 29455d3..2700097 100644
--- a/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/lime_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
+                        xaib.explainers.feature_importance.lime_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.lime_explainer
 import numpy as np
 from lime import lime_tabular
 
diff --git a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
index a8d4835..00cf00a 100644
--- a/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/linear_regression_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
+                        xaib.explainers.feature_importance.linear_regression_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.linear_regression_explainer
 import numpy as np
 
 from ...base import Explainer
diff --git a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
index c253074..d1c2825 100644
--- a/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/random_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
+                        xaib.explainers.feature_importance.random_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.random_explainer
 from typing import Any
 
 import numpy as np
diff --git a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
index 31a172a..454f305 100644
--- a/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
+++ b/docs/_modules/xaib/explainers/feature_importance/shap_explainer.html
@@ -1,4 +1,4 @@
-                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
+                        xaib.explainers.feature_importance.shap_explainer — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.explainers.feature_importance.shap_explainer
 import numpy as np
 import shap
 
diff --git a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
index 8ca5981..6907164 100644
--- a/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
+++ b/docs/_modules/xaib/metrics/example_selection/covariate_regularity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.covariate_regularity
+                        xaib.metrics.example_selection.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.covariate_regularity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
index 5ce0bc2..f3cfa1e 100644
--- a/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
+++ b/docs/_modules/xaib/metrics/example_selection/parameter_randomization_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.parameter_randomization_check
+                        xaib.metrics.example_selection.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.parameter_randomization_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
index eba1101..223979f 100644
--- a/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
+++ b/docs/_modules/xaib/metrics/example_selection/small_noise_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.small_noise_check
+                        xaib.metrics.example_selection.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.small_noise_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
index 6f3b3f3..8de255a 100644
--- a/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
+++ b/docs/_modules/xaib/metrics/example_selection/target_discriminativeness.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.example_selection.target_discriminativeness — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.target_discriminativeness
+                        xaib.metrics.example_selection.target_discriminativeness — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.example_selection.target_discriminativeness
 from cascade import data as cdd
 from tqdm import tqdm
 
diff --git a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
index 985732a..7e50b68 100644
--- a/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
+++ b/docs/_modules/xaib/metrics/feature_importance/covariate_regularity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.covariate_regularity
+                        xaib.metrics.feature_importance.covariate_regularity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.covariate_regularity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/label_difference.html b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
index 16b70ed..168691c 100644
--- a/docs/_modules/xaib/metrics/feature_importance/label_difference.html
+++ b/docs/_modules/xaib/metrics/feature_importance/label_difference.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.label_difference — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.label_difference
+                        xaib.metrics.feature_importance.label_difference — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.label_difference
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
index 77e34e1..a9c3190 100644
--- a/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
+++ b/docs/_modules/xaib/metrics/feature_importance/other_disagreement.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.other_disagreement — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.other_disagreement
+                        xaib.metrics.feature_importance.other_disagreement — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.other_disagreement
 from typing import Any, Dict, List, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
index 5fb5d17..ead5a4e 100644
--- a/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
+++ b/docs/_modules/xaib/metrics/feature_importance/parameter_randomization_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.parameter_randomization_check
+                        xaib.metrics.feature_importance.parameter_randomization_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.parameter_randomization_check
 from typing import Any, Dict, Union
 
 import numpy as np
@@ -34,7 +34,7 @@
         super().__init__(ds, model, **kwargs)
         self._noisy_model = noisy_model
         self.name = "parameter_randomization_check"
-        self.direction = "down"

+        self.direction = "up"

 
 
[docs]    def compute(
         self,
diff --git a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
index 89b01ac..a46af42 100644
--- a/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
+++ b/docs/_modules/xaib/metrics/feature_importance/small_noise_check.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.small_noise_check
+                        xaib.metrics.feature_importance.small_noise_check — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.small_noise_check
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/metrics/feature_importance/sparsity.html b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
index 6093569..ff5fa78 100644
--- a/docs/_modules/xaib/metrics/feature_importance/sparsity.html
+++ b/docs/_modules/xaib/metrics/feature_importance/sparsity.html
@@ -1,4 +1,4 @@
-                        xaib.metrics.feature_importance.sparsity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.sparsity
+                        xaib.metrics.feature_importance.sparsity — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.metrics.feature_importance.sparsity
 from typing import Any, Dict, Union
 
 import numpy as np
diff --git a/docs/_modules/xaib/utils/utils.html b/docs/_modules/xaib/utils/utils.html
index 1e1e5db..a91ba0e 100644
--- a/docs/_modules/xaib/utils/utils.html
+++ b/docs/_modules/xaib/utils/utils.html
@@ -1,4 +1,4 @@
-                        xaib.utils.utils — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
+                        xaib.utils.utils — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Source code for xaib.utils.utils
 from typing import Dict, List
 
 import numpy as np
diff --git a/docs/cases.html b/docs/cases.html
index 7ab558d..0efdf26 100644
--- a/docs/cases.html
+++ b/docs/cases.html
@@ -1 +1 @@
-                        Cases — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
+                        Cases — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Cases
 
Coming soon
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/contribute.html b/docs/contribute.html
index 541726a..02d6623 100644
--- a/docs/contribute.html
+++ b/docs/contribute.html
@@ -1 +1 @@
-                        How to contribute — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
How to contribute
 
Any contributions are welcome! You can help to extend the picture of XAI-methods quality by adding:
 
  
 
 
Note
 
If you want to extend the benchmark by adding new Case, please fill the issue
 
 
 
 
 
 
 
   
\ No newline at end of file
+                        How to contribute — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
How to contribute
 
Any contributions are welcome! You can help to extend the picture of XAI-methods quality by adding:
 
  
 
 
Note
 
If you want to extend the benchmark by adding new Case, please fill the issue
 
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/contribute/add_dataset.html b/docs/contribute/add_dataset.html
index 4ebf643..1b55fe3 100644
--- a/docs/contribute/add_dataset.html
+++ b/docs/contribute/add_dataset.html
@@ -1,4 +1,4 @@
-                        Add dataset — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add dataset
 
New datasets may extend our understanding of how different explainers behave in context of different domains and tasks. To add your dataset, you should provide a Wrapper, which will download or access prepared data from disk.
 
 
Create data wrapper
 
First you need to create a wrapper with required interface and fields
 
import numpy as np
+                        Add dataset — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add dataset
 
New datasets may extend our understanding of how different explainers behave in context of different domains and tasks. To add your dataset, you should provide a Wrapper, which will download or access prepared data from disk.
 
 
Create data wrapper
 
First you need to create a wrapper with required interface and fields
 
import numpy as np
 from xaib import Dataset
 
 
diff --git a/docs/contribute/add_explainer.html b/docs/contribute/add_explainer.html
index d00e3ab..7c73955 100644
--- a/docs/contribute/add_explainer.html
+++ b/docs/contribute/add_explainer.html
@@ -1,4 +1,4 @@
-                        Add explainer — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add explainer
 
Explainers are the heart of this benchmark, they are being thorougly tested and the more of them added, the more we know
 
 
Create explainer wrapper
 
Explainers wrappers are less demanding than model’s which makes them easier to be implemented
 
import numpy as np
+                        Add explainer — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add explainer
 
Explainers are the heart of this benchmark, they are being thorougly tested and the more of them added, the more we know
 
 
Create explainer wrapper
 
Explainers wrappers are less demanding than model’s which makes them easier to be implemented
 
import numpy as np
 from xaib import Explainer
 
 
diff --git a/docs/contribute/add_metric.html b/docs/contribute/add_metric.html
index 4d309ef..715ad0d 100644
--- a/docs/contribute/add_metric.html
+++ b/docs/contribute/add_metric.html
@@ -1,4 +1,4 @@
-                        Add metric — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add metric
 
Metrics are ways to numerically assess the quality of explainers and are parts of Cases
 
 
Create metric
 
First you need to create a Metric object - which will accept and explainer and data and return some value
 
 
Note
 
You should define self.name and self.direction of the metric in order for it to be displayed correctly in results. self.name is the short name of what is measured and self.direction denotes what values are considered better - greater or less.
 
 
from xaib import Metric
+                        Add metric — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add metric
 
Metrics are ways to numerically assess the quality of explainers and are parts of Cases
 
 
Create metric
 
First you need to create a Metric object - which will accept and explainer and data and return some value
 
 
Note
 
You should define self.name and self.direction of the metric in order for it to be displayed correctly in results. self.name is the short name of what is measured and self.direction denotes what values are considered better - greater or less.
 
 
from xaib import Metric
 
 class NewMetric(Metric):
     def __init__(self, ds, model *args, **kwargs):
diff --git a/docs/contribute/add_model.html b/docs/contribute/add_model.html
index 22c4e61..4ae5327 100644
--- a/docs/contribute/add_model.html
+++ b/docs/contribute/add_model.html
@@ -1,4 +1,4 @@
-                        Add model — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add model
 
New models and model classes provide information on how good explainers are in some particular cases.
 
 
Create model wrapper
 
First model wrapper should be implemented. It has many required methods that should be implemented. For example fit and evaluate methods are needed to be able to train the model on different datasets see specification in xaib/base and examples in xaib/evaluation/model_factory.py
 
import numpy as np
+                        Add model — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Add model
 
New models and model classes provide information on how good explainers are in some particular cases.
 
 
Create model wrapper
 
First model wrapper should be implemented. It has many required methods that should be implemented. For example fit and evaluate methods are needed to be able to train the model on different datasets see specification in xaib/base and examples in xaib/evaluation/model_factory.py
 
import numpy as np
 from xaib import Model
 
 
diff --git a/docs/datasets.html b/docs/datasets.html
index fa47cf3..6dc70c0 100644
--- a/docs/datasets.html
+++ b/docs/datasets.html
@@ -1 +1 @@
-                        Datasets — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
Until that, consider reading reference on datasets.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Datasets — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Datasets
 
Coming soon
 
Until that, consider reading reference on datasets.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/genindex.html b/docs/genindex.html
index 832b5fb..0f151ba 100644
--- a/docs/genindex.html
+++ b/docs/genindex.html
@@ -1 +1 @@
-                        Index — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | K | L | M | O | P | R | S | T 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
K
 
  
 
L
 
   
 
M
 
   
 
O
 
  
 
P
 
  
 
R
 
  
 
S
 
   
 
T
 
  
 
 
 
 
 
   
\ No newline at end of file
+                        Index — xai-benchmark 0.2.0 documentation                     Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
Index
 
 _ | A | B | C | D | E | G | K | L | M | O | P | R | S | T 
 
_
 
  
 
A
 
  
 
B
 
   
 
C
 
   
 
D
 
  
 
E
 
   
 
G
 
  
 
K
 
  
 
L
 
   
 
M
 
   
 
O
 
  
 
P
 
  
 
R
 
  
 
S
 
   
 
T
 
  
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
index 7083639..61d8d8a 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -1 +1 @@
-                        Welcome to XAIB! — xai-benchmark 0.2.0 documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        Welcome to XAIB! — xai-benchmark 0.2.0 documentation                      Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/installation.html b/docs/installation.html
index e8ea378..ae767bc 100644
--- a/docs/installation.html
+++ b/docs/installation.html
@@ -1,4 +1,4 @@
-                        Installation — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Installation
 
To install latest version of a package, run:
 
pip3 install xai-benchmark
+                        Installation — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Installation
 
To install latest version of a package, run:
 
pip3 install xai-benchmark
 
 
 
Remember to create virtual environment if you need one.
 
After the installation you can verify the package by printing out its version:
 
import xaib
 print(xaib.__version__)
 
 
 
To use all explainers you should also install explainers_requirements.txt which can be done directly
 
pip3 install -r https://raw.githubusercontent.com/oxid15/xai-benchmark/master/explainers_requirements.txt
diff --git a/docs/metrics.html b/docs/metrics.html
index 88a3a30..bcaf65c 100644
--- a/docs/metrics.html
+++ b/docs/metrics.html
@@ -1 +1 @@
-                        Metrics — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
Until that, consider reading reference on metrics.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Metrics — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Metrics
 
Coming soon
 
Until that, consider reading reference on metrics.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/models.html b/docs/models.html
index 8b112d4..f79d972 100644
--- a/docs/models.html
+++ b/docs/models.html
@@ -1 +1 @@
-                        Models — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
Until that, consider reading reference on models.
 
 
 
 
 
 
   
\ No newline at end of file
+                        Models — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
Models
 
Coming soon
 
Until that, consider reading reference on models.
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/modules.html b/docs/modules.html
index 6dce53a..4e00660 100644
--- a/docs/modules.html
+++ b/docs/modules.html
@@ -1 +1 @@
-                        XAIB modules reference — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
+                        XAIB modules reference — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
 
XAIB modules reference
 
  
 
 
 
 
 
 
   
\ No newline at end of file
diff --git a/docs/objects.inv b/docs/objects.inv
index 816eb05..afb5ad3 100644
Binary files a/docs/objects.inv and b/docs/objects.inv differ
diff --git a/docs/results.html b/docs/results.html
index f0005aa..d99e7e6 100644
--- a/docs/results.html
+++ b/docs/results.html
@@ -1 +1 @@
-                        Evaluation results — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
+                        Evaluation results — xai-benchmark 0.2.0 documentation                       Skip to content  
  
 
  
 
 
 
 
  
 
 
 
 
 
  
 
 
 
  
 
 
 
   
\ No newline at end of file
diff --git a/docs/results/example_selection.html b/docs/results/example_selection.html
index 823cd94..227667f 100644
--- a/docs/results/example_selection.html
+++ b/docs/results/example_selection.html
@@ -68,12 +68,7 @@
       

        
       

@@ -169,12 +164,7 @@