transformers 4.46 compatibility (#960)

* transformers 4.46 compatibility

* test

* upgrade python

* bump python

* Apply suggestions from code review

* fix trainer

* update examples requirements

* bump ipex version in tests

* update setup

* no ipex whl for python 3.12

* bump tests transformers version

* bump tests transformers version

* fix

---------

Co-authored-by: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com>

.github/workflows/check_code_quality.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8]
+        python-version: ["3.9"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}

.github/workflows/test_generation.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.9", "3.12"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}

.github/workflows/test_inc.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.9", "3.11"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -35,8 +35,8 @@ jobs:
         python -m pip install --upgrade pip
         pip install cmake
         pip install py-cpuinfo
-        pip install torch==2.3.0 torchaudio==2.3.0 torchvision==0.18 --index-url https://download.pytorch.org/whl/cpu
-        pip install intel-extension-for-pytorch==2.3.0
+        pip install torch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 --index-url https://download.pytorch.org/whl/cpu
+        pip install intel-extension-for-pytorch==2.4.0
         pip install datasets==2.19.0
         pip install .[neural-compressor,diffusers,tests]
         pip install peft

.github/workflows/test_ipex.yml

@@ -21,13 +21,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9"]
         transformers-version: ["4.39.0", "4.44.*"]
         ipex-version: ["2.2.0", "2.3.*"]
         include:
-          - python-version: 3.8
-            transformers-version: 4.39.0
-            ipex-version: 2.2.0
+          - python-version: "3.10"
+            transformers-version: "4.39.0"
+            ipex-version: "2.2.0"
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/test_offline.yaml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}

.github/workflows/test_openvino.yml

@@ -20,8 +20,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.12"]
-        transformers-version: ["4.36.0", "4.45.*"]
+        python-version: ["3.9", "3.12"]
+        transformers-version: ["4.36.0", "latest"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -32,16 +32,16 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install lowest compatible transformers version
+        if: ${{ matrix.transformers-version != 'latest' }}
+        run: pip install transformers==${{ matrix.transformers-version }} accelerate==0.*
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           # install PyTorch CPU version to avoid installing CUDA packages on GitHub runner without GPU
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install .[openvino,openvino-tokenizers,tests,diffusers] onnxruntime
-          pip install transformers==${{ matrix.transformers-version }}
-
-      - if: ${{ matrix.transformers-version == '4.36.0' }}
-        run: pip install accelerate==0.*
 
       - name: Test with Pytest
         env:

.github/workflows/test_openvino_basic.yml

@@ -24,7 +24,7 @@ jobs:
         # This also ensures that the test fails if dependencies break for Python 3.7
         python-version: ["3.9", "3.12"]
         os: ["ubuntu-22.04", "windows-latest"]
-        transformers-version: ["4.45.*"]
+        transformers-version: ["latest"]
         openvino: ["openvino openvino-tokenizers"]
         nncf: ["nncf"]
         include:
@@ -35,12 +35,12 @@ jobs:
             nncf: "nncf"
           - python-version: "3.12"
             os: "ubuntu-22.04"
-            transformers-version: "4.45.*"
+            transformers-version: "latest"
             openvino: "--pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
             nncf: "nncf"
           - python-version: "3.12"
             os: "ubuntu-22.04"
-            transformers-version: "4.45.*"
+            transformers-version: "latest"
             openvino: "--pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
             nncf: "git+https://github.com/openvinotoolkit/nncf.git"
 
@@ -53,6 +53,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install lowest compatible transformers version
+        if: ${{ matrix.transformers-version != 'latest' }}
+        run: pip install transformers==${{ matrix.transformers-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -61,7 +65,6 @@ jobs:
           # Install openvino manually to prevent dependency conflicts when .[openvino] pins
           # optimum or transformers to a specific version
           pip install ${{ matrix.openvino }}
-          pip install transformers==${{ matrix.transformers-version }}
           pip install .[tests]
 
       - name: Pip freeze

.github/workflows/test_openvino_examples.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.12"]
+        python-version: ["3.9", "3.12"]
 
     runs-on: ubuntu-22.04
 

.github/workflows/test_openvino_notebooks.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.12"]
+        python-version: ["3.9", "3.12"]
 
     runs-on: ubuntu-22.04
 

examples/openvino/audio-classification/requirements.txt

@@ -1,3 +1,4 @@
+transformers>=4.36.0,<4.46.0
 datasets>=1.14.0,<2.20.0
 evaluate
 librosa

examples/openvino/image-classification/requirements.txt

@@ -1,3 +1,4 @@
+transformers>=4.36.0,<4.46.0
 datasets>=1.14.0,<2.20.0
 torch >= 1.9.0
 torchvision>=0.6.0

examples/openvino/question-answering/requirements.txt

@@ -1,3 +1,4 @@
+transformers>=4.36.0,<4.46.0
 datasets>=1.14.0,<2.20.0
 torch >= 1.9.0
 evaluate

examples/openvino/text-classification/requirements.txt

@@ -1,3 +1,4 @@
+transformers>=4.36.0,<4.46.0
 datasets>=1.14.0,<2.20.0
 sentencepiece != 0.1.92
 scipy

optimum/intel/neural_compressor/trainer.py

@@ -39,6 +39,7 @@
 from transformers import Trainer
 from transformers.data.data_collator import DataCollator
 from transformers.debug_utils import DebugOption, DebugUnderflowOverflow
+from transformers.feature_extraction_utils import FeatureExtractionMixin
 from transformers.modeling_utils import PreTrainedModel, get_parameter_dtype, unwrap_model
 from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase
@@ -104,7 +105,7 @@
     from neural_compressor.config import _BaseQuantizationConfig
 
 
-__version__ = "4.22.2"
+__version__ = "4.46.0"
 
 
 logger = logging.get_logger(__name__)
@@ -122,8 +123,9 @@ def __init__(
         data_collator: Optional[DataCollator] = None,
         train_dataset: Optional[Dataset] = None,
         eval_dataset: Optional[Dataset] = None,
-        tokenizer: Optional[PreTrainedTokenizerBase] = None,
+        processing_class: Optional[Union[PreTrainedTokenizerBase, FeatureExtractionMixin]] = None,
         model_init: Callable[[], PreTrainedModel] = None,
+        compute_loss_func: Optional[Callable] = None,
         compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,
         callbacks: Optional[List[TrainerCallback]] = None,
         optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None),
@@ -132,6 +134,7 @@ def __init__(
         pruning_config: Optional[_BaseQuantizationConfig] = None,
         distillation_config: Optional[_BaseQuantizationConfig] = None,
         task: Optional[str] = None,
+        **kwargs,
     ):
         self.neftune_noise_alpha = None
 
@@ -141,12 +144,12 @@ def __init__(
             data_collator,
             train_dataset,
             eval_dataset,
-            tokenizer,
-            model_init,
-            compute_metrics,
-            callbacks,
-            optimizers,
-            preprocess_logits_for_metrics,
+            processing_class or kwargs.get("tokenizer", None),
+            model_init=model_init,
+            compute_metrics=compute_metrics,
+            callbacks=callbacks,
+            optimizers=optimizers,
+            preprocess_logits_for_metrics=preprocess_logits_for_metrics,
         )
 
         if self.args.device.type == "cuda" and not is_neural_compressor_version(">", "2.0.0"):
@@ -766,7 +769,7 @@ def _get_logits(model_outputs):
         output_names = ["logits", "start_logits", "end_logits"]
         return tuple(model_outputs.get(name) for name in output_names if name in model_outputs)
 
-    def compute_loss(self, model, inputs, return_outputs=False):
+    def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
         """
         How the loss is computed by Trainer. By default, all models return the loss in the first element.
         """

optimum/intel/openvino/trainer.py

@@ -216,6 +216,11 @@ def __init__(
         logger.warning("OVTrainer is deprecated and will be removed in optimum-intel v1.22.0.")
 
         if is_transformers_version(">=", "4.45.0"):
+            if is_transformers_version(">=", "4.46.0"):
+                raise ImportError(
+                    f"Unsupported transformers version found is {_transformers_version} which is not supported by the OVTrainer. Please downgrade to v4.44"
+                )
+
             logger.warning(
                 f"The transformers version found is {_transformers_version} which is not officially supported by the OVTrainer, use at your own risk"
             )

setup.py

@@ -28,7 +28,7 @@
 
 INSTALL_REQUIRE = [
     "torch>=1.11",
-    "transformers>=4.36,<4.46",
+    "transformers>=4.36,<4.47",
     "optimum~=1.23",
     "datasets>=1.4.0",
     "sentencepiece",
@@ -60,7 +60,7 @@
 QUALITY_REQUIRE = ["black~=23.1", "ruff==0.4.4"]
 
 EXTRAS_REQUIRE = {
-    "neural-compressor": ["neural-compressor[pt]>3.0", "accelerate"],
+    "neural-compressor": ["neural-compressor[pt]>3.0", "accelerate", "transformers<4.46"],
     "openvino": [
         "openvino==2024.4.1.dev20240926",
         "nncf>=2.11.0",

tests/openvino/test_quantization.py

@@ -786,6 +786,9 @@ class OVTrainerTest(unittest.TestCase):
     SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = (("albert", 64, 39),)
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS)
+    @unittest.skipIf(
+        is_transformers_version(">=", "4.46"), reason="OVTrainer is not compatible with transformers>=v4.46"
+    )
     def test_aware_training_quantization(self, model_name, expected_fake_quantize, expected_int8):
         model_id = MODEL_NAMES[model_name]
         model = AutoModelForSequenceClassification.from_pretrained(model_id, attn_implementation="eager")

tests/openvino/test_training.py

@@ -475,7 +475,10 @@ class OVTrainerTextClassificationTrainingTest(OVTrainerBaseTrainingTest):
     task = "sequence-classification"
 
     @parameterized.expand(OVTRAINER_TEXT_CLASSIFICATION_TEST_DESCRIPTORS.items())
-    @unittest.skipIf(is_transformers_version("<", "4.41.0"), reason="Mismatch in expected fake quantized op")
+    @unittest.skipIf(
+        is_transformers_version("<", "4.41") or is_transformers_version(">=", "4.46"),
+        reason="Mismatch in expected fake quantized op and incompatible with transformers v4.46",
+    )
     def test_training(self, _, desc: OVTrainerTestDescriptor):
         self.run_ovtrainer_training_checks(desc)
 
@@ -627,7 +630,10 @@ class OVTrainerImageClassificationTrainingTest(OVTrainerBaseTrainingTest):
     @parameterized.expand(OVTRAINER_IMAGE_CLASSIFICATION_TEST_DESCRIPTORS.items())
     @pytest.mark.run_slow
     @slow
-    @unittest.skipIf(is_transformers_version("<", "4.41.0"), reason="Mismatch in expected fake quantized op")
+    @unittest.skipIf(
+        is_transformers_version("<", "4.41") or is_transformers_version(">=", "4.46"),
+        reason="Mismatch in expected fake quantized op and incompatible with transformers v4.46",
+    )
     def test_training(self, _, desc: OVTrainerTestDescriptor):
         self.run_ovtrainer_training_checks(desc)
 
@@ -808,6 +814,9 @@ class OVTrainerAudioClassificationTrainingTest(OVTrainerBaseTrainingTest):
     @parameterized.expand(OVTRAINER_AUDIO_CLASSIFICATION_TEST_DESCRIPTORS.items())
     @pytest.mark.run_slow
     @slow
+    @unittest.skipIf(
+        is_transformers_version(">=", "4.46"), reason="OVTrainer is not compatible with transformers>=v4.46"
+    )
     def test_training(self, _, desc: OVTrainerTestDescriptor):
         self.run_ovtrainer_training_checks(desc)