Fix smoke tests

.github/workflows/python-app.yml

@@ -4,14 +4,15 @@ on:
   push:
     branches: [ "main" ]
   pull_request:
+  workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
+    environment: Scheduled Testing
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
@@ -20,6 +21,55 @@ jobs:
 
     - uses: actions/checkout@v4
 
+    - name: Vertex AI authentication
+      uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.VERTEX_AI_CREDENTIALS_JSON }}'
+
+    - name: Write secrets
+      env:
+        SECRETS_CONFIG: |
+          [anthropic]
+          api_key = "${{ secrets.ANTHROPIC_API_KEY }}"
+
+          [together]
+          api_key = "${{ secrets.TOGETHER_API_KEY }}"
+
+          [openai]
+          api_key = "${{ secrets.OPENAI_API_KEY }}"
+
+          [hugging_face]
+          token = "${{ secrets.HUGGING_FACE_TOKEN }}"
+
+          [google_ai]
+          api_key = "${{ secrets.GOOGLE_AI_API_KEY }}"
+
+          [mistralai]
+          api_key = "${{ secrets.MISTRAL_AI_API_KEY }}"
+
+          [vertexai]
+          project_id = "${{ secrets.GOOGLE_PROJECT_ID }}"
+          region = "us-central1"
+
+          [azure_phi_3_5_mini_endpoint]
+          api_key = "${{ secrets.AZURE_PHI_3_5_MINI_API_KEY }}"
+
+          [azure_phi_3_5_moe_endpoint]
+          api_key = "${{ secrets.AZURE_PHI_3_5_MOE_API_KEY }}"
+
+          [modellab_files]
+          token = "${{ secrets.MODELLAB_FILE_DOWNLOAD_TOKEN }}"
+
+          [nvidia-nim-api]
+          api_key = "${{ secrets.NVIDIA_API_KEY }}"
+
+          [demo]
+          api_key="12345"
+
+      run: |
+        mkdir -p config
+        echo "$SECRETS_CONFIG" > config/secrets.toml
+
     - name: Install poetry
       run: pipx install poetry
 
@@ -41,4 +91,3 @@ jobs:
 
     - name: Run mypy
       run: poetry run mypy --follow-imports silent --exclude modelbench src/modelgauge
-

.github/workflows/scheduled-smoke-test.yml

@@ -55,6 +55,11 @@ jobs:
     - name: Install with plugins
       run: poetry install --no-interaction --sync --extras all_plugins
 
+    - name: Vertex AI authentication
+      uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.VERTEX_AI_CREDENTIALS_JSON }}'
+
     - name: Write secrets
       env:
         SECRETS_CONFIG: |
@@ -69,12 +74,32 @@ jobs:
 
           [hugging_face]
           token = "${{ secrets.HUGGING_FACE_TOKEN }}"
-          
+
           [google_ai]
           api_key = "${{ secrets.GOOGLE_AI_API_KEY }}"
 
+          [mistralai]
+          api_key = "${{ secrets.MISTRAL_AI_API_KEY }}"
+
+          [vertexai]
+          project_id = "${{ secrets.GOOGLE_PROJECT_ID }}"
+          region = "us-central1"
+
+          [azure_phi_3_5_mini_endpoint]
+          api_key = "${{ secrets.AZURE_PHI_3_5_MINI_API_KEY }}"
+
+          [azure_phi_3_5_moe_endpoint]
+          api_key = "${{ secrets.AZURE_PHI_3_5_MOE_API_KEY }}"
+
+          [modellab_files]
+          token = "${{ secrets.MODELLAB_FILE_DOWNLOAD_TOKEN }}"
+
+          [nvidia-nim-api]
+          api_key = "${{ secrets.NVIDIA_API_KEY }}"
+
           [demo]
           api_key="12345"
+
       run: |
         mkdir -p config
         echo "$SECRETS_CONFIG" > config/secrets.toml

plugins/validation_tests/test_object_creation.py

@@ -20,7 +20,8 @@
 
 # Ensure all the plugins are available during testing.
 load_plugins()
-_FAKE_SECRETS = fake_all_secrets()
+# Some tests need to download a file from modellab, which requires a real auth token
+_FAKE_SECRETS = fake_all_secrets(use_real_secrets_for=("modellab_files",))
 
 
 @pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items()])
@@ -70,7 +71,14 @@ def test_all_suts_construct_and_record_init(sut_name):
     assert isinstance(sut.initialization_record, InitializationRecord)
 
 
-SUTS_THAT_WE_DONT_CARE_ABOUT_FAILING = {"StripedHyena-Nous-7B", "olmo-7b-0724-instruct-hf"}
+# mistral-nemo-instruct-2407-hf removed from test because of a bug in HF's date parsing code
+# https://github.com/huggingface/huggingface_hub/issues/2671
+# Remove mistral-nemo-instruct-2407-hf from this set once it's fixed.
+SUTS_THAT_WE_DONT_CARE_ABOUT_FAILING = {
+    "StripedHyena-Nous-7B",
+    "olmo-7b-0724-instruct-hf",
+    "mistral-nemo-instruct-2407-hf",
+}
 
 
 # This test can take a while, and we don't want a test run to fail

src/modelgauge/secret_values.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+from typing import Generic, List, Mapping, Optional, Sequence, Type, TypeVar
+
 from modelgauge.general import get_concrete_subclasses
 from pydantic import BaseModel
-from typing import Generic, List, Mapping, Optional, Sequence, Type, TypeVar
 
 
 class SecretDescription(BaseModel):

tests/modelgauge_tests/fake_secrets.py

@@ -1,23 +1,30 @@
-from modelgauge.secret_values import (
-    RawSecrets,
-    RequiredSecret,
-    SecretDescription,
-    get_all_secrets,
-)
 from typing import Dict
 
+from modelgauge.config import load_secrets_from_config
+
+from modelgauge.secret_values import get_all_secrets, RawSecrets, RequiredSecret, SecretDescription
+
 
 class FakeRequiredSecret(RequiredSecret):
     @classmethod
     def description(cls) -> SecretDescription:
         return SecretDescription(scope="some-scope", key="some-key", instructions="some-instructions")
 
 
-def fake_all_secrets(value="some-value") -> RawSecrets:
+def fake_all_secrets(value="some-value", use_real_secrets_for: list[str] | None = None) -> RawSecrets:
     secrets = get_all_secrets()
     raw_secrets: Dict[str, Dict[str, str]] = {}
+    if use_real_secrets_for:
+        real_secrets = load_secrets_from_config()
+    else:
+        real_secrets = {}
+
     for secret in secrets:
         if secret.scope not in raw_secrets:
             raw_secrets[secret.scope] = {}
-        raw_secrets[secret.scope][secret.key] = value
+        if use_real_secrets_for and secret.scope in use_real_secrets_for:
+            raw_secrets[secret.scope][secret.key] = real_secrets[secret.scope][secret.key]
+        else:
+            raw_secrets[secret.scope][secret.key] = value
+
     return raw_secrets