Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix caps to expected caps for LangChain provider (and to match docs) #1456

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .grit/dealias_trulens_eval.grit
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ any {
`from trulens_eval import Huggingface as $alias` => `from trulens.providers.huggingface.provider import Huggingface as $alias`,
`from trulens_eval import HuggingfaceLocal` => `from trulens.providers.huggingface.provider import HuggingfaceLocal`,
`from trulens_eval import HuggingfaceLocal as $alias` => `from trulens.providers.huggingface.provider import HuggingfaceLocal as $alias`,
`from trulens_eval import Langchain` => `from trulens.providers.langchain.provider import Langchain`,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we change these? I thought this is referencing the old trulens_eval API?

`from trulens_eval import Langchain as $alias` => `from trulens.providers.langchain.provider import Langchain as $alias`,
`from trulens_eval import LangChain` => `from trulens.providers.langchain.provider import LangChain`,
`from trulens_eval import LangChain as $alias` => `from trulens.providers.langchain.provider import LangChain as $alias`,
`from trulens_eval import TruChain` => `from trulens.apps.langchain.tru_chain import TruChain`,
`from trulens_eval import TruChain as $alias` => `from trulens.apps.langchain.tru_chain import TruChain as $alias`,
`from trulens_eval import TruLlama` => `from trulens.apps.llamaindex.tru_llama import TruLlama`,
Expand Down Expand Up @@ -204,8 +204,8 @@ any {
`from trulens_eval.feedback import Huggingface as $alias` => `from trulens.providers.huggingface.provider import Huggingface as $alias`,
`from trulens_eval.feedback import HuggingfaceLocal` => `from trulens.providers.huggingface.provider import HuggingfaceLocal`,
`from trulens_eval.feedback import HuggingfaceLocal as $alias` => `from trulens.providers.huggingface.provider import HuggingfaceLocal as $alias`,
`from trulens_eval.feedback import Langchain` => `from trulens.providers.langchain.provider import Langchain`,
`from trulens_eval.feedback import Langchain as $alias` => `from trulens.providers.langchain.provider import Langchain as $alias`,
`from trulens_eval.feedback import LangChain` => `from trulens.providers.langchain.provider import LangChain`,
`from trulens_eval.feedback import LangChain as $alias` => `from trulens.providers.langchain.provider import LangChain as $alias`,
`from trulens_eval.feedback import Cortex` => `from trulens.providers.cortex.provider import Cortex`,
`from trulens_eval.feedback import Cortex as $alias` => `from trulens.providers.cortex.provider import Cortex as $alias`,
`from trulens_eval.feedback.embeddings import Embeddings` => `from trulens.feedback.embeddings import Embeddings`,
Expand Down Expand Up @@ -308,8 +308,8 @@ any {
`from trulens_eval.feedback.provider import Huggingface as $alias` => `from trulens.providers.huggingface.provider import Huggingface as $alias`,
`from trulens_eval.feedback.provider import HuggingfaceLocal` => `from trulens.providers.huggingface.provider import HuggingfaceLocal`,
`from trulens_eval.feedback.provider import HuggingfaceLocal as $alias` => `from trulens.providers.huggingface.provider import HuggingfaceLocal as $alias`,
`from trulens_eval.feedback.provider import Langchain` => `from trulens.providers.langchain.provider import Langchain`,
`from trulens_eval.feedback.provider import Langchain as $alias` => `from trulens.providers.langchain.provider import Langchain as $alias`,
`from trulens_eval.feedback.provider import LangChain` => `from trulens.providers.langchain.provider import LangChain`,
`from trulens_eval.feedback.provider import LangChain as $alias` => `from trulens.providers.langchain.provider import LangChain as $alias`,
`from trulens_eval.feedback.provider.base import Provider` => `from trulens.core.feedback.provider import Provider`,
`from trulens_eval.feedback.provider.base import Provider as $alias` => `from trulens.core.feedback.provider import Provider as $alias`,
`from trulens_eval.feedback.provider.base import LLMProvider` => `from trulens.feedback.llm_provider import LLMProvider`,
Expand Down Expand Up @@ -406,8 +406,8 @@ any {
`from trulens_eval.feedback.provider.hugs import HuggingfaceBase as $alias` => `from trulens.providers.huggingface.provider import HuggingfaceBase as $alias`,
`from trulens_eval.feedback.provider.hugs import HuggingfaceLocal` => `from trulens.providers.huggingface.provider import HuggingfaceLocal`,
`from trulens_eval.feedback.provider.hugs import HuggingfaceLocal as $alias` => `from trulens.providers.huggingface.provider import HuggingfaceLocal as $alias`,
`from trulens_eval.feedback.provider.langchain import Langchain` => `from trulens.providers.langchain.provider import Langchain`,
`from trulens_eval.feedback.provider.langchain import Langchain as $alias` => `from trulens.providers.langchain.provider import Langchain as $alias`,
`from trulens_eval.feedback.provider.langchain import LangChain` => `from trulens.providers.langchain.provider import LangChain`,
`from trulens_eval.feedback.provider.langchain import LangChain as $alias` => `from trulens.providers.langchain.provider import LangChain as $alias`,
`from trulens_eval.feedback.provider.litellm import LiteLLM` => `from trulens.providers.litellm.provider import LiteLLM`,
`from trulens_eval.feedback.provider.litellm import LiteLLM as $alias` => `from trulens.providers.litellm.provider import LiteLLM as $alias`,
`from trulens_eval.feedback.provider.openai import CLASS_INFO` => `from trulens.providers.openai.provider import CLASS_INFO`,
Expand Down
26 changes: 13 additions & 13 deletions .grit/patterns/trulens_eval_migration.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ any {
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.langchain`,
from_name=`Langchain`,
from_name=`LangChain`,
to_package=`trulens.providers.langchain.provider`,
to_name=`Langchain`
to_name=`LangChain`
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.litellm`,
Expand Down Expand Up @@ -350,15 +350,15 @@ any {
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.langchain`,
from_name=`Langchain`,
from_name=`LangChain`,
to_package=`trulens.providers.langchain.provider`,
to_name=`Langchain`
to_name=`LangChain`
),
py_find_replace_import(
from_package=`trulens_eval`,
from_name=`Langchain`,
from_name=`LangChain`,
to_package=`trulens.providers.langchain.provider`,
to_name=`Langchain`
to_name=`LangChain`
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.litellm`,
Expand Down Expand Up @@ -476,9 +476,9 @@ any {
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.langchain`,
from_name=`Langchain`,
from_name=`LangChain`,
to_package=`trulens.providers.langchain.provider`,
to_name=`Langchain`
to_name=`LangChain`
),
py_find_replace_import(
from_package=`trulens_eval.feedback.provider.litellm`,
Expand Down Expand Up @@ -562,7 +562,7 @@ from trulens.apps.custom import TruCustomApp
from trulens.apps.basic import TruBasicApp
```

## Updates to working with Langchain
## Updates to working with LangChain

```python
from trulens_eval import TruChain
Expand Down Expand Up @@ -680,18 +680,18 @@ from trulens_eval.feedback.provider.cortex import Cortex
from trulens.providers.cortex.provider import Cortex
```

## Using Langchain Provider
## Using LangChain Provider
```python
from trulens_eval import Cortex
```
```python
from trulens.providers.cortex.provider import Cortex
```

## Using Langchain Provider Full Import Path
## Using LangChain Provider Full Import Path
```python
from trulens_eval.feedback.provider.langchain import Langchain
from trulens_eval.feedback.provider.langchain import LangChain
```
```python
from trulens.providers.langchain.provider import Langchain
from trulens.providers.langchain.provider import LangChain
```
4 changes: 2 additions & 2 deletions docs/reference/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ types and providers:
- 📦 [Cortex][trulens.providers.cortex.provider.Cortex] in the package
`trulens-providers-cortex` for using Snowflake Cortex models.

- 📦 [Langchain][trulens.providers.langchain.provider.Langchain] in the package
`trulens-providers-langchain` for using models via Langchain.
- 📦 [LangChain][trulens.providers.langchain.provider.LangChain] in the package
`trulens-providers-langchain` for using models via LangChain.

- 📦 [Bedrock][trulens.providers.bedrock.provider.Bedrock] in the package
`trulens-providers-bedrock` for using Amazon Bedrock models.
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/providers/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ some derive from [LLMProvider][trulens.feedback.llm_provider.LLMProvider].
pip install trulens-providers-cortex
```

- [Langchain][trulens.providers.langchain.provider.Langchain] in package
- [LangChain][trulens.providers.langchain.provider.LangChain] in package
`trulens-providers-langchain`.

```bash
Expand Down
2 changes: 1 addition & 1 deletion docs/trulens/evaluation/feedback_providers/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Providers which use large language models for feedback evaluation:
[AzureOpenAI provider][trulens.providers.openai.AzureOpenAI]
- [Bedrock provider][trulens.providers.bedrock.Bedrock]
- [LiteLLM provider][trulens.providers.litellm.LiteLLM]
- [_LangChain_ provider][trulens.providers.langchain.Langchain]
- [_LangChain_ provider][trulens.providers.langchain.LangChain]

Feedback functions in common across these providers are in their abstract class
[LLMProvider][trulens.feedback.LLMProvider].
Expand Down
4 changes: 2 additions & 2 deletions docs/trulens/guardrails/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ If we consider a RAG, context filter guardrails can be used to evaluate the *con

## Using *TruLens* guardrails

*TruLens* context filter guardrails are easy to add to your app built with custom python, *Langchain*, or *Llama-Index*.
*TruLens* context filter guardrails are easy to add to your app built with custom python, *LangChain*, or *Llama-Index*.

!!! example "Using context filter guardrails"

Expand Down Expand Up @@ -78,4 +78,4 @@ If we consider a RAG, context filter guardrails can be used to evaluate the *con

Feedback function used as a guardrail must only return a float score, and cannot also return reasons.

TruLens has native python and framework-specific tooling for implementing guardrails. Read more about the available guardrails in [native python][trulens.core.guardrails.base], [Langchain][trulens.apps.langchain.guardrails] and [Llama-Index][trulens.apps.llamaindex.guardrails].
TruLens has native python and framework-specific tooling for implementing guardrails. Read more about the available guardrails in [native python][trulens.core.guardrails.base], [LangChain][trulens.apps.langchain.guardrails] and [Llama-Index][trulens.apps.llamaindex.guardrails].
2 changes: 1 addition & 1 deletion docs/trulens/release_blog_1dot.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ To see the core re-architecture changes in action, we've included some usage exa
ca = CustomApp()
```

=== "Langchain"
=== "LangChain"

```bash
pip install trulens-apps-langchain
Expand Down
4 changes: 2 additions & 2 deletions docs/trulens/tracking/instrumentation/langchain.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ apps.

## Example Usage

To demonstrate usage, we'll create a standard RAG defined with Langchain Expression Language (LCEL).
To demonstrate usage, we'll create a standard RAG defined with LangChain Expression Language (LCEL).

First, this requires loading data into a vector store.

Expand Down Expand Up @@ -81,7 +81,7 @@ us to evaluate for metrics including context relevance and groundedness.
For LangChain applications where the BaseRetriever is used, `select_context` can
be used to access the retrieved text for evaluation.

!!! example "Evaluating retrieved context in Langchain"
!!! example "Evaluating retrieved context in LangChain"

```python
import numpy as np
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
"source": [
"### Create Simple LLM Application\n",
"\n",
"This example uses Langchain with an OpenAI LLM."
"This example uses LangChain with an OpenAI LLM."
]
},
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import from Langchain and TruLens"
"### Import from LangChain and TruLens"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/expositional/index.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# 🧑‍🍳 TruLens Cookbook

Examples for tracking and evaluating apps with _TruLens_. Examples are organized by different frameworks (such as Langchain or Llama-Index), model (including Azure, OSS models and more), vector store, and use case.
Examples for tracking and evaluating apps with _TruLens_. Examples are organized by different frameworks (such as LangChain or Llama-Index), model (including Azure, OSS models and more), vector store, and use case.

The examples in this cookbook are more focused on applying core concepts to external libraries or end to end applications than the [quickstarts](../trulens/getting_started/quickstarts/quickstart/).
2 changes: 1 addition & 1 deletion examples/expositional/models/bedrock/bedrock.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import from TruLens, Langchain and Boto3"
"### Import from TruLens, LangChain and Boto3"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion src/apps/langchain/trulens/apps/langchain/guardrails.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class WithFeedbackFilterDocuments(VectorStoreRetriever):

threshold (float): and keep documents only if their feedback value is at least this threshold.

Example: "Using TruLens guardrail context filters with Langchain"
Example: "Using TruLens guardrail context filters with LangChain"

```python
from trulens.apps.langchain import WithFeedbackFilterDocuments
Expand Down
2 changes: 1 addition & 1 deletion src/core/trulens/core/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ class TruSession(
referred to by `database_url`.

Supported App Types:
[TruChain][trulens.apps.langchain.TruChain]: Langchain
[TruChain][trulens.apps.langchain.TruChain]: LangChain
apps.

[TruLlama][trulens.apps.llamaindex.TruLlama]: Llama Index
Expand Down
2 changes: 1 addition & 1 deletion src/feedback/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ The components of this specifications are:

### langchain-specific selectors

- `Langchain.select_context()` -- outputs the selector for retrieved context
- `LangChain.select_context()` -- outputs the selector for retrieved context
from the app's internal `get_relevant_documents` method.

### NeMo-specific selectors
Expand Down
10 changes: 5 additions & 5 deletions src/feedback/trulens/feedback/llm_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class LLMProvider(Provider):
interface to a [wide range of
models](https://docs.litellm.ai/docs/providers).

* [Langchain][trulens.providers.langchain.Langchain].
* [LangChain][trulens.providers.langchain.LangChain].

"""

Expand Down Expand Up @@ -642,7 +642,7 @@ def model_agreement(self, prompt: str, response: str) -> float:
def _langchain_evaluate(self, text: str, criteria: str) -> float:
"""
Uses chat completion model. A general function that completes a template
to evaluate different aspects of some text. Prompt credit to Langchain.
to evaluate different aspects of some text. Prompt credit to LangChain.

Args:
text (str): A prompt to an agent.
Expand All @@ -667,7 +667,7 @@ def _langchain_evaluate_with_cot_reasons(
) -> Tuple[float, Dict]:
"""
Uses chat completion model. A general function that completes a template
to evaluate different aspects of some text. Prompt credit to Langchain.
to evaluate different aspects of some text. Prompt credit to LangChain.

Args:
text (str): A prompt to an agent.
Expand Down Expand Up @@ -936,7 +936,7 @@ def helpfulness_with_cot_reasons(self, text: str) -> Tuple[float, Dict]:
def controversiality(self, text: str) -> float:
"""
Uses chat completion model. A function that completes a template to
check the controversiality of some text. Prompt credit to Langchain
check the controversiality of some text. Prompt credit to LangChain
Eval.

Example:
Expand All @@ -960,7 +960,7 @@ def controversiality_with_cot_reasons(
) -> Tuple[float, Dict]:
"""
Uses chat completion model. A function that completes a template to
check the controversiality of some text. Prompt credit to Langchain
check the controversiality of some text. Prompt credit to LangChain
Eval. Also uses chain of thought methodology and emits the reasons.

Example:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
from importlib.metadata import version

from trulens.core.utils.imports import safe_importlib_package_name
from trulens.providers.langchain.provider import Langchain
from trulens.providers.langchain.provider import LangChain

__version__ = version(safe_importlib_package_name(__package__ or __name__))


__all__ = [
"Langchain",
"LangChain",
]
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def _convert_message(message: Dict) -> BaseMessage:
return AIMessage(content=message["content"])


class Langchain(LLMProvider):
class LangChain(LLMProvider):
"""Out of the box feedback functions using LangChain LLMs and ChatModels

Create a LangChain Provider with out of the box feedback functions.
Expand Down
4 changes: 2 additions & 2 deletions src/trulens_eval/trulens_eval/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
with imports_utils.OptionalImports(
messages=eval_optional_utils.REQUIREMENT_PROVIDER_LANGCHAIN
):
from trulens.providers.langchain.provider import Langchain
from trulens.providers.langchain.provider import LangChain

with imports_utils.OptionalImports(
messages=optional_utils.REQUIREMENT_APPS_LANGCHAIN
Expand Down Expand Up @@ -92,7 +92,7 @@
"Provider",
"AzureOpenAI",
"OpenAI",
"Langchain",
"LangChain",
"LiteLLM",
"Bedrock",
"Huggingface",
Expand Down
4 changes: 2 additions & 2 deletions src/trulens_eval/trulens_eval/feedback/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
with imports_utils.OptionalImports(
messages=optional_utils.REQUIREMENT_PROVIDER_LANGCHAIN
):
from trulens.providers.langchain.provider import Langchain
from trulens.providers.langchain.provider import LangChain

with imports_utils.OptionalImports(
messages=optional_utils.REQUIREMENT_PROVIDER_CORTEX
Expand All @@ -62,7 +62,7 @@
"HuggingfaceLocal",
"LiteLLM",
"Bedrock",
"Langchain",
"LangChain",
"Cortex",
]

Expand Down
4 changes: 2 additions & 2 deletions src/trulens_eval/trulens_eval/feedback/provider/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
with imports_utils.OptionalImports(
messages=optional_utils.REQUIREMENT_PROVIDER_LANGCHAIN
):
from trulens.providers.langchain.provider import Langchain
from trulens.providers.langchain.provider import LangChain
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also alias these


__all__ = [
"Provider",
Expand All @@ -54,7 +54,7 @@
"HuggingfaceLocal",
"LiteLLM",
"Bedrock",
"Langchain",
"LangChain",
"Cortex",
]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@

deprecation_utils.packages_dep_warn()

from trulens.providers.langchain.provider import Langchain
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

may need to alias for compatability

from trulens.... import LangChain as Langchain

from trulens.providers.langchain.provider import LangChain
Loading