-
Notifications
You must be signed in to change notification settings - Fork 195
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add chatqna wrapper for multiple model selection (#1144)
Signed-off-by: lvliang-intel <liang1.lv@intel.com> Co-authored-by: Ying Hu <ying.hu@intel.com> Co-authored-by: chen, suyue <suyue.chen@intel.com>
- Loading branch information
3 people
authored
Nov 18, 2024
1 parent
b1bb6db
commit fb514bb
Showing
6 changed files
with
262 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| # Copyright (C) 2024 Intel Corporation | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| FROM python:3.11-slim | ||
|
|
||
| RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ | ||
| libgl1-mesa-glx \ | ||
| libjemalloc-dev \ | ||
| git | ||
|
|
||
| RUN useradd -m -s /bin/bash user && \ | ||
| mkdir -p /home/user && \ | ||
| chown -R user /home/user/ | ||
|
|
||
| WORKDIR /home/user/ | ||
| RUN git clone https://github.com/opea-project/GenAIComps.git | ||
|
|
||
| WORKDIR /home/user/GenAIComps | ||
| RUN pip install --no-cache-dir --upgrade pip && \ | ||
| pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt | ||
|
|
||
| COPY ./chatqna_wrapper.py /home/user/chatqna.py | ||
|
|
||
| ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps | ||
|
|
||
| USER user | ||
|
|
||
| WORKDIR /home/user | ||
|
|
||
| RUN echo 'ulimit -S -n 999999' >> ~/.bashrc | ||
|
|
||
| ENTRYPOINT ["python", "chatqna.py"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| # Copyright (C) 2024 Intel Corporation | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import os | ||
|
|
||
| from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType | ||
|
|
||
| MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0") | ||
| MEGA_SERVICE_PORT = int(os.getenv("MEGA_SERVICE_PORT", 8888)) | ||
| EMBEDDING_SERVICE_HOST_IP = os.getenv("EMBEDDING_SERVICE_HOST_IP", "0.0.0.0") | ||
| EMBEDDING_SERVICE_PORT = int(os.getenv("EMBEDDING_SERVICE_PORT", 6000)) | ||
| RETRIEVER_SERVICE_HOST_IP = os.getenv("RETRIEVER_SERVICE_HOST_IP", "0.0.0.0") | ||
| RETRIEVER_SERVICE_PORT = int(os.getenv("RETRIEVER_SERVICE_PORT", 7000)) | ||
| RERANK_SERVICE_HOST_IP = os.getenv("RERANK_SERVICE_HOST_IP", "0.0.0.0") | ||
| RERANK_SERVICE_PORT = int(os.getenv("RERANK_SERVICE_PORT", 8000)) | ||
| LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0") | ||
| LLM_SERVICE_PORT = int(os.getenv("LLM_SERVICE_PORT", 9000)) | ||
|
|
||
|
|
||
| class ChatQnAService: | ||
| def __init__(self, host="0.0.0.0", port=8000): | ||
| self.host = host | ||
| self.port = port | ||
| self.megaservice = ServiceOrchestrator() | ||
|
|
||
| def add_remote_service(self): | ||
| embedding = MicroService( | ||
| name="embedding", | ||
| host=EMBEDDING_SERVICE_HOST_IP, | ||
| port=EMBEDDING_SERVICE_PORT, | ||
| endpoint="/v1/embeddings", | ||
| use_remote_service=True, | ||
| service_type=ServiceType.EMBEDDING, | ||
| ) | ||
| retriever = MicroService( | ||
| name="retriever", | ||
| host=RETRIEVER_SERVICE_HOST_IP, | ||
| port=RETRIEVER_SERVICE_PORT, | ||
| endpoint="/v1/retrieval", | ||
| use_remote_service=True, | ||
| service_type=ServiceType.RETRIEVER, | ||
| ) | ||
| rerank = MicroService( | ||
| name="rerank", | ||
| host=RERANK_SERVICE_HOST_IP, | ||
| port=RERANK_SERVICE_PORT, | ||
| endpoint="/v1/reranking", | ||
| use_remote_service=True, | ||
| service_type=ServiceType.RERANK, | ||
| ) | ||
| llm = MicroService( | ||
| name="llm", | ||
| host=LLM_SERVICE_HOST_IP, | ||
| port=LLM_SERVICE_PORT, | ||
| endpoint="/v1/chat/completions", | ||
| use_remote_service=True, | ||
| service_type=ServiceType.LLM, | ||
| ) | ||
| self.megaservice.add(embedding).add(retriever).add(rerank).add(llm) | ||
| self.megaservice.flow_to(embedding, retriever) | ||
| self.megaservice.flow_to(retriever, rerank) | ||
| self.megaservice.flow_to(rerank, llm) | ||
| self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| chatqna = ChatQnAService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT) | ||
| chatqna.add_remote_service() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.