llama3_deploy_lmi_v9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "16a5bb77-e829-4040-97fc-ba2f87b8694a",
   "metadata": {},
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "932f0762-ca08-417f-aba1-5f20907805a1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%pip install sagemaker huggingface_hub --upgrade  --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f61a483-1d76-4f9d-b014-d8c88d8fdc27",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import login\n",
    "## input your hf token\n",
    "login(token='INPUT_YOUR_HF_MODEL_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eda7a1e3-c12e-4f55-a2a9-a6269d38fd67",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "local_model_path = Path(\"./LLM_llama3_8b_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "\n",
    "model_name = \"meta-llama/Meta-Llama-3-8B-Instruct\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d55d5b3-910d-41a5-a568-8052c312c26d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "snapshot_download(repo_id=model_name, cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88b5253d-2f5a-47dc-a645-9e764932edb5",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a20fa62-7d7b-46b3-92bc-799bd6643991",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker import Model, image_uris, serializers, deserializers\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "region = sess._region_name  # region name of the current SageMaker Studio environment\n",
    "account_id = sess.account_id()  # account_id of the current SageMaker Studio environment\n",
    "bucket = sess.default_bucket()\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b17d507a-aaf8-4d08-a8df-6c69efa06e0f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_model_prefix = f\"llm-models/{model_name}\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = f\"llm-models/{model_name}_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0a0fd1e-8d83-4a86-aa2e-3b1f316babd1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_path = f\"s3://{bucket}/{s3_model_prefix}/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67a73b68-a0d0-43b2-9c9f-d5a0e3434fcd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws s3 cp --recursive --exclude \"*.pth\" {model_snapshot_path} {s3_path}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6cf5687-0669-464d-9c63-6786d3ffbca3",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "020a4da8-62f6-40e3-8bd4-b59eead2a322",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inference_image_uri = image_uris.retrieve(\n",
    "        framework=\"djl-deepspeed\",\n",
    "        region=sess.boto_session.region_name,\n",
    "        version=\"0.27.0\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30c59e5a-de54-4bf2-a60f-74d7d0df5e62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "local_code_dir = s3_code_prefix.split('/')[-1]\n",
    "!mkdir -p {local_code_dir}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d7fca02-3d9f-4fc8-a59a-7734e5041baf",
   "metadata": {},
   "source": [
    "#### Note: option.model_id 需要改成模型下载的s3_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5284740-ecb9-4f2c-812e-32c94b409e45",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile {local_code_dir}/serving.properties\n",
    "engine=Python\n",
    "option.model_id=S3PATH\n",
    "option.dtype=bf16\n",
    "option.task=text-generation\n",
    "option.rolling_batch=vllm\n",
    "option.tensor_parallel_degree=1\n",
    "option.device_map=auto\n",
    "option.gpu_memory_utilization=0.85\n",
    "option.max_model_len=8192\n",
    "option.max_tokens=8192\n",
    "option.output_formatter = json\n",
    "option.model_loading_timeout = 1200\n",
    "option.enforce_eager=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c1d94f3-f384-4085-b90d-4dcebb319dba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!sed -i \"s|option.model_id=S3PATH|option.model_id={s3_path}|\" {local_code_dir}/serving.properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "263e7028-f2f9-42c7-9328-24bbd887183d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm model.tar.gz\n",
    "!cd {local_code_dir} && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz {local_code_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09cd6483-a677-4ecf-ab46-91dce63fe85f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "397ae979-320c-41d1-9fc5-010e4f6fbe92",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fb12885-6a96-4b50-be47-acfabcf5b770",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"llama-8b-instruct\") \n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff327df2-84b9-4908-96e3-4cc272edcaef",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.2xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 10*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37884ab7-69f8-476f-bc8a-f3b54661592e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83d4721f-05ce-4f76-9294-7a0c320afd9b",
   "metadata": {},
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8e8dc98-e43a-4352-872f-ac3a49681da3",
   "metadata": {},
   "source": [
    "## No stream "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11de5b96-7a78-4a27-9523-73230a7fa184",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "import json\n",
    "import boto3\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "\n",
    "parameters = {\n",
    "  \"max_new_tokens\": 8192,\n",
    "  \"temperature\": 0.9,\n",
    "  \"top_p\":0.8\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4564bdfc-e656-4620-bd60-3709e624546b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompts1 = \"\"\"今天天气，很不错哦\"\"\"\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters            \n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "resp = response_model['Body'].read()\n",
    "print(json.loads(resp)['generated_text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d63bf3f9-6dd7-4a82-94e9-ba8a0ffc15e7",
   "metadata": {},
   "source": [
    "## stream "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de12d067-667a-4018-9e55-8dd7efa1e281",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import io\n",
    "import re\n",
    "\n",
    "NEWLINE = re.compile(r'\\\\n')  \n",
    "DOUBLE_NEWLINE = re.compile(r'\\\\n\\\\n')\n",
    "\n",
    "class LineIterator:\n",
    "    \n",
    "    def __init__(self, stream):\n",
    "        self.byte_iterator = iter(stream)\n",
    "        self.buffer = io.BytesIO()\n",
    "        self.read_pos = 0\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        start_sequence = b'{\"generated_text\": \"'\n",
    "        stop_sequence = b'\"}'\n",
    "        new_line = '\\n'\n",
    "        double_new_line = '\\n\\n'\n",
    "        while True:\n",
    "            self.buffer.seek(self.read_pos)\n",
    "            line = self.buffer.readline()\n",
    "            if line:\n",
    "                self.read_pos += len(line)\n",
    "                if line.startswith(start_sequence):# in :\n",
    "                    line = line.lstrip(start_sequence)\n",
    "\n",
    "                if line.endswith(stop_sequence):\n",
    "                    line =line.rstrip(stop_sequence)\n",
    "                line = line.decode('utf-8')\n",
    "                line = NEWLINE.sub(new_line, line)\n",
    "                line = DOUBLE_NEWLINE.sub(double_new_line, line)\n",
    "                return line\n",
    "            try:\n",
    "                chunk = next(self.byte_iterator)\n",
    "            except StopIteration:\n",
    "                if self.read_pos < self.buffer.getbuffer().nbytes:\n",
    "                    continue\n",
    "                raise\n",
    "            if 'PayloadPart' not in chunk:\n",
    "                print('Unknown event type:' + chunk)\n",
    "                continue\n",
    "            self.buffer.seek(0, io.SEEK_END)\n",
    "            self.buffer.write(chunk['PayloadPart']['Bytes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9981efd-6319-47ca-927a-7293a3b96e90",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import boto3\n",
    "\n",
    "input_text = \"\"\"今天天气，很不错哦\"\"\"\n",
    "\n",
    "\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")\n",
    "response_model = smr_client.invoke_endpoint_with_response_stream(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": input_text,\n",
    "                \"parameters\": parameters,\n",
    "                \"stream\" : True\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "def print_response_stream(response_stream):\n",
    "    event_stream = response_stream.get('Body')\n",
    "    for line in LineIterator(event_stream):\n",
    "        print(line, end='')\n",
    "        \n",
    "print_response_stream(response_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7f01e66-d807-40de-adf9-8ebe92585491",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name {endpoint_name}\n",
    "!aws sagemaker delete-endpoint-config --endpoint-config-name {endpoint_config_name}\n",
    "!aws sagemaker delete-model --model-name {model_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1878af7a-5e5c-4abc-85ab-6363604a1858",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}