bengali-news-summarization-mt5.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":1527319,"sourceType":"datasetVersion","datasetId":900472},{"sourceId":7370632,"sourceType":"datasetVersion","datasetId":4282380}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/mukaffimoin/bengali-news-summarization-mt5?scriptVersionId=158333739\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"code","source":"import pandas as pd\nfrom datasets import Dataset\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-09T14:42:59.601783Z","iopub.execute_input":"2024-01-09T14:42:59.602512Z","iopub.status.idle":"2024-01-09T14:42:59.607276Z","shell.execute_reply.started":"2024-01-09T14:42:59.602473Z","shell.execute_reply":"2024-01-09T14:42:59.606195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/bengali-news-summarization-dataset/Bengali-News-Summarization-Dataset/article.txt\", \"r\") as f:\n    articles = f.read().splitlines()\n    \n    \nwith open(\"/kaggle/input/bengali-news-summarization-dataset/Bengali-News-Summarization-Dataset/summary.txt\", \"r\") as f:\n    summaries = f.read().splitlines()","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:00.653435Z","iopub.execute_input":"2024-01-09T14:43:00.65412Z","iopub.status.idle":"2024-01-09T14:43:00.736559Z","shell.execute_reply.started":"2024-01-09T14:43:00.65408Z","shell.execute_reply":"2024-01-09T14:43:00.735661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(list(zip(articles,summaries)),columns=[\"article\",\"summary\"])","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:01.553508Z","iopub.execute_input":"2024-01-09T14:43:01.553896Z","iopub.status.idle":"2024-01-09T14:43:01.569888Z","shell.execute_reply.started":"2024-01-09T14:43:01.553865Z","shell.execute_reply":"2024-01-09T14:43:01.56895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:02.031218Z","iopub.execute_input":"2024-01-09T14:43:02.031631Z","iopub.status.idle":"2024-01-09T14:43:02.046646Z","shell.execute_reply.started":"2024-01-09T14:43:02.031601Z","shell.execute_reply":"2024-01-09T14:43:02.045545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dropna()","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:02.373506Z","iopub.execute_input":"2024-01-09T14:43:02.374278Z","iopub.status.idle":"2024-01-09T14:43:02.392468Z","shell.execute_reply.started":"2024-01-09T14:43:02.374219Z","shell.execute_reply":"2024-01-09T14:43:02.391482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:02.613694Z","iopub.execute_input":"2024-01-09T14:43:02.614826Z","iopub.status.idle":"2024-01-09T14:43:02.62649Z","shell.execute_reply.started":"2024-01-09T14:43:02.614787Z","shell.execute_reply":"2024-01-09T14:43:02.625346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndf_train, df_test = train_test_split(df, test_size=0.30, shuffle=True)\ndf_val, df_test = train_test_split(df_test, test_size=0.65,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:02.939424Z","iopub.execute_input":"2024-01-09T14:43:02.939796Z","iopub.status.idle":"2024-01-09T14:43:02.950785Z","shell.execute_reply.started":"2024-01-09T14:43:02.939769Z","shell.execute_reply":"2024-01-09T14:43:02.949902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:04.245378Z","iopub.execute_input":"2024-01-09T14:43:04.245794Z","iopub.status.idle":"2024-01-09T14:43:04.257685Z","shell.execute_reply.started":"2024-01-09T14:43:04.245758Z","shell.execute_reply":"2024-01-09T14:43:04.256741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib.font_manager import FontProperties\n\ndef visualize_text_length(data, title):\n\n    data['News_article_text_length'] = data['article'].apply(len)\n    data['News_summary_text_length'] = data['summary'].apply(len)\n\n   \n    plt.figure(figsize=(8, 4))\n\n    \n    custom_font = FontProperties(family='serif', style='normal', size=14, weight='bold')\n\n    \n    plt.subplot(1, 2, 1)\n    plt.hist(data['News_article_text_length'], bins=40, color='cornflowerblue', edgecolor='black', alpha=0.7, label='Bangla News Article')\n    plt.grid(linestyle='--', alpha=0.6)\n    plt.xlabel(\"Bangla News Article Text Length\", fontsize=10, fontproperties=custom_font, color='black')\n    plt.ylabel(\"Frequency\", fontsize=10, fontproperties=custom_font, color='black')\n\n    \n    plt.subplot(1, 2, 2)\n    plt.hist(data['News_summary_text_length'], bins=40, color='firebrick', edgecolor='black', alpha=0.7, label='Bangla News Summary')\n    plt.grid(linestyle='--', alpha=0.6)\n    plt.xlabel(\"News Summary Text Length\", fontsize=10, fontproperties=custom_font, color='black')\n    plt.ylabel(\"Frequency\", fontsize=10, fontproperties=custom_font, color='black')\n\n    \n    plt.suptitle(f'Text Length Distribution for {title}', fontsize=12, fontproperties=custom_font, color='black')\n    \n    plt.tight_layout()\n\n    # Show the plot\n    plt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:52:55.919732Z","iopub.execute_input":"2024-01-09T14:52:55.920466Z","iopub.status.idle":"2024-01-09T14:52:55.932265Z","shell.execute_reply.started":"2024-01-09T14:52:55.920432Z","shell.execute_reply":"2024-01-09T14:52:55.931007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Visualize text length distribution for each dataset**","metadata":{}},{"cell_type":"code","source":"visualize_text_length(df_train, 'Training Dataset')","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:52:56.7829Z","iopub.execute_input":"2024-01-09T14:52:56.783724Z","iopub.status.idle":"2024-01-09T14:52:57.288576Z","shell.execute_reply.started":"2024-01-09T14:52:56.783692Z","shell.execute_reply":"2024-01-09T14:52:57.287556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_text_length(df_test , 'Test Dataset')","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:53:23.257695Z","iopub.execute_input":"2024-01-09T14:53:23.25845Z","iopub.status.idle":"2024-01-09T14:53:23.891609Z","shell.execute_reply.started":"2024-01-09T14:53:23.258416Z","shell.execute_reply":"2024-01-09T14:53:23.890624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_text_length(df_val , 'Validation Dataset')","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:53:53.993449Z","iopub.execute_input":"2024-01-09T14:53:53.993833Z","iopub.status.idle":"2024-01-09T14:53:54.704575Z","shell.execute_reply.started":"2024-01-09T14:53:53.9938Z","shell.execute_reply":"2024-01-09T14:53:54.703415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud\n\ndef create_wordcloud(data, column, title):\n    # Specify a Bangla-supported font, such as \"Siyam Rupali\"\n    font_path = \"/kaggle/input/fonts-paths/Siyam Rupali Regular.ttf\"\n\n\n    #This setting disables the detection of collocations (multi-word phrases) to focus on individual words.\n    wordcloud = WordCloud(width=800, height=400, background_color='black', font_path=font_path,\n                          colormap='rainbow', collocations=False).generate(' '.join(data[column]))\n\n    plt.figure(figsize=(8, 4))\n    plt.imshow(wordcloud, interpolation='bilinear')\n    plt.title(f'Word Cloud for {column} in {title}', fontsize=16, color='black')\n    plt.axis('off')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T15:06:44.566446Z","iopub.execute_input":"2024-01-09T15:06:44.566827Z","iopub.status.idle":"2024-01-09T15:06:44.574298Z","shell.execute_reply.started":"2024-01-09T15:06:44.566796Z","shell.execute_reply":"2024-01-09T15:06:44.573228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_wordcloud(df_train, 'article', 'Training Set')","metadata":{"execution":{"iopub.status.busy":"2024-01-09T15:06:45.501119Z","iopub.execute_input":"2024-01-09T15:06:45.501578Z","iopub.status.idle":"2024-01-09T15:06:48.660009Z","shell.execute_reply.started":"2024-01-09T15:06:45.501522Z","shell.execute_reply":"2024-01-09T15:06:48.659068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_wordcloud(df_train, 'article', 'Training Set')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_wordcloud(df_train, 'summary', 'Training Set')","metadata":{"execution":{"iopub.status.busy":"2024-01-09T15:07:20.597115Z","iopub.execute_input":"2024-01-09T15:07:20.598003Z","iopub.status.idle":"2024-01-09T15:07:22.593695Z","shell.execute_reply.started":"2024-01-09T15:07:20.597966Z","shell.execute_reply":"2024-01-09T15:07:22.592727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/csebuetnlp/normalizer","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:04.629879Z","iopub.execute_input":"2024-01-09T14:43:04.630241Z","iopub.status.idle":"2024-01-09T14:43:18.419874Z","shell.execute_reply.started":"2024-01-09T14:43:04.630211Z","shell.execute_reply":"2024-01-09T14:43:18.418329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\nfrom normalizer import normalize # pip install git+https://github.com/csebuetnlp/normalizer","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:43:18.42303Z","iopub.execute_input":"2024-01-09T14:43:18.423938Z","iopub.status.idle":"2024-01-09T14:43:18.430357Z","shell.execute_reply.started":"2024-01-09T14:43:18.423888Z","shell.execute_reply":"2024-01-09T14:43:18.428818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:09.795429Z","iopub.execute_input":"2024-01-09T12:24:09.795827Z","iopub.status.idle":"2024-01-09T12:24:09.800711Z","shell.execute_reply.started":"2024-01-09T12:24:09.795795Z","shell.execute_reply":"2024-01-09T12:24:09.799694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom normalizer import normalize\nfrom transformers import MT5ForConditionalGeneration, AutoTokenizer ,DataCollatorForSeq2Seq, Trainer, TrainingArguments\nimport os\n\n\nmodel_name = \"google/mt5-small\" \nmodel = MT5ForConditionalGeneration.from_pretrained(model_name)\ntokenizer = AutoTokenizer.from_pretrained(model_name)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:10.531363Z","iopub.execute_input":"2024-01-09T12:24:10.532077Z","iopub.status.idle":"2024-01-09T12:24:18.069671Z","shell.execute_reply.started":"2024-01-09T12:24:10.532045Z","shell.execute_reply":"2024-01-09T12:24:18.06873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nclass Seq2SeqDataset(Dataset):\n    def __init__(self, data, tokenizer, max_length=128):\n        self.input_text = data['article'].apply(normalize).tolist()\n        self.labels = data['summary'].apply(normalize).tolist()\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.input_text)\n\n    def __getitem__(self, idx):\n        input_text = self.input_text[idx]\n        label_text = self.labels[idx]\n\n        # Tokenize the input text\n        input_encodings = self.tokenizer(\n            input_text,\n            truncation=True,\n            padding='max_length',\n            max_length=self.max_length,\n            return_tensors='pt'\n        )\n\n        # Tokenize the label text to get its 'input_ids' and 'attention_mask'\n        label_encodings = self.tokenizer(\n            label_text,\n            truncation=True,\n            padding='max_length',\n            max_length=self.max_length,\n            return_tensors='pt'\n        )\n\n        return {\n            'input_ids': input_encodings['input_ids'].squeeze(),\n            'attention_mask': input_encodings['attention_mask'].squeeze(),\n            'labels': label_encodings['input_ids'].squeeze(),\n        }","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:24.321785Z","iopub.execute_input":"2024-01-09T12:24:24.322151Z","iopub.status.idle":"2024-01-09T12:24:24.331621Z","shell.execute_reply.started":"2024-01-09T12:24:24.32212Z","shell.execute_reply":"2024-01-09T12:24:24.330534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Modify the data collation process to handle PyTorch tensors correctly\nclass MyDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):\n    def __call__(self, features):\n        batch = {}\n        batch[\"input_ids\"] = torch.stack([feature[\"input_ids\"] for feature in features])\n        batch[\"attention_mask\"] = torch.stack([feature[\"attention_mask\"] for feature in features])\n\n        # Labels should be processed differently for PyTorch tensors\n        if isinstance(features[0][\"labels\"], torch.Tensor):\n            batch[\"labels\"] = torch.stack([feature[\"labels\"] for feature in features])\n        else:\n            # Convert the list of lists to a PyTorch tensor\n            batch[\"labels\"] = torch.tensor([feature[\"labels\"] for feature in features])\n\n        return batch","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:26.313795Z","iopub.execute_input":"2024-01-09T12:24:26.314673Z","iopub.status.idle":"2024-01-09T12:24:26.880734Z","shell.execute_reply.started":"2024-01-09T12:24:26.314639Z","shell.execute_reply":"2024-01-09T12:24:26.879641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create train , test and validation datasets\ntrain_dataset = Seq2SeqDataset(df_train, tokenizer)\ntest_dataset = Seq2SeqDataset(df_test, tokenizer)\nvalidation_dataset = Seq2SeqDataset(df_val, tokenizer)\n\n# Create train , test and validation dataloaders\ntrain_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True)  #batch_size=32\ntest_dataloader = DataLoader(test_dataset, batch_size=16) #batch_size=32\nvalidation_dataloader = DataLoader(validation_dataset, batch_size=16) #batch_size=32\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:32.536614Z","iopub.execute_input":"2024-01-09T12:24:32.536992Z","iopub.status.idle":"2024-01-09T12:24:44.180989Z","shell.execute_reply.started":"2024-01-09T12:24:32.536961Z","shell.execute_reply":"2024-01-09T12:24:44.179638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move the model to the device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:44.182957Z","iopub.execute_input":"2024-01-09T12:24:44.18339Z","iopub.status.idle":"2024-01-09T12:24:44.217947Z","shell.execute_reply.started":"2024-01-09T12:24:44.183354Z","shell.execute_reply":"2024-01-09T12:24:44.216647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:53.042728Z","iopub.execute_input":"2024-01-09T12:24:53.043445Z","iopub.status.idle":"2024-01-09T12:24:53.049789Z","shell.execute_reply.started":"2024-01-09T12:24:53.043413Z","shell.execute_reply":"2024-01-09T12:24:53.048613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a custom optimizer using torch.optim.AdamW\ncustom_optimizer = torch.optim.AdamW(\n    model.parameters(),\n    lr=1e-3,\n    eps=1e-8,\n    weight_decay=0.01,\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:54.160144Z","iopub.execute_input":"2024-01-09T12:24:54.160989Z","iopub.status.idle":"2024-01-09T12:24:54.167231Z","shell.execute_reply.started":"2024-01-09T12:24:54.160956Z","shell.execute_reply":"2024-01-09T12:24:54.166299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the TrainingArguments for fine-tuning\ntraining_args = TrainingArguments(\n    output_dir='/kaggle/working/',\n    num_train_epochs=15,\n    per_device_train_batch_size=5,\n    gradient_accumulation_steps=8,\n    evaluation_strategy=\"epoch\",\n    save_total_limit=1,\n    save_steps=5000,\n    learning_rate=1e-3,\n    do_train=True,\n    do_eval=True,\n    remove_unused_columns=False,\n    push_to_hub=False,\n    report_to=\"none\",\n    load_best_model_at_end=False,\n    lr_scheduler_type=\"cosine_with_restarts\",\n    warmup_steps=100,\n    weight_decay=0.01,\n    logging_dir='/kaggle/working/',\n    logging_steps=200,\n    \n)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:55.31312Z","iopub.execute_input":"2024-01-09T12:24:55.31354Z","iopub.status.idle":"2024-01-09T12:24:55.323032Z","shell.execute_reply.started":"2024-01-09T12:24:55.31351Z","shell.execute_reply":"2024-01-09T12:24:55.322127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a data collator for sequence-to-sequence tasks\ndata_collator = MyDataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    padding=False,\n    max_length=80,\n    label_pad_token_id=tokenizer.pad_token_id,\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:55.888718Z","iopub.execute_input":"2024-01-09T12:24:55.889075Z","iopub.status.idle":"2024-01-09T12:24:55.894119Z","shell.execute_reply.started":"2024-01-09T12:24:55.889032Z","shell.execute_reply":"2024-01-09T12:24:55.893019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Trainer\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    data_collator=data_collator,\n    train_dataset=train_dataset,\n    eval_dataset=validation_dataset,\n    optimizers=(custom_optimizer, None),\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:58.040703Z","iopub.execute_input":"2024-01-09T12:24:58.041109Z","iopub.status.idle":"2024-01-09T12:24:58.537126Z","shell.execute_reply.started":"2024-01-09T12:24:58.041079Z","shell.execute_reply":"2024-01-09T12:24:58.536068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine-tune the model\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2024-01-09T12:24:59.969088Z","iopub.execute_input":"2024-01-09T12:24:59.969993Z","iopub.status.idle":"2024-01-09T14:20:19.262226Z","shell.execute_reply.started":"2024-01-09T12:24:59.969957Z","shell.execute_reply":"2024-01-09T14:20:19.261236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSeq2SeqLM\n\n# Save the model\nmodel.save_pretrained(\"/kaggle/working/mt5_model.pt\")\n\n# Save the tokenizer\ntokenizer.save_pretrained(\"/kaggle/working/mt5_tokenizer.json\")","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:24:31.263655Z","iopub.execute_input":"2024-01-09T14:24:31.264442Z","iopub.status.idle":"2024-01-09T14:24:35.147726Z","shell.execute_reply.started":"2024-01-09T14:24:31.264402Z","shell.execute_reply":"2024-01-09T14:24:35.146623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n# Load the saved model\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"/kaggle/working/mt5_model.pt\")\n\n# Load the tokenizer\ntokenizer = AutoTokenizer.from_pretrained(\"/kaggle/working/mt5_tokenizer.json\")\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:24:35.149122Z","iopub.execute_input":"2024-01-09T14:24:35.150218Z","iopub.status.idle":"2024-01-09T14:24:38.693162Z","shell.execute_reply.started":"2024-01-09T14:24:35.150186Z","shell.execute_reply":"2024-01-09T14:24:38.692063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:24:38.695016Z","iopub.execute_input":"2024-01-09T14:24:38.695368Z","iopub.status.idle":"2024-01-09T14:24:51.119061Z","shell.execute_reply.started":"2024-01-09T14:24:38.69534Z","shell.execute_reply":"2024-01-09T14:24:51.117764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install python-Levenshtein","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:24:51.120737Z","iopub.execute_input":"2024-01-09T14:24:51.121056Z","iopub.status.idle":"2024-01-09T14:25:03.310709Z","shell.execute_reply.started":"2024-01-09T14:24:51.121027Z","shell.execute_reply":"2024-01-09T14:25:03.309344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install jiwer","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:25:03.312368Z","iopub.execute_input":"2024-01-09T14:25:03.31269Z","iopub.status.idle":"2024-01-09T14:25:15.77172Z","shell.execute_reply.started":"2024-01-09T14:25:03.312662Z","shell.execute_reply":"2024-01-09T14:25:15.770272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move the model to the device (CPU or GPU)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:25:15.773922Z","iopub.execute_input":"2024-01-09T14:25:15.774419Z","iopub.status.idle":"2024-01-09T14:25:16.131444Z","shell.execute_reply.started":"2024-01-09T14:25:15.774378Z","shell.execute_reply":"2024-01-09T14:25:16.130425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rouge-score","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:25:16.132886Z","iopub.execute_input":"2024-01-09T14:25:16.133299Z","iopub.status.idle":"2024-01-09T14:25:28.463582Z","shell.execute_reply.started":"2024-01-09T14:25:16.133239Z","shell.execute_reply":"2024-01-09T14:25:28.46247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install evaluate","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:25:28.465045Z","iopub.execute_input":"2024-01-09T14:25:28.465384Z","iopub.status.idle":"2024-01-09T14:25:40.938736Z","shell.execute_reply.started":"2024-01-09T14:25:28.465356Z","shell.execute_reply":"2024-01-09T14:25:40.937605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport Levenshtein\nfrom evaluate import load\n# Define the move_to_device function\ndef move_to_device(batch, device):\n    if isinstance(batch, torch.Tensor):\n        return batch.to(device)\n    elif isinstance(batch, list):\n        return [move_to_device(item, device) for item in batch]\n    elif isinstance(batch, dict):\n        return {key: move_to_device(value, device) for key, value in batch.items()}\n    else:\n        return batch  # If it's not a tensor, list, or dict, leave it as is\n\n# Load the evaluation metric for Character Error Rate (CER) and Word Error Rate (WER) and Exact Match(em)\ncer_metric = load(\"cer\")\nwer_metric = load(\"wer\")\n\nexact_match_metric = load(\"exact_match\")\n\n# Load BLEU and ROUGE metrics\nbleu_metric = load(\"bleu\")\nrouge_metric = load('rouge')\n\n# Initialize lists to store generated summarisaions and references\ngenerated_summarisaions = []\nreferences = []\n\n# Generate summarisaions for the test dataset\nfor batch in test_dataloader:\n    # Move the batch to CUDA\n    batch = move_to_device(batch, 'cuda')\n\n    input_text = batch['input_ids']  # Access the input_text using the correct key\n    labels = batch['labels']  # Access the labels using the correct key\n\n    # Generate summarisaions\n    summarisaion_ids = model.generate(input_text, max_length=512, num_beams=4, length_penalty=2.0, early_stopping=True)\n\n    # Move the summarisaion_ids to CPU to decode\n    summarisaion_ids = summarisaion_ids.to('cpu')\n\n    generated_summarisaion = tokenizer.batch_decode(summarisaion_ids, skip_special_tokens=True)\n\n    generated_summarisaions.extend(generated_summarisaion)\n    references.extend(tokenizer.batch_decode(labels, skip_special_tokens=True))  # Decoding the label IDs\n\n# Make sure to move generated_summarisaions back to CPU for evaluation if necessary\ngenerated_summarisaions = [summarisaion if not isinstance(summarisaion, str) else summarisaion for summarisaion in generated_summarisaions]\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:25:40.940386Z","iopub.execute_input":"2024-01-09T14:25:40.940758Z","iopub.status.idle":"2024-01-09T14:28:22.853043Z","shell.execute_reply.started":"2024-01-09T14:25:40.940721Z","shell.execute_reply":"2024-01-09T14:28:22.851993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of generated summarisaions:\", len(generated_summarisaions))\nprint(\"Number of references:\", len(references))","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:22.854374Z","iopub.execute_input":"2024-01-09T14:28:22.854684Z","iopub.status.idle":"2024-01-09T14:28:22.859977Z","shell.execute_reply.started":"2024-01-09T14:28:22.854657Z","shell.execute_reply":"2024-01-09T14:28:22.859059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate Character Error Rate (CER) and Word Error Rate (WER)\nresults_CER = cer_metric.compute(predictions=generated_summarisaions, references=references)\nresults_WER = wer_metric.compute(predictions=generated_summarisaions, references=references)\n\n# Calculate Exact Match (EM)\nresults_em = exact_match_metric.compute(predictions=generated_summarisaions, references=references)\n\n\n# Calculate Bilingual Evaluation Understudy (BLEU)\nresults_bleu = bleu_metric.compute(predictions=generated_summarisaions, references=references)\n\n\n# Calculate Levenshtein Distance\nlevenshtein_distances = [Levenshtein.distance(generated, reference) for generated, reference in zip(generated_summarisaions, references)]\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:22.861207Z","iopub.execute_input":"2024-01-09T14:28:22.861606Z","iopub.status.idle":"2024-01-09T14:28:24.143187Z","shell.execute_reply.started":"2024-01-09T14:28:22.861574Z","shell.execute_reply":"2024-01-09T14:28:24.142364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(results_CER)\nprint(results_WER)\nprint(results_em)\nprint(results_bleu)\n# print(levenshtein_distances)","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:24.144335Z","iopub.execute_input":"2024-01-09T14:28:24.144594Z","iopub.status.idle":"2024-01-09T14:28:24.149905Z","shell.execute_reply.started":"2024-01-09T14:28:24.14457Z","shell.execute_reply":"2024-01-09T14:28:24.149002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install unidecode","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:24.151088Z","iopub.execute_input":"2024-01-09T14:28:24.151378Z","iopub.status.idle":"2024-01-09T14:28:36.527192Z","shell.execute_reply.started":"2024-01-09T14:28:24.151353Z","shell.execute_reply":"2024-01-09T14:28:36.526186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from rouge_score import rouge_scorer\nfrom unidecode import unidecode\n\n# Initialize the Rouge scorer\nscorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=False)\n# Define a function to preprocess and tokenize Bengali text\ndef preprocess_text(text):\n    text = unidecode(text)\n    tokens = text.split()\n    return ' '.join(tokens)\n\n# Create lists to store individual scores\nrouge1_f1_scores = []\nrouge1_precision_scores = []\nrouge1_recall_scores = []\nrouge2_f1_scores = []\nrouge2_precision_scores = []\nrouge2_recall_scores = []\nrougeL_f1_scores = []\nrougeL_precision_scores = []\nrougeL_recall_scores = []\n\nfor ref, pred in zip(references, generated_summarisaions):\n    candidate = preprocess_text(pred)\n    reference = preprocess_text(' '.join(ref))\n    scores = scorer.score(reference, candidate)\n\n    rouge1_f1_scores.append(scores['rouge1'].fmeasure)\n    rouge1_precision_scores.append(scores['rouge1'].precision)\n    rouge1_recall_scores.append(scores['rouge1'].recall)\n    rouge2_f1_scores.append(scores['rouge2'].fmeasure)\n    rouge2_precision_scores.append(scores['rouge2'].precision)\n    rouge2_recall_scores.append(scores['rouge2'].recall)\n    rougeL_f1_scores.append(scores['rougeL'].fmeasure)\n    rougeL_precision_scores.append(scores['rougeL'].precision)\n    rougeL_recall_scores.append(scores['rougeL'].recall)\n\n# Calculate the average scores\navg_rouge1_f1 = sum(rouge1_f1_scores) / len(rouge1_f1_scores)\navg_rouge1_precision = sum(rouge1_precision_scores) / len(rouge1_precision_scores)\navg_rouge1_recall = sum(rouge1_recall_scores) / len(rouge1_recall_scores)\navg_rouge2_f1 = sum(rouge2_f1_scores) / len(rouge2_f1_scores)\navg_rouge2_precision = sum(rouge2_precision_scores) / len(rouge2_precision_scores)\navg_rouge2_recall = sum(rouge2_recall_scores) / len(rouge2_recall_scores)\navg_rougeL_f1 = sum(rougeL_f1_scores) / len(rougeL_f1_scores)\navg_rougeL_precision = sum(rougeL_precision_scores) / len(rougeL_precision_scores)\navg_rougeL_recall = sum(rougeL_recall_scores) / len(rougeL_recall_scores)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:36.52912Z","iopub.execute_input":"2024-01-09T14:28:36.530065Z","iopub.status.idle":"2024-01-09T14:28:38.144572Z","shell.execute_reply.started":"2024-01-09T14:28:36.530024Z","shell.execute_reply":"2024-01-09T14:28:38.143718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the average scores\nprint(\"Average Rouge-1 F1 Score:\", avg_rouge1_f1)\nprint(\"Average Rouge-1 Precision:\", avg_rouge1_precision)\nprint(\"Average Rouge-1 Recall:\", avg_rouge1_recall)\n\nprint(\"Average Rouge-2 F1 Score:\", avg_rouge2_f1)\nprint(\"Average Rouge-2 Precision:\", avg_rouge2_precision)\nprint(\"Average Rouge-2 Recall:\", avg_rouge2_recall)\n\nprint(\"Average Rouge-L F1 Score:\", avg_rougeL_f1)\nprint(\"Average Rouge-L Precision:\", avg_rougeL_precision)\nprint(\"Average Rouge-L Recall:\", avg_rougeL_recall)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-09T14:28:38.151131Z","iopub.execute_input":"2024-01-09T14:28:38.151508Z","iopub.status.idle":"2024-01-09T14:28:38.157747Z","shell.execute_reply.started":"2024-01-09T14:28:38.151482Z","shell.execute_reply":"2024-01-09T14:28:38.156791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}