dataset_creation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2223e3e7-3c32-4899-bb7b-949d65e6cf72",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, load_metric, Dataset\n",
    "import pandas as pd\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3cc8e0e-82c4-4d9a-b1eb-529b3aff1133",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_text_path = \"D:/SciPaper/arxiv-dataset/ds_train_article.txt\"\n",
    "train_summary_path = \"D:/SciPaper/arxiv-dataset/ds_train_abstract.txt\"\n",
    "\n",
    "test_text_path = \"D:/SciPaper/arxiv-dataset/ds_test_article.txt\"\n",
    "test_summary_path = \"D:/SciPaper/arxiv-dataset/ds_test_abstract.txt\"\n",
    "\n",
    "val_text_path = \"D:/SciPaper/arxiv-dataset/ds_val_article.txt\"\n",
    "val_summary_path = \"D:/SciPaper/arxiv-dataset/ds_val_abstract.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6fd917d8-a09b-45c1-8ee3-de70dcf9d8d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(train_text_path, 'r') as f:\n",
    "    train_text = f.readlines()\n",
    "    \n",
    "with open(train_summary_path, 'r') as f:\n",
    "    train_summary = f.readlines()\n",
    "\n",
    "# Load the test data\n",
    "with open(test_text_path, 'r') as f:\n",
    "    test_text = f.readlines()\n",
    "    \n",
    "with open(test_summary_path, 'r') as f:\n",
    "    test_summary = f.readlines()\n",
    "\n",
    "# Load the validation data\n",
    "with open(val_text_path, 'r') as f:\n",
    "    val_text = f.readlines()\n",
    "    \n",
    "with open(val_summary_path, 'r') as f:\n",
    "    val_summary = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "23368caf-afb1-4b66-a245-2bb70ed24fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_latex_formula(text):\n",
    "    # Remove inline LaTeX formulas\n",
    "    text = re.sub(r'@\\w+', '', text)\n",
    "    text = re.sub(r'\\$[^$]*\\$', '', text)\n",
    "    # Remove displayed LaTeX formulas\n",
    "    text = re.sub(r'\\\\begin\\{equation\\}.*?\\\\end\\{equation\\}', '', text, flags=re.DOTALL)\n",
    "    text = re.sub(r'\\\\begin\\{align\\}.*?\\\\end\\{align\\}', '', text, flags=re.DOTALL)\n",
    "    text = re.sub(r'\\\\begin\\{eqnarray\\}.*?\\\\end\\{eqnarray\\}', '', text, flags=re.DOTALL)\n",
    "    text = text.replace('  ',' ')\n",
    "    text = text.replace('\\n','')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3dabe29f-ab8d-4576-bea9-6155d08a539e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(train_summary)):\n",
    "    train_text[i] = remove_latex_formula(train_text[i])\n",
    "    train_summary[i] = remove_latex_formula(train_summary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "69906b62-ff30-4fdb-a154-9a816f2b46fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_text)):\n",
    "    test_text[i] = remove_latex_formula(test_text[i])\n",
    "    test_summary[i] = remove_latex_formula(test_summary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "34df18c7-e28f-4162-927b-4369d4bcd80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(val_text)):\n",
    "    val_text[i] = remove_latex_formula(val_text[i])\n",
    "    val_summary[i] = remove_latex_formula(val_summary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "63c42869-c483-425e-91ff-3b566639c724",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame({'article':train_text,'abstract':train_summary})\n",
    "test_df = pd.DataFrame({'article':test_text,'abstract':test_summary})\n",
    "val_df = pd.DataFrame({'article':val_text,'abstract':val_summary})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "984ac1cd-0878-448d-a91a-5e49fc7deb94",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = Dataset.from_pandas(train_df, split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b61a753f-8ef3-451e-8e3e-c08d3dede57b",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ds = Dataset.from_pandas(test_df, split=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6b8fa0a5-fa4c-4b35-b873-775f8ab208a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_ds = Dataset.from_pandas(val_df, split=\"validation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "38dd4c36-e919-4c52-a38e-b7734fd91506",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('D:/SciPaper/arxiv-dataset/train.jsonl','w') as f:\n",
    "    for line in train_ds:\n",
    "        f.write(json.dumps(line) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "51f1fb55-c93d-43ad-a5f8-611508551d54",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('D:/SciPaper/arxiv-dataset/test.jsonl','w') as f:\n",
    "    for line in test_ds:\n",
    "        f.write(json.dumps(line) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a5fd87b2-c751-4add-90c8-b81b37bb9a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('D:/SciPaper/arxiv-dataset/validation.jsonl','w') as f:\n",
    "    for line in val_ds:\n",
    "        f.write(json.dumps(line) + '\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}