scrape_discours.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b545700d-714c-48a8-b05a-dcec638d3f16",
   "metadata": {},
   "source": [
    "# Scraper des Discours\n",
    "\n",
    "This notebook is foreked from:\n",
    "https://github.com/Araxide/french_speech_scraper/blob/master/discours_scraper.py\n",
    "I have updated the calls selenium's find_elements_by* functions.\n",
    "Many thanks!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e08927c-e044-4d7b-93ac-bf73360a87aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mettre à jour Chromium ou Chrome version=Version 114.0.5735.90 (Official Build) Arch Linux (64-bit)\n",
    "#!pip install webdriver-manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "252a1de7-a3a6-45e7-b4e9-eecb662c6961",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install selenium # donc on utilise selenium 4!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "160e6298-ea88-48b5-a789-be6cdccb24a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from time import sleep\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import NoSuchElementException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dc3214e9-093f-400c-985f-9d4e055398a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "class DiscoursScraper():\n",
    "  \n",
    "    def __init__(self):\n",
    "        #self.driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "        service = Service(\"/usr/bin/chromium\")\n",
    "        self.driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()))\n",
    "        # à changer si tu as des critères de recherche\n",
    "        #self.main_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=&field_date_prononciation_discour_interval%5Bmax%5D=&field_type_emetteur=&field_type_de_document='\n",
    "        #self.child_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=&field_date_prononciation_discour_interval%5Bmax%5D=&field_type_emetteur=&field_type_de_document=&page=1'\n",
    "    \n",
    "        #ça marche pour la plupart de discours 2019-2023\n",
    "        '''\n",
    "        self.main_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=&field_date_prononciation_discour_interval%5Bmax%5D=&field_type_de_document%5B%5D=&field_type_emetteur%5B%5D=9340&form_build_id=form-5OkWQR9wk6heLS7UrXWRSOMkhi_8G94YkOvZzq_l8fg&form_id=views_exposed_form'\n",
    "        self.child_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=&field_date_prononciation_discour_interval%5Bmax%5D=&field_type_de_document%5B0%5D=&field_type_emetteur%5B0%5D=9340&form_build_id=form-5OkWQR9wk6heLS7UrXWRSOMkhi_8G94YkOvZzq_l8fg&form_id=views_exposed_form&page='\n",
    "        '''\n",
    "\n",
    "        #mieux pour les discours  avant 2019, on ne rate rien mais on doit nettoyer les html après\n",
    "        self.main_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=1974-05-27&field_date_prononciation_discour_interval%5Bmax%5D=2018-12-31&field_type_emetteur%5B0%5D=9340&field_type_de_document%5B0%5D='\n",
    "        self.child_page_url = 'https://www.vie-publique.fr/discours/recherche?search_api_fulltext_discours=&sort_by=field_date_prononciation_discour&field_intervenant_title=&field_intervenant_qualite=&field_date_prononciation_discour_interval%5Bmin%5D=1974-05-27&field_date_prononciation_discour_interval%5Bmax%5D=2018-12-31&field_type_emetteur%5B0%5D=9340&field_type_de_document%5B0%5D=&page='\n",
    "        \n",
    "        # à changer selon vos besoins\n",
    "        self.pages_begin = 816\n",
    "        self.pages_end = 823 #937 #100 \n",
    "\n",
    "    def scrap_speeches_urls(self, page_url):\n",
    "        self.driver.get(page_url)\n",
    "        sleep(3) # wait until all elements have loaded\n",
    "        articles = self.driver.find_elements(By.CSS_SELECTOR, \"h3.fr-card__title > a\")\n",
    "        articles = [article.get_attribute('href') for article in articles]\n",
    "        return articles\n",
    "\n",
    "    def record_speeches_of_page(self, writer, speeches_urls):\n",
    "        for speech_url in speeches_urls:\n",
    "            self.driver.get(speech_url)\n",
    "            try:                \n",
    "                sleep(1) # wait until all elements have loaded\n",
    "                discours = self.driver.find_elements(By.CSS_SELECTOR, \"div.vp-discours-content > div > p\")\n",
    "                discoursText = \"\"            \n",
    "                for d in discours:\n",
    "                    discoursText += self.remove_line_breaks(d)\n",
    "                # discours title  \n",
    "                title = self.driver.find_element(By.CSS_SELECTOR, \"h1\")\n",
    "                # discours date\n",
    "                date_discours = self.driver.find_element(By.CSS_SELECTOR, \"time\")\n",
    "                # append row on csv file\n",
    "                writer.writerow({'titre': title.text, 'date': date_discours.text, 'discours': discoursText})\n",
    "            except (NoSuchElementException):\n",
    "                pass #pour l'instant on l'ignore\n",
    "            \n",
    "    def record_speeches_of_page_before_2019(self, writer, speeches_urls):\n",
    "        for speech_url in speeches_urls:\n",
    "            self.driver.get(speech_url)\n",
    "            try:                \n",
    "                sleep(2) # wait until all elements have loaded\n",
    "                discoursText = self.driver.find_element(By.CSS_SELECTOR, \"div.vp-discours-content > div\").get_attribute('innerHTML')\n",
    "                # discours title  \n",
    "                title = self.driver.find_element(By.CSS_SELECTOR, \"h1\")\n",
    "                # discours date\n",
    "                date_discours = self.driver.find_element(By.CSS_SELECTOR, \"time\")\n",
    "                # append row on csv file\n",
    "                writer.writerow({'titre': title.text, 'date': date_discours.text, 'discours': discoursText})\n",
    "            except (NoSuchElementException):\n",
    "                pass #pour l'instant on l'ignore\n",
    "    \n",
    "    def remove_line_breaks(self, element):\n",
    "        element = element.text.replace('\\n', ' ').replace('\\r', '') # remove line breaks in text\n",
    "        return str(element)\n",
    "  \n",
    "    def main(self):\n",
    "        # create csv\n",
    "        with open('scaped_discours-Président_2018-1974.csv', 'a', encoding='utf-8', newline='') as csvfile:\n",
    "            fieldnames = ['titre', 'date', 'discours']\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "            writer.writeheader()\n",
    "            \n",
    "            # scrap first page -- à enlever si on ne commence pas par le page 0\n",
    "            speeches_urls_first_page = self.scrap_speeches_urls(self.main_page_url) #2019-2023\n",
    "            #self.record_speeches_of_page(writer, speeches_urls_first_page)   \n",
    "            self.record_speeches_of_page_before_2019(writer, speeches_urls_first_page)\n",
    "            \n",
    "            # get speeches urls for each next page until reach pages_total\n",
    "            for i in range(self.pages_begin, self.pages_end):\n",
    "                speeches_urls = self.scrap_speeches_urls(self.child_page_url+str(i))\n",
    "                # scrap speech text\n",
    "                #self.record_speeches_of_page(writer, speeches_urls) #2019-2023\n",
    "                self.record_speeches_of_page_before_2019(writer, speeches_urls)\n",
    "\n",
    "        # end webdriver session    \n",
    "        self.driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5b5ddca1-3b3e-49dd-ac0e-44862794d900",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - Downloading: 100%|██████████| 7.06M/7.06M [00:00<00:00, 52.7MB/s]\n"
     ]
    }
   ],
   "source": [
    "DiscoursScraper().main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f12a468-6280-4de3-beaa-761f7e448458",
   "metadata": {},
   "source": [
    "# Read the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e1a47dce-6d9c-4dd0-9bba-81deb8e53110",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e2cb4b2-bcbb-49a1-9b5d-218acf0498ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1974_2018 = pd.read_csv('scaped_discours-Président_2018-1974.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ab5dc4e9-d0dd-4a4b-a38e-36278d3cb299",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019_2023 = pd.read_csv('scraped_discours-Président_2023-2019.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dc1e1884-d857-47b8-a823-a843207eb36e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>titre</th>\n",
       "      <th>date</th>\n",
       "      <th>discours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>317</td>\n",
       "      <td>317</td>\n",
       "      <td>316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>317</td>\n",
       "      <td>253</td>\n",
       "      <td>316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, président d...</td>\n",
       "      <td>30 janvier 2020</td>\n",
       "      <td>Merci beaucoup Monsieur le Président, je suis ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    titre             date  \\\n",
       "count                                                 317              317   \n",
       "unique                                                317              253   \n",
       "top     Déclaration de M. Emmanuel Macron, président d...  30 janvier 2020   \n",
       "freq                                                    1                4   \n",
       "\n",
       "                                                 discours  \n",
       "count                                                 316  \n",
       "unique                                                316  \n",
       "top     Merci beaucoup Monsieur le Président, je suis ...  \n",
       "freq                                                    1  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_2023 = df_2019_2023.drop_duplicates(subset=['discours'])\n",
    "df_2019_2023.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5812e060-6b7c-4a89-bccc-b9894a784bcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_2023['discours'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "bc097ec8-135f-45fb-9849-f777332bca47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_2023.dropna(subset=['discours'], inplace=True)\n",
    "df_2019_2023['discours'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "824c1345-9bf0-47c3-8f6f-7ebcb3665b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019_2023.to_csv('cleaned_discours-Président_2023-2019.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f183461a-155a-4e3e-8a90-bee049574f8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'fr_FR.UTF-8'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import locale\n",
    "from dateutil.parser import parse as parse_dt\n",
    "locale.setlocale(locale.LC_TIME, 'fr_FR.UTF-8')   ## first I set locale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "0ea29962-fc42-4126-add1-48f5d07c388f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1974_2018 = pd.read_csv('cleaned_discours-Président_2018-1974.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7857eb89-1353-4266-ac0b-74c92125764b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2019_2023 = pd.read_csv('cleaned_discours-Président_2023-2019.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "afe58dec-1a5e-46fa-b583-0c5e486b8975",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>titre</th>\n",
       "      <th>date</th>\n",
       "      <th>discours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>ALLOCUTION RADIOTELEVISEE PRONONCEE PAR M. VAL...</td>\n",
       "      <td>31 décembre 1974</td>\n",
       "      <td>BONNE ANNEE POUR CHACUNE DE VOUS, BONNE ANNEE ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>23 décembre 1974</td>\n",
       "      <td>&lt;p&gt;MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>23 décembre 1974</td>\n",
       "      <td>&lt;p&gt;MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>23 décembre 1974</td>\n",
       "      <td>&lt;p&gt;MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>ALLOCUTION DE M. VALERY GISCARD D'ESTAING A L'...</td>\n",
       "      <td>15 décembre 1974</td>\n",
       "      <td>&lt;p&gt;MONSIEUR LE PRESIDENT, MESDAMES, MESSIEURS,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                titre              date  \\\n",
       "69  ALLOCUTION RADIOTELEVISEE PRONONCEE PAR M. VAL...  31 décembre 1974   \n",
       "70  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...  23 décembre 1974   \n",
       "71  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...  23 décembre 1974   \n",
       "72  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...  23 décembre 1974   \n",
       "73  ALLOCUTION DE M. VALERY GISCARD D'ESTAING A L'...  15 décembre 1974   \n",
       "\n",
       "                                             discours  \n",
       "69  BONNE ANNEE POUR CHACUNE DE VOUS, BONNE ANNEE ...  \n",
       "70  <p>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...  \n",
       "71  <p>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...  \n",
       "72  <p>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEU...  \n",
       "73  <p>MONSIEUR LE PRESIDENT, MESDAMES, MESSIEURS,...  "
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1974_2018.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5386e731-0cb7-43f4-a56d-f3b0e40b6c17",
   "metadata": {},
   "source": [
    "# Remove html tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "aae85d5a-ab32-40bf-9a14-f4e2620fc7c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1974_2018['discours'] = df_1974_2018['discours'].str.replace(\"<br>\",'\\n')\n",
    "df_1974_2018['discours'] = df_1974_2018['discours'].str.replace(r'<[^<>]*>', '', regex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0aed94c0-efda-4f21-9bcf-1d0e77ed8440",
   "metadata": {},
   "source": [
    "# Parse French date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "e444f9ab-c02d-410c-9785-024aaa36d893",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.date(1974, 12, 31)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!pip install dateparser\n",
    "import dateparser\n",
    "dateparser.parse('31 décembre 1974').date()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ce597cfd-e37b-4ff7-9fbb-ff9e767330fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>titre</th>\n",
       "      <th>date</th>\n",
       "      <th>discours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, Président d...</td>\n",
       "      <td>2018-01-19</td>\n",
       "      <td>Mesdames les ministres, Mesdames et messieurs ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, Président d...</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>Monsieur le Premier Ministre, Mesdames, Messie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, Président d...</td>\n",
       "      <td>2017-12-06</td>\n",
       "      <td>Bonjour Messieurs Dames, Je m'exprimerai plus ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, Président d...</td>\n",
       "      <td>2017-11-25</td>\n",
       "      <td>Monsieur le Premier ministre, Mesdames, Messie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>Déclaration de M. Emmanuel Macron, Président d...</td>\n",
       "      <td>2017-11-08</td>\n",
       "      <td>Mesdames, messieurs les Ministres, Monsieur l'...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 titre        date  \\\n",
       "439  Déclaration de M. Emmanuel Macron, Président d...  2018-01-19   \n",
       "452  Déclaration de M. Emmanuel Macron, Président d...  2018-01-04   \n",
       "478  Déclaration de M. Emmanuel Macron, Président d...  2017-12-06   \n",
       "484  Déclaration de M. Emmanuel Macron, Président d...  2017-11-25   \n",
       "499  Déclaration de M. Emmanuel Macron, Président d...  2017-11-08   \n",
       "\n",
       "                                              discours  \n",
       "439  Mesdames les ministres, Mesdames et messieurs ...  \n",
       "452  Monsieur le Premier Ministre, Mesdames, Messie...  \n",
       "478  Bonjour Messieurs Dames, Je m'exprimerai plus ...  \n",
       "484  Monsieur le Premier ministre, Mesdames, Messie...  \n",
       "499  Mesdames, messieurs les Ministres, Monsieur l'...  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_2023['date'] = df_2019_2023['date'].map(lambda dt: dateparser.parse(dt).date())\n",
    "df_2019_2023.to_csv('cleaned_discours-Président_2023-2019_without_HTML.csv',index=False)\n",
    "df_2019_2023.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d95d775f-54c1-43fe-8d11-435c972c40cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1974_2018['date'] = df_1974_2018['date'].map(lambda dt: dateparser.parse(dt).date())\n",
    "df_1974_2018.to_csv('cleaned_discours-Président_2018-1974_without_HTML.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d9963fea-b9f0-4bc6-ac8b-ef2b337c0fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# now you can read the csv with \n",
    "df_1974_2018_clean = pd.read_csv('cleaned_discours-Président_2018-1974_without_HTML.csv', parse_dates=['date'])\n",
    "#df_2019_2023_clean = pd.read_csv('cleaned_discours-Président_2023-2019_without_HTML.csv', parse_dates=['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "acfba5ba-8888-4460-9e91-f15a454e9034",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>titre</th>\n",
       "      <th>date</th>\n",
       "      <th>discours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9146</th>\n",
       "      <td>ALLOCUTION RADIOTELEVISEE PRONONCEE PAR M. VAL...</td>\n",
       "      <td>1974-12-31</td>\n",
       "      <td>BONNE ANNEE POUR CHACUNE DE VOUS, BONNE ANNEE ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9147</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>1974-12-23</td>\n",
       "      <td>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9148</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>1974-12-23</td>\n",
       "      <td>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9149</th>\n",
       "      <td>ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE...</td>\n",
       "      <td>1974-12-23</td>\n",
       "      <td>MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9150</th>\n",
       "      <td>ALLOCUTION DE M. VALERY GISCARD D'ESTAING A L'...</td>\n",
       "      <td>1974-12-15</td>\n",
       "      <td>MONSIEUR LE PRESIDENT, MESDAMES, MESSIEURS, =P...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  titre       date  \\\n",
       "9146  ALLOCUTION RADIOTELEVISEE PRONONCEE PAR M. VAL... 1974-12-31   \n",
       "9147  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE... 1974-12-23   \n",
       "9148  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE... 1974-12-23   \n",
       "9149  ALLOCUTION DE M. LE PRESIDENT DE LA REPUBLIQUE... 1974-12-23   \n",
       "9150  ALLOCUTION DE M. VALERY GISCARD D'ESTAING A L'... 1974-12-15   \n",
       "\n",
       "                                               discours  \n",
       "9146  BONNE ANNEE POUR CHACUNE DE VOUS, BONNE ANNEE ...  \n",
       "9147  MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...  \n",
       "9148  MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...  \n",
       "9149  MONSIEUR L'AMBASSADEUR, =POLITIQUE EXTERIEURE=...  \n",
       "9150  MONSIEUR LE PRESIDENT, MESDAMES, MESSIEURS, =P...  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1974_2018_clean.tail()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}