Merge pull request #58 from JohnPaton/request-token

request URLs with custom header

airbase/airbase.py

@@ -12,7 +12,7 @@
     fetch_to_file,
     fetch_unique_lines,
 )
-from .resources import CURRENT_YEAR, METADATA_URL
+from .resources import CURRENT_YEAR, LIST_URL_HEADERS, METADATA_URL
 from .summary import DB
 from .util import link_list_url, string_safe_list
 
@@ -323,6 +323,7 @@ def _get_csv_links(self, force: bool = False) -> None:
             self._download_links,
             progress=self.verbose,
             encoding="utf-8-sig",
+            headers=LIST_URL_HEADERS,
         )
 
         # list of links (no duplicates)

airbase/fetch.py

@@ -77,6 +77,7 @@ def fetcher(
     progress: bool = DEFAULT.progress,
     raise_for_status: bool = DEFAULT.raise_for_status,
     max_concurrent: int = DEFAULT.max_concurrent,
+    headers: dict[str, str] | None = None,
 ) -> AsyncIterator[str]:  # pragma: no cover
     ...
 
@@ -89,6 +90,7 @@ def fetcher(
     progress: bool = DEFAULT.progress,
     raise_for_status: bool = DEFAULT.raise_for_status,
     max_concurrent: int = DEFAULT.max_concurrent,
+    headers: dict[str, str] | None = None,
 ) -> AsyncIterator[Path]:  # pragma: no cover
     ...
 
@@ -100,6 +102,7 @@ async def fetcher(
     progress: bool = DEFAULT.progress,
     raise_for_status: bool = DEFAULT.raise_for_status,
     max_concurrent: int = DEFAULT.max_concurrent,
+    headers: dict[str, str] | None = None,
 ) -> AsyncIterator[str | Path]:
     """Request multiple urls and write request text into individual paths
     it a `dict[url, path]` is provided, or return the decoded text from each request
@@ -115,8 +118,7 @@ async def fetcher(
 
     :return: url text or path to downloaded text, one by one as the requests are completed
     """
-
-    async with aiohttp.ClientSession() as session:
+    async with aiohttp.ClientSession(headers=headers) as session:
         semaphore = asyncio.Semaphore(max_concurrent)
 
         async def fetch(url: str) -> str:
@@ -162,6 +164,7 @@ def fetch_unique_lines(
     progress: bool = DEFAULT.progress,
     raise_for_status: bool = DEFAULT.raise_for_status,
     max_concurrent: int = DEFAULT.max_concurrent,
+    headers: dict[str, str] | None = None,
 ) -> set[str]:
     """Request a list of url and return only the unique lines among all the responses
 
@@ -184,6 +187,7 @@ async def fetch() -> set[str]:
             progress=progress,
             raise_for_status=raise_for_status,
             max_concurrent=max_concurrent,
+            headers=headers,
         ):
             lines.update(text.splitlines())
         return lines

airbase/resources.py

@@ -5,7 +5,7 @@
 E1A_SUMMARY_URL = "http://discomap.eea.europa.eu/map/fme/E1a/summaryE1a.js"
 
 LINK_LIST_URL_TEMPLATE = (
-    "http://fme.discomap.eea.europa.eu/fmedatastreaming/"
+    "https://fme.discomap.eea.europa.eu/fmedatastreaming/"
     "AirQualityDownload/AQData_Extract.fmw"
     "?CountryCode={country}"
     "&CityName="
@@ -18,6 +18,11 @@
     "&Output=TEXT"
     "&UpdateDate={update_date}"
 )
+FME_TOKEN = "8f3a54b3e7054080813237004b35694fbff43580"
+LIST_URL_HEADERS = {
+    "Authorization": f"fmetoken token={FME_TOKEN}",
+    "Content-Type": "application/json",
+}
 
 METADATA_URL = (
     "http://discomap.eea.europa.eu/map/fme/metadata/PanEuropean_metadata.csv"

tests/conftest.py

@@ -19,7 +19,7 @@ def csv_links_response(response: aioresponses):
     """mock response from station data links url"""
     response.get(
         re.compile(
-            r"http://fme\.discomap\.eea\.europa\.eu/fmedatastreaming/"
+            r"https://fme\.discomap\.eea\.europa\.eu/fmedatastreaming/"
             r"AirQualityDownload/AQData_Extract\.fmw.*"
         ),
         body=resources.CSV_LINKS_RESPONSE_TEXT,