diff --git a/README.md b/README.md index 9b7bca1..c82be05 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,45 @@ If a param is provided both in the config file and in the environement, the valu > [!TIP] > The paths in `files` can either be absolute, relative to the current user homedir, or relative the current working directory. +#### HTTP call + +```json +{ + "type": "http", + "name": "http", + "url": "https://httpbin.org/json", + "method": "GET", + "request_params": { + "headers": { + "Authorization": "Bearer THIS_IS_A_TOKEN" + } + } +} +``` + +| Key | Description | Required | Default value | +| ----------------- | ------------------------------------------------ | -------- | ------------- | +| `type` | Type of source | Yes | `"http"` | +| `name` | Name of source | Yes | | +| `url` | URL to call | Yes |   | +| `method` | HTTP method to use to do the API call | Yes | | +| `request_params` | Any extra param required for the call. | No | | + +> [!TIP] +> The `request_params` is provided as kwargs to the [`requests.request`](https://requests.readthedocs.io/en/latest/api/#requests.request). Check out the documentation to see which parameter is available. + +> [!TIP] +> The archive created after the backup is a JSON with the following format: +> ```json +> { +> "url": "", +> "timestamp": "", +> "result": "", +> "detail": "", +> "msg": "" +> } +> ``` + ### Available destinations #### S3 diff --git a/backup_me/sources/__init__.py b/backup_me/sources/__init__.py index 591bcce..ee79eed 100644 --- a/backup_me/sources/__init__.py +++ b/backup_me/sources/__init__.py @@ -4,18 +4,21 @@ from .base import BaseSource from .db import MySQLDB, PostgresDB from .files import ArchiveType, RawFiles +from .http import HTTP class SourceTypes(str, Enum): mysql = "mysql" postgres = "postgres" files = "files" + http = "http" SOURCES: t.Dict[SourceTypes, BaseSource] = { SourceTypes.mysql: MySQLDB, SourceTypes.postgres: PostgresDB, SourceTypes.files: RawFiles, + SourceTypes.http: HTTP, } __all__ = [ @@ -25,4 +28,5 @@ class SourceTypes(str, Enum): "RawFiles", "MySQLDB", "PostgresDB", + "HTTP", ] diff --git a/backup_me/sources/http.py b/backup_me/sources/http.py new file mode 100644 index 0000000..4a8ca44 --- /dev/null +++ b/backup_me/sources/http.py @@ -0,0 +1,42 @@ +import json +import os +import typing as t + +import requests + +from .base import BaseSource + + +class HTTP(BaseSource): + url: str + method: str + request_params: t.Dict[str, t.Any] + + def backup(self, output_dir: str) -> str: + resp = requests.request(self.method, self.url, **self.request_params) + + result = { + "url": self.url, + "timestamp": self.now(), + "result": resp.status_code, + } + is_json = resp.headers.get("Content-Type", "").lower() == "application/json" + if is_json: + result["detail"] = resp.json() + else: + result["msg"] = resp.text + + backup_filename = ( + f"{os.path.join(output_dir, self.backup_filename)}_{self.now()}.json" + ) + with open(backup_filename, "w") as f: + json.dump(result, f) + + if resp.ok: + print(f"HTTP call for backup completed successfully.") + else: + print( + f"HTTP call for backup failed with return code {resp.status_code} {resp.text}." + ) + + return backup_filename diff --git a/poetry.lock b/poetry.lock index 5063187..4892fca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -56,7 +56,7 @@ crt = ["awscrt (==0.19.19)"] name = "certifi" version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -133,7 +133,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "dev" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -380,7 +380,7 @@ test-randomorder = ["pytest-randomly"] name = "idna" version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -893,14 +893,14 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.3" description = "Python HTTP for Humans." -category = "dev" +category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -913,6 +913,24 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-mock" +version = "1.12.1" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "requests-mock-1.12.1.tar.gz", hash = "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"}, + {file = "requests_mock-1.12.1-py2.py3-none-any.whl", hash = "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563"}, +] + +[package.dependencies] +requests = ">=2.22,<3" + +[package.extras] +fixture = ["fixtures"] + [[package]] name = "responses" version = "0.25.0" @@ -1082,4 +1100,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "450a5a0f66f80d1541485f1e927302a438ac235b070d58b2d8da042c99989ac2" +content-hash = "b6b384ec437fc5fe3838bb7b29f4c39fe76adec283683aac120571da8f258135" diff --git a/pyproject.toml b/pyproject.toml index 0489408..9134613 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ python = "^3.11" boto3 = "^1.34.14" typer = { extras = ["all"], version = "^0.9.0" } pydantic = "^2.6.1" +requests = "^2.32.3" [tool.poetry.group.tests.dependencies] pytest = "^8.0.0" @@ -17,6 +18,7 @@ coverage = "^7.4.1" pytest-subprocess = "^1.5.0" pytest-mock = "^3.12.0" moto = { extras = ["s3"], version = "^5.0.9" } +requests-mock = "^1.12.1" [tool.poetry.scripts] backup-me = 'backup_me.main:console' diff --git a/tests/test_sources.py b/tests/test_sources.py index 2ac8cee..341e896 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1,9 +1,12 @@ +import json import os import tarfile import zipfile from datetime import datetime -from backup_me.sources import ArchiveType, MySQLDB, PostgresDB, RawFiles +import pytest + +from backup_me.sources import HTTP, ArchiveType, MySQLDB, PostgresDB, RawFiles def test_rawfiles_source(tmp_path): @@ -114,3 +117,52 @@ def test_postgres_source(tmp_path, fake_process, mocker): fake_process.register(cmd, stdout="") backup = postgres_source.backup(tmp_path) assert fake_process.call_count(cmd) == 2 + + +@pytest.mark.parametrize( + "status,json_body,text_body", + [ + (200, {"status": "OK", "msg": "Success"}, ""), + (200, {}, "Sucess"), + (500, {"status": "KO", "msg": "Error"}, ""), + (500, {}, "Error"), + ], +) +def test_http_source(status, json_body, text_body, tmp_path, requests_mock, mocker): + now = datetime.now() + mock_date = mocker.patch("backup_me.sources.base.datetime") + mock_date.now.return_value = now + + url = "https://backup.me" + args = {"status_code": status} + if json_body: + args["json"] = json_body + args["headers"] = {"Content-Type": "application/json"} + else: + args["text"] = text_body + requests_mock.post(url, **args) + + backup_filename = "backup" + source = HTTP( + backup_filename=backup_filename, + name="http", + url=url, + method="POST", + request_params={"headers": {"Authorization": "Bearer XYZ"}}, + ) + filename = f"{os.path.join(tmp_path, backup_filename)}_{source.now()}.json" + backup = source.backup(tmp_path) + assert backup == filename + with open(backup) as f: + result = json.load(f) + expected = { + "url": url, + "timestamp": source.now(), + "result": status, + } + if json_body: + expected["detail"] = json_body + else: + expected["msg"] = text_body + + assert result == expected