diff --git a/.coveragerc b/.coveragerc index e803fc6..06ea706 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,4 +6,5 @@ omit = setup.py zeroae/goblet/__main__.py zeroae/goblet/_version.py - tests/* \ No newline at end of file + zeroae/smee/__main__.py + tests/* diff --git a/environment.yml b/environment.yml index 524ba24..969b93b 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - conda-forge - defaults dependencies: + - ipython - python - pip @@ -16,6 +17,7 @@ dependencies: - chalice >=1.13 - click >=7.0 - click-plugins + - click-log - entrypoints - environs - jinja2 >=2.9,<3 @@ -27,6 +29,7 @@ dependencies: - pytest >=3 - pytest-chalice - pytest-cov + - requests-mock # Documentation Requirements - sphinx @@ -36,5 +39,5 @@ dependencies: - pre-commit - black - flake8 + - rsa - twine - diff --git a/setup.py b/setup.py index 059cf76..78edc3c 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ "chalice>=1.13", "click>=7.0", "click-plugins", + "click-log", "entrypoints", "environs", "jinja2>=2.9,<3", @@ -37,6 +38,7 @@ "pytest>=3", "pytest-chalice", "pytest-cov", + "requests-mock", # fmt: on ] @@ -64,6 +66,9 @@ description="A Chalice blueprint for creating GitHub Apps", # fmt: off entry_points={ + "console_scripts": [ + "smee=zeroae.smee.cli:smee" + ], "zeroae.cli": [ "goblet=zeroae.goblet.cli:goblet", ], diff --git a/tests/smee/__init__.py b/tests/smee/__init__.py new file mode 100644 index 0000000..26fe968 --- /dev/null +++ b/tests/smee/__init__.py @@ -0,0 +1,15 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ diff --git a/tests/smee/test_event_stream.py b/tests/smee/test_event_stream.py new file mode 100644 index 0000000..8d604ae --- /dev/null +++ b/tests/smee/test_event_stream.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ + +import textwrap +from http import HTTPStatus + +import pytest + +from zeroae.smee import event_stream +from zeroae.smee.event_stream import Event + + +def test_session(): + s = event_stream.session() + assert s.headers["accept"] == "text/event-stream" + assert s.stream + + +def test_get(requests_mock): + requests_mock.get( + "mock://smee.io/new", text="Hello World", status_code=HTTPStatus.OK + ) + r = event_stream.get("mock://smee.io/new") + assert r.text == "Hello World" + + +@pytest.mark.parametrize("stream,data", [(":this is a test\n\n", ""), (":🥇\n\n", "🥇")]) +def test_no_event(stream, data, requests_mock): + requests_mock.get("mock://smee.io/new", text=textwrap.dedent(stream)) + r = event_stream.get("mock://smee.io/new") + events: list[Event] = list(r.iter_events()) + assert len(events) == 0 + + +@pytest.mark.parametrize( + "stream,data", + [ + ("data\n\n", ""), + ("data\ndata\n\n", "\n"), + ("data:test\n\n", "test"), + ("data: test\n\n", "test"), + ("data: test\n\n", " test"), + ("data:test\n: This is a comment, ignore me!!!:::\n\n", "test"), + ("data: YHOO\ndata: +2\ndata: 10\n\n", "YHOO\n+2\n10"), + ], +) +def test_one_event(stream, data, requests_mock): + requests_mock.get("mock://smee.io/new", text=textwrap.dedent(stream)) + r = event_stream.get("mock://smee.io/new") + events: list[Event] = list(r.iter_events()) + assert len(events) == 1 + assert events[0].data == data + assert events[0].type == "message" + + +@pytest.mark.parametrize( + "stream,data", [("data: {}\n\n", {}), ('data: {"key":"🥇"}\n\n', {"key": "🥇"})], +) +def test_one_json_event(stream, data, requests_mock): + requests_mock.get("mock://smee.io/new", text=textwrap.dedent(stream)) + r = event_stream.get("mock://smee.io/new") + events: list[Event] = list(r.iter_events()) + assert len(events) == 1 + assert events[0].type == "message" + assert events[0].json() == data diff --git a/tests/smee/test_smee.py b/tests/smee/test_smee.py new file mode 100644 index 0000000..f72dc0e --- /dev/null +++ b/tests/smee/test_smee.py @@ -0,0 +1,75 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ +import pytest +from click.testing import CliRunner + +from zeroae.smee import cli + + +@pytest.fixture() +def smee_server_mock(requests_mock): + url = "mock://smee.io/new" + requests_mock.get( + url, + text="\n".join( + [ + "event:ready\ndata:{}\n", + "event:ping\ndata:{}\n", + 'data:{"body":{},"timestamp":1,"query":{}}\n\n', + ] + ), + ) + return url + + +def test_command_line_interface(smee_server_mock, requests_mock): + """Test the SMEE CLI.""" + runner = CliRunner() + args = [f"--url={smee_server_mock}"] + + target_url = "mock://target.io/events" + requests_mock.post(target_url) + args += [f"--target={target_url}"] + + help_result = runner.invoke(cli.smee, args) + assert help_result.exit_code == 0 + assert f"Connected {smee_server_mock}" in help_result.output + + +@pytest.mark.parametrize( + "port,path", [(None, None), (6000, None), (None, "/events"), (6000, "/events")] +) +def test_command_line_interface_port_path(port, path, smee_server_mock, requests_mock): + """Test the SMEE CLI.""" + runner = CliRunner() + args = [f"--url={smee_server_mock}"] + + if port is None: + port = 3000 + else: + args += [f"--port={port}"] + + if path is None: + path = "/" + else: + args += [f"--path={path}"] + + target_url = f"http://127.0.0.1:{port}{path}" + requests_mock.post(target_url) + + help_result = runner.invoke(cli.smee, args) + assert help_result.exit_code == 0 + assert f"Connected {smee_server_mock}" in help_result.output diff --git a/tests/test_goblet.py b/tests/test_goblet.py index e5bfdc8..d010610 100644 --- a/tests/test_goblet.py +++ b/tests/test_goblet.py @@ -21,7 +21,7 @@ def response(): def test_content(response): """Sample pytest test function with the pytest fixture as an argument.""" # from bs4 import BeautifulSoup - # assert 'GitHub' in BeautifulSoup(response.content).title.string + # assert 'GitHub' in BeautifulSoup(source.content).title.string def test_command_line_interface(): diff --git a/zeroae/smee/__init__.py b/zeroae/smee/__init__.py new file mode 100644 index 0000000..4e9c935 --- /dev/null +++ b/zeroae/smee/__init__.py @@ -0,0 +1,63 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ + +"""Webhook data delivery client. Please visit https://smee.io for more information.""" +import logging +from dataclasses import dataclass + +import requests + +from . import event_stream + +logger = logging.getLogger(__name__) + + +@dataclass +class SmeeClient(object): + source: str + target: str + + _events: requests.Response + + def __init__(self, source, target): + self.target = target + + self._events = event_stream.get(source) + self._events.raise_for_status() + + self.source = self._events.url + + def run(self): + for event in self._events.iter_events(): + self.__getattribute__(f"on_{event.type}")(event) + + def on_ping(self, event): + logger.debug(f"{self.source} is still alive...") + + def on_ready(self, event): + logger.info(f"Connected {self.source}") + + def on_message(self, event: event_stream.Event): + smee_event = event.json() + body = smee_event.pop("body") + _ = smee_event.pop("query") + _ = smee_event.pop("timestamp") + try: + requests.post(self.target, json=body, headers=smee_event) + except requests.exceptions.ConnectionError: + logger.warning( + f"Event {event.id} was not delivered. Target did not respond." + ) diff --git a/zeroae/smee/__main__.py b/zeroae/smee/__main__.py new file mode 100644 index 0000000..729d64a --- /dev/null +++ b/zeroae/smee/__main__.py @@ -0,0 +1,19 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ + +from .cli import smee + +smee() diff --git a/zeroae/smee/cli.py b/zeroae/smee/cli.py new file mode 100644 index 0000000..e884691 --- /dev/null +++ b/zeroae/smee/cli.py @@ -0,0 +1,65 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ + +from urllib.parse import urljoin + +import click +import click_log + +from zeroae.smee import SmeeClient, logger + +click_log.basic_config(logger) + + +@click.command(name="smee") +@click.version_option("0.0.1") +@click.option( + "-u", + "--url", + help="URL of the webhook proxy service.", + default="https://smee.io/new", + show_default=True, + envvar="WEBHOOK_PROXY", +) +@click.option( + "-t", + "--target", + help="Full URL (including protocol and path) of the target service the events will be " + "forwarded to. [default: http://127.0.0.1:{port}/{path}]", +) +@click.option( + "-p", "--port", help="Local HTTP server port", default=3000, show_default=True +) +@click.option( + "-P", + "--path", + help="URL path to post proxied requests to", + default="/", + show_default=True, +) +@click_log.simple_verbosity_option(logger, "-l", "--logging", show_default=True) +def smee(url, target, port, path): + """ + Webhook data delivery client. + + Please visit https://smee.io for more information. + """ + if target is None: + target = urljoin(f"http://127.0.0.1:{port}/", path) + + client = SmeeClient(source=url, target=target) + logger.info(f"Forwarding {client.source} to {target}") + client.run() diff --git a/zeroae/smee/event_stream.py b/zeroae/smee/event_stream.py new file mode 100644 index 0000000..9982493 --- /dev/null +++ b/zeroae/smee/event_stream.py @@ -0,0 +1,91 @@ +# ------------------------------------------------------------------------------ +# Copyright (c) 2020 Zero A.E., LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ + +from dataclasses import dataclass + +import requests +from requests.compat import json as complexjson + + +@dataclass +class Event: + """ + Event class for text/event-stream. + + ref: https://html.spec.whatwg.org/multipage/server-sent-events.html + """ + + id: str = None + event: str = "message" + retry: int = 2 * 1_000 + _data: str = None + + @property + def type(self) -> str: + return self.event + + @property + def data(self) -> str: + return self._data + + @data.setter + def data(self, v: str): + self._data = v if self._data is None else "\n".join([self._data, v]) + + def json(self, **kwargs): + return complexjson.loads(self._data, **kwargs) + + +def session() -> requests.Session: + """Return a request Session configured for processing an Event-Stream.""" + s = requests.Session() + s.headers["Accept"] = "text/event-stream" + s.stream = True + return s + + +def get(url: str, **kwargs): + """Sends a GET request that only accepts a text/event-stream source.""" + with session() as s: + return s.get(url, **kwargs) + + +def iter_events(r: requests.models.Response) -> Event: + """Iterates over the source data, one Event at a time. + When stream=True is set on the request, this avoids reading the + content at once into memory for large responses. + .. note:: This method is not reentrant safe. + """ + + event = None + r.encoding = r.encoding if r.encoding else "utf-8" + for line in r.iter_lines(chunk_size=128, decode_unicode=True): + if not line: + if event is not None and event.data is not None: + yield event + event = None + elif line.startswith(":"): + # Ignore comments + pass + else: + if event is None: + event = Event() + k, v = line.split(":", maxsplit=1) if ":" in line else (line, "") + if hasattr(event, k): + event.__setattr__(k, v[1:] if v.startswith(" ") else v) + + +requests.models.Response.iter_events = iter_events