diff --git a/.github/workflows/layout_validity.yaml b/.github/workflows/layout_validity.yaml new file mode 100644 index 0000000..b23858f --- /dev/null +++ b/.github/workflows/layout_validity.yaml @@ -0,0 +1,62 @@ +name: Validity + +on: + workflow_run: + workflows: + - CI + branches: + - main + types: + - completed + +jobs: + push_to_hub: + runs-on: ubuntu-latest + + env: + REPO_NAME: "layout-validity" + DIR_NAME: "layout_validity" + + SRC_DIR: ./github-repo + DST_DIR: ./huggingface-repo + + steps: + - name: Checkout GitHub repository + uses: actions/checkout@v3 + with: + path: ${{ env.SRC_DIR }} + + - name: Checkout Huggingface repository + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_USERNAME: ${{ secrets.HF_USERNAME }} + run: | + git clone https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${REPO_NAME} ${DST_DIR} + + - name: Export requirements.txt + run: | + pip install poetry + poetry -C ${SRC_DIR} export -f requirements.txt --output ${DST_DIR}/requirements.txt --without-hashes + + - name: Copy files to Huggingface repository + env: + HF_USERNAME: ${{ secrets.HF_USERNAME }} + HF_EMAIL: ${{ secrets.HF_EMAIL }} + run: | + SCRIPT_NAME=${REPO_NAME}.py + + cp ${SRC_DIR}/${DIR_NAME}/README.md ${DST_DIR}/README.md + cp ${SRC_DIR}/${DIR_NAME}/${SCRIPT_NAME} ${DST_DIR}/${SCRIPT_NAME} + + git -C ${DST_DIR} config user.name "${HF_USERNAME}" + git -C ${DST_DIR} config user.email "${HF_EMAIL}" + + git -C ${DST_DIR} add README.md requirements.txt ${SCRIPT_NAME} + + if git -C ${DST_DIR} diff --cached --quiet; then + echo "No changes to commit" + else + msg=$(git -C ${SRC_DIR} rev-parse HEAD) + git -C ${DST_DIR} commit -m "deploy: ${msg}" + git -C ${DST_DIR} push -u origin main + fi diff --git a/README.md b/README.md index 2fb3782..c204389 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,4 @@ alignment_score.compute() - Kikuchi, Kotaro, et al. "[Constrained graphic layout generation via latent optimization.](https://arxiv.org/abs/2108.00871)" Proceedings of the 29th ACM International Conference on Multimedia. 2021. - Arroyo, Diego Martin, Janis Postels, and Federico Tombari. "[Variational transformer networks for layout generation.](https://arxiv.org/abs/2104.02416)" Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2021. - Kong, Xiang, et al. "[BLT: bidirectional layout transformer for controllable layout generation.](https://arxiv.org/abs/2112.05112)" European Conference on Computer Vision. Cham: Springer Nature Switzerland, 2022. +- Hsu, Hsiao Yuan, et al. "[Posterlayout: A new benchmark and approach for content-aware visual-textual presentation layout.](https://arxiv.org/abs/2303.15937)" Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2023. diff --git a/layout_validity/layout-validity.py b/layout_validity/layout-validity.py new file mode 100644 index 0000000..76ecf29 --- /dev/null +++ b/layout_validity/layout-validity.py @@ -0,0 +1,86 @@ +from typing import List, Union + +import datasets as ds +import evaluate +import numpy as np +import numpy.typing as npt + +_DESCRIPTION = r"""\ +Computes the ratio of valid elements to all elements in the layout, where the area within the canvas of a valid element must be greater than 0.1% of the canvas. +""" + +_KWARGS_DESCRIPTION = """\ +""" + +_CITATION = """\ +@inproceedings{hsu2023posterlayout, + title={Posterlayout: A new benchmark and approach for content-aware visual-textual presentation layout}, + author={Hsu, Hsiao Yuan and He, Xiangteng and Peng, Yuxin and Kong, Hao and Zhang, Qing}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + pages={6018--6026}, + year={2023} +} +""" + + +class LayoutValidity(evaluate.Metric): + def __init__( + self, + canvas_width: int, + canvas_height: int, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.canvas_width = canvas_width + self.canvas_height = canvas_height + + def _info(self) -> evaluate.EvaluationModuleInfo: + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=ds.Features( + { + "predictions": ds.Sequence(ds.Sequence(ds.Value("float64"))), + "gold_labels": ds.Sequence(ds.Sequence(ds.Value("int64"))), + } + ), + codebase_urls=[ + "https://github.com/PKU-ICST-MIPL/PosterLayout-CVPR2023/blob/main/eval.py#L105-L127" + ], + ) + + def _compute( + self, + *, + predictions: Union[npt.NDArray[np.float64], List[List[float]]], + gold_labels: Union[npt.NDArray[np.int64], List[int]], + ) -> float: + predictions = np.array(predictions) + gold_labels = np.array(gold_labels) + + predictions[:, :, ::2] *= self.canvas_width + predictions[:, :, 1::2] *= self.canvas_height + + total_elements, empty_elements = 0, 0 + + w = self.canvas_width / 100 + h = self.canvas_height / 100 + + assert len(predictions) == len(gold_labels) + + for gold_label, prediction in zip(gold_labels, predictions): + mask = (gold_label > 0).reshape(-1) + mask_prediction = prediction[mask] + total_elements += len(mask_prediction) + for mp in mask_prediction: + xl, yl, xr, yr = mp + xl = max(0, xl) + yl = max(0, yl) + xr = min(self.canvas_width, xr) + yr = min(self.canvas_height, yr) + + if abs((xr - xl) * (yr - yl)) < w * h * 10: + empty_elements += 1 + + return 1 - empty_elements / total_elements diff --git a/test_fixtures/poster_layout_boxes.pt b/test_fixtures/poster_layout_boxes.pt new file mode 100644 index 0000000..98a3ec8 Binary files /dev/null and b/test_fixtures/poster_layout_boxes.pt differ diff --git a/test_fixtures/poster_layout_clses.pt b/test_fixtures/poster_layout_clses.pt new file mode 100644 index 0000000..d643ad9 Binary files /dev/null and b/test_fixtures/poster_layout_clses.pt differ diff --git a/tests/layout_validity_test.py b/tests/layout_validity_test.py new file mode 100644 index 0000000..9197cbf --- /dev/null +++ b/tests/layout_validity_test.py @@ -0,0 +1,54 @@ +import os +import pathlib + +import evaluate +import pytest +import torch + + +@pytest.fixture +def base_dir() -> str: + return "layout_validity" + + +@pytest.fixture +def metric_path(base_dir: str) -> str: + return os.path.join(base_dir, "layout-validity.py") + + +@pytest.fixture +def test_fixture_dir() -> pathlib.Path: + return pathlib.Path(__file__).parents[1] / "test_fixtures" + + +@pytest.fixture +def poster_width() -> int: + return 513 + + +@pytest.fixture +def poster_height() -> int: + return 750 + + +def test_metric( + metric_path: str, + test_fixture_dir: pathlib.Path, + poster_width: int, + poster_height: int, + # https://github.com/PKU-ICST-MIPL/PosterLayout-CVPR2023/blob/main/output/results.txt#L2C14-L2C31 + expected_score: float = 0.878844169246646, +): + # shape: (batch_size, max_elements, 4) + predictions = torch.load(test_fixture_dir / "poster_layout_boxes.pt") + # shape: (batch_size, max_elements, 1) + gold_labels = torch.load(test_fixture_dir / "poster_layout_clses.pt") + + metric = evaluate.load( + path=metric_path, + canvas_width=poster_width, + canvas_height=poster_height, + ) + metric.add_batch(predictions=predictions, gold_labels=gold_labels) + score = metric.compute() + assert score is not None and score == expected_score