add layout-validity (#16)

* add files

* update

* update README.md

* update

.github/workflows/layout_validity.yaml

@@ -0,0 +1,62 @@
+name: Validity
+
+on:
+  workflow_run:
+    workflows:
+      - CI
+    branches:
+      - main
+    types:
+      - completed
+
+jobs:
+  push_to_hub:
+    runs-on: ubuntu-latest
+
+    env:
+      REPO_NAME: "layout-validity"
+      DIR_NAME: "layout_validity"
+
+      SRC_DIR: ./github-repo
+      DST_DIR: ./huggingface-repo
+
+    steps:
+      - name: Checkout GitHub repository
+        uses: actions/checkout@v3
+        with:
+          path: ${{ env.SRC_DIR }}
+
+      - name: Checkout Huggingface repository
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+        run: |
+          git clone https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/${REPO_NAME} ${DST_DIR}
+
+      - name: Export requirements.txt
+        run: |
+          pip install poetry
+          poetry -C ${SRC_DIR} export -f requirements.txt --output ${DST_DIR}/requirements.txt --without-hashes
+
+      - name: Copy files to Huggingface repository
+        env:
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+        run: |
+          SCRIPT_NAME=${REPO_NAME}.py
+
+          cp ${SRC_DIR}/${DIR_NAME}/README.md ${DST_DIR}/README.md
+          cp ${SRC_DIR}/${DIR_NAME}/${SCRIPT_NAME} ${DST_DIR}/${SCRIPT_NAME}
+
+          git -C ${DST_DIR} config user.name "${HF_USERNAME}"
+          git -C ${DST_DIR} config user.email "${HF_EMAIL}"
+
+          git -C ${DST_DIR} add README.md requirements.txt ${SCRIPT_NAME}
+
+          if git -C ${DST_DIR} diff --cached --quiet; then
+              echo "No changes to commit"
+          else
+              msg=$(git -C ${SRC_DIR} rev-parse HEAD)
+              git -C ${DST_DIR} commit -m "deploy: ${msg}"
+              git -C ${DST_DIR} push -u origin main
+          fi

README.md

@@ -51,3 +51,4 @@ alignment_score.compute()
 - Kikuchi, Kotaro, et al. "[Constrained graphic layout generation via latent optimization.](https://arxiv.org/abs/2108.00871)" Proceedings of the 29th ACM International Conference on Multimedia. 2021.
 - Arroyo, Diego Martin, Janis Postels, and Federico Tombari. "[Variational transformer networks for layout generation.](https://arxiv.org/abs/2104.02416)" Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2021.
 - Kong, Xiang, et al. "[BLT: bidirectional layout transformer for controllable layout generation.](https://arxiv.org/abs/2112.05112)" European Conference on Computer Vision. Cham: Springer Nature Switzerland, 2022.
+- Hsu, Hsiao Yuan, et al. "[Posterlayout: A new benchmark and approach for content-aware visual-textual presentation layout.](https://arxiv.org/abs/2303.15937)" Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2023.

layout_validity/layout-validity.py

@@ -0,0 +1,86 @@
+from typing import List, Union
+
+import datasets as ds
+import evaluate
+import numpy as np
+import numpy.typing as npt
+
+_DESCRIPTION = r"""\
+Computes the ratio of valid elements to all elements in the layout, where the area within the canvas of a valid element must be greater than 0.1% of the canvas.
+"""
+
+_KWARGS_DESCRIPTION = """\
+"""
+
+_CITATION = """\
+@inproceedings{hsu2023posterlayout,
+  title={Posterlayout: A new benchmark and approach for content-aware visual-textual presentation layout},
+  author={Hsu, Hsiao Yuan and He, Xiangteng and Peng, Yuxin and Kong, Hao and Zhang, Qing},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  pages={6018--6026},
+  year={2023}
+}
+"""
+
+
+class LayoutValidity(evaluate.Metric):
+    def __init__(
+        self,
+        canvas_width: int,
+        canvas_height: int,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.canvas_width = canvas_width
+        self.canvas_height = canvas_height
+
+    def _info(self) -> evaluate.EvaluationModuleInfo:
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=ds.Features(
+                {
+                    "predictions": ds.Sequence(ds.Sequence(ds.Value("float64"))),
+                    "gold_labels": ds.Sequence(ds.Sequence(ds.Value("int64"))),
+                }
+            ),
+            codebase_urls=[
+                "https://github.com/PKU-ICST-MIPL/PosterLayout-CVPR2023/blob/main/eval.py#L105-L127"
+            ],
+        )
+
+    def _compute(
+        self,
+        *,
+        predictions: Union[npt.NDArray[np.float64], List[List[float]]],
+        gold_labels: Union[npt.NDArray[np.int64], List[int]],
+    ) -> float:
+        predictions = np.array(predictions)
+        gold_labels = np.array(gold_labels)
+
+        predictions[:, :, ::2] *= self.canvas_width
+        predictions[:, :, 1::2] *= self.canvas_height
+
+        total_elements, empty_elements = 0, 0
+
+        w = self.canvas_width / 100
+        h = self.canvas_height / 100
+
+        assert len(predictions) == len(gold_labels)
+
+        for gold_label, prediction in zip(gold_labels, predictions):
+            mask = (gold_label > 0).reshape(-1)
+            mask_prediction = prediction[mask]
+            total_elements += len(mask_prediction)
+            for mp in mask_prediction:
+                xl, yl, xr, yr = mp
+                xl = max(0, xl)
+                yl = max(0, yl)
+                xr = min(self.canvas_width, xr)
+                yr = min(self.canvas_height, yr)
+
+                if abs((xr - xl) * (yr - yl)) < w * h * 10:
+                    empty_elements += 1
+
+        return 1 - empty_elements / total_elements

tests/layout_validity_test.py

@@ -0,0 +1,54 @@
+import os
+import pathlib
+
+import evaluate
+import pytest
+import torch
+
+
+@pytest.fixture
+def base_dir() -> str:
+    return "layout_validity"
+
+
+@pytest.fixture
+def metric_path(base_dir: str) -> str:
+    return os.path.join(base_dir, "layout-validity.py")
+
+
+@pytest.fixture
+def test_fixture_dir() -> pathlib.Path:
+    return pathlib.Path(__file__).parents[1] / "test_fixtures"
+
+
+@pytest.fixture
+def poster_width() -> int:
+    return 513
+
+
+@pytest.fixture
+def poster_height() -> int:
+    return 750
+
+
+def test_metric(
+    metric_path: str,
+    test_fixture_dir: pathlib.Path,
+    poster_width: int,
+    poster_height: int,
+    # https://github.com/PKU-ICST-MIPL/PosterLayout-CVPR2023/blob/main/output/results.txt#L2C14-L2C31
+    expected_score: float = 0.878844169246646,
+):
+    # shape: (batch_size, max_elements, 4)
+    predictions = torch.load(test_fixture_dir / "poster_layout_boxes.pt")
+    # shape: (batch_size, max_elements, 1)
+    gold_labels = torch.load(test_fixture_dir / "poster_layout_clses.pt")
+
+    metric = evaluate.load(
+        path=metric_path,
+        canvas_width=poster_width,
+        canvas_height=poster_height,
+    )
+    metric.add_batch(predictions=predictions, gold_labels=gold_labels)
+    score = metric.compute()
+    assert score is not None and score == expected_score