update for metrics (#2)

* update for metrics * add overlap metrics * update * update * update * update README.md * update * update README.md * update * update * update * update * update * update * update README.md
creative-graphic-design · Feb 10, 2024 · 5c15b29 · 5c15b29
1 parent 5077e81
commit 5c15b29
Show file tree

Hide file tree

Showing 11 changed files with 422 additions and 24 deletions.
diff --git a/.github/workflows/push_to_space.yaml → .github/workflows/layout_alignment.yaml b/.github/workflows/push_to_space.yaml → .github/workflows/layout_alignment.yaml
@@ -1,4 +1,4 @@
-name: Push to Hugging Face Space
+name: Alignment
 
 on:
   workflow_run:
@@ -14,8 +14,8 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      SRC_DIR: github-repo
-      DST_DIR: huggingface-repo
+      SRC_DIR: ./github-repo
+      DST_DIR: ./huggingface-repo
 
     steps:
       - name: Checkout GitHub repository
@@ -40,8 +40,8 @@ jobs:
           HF_USERNAME: ${{ secrets.HF_USERNAME }}
           HF_EMAIL: ${{ secrets.HF_EMAIL }}
         run: |
-          cp ${SRC_DIR}/README.md ${DST_DIR}/README.md
-          cp ${SRC_DIR}/layout_alignment.py ${DST_DIR}/layout_alignment.py
+          cp ${SRC_DIR}/layout_alignment/README.md ${DST_DIR}/README.md
+          cp ${SRC_DIR}/layout_alignment/layout_alignment.py ${DST_DIR}/layout_alignment.py
 
           msg=$(git -C ${SRC_DIR} rev-parse HEAD)
           git -C ${DST_DIR} add README.md layout_alignment.py requirements.txt

diff --git a/.github/workflows/layout_overlap.yaml b/.github/workflows/layout_overlap.yaml
@@ -0,0 +1,53 @@
+name: Overlap
+
+on:
+  workflow_run:
+    workflows:
+      - CI
+    branches:
+      - main
+    types:
+      - completed
+
+jobs:
+  push_to_hub:
+    runs-on: ubuntu-latest
+
+    env:
+      SRC_DIR: ./github-repo
+      DST_DIR: ./huggingface-repo
+
+    steps:
+      - name: Checkout GitHub repository
+        uses: actions/checkout@v3
+        with:
+          path: ${{ env.SRC_DIR }}
+
+      - name: Checkout Huggingface repository
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+        run: |
+          git clone https://${HF_USERNAME}:${HF_TOKEN}@huggingface.co/spaces/${HF_USERNAME}/layout_overlap ${DST_DIR}
+
+      - name: Export requirements.txt
+        run: |
+          pip install poetry
+          poetry -C ${SRC_DIR} --without-hashes export -f requirements.txt --output ${DST_DIR}/requirements.txt
+
+      - name: Copy files to Huggingface repository
+        env:
+          HF_USERNAME: ${{ secrets.HF_USERNAME }}
+          HF_EMAIL: ${{ secrets.HF_EMAIL }}
+        run: |
+          cp ${SRC_DIR}/layout_overlap/README.md ${DST_DIR}/README.md
+          cp ${SRC_DIR}/layout_overlap/layout_overlap.py ${DST_DIR}/layout_overlap.py
+
+          msg=$(git -C ${SRC_DIR} rev-parse HEAD)
+          git -C ${DST_DIR} add README.md layout_overlap.py requirements.txt
+
+          git -C ${DST_DIR} config user.name "${HF_USERNAME}"
+          git -C ${DST_DIR} config user.email "${HF_EMAIL}"
+
+          git -C ${DST_DIR} commit -m "deploy: ${msg}"
+          git -C ${DST_DIR} push -u origin main
diff --git a/README.md b/README.md
@@ -1,12 +1,44 @@
----
-title: Layout Alignment
-emoji: 📊
-colorFrom: pink
-colorTo: purple
-sdk: gradio
-sdk_version: 4.17.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🤗 Layout Evaluation Metrics by Huggingface Evaluate
+[![CI](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/ci.yaml/badge.svg)](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/ci.yaml)
+
+
+
+| 📊 Metric | 🤗 Space |
+|:---------:|:---------:|
+| [![Alignment](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/layout_alignment.yaml/badge.svg)](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/layout_alignment.yaml) | [`pytorch-layout-generation/layout_alignment`](https://huggingface.co/spaces/pytorch-layout-generation/layout_alignment) |
+| [![Overlap](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/layout_overlap.yaml/badge.svg)](https://github.com/shunk031/huggingface-evaluate_layout-metrics/actions/workflows/layout_overlap.yaml) | [`pytorch-layout-generation/layout_overlap`](https://huggingface.co/spaces/pytorch-layout-generation/layout_overlap) |
+
+# How to use
+
+- Install [`evaluate`](https://huggingface.co/docs/evaluate/index) library
+
+```shell
+pip install evaluate
+```
+
+- Load the layout metric and then compute the score
+
+```python
+import evaluate
+import numpy as np
+
+# Load the evaluation metric named "pytorch-layout-generation/layout_alignment"
+alignment_score = evaluate.load("pytorch-layout-generation/layout_alignment")
+
+# `batch_bbox` is a tensor representing (batch_size, max_num_elements, coordinates) 
+# and `batch_mask` is a tensor representing (batch_size, max_num_elements).
+batch_bbox = np.random.rand(512, 25, 4)
+batch_mask = np.random.rand(512, 25)
+
+# Add the batch of bboxes and masks to the metric
+alignment_score.add_batch(batch_bbox=batch_bbox, batch_mask=batch_mask)
+# Perform the computation of the evaluation metric
+alignment_score.compute()
+```
+
+## Reference
+
+- Li, Jianan, et al. "[LayoutGAN: Generating Graphic Layouts with Wireframe Discriminators.](https://arxiv.org/abs/1901.06767)" International Conference on Learning Representations. 2019.
+- Lee, Hsin-Ying, et al. "[Neural design network: Graphic layout generation with constraints.](https://arxiv.org/abs/1912.09421)" Computer Vision–ECCV 2020: 16th European Conference, Glasgow, UK, August 23–28, 2020, Proceedings, Part III 16. Springer International Publishing, 2020.
+- Li, Jianan, et al. "[Attribute-conditioned layout gan for automatic graphic design.](https://arxiv.org/abs/2009.05284)" IEEE Transactions on Visualization and Computer Graphics 27.10 (2020): 4039-4048.
+- Kikuchi, Kotaro, et al. "[Constrained graphic layout generation via latent optimization.](https://arxiv.org/abs/2108.00871)" Proceedings of the 29th ACM International Conference on Multimedia. 2021.
diff --git a/layout_alignment/README.md b/layout_alignment/README.md
@@ -0,0 +1,12 @@
+---
+title: Layout Alignment
+emoji: 📊
+colorFrom: pink
+colorTo: purple
+sdk: gradio
+sdk_version: 4.17.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
diff --git a/layout_alignment.py → layout_alignment/layout_alignment.py b/layout_alignment.py → layout_alignment/layout_alignment.py
@@ -96,8 +96,8 @@ def _compute_ac_layout_gan(
         X = np.abs(X).transpose(0, 2, 1, 3)
         X[~batch_mask] = 1.0
 
-        # shape: (B, S)
-        X = X.min(axis=-1).min(axis=-1)
+        # shape: (B, S, 6, S) -> (B, S)
+        X = X.min(axis=(2, 3))
         X[X == 1.0] = 0.0
         X = -np.log(1 - X)
 
@@ -141,7 +141,7 @@ def _compute_neural_design_network(
         Y[batch_mask] = 1.0
 
         # shape: (B, 3, S, S) -> (B, S, S) -> (B, S)
-        Y = np.abs(Y).min(axis=1).min(axis=2)
+        Y = np.abs(Y).min(axis=(1, 2))
         Y[Y == 1.0] = 0.0
 
         # shape: (B, S) -> (B,)

diff --git a/layout_overlap/README.md b/layout_overlap/README.md
@@ -0,0 +1,12 @@
+---
+title: Layout Overlap
+emoji: 📊
+colorFrom: pink
+colorTo: purple
+sdk: gradio
+sdk_version: 4.17.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
diff --git a/layout_overlap/__init__.py b/layout_overlap/__init__.py
diff --git a/layout_overlap/layout_overlap.py b/layout_overlap/layout_overlap.py
@@ -0,0 +1,186 @@
+from typing import Dict, List, Tuple, TypedDict, Union
+
+import datasets as ds
+import evaluate
+import numpy as np
+import numpy.typing as npt
+
+_DESCRIPTION = """\
+Some overlap metrics that are different to each other in previous works.
+"""
+
+_CITATION = """\
+@inproceedings{li2018layoutgan,
+  title={LayoutGAN: Generating Graphic Layouts with Wireframe Discriminators},
+  author={Li, Jianan and Yang, Jimei and Hertzmann, Aaron and Zhang, Jianming and Xu, Tingfa},
+  booktitle={International Conference on Learning Representations},
+  year={2019}
+}
+
+@article{li2020attribute,
+  title={Attribute-conditioned layout gan for automatic graphic design},
+  author={Li, Jianan and Yang, Jimei and Zhang, Jianming and Liu, Chang and Wang, Christina and Xu, Tingfa},
+  journal={IEEE Transactions on Visualization and Computer Graphics},
+  volume={27},
+  number={10},
+  pages={4039--4048},
+  year={2020},
+  publisher={IEEE}
+}
+
+@inproceedings{kikuchi2021constrained,
+  title={Constrained graphic layout generation via latent optimization},
+  author={Kikuchi, Kotaro and Simo-Serra, Edgar and Otani, Mayu and Yamaguchi, Kota},
+  booktitle={Proceedings of the 29th ACM International Conference on Multimedia},
+  pages={88--96},
+  year={2021}
+}
+"""
+
+
+def convert_xywh_to_ltrb(
+    batch_bbox: npt.NDArray[np.float64],
+) -> Tuple[
+    npt.NDArray[np.float64],
+    npt.NDArray[np.float64],
+    npt.NDArray[np.float64],
+    npt.NDArray[np.float64],
+]:
+    xc, yc, w, h = batch_bbox
+    x1 = xc - w / 2
+    y1 = yc - h / 2
+    x2 = xc + w / 2
+    y2 = yc + h / 2
+    return (x1, y1, x2, y2)
+
+
+class A(TypedDict):
+    a1: npt.NDArray[np.float64]
+    ai: npt.NDArray[np.float64]
+
+
+class LayoutOverlap(evaluate.Metric):
+    def _info(self) -> evaluate.EvaluationModuleInfo:
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            features=ds.Features(
+                {
+                    "batch_bbox": ds.Sequence(ds.Sequence(ds.Value("float64"))),
+                    "batch_mask": ds.Sequence(ds.Value("bool")),
+                }
+            ),
+            codebase_urls=[
+                "https://github.com/ktrk115/const_layout/blob/master/metric.py#L138-L164",
+                "https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L150-L203",
+            ],
+        )
+
+    def __calculate_a1_ai(self, batch_bbox: npt.NDArray[np.float64]) -> A:
+
+        l1, t1, r1, b1 = convert_xywh_to_ltrb(batch_bbox[:, :, :, None])
+        l2, t2, r2, b2 = convert_xywh_to_ltrb(batch_bbox[:, :, None, :])
+        a1 = (r1 - l1) * (b1 - t1)
+
+        # shape: (B, S, S)
+        l_max = np.maximum(l1, l2)
+        r_min = np.minimum(r1, r2)
+        t_max = np.maximum(t1, t2)
+        b_min = np.minimum(b1, b2)
+        cond = (l_max < r_min) & (t_max < b_min)
+        ai = np.where(cond, (r_min - l_max) * (b_min - t_max), 0.0)
+
+        return {"a1": a1, "ai": ai}
+
+    def _compute_ac_layout_gan(
+        self,
+        S: int,
+        ai: npt.NDArray[np.float64],
+        a1: npt.NDArray[np.float64],
+        batch_mask: npt.NDArray[np.bool_],
+    ) -> npt.NDArray[np.float64]:
+
+        # shape: (B, S) -> (B, S, S)
+        batch_mask = ~batch_mask[:, None, :] | ~batch_mask[:, :, None]
+        indices = np.arange(S)
+        batch_mask[:, indices, indices] = True
+        ai[batch_mask] = 0.0
+
+        # shape: (B, S, S)
+        ar = np.nan_to_num(ai / a1)
+        score = ar.sum(axis=(1, 2))
+
+        return score
+
+    def _compute_layout_gan_pp(
+        self,
+        score_ac_layout_gan: npt.NDArray[np.float64],
+        batch_mask: npt.NDArray[np.bool_],
+    ) -> npt.NDArray[np.float64]:
+
+        # shape: (B, S) -> (B,)
+        batch_mask = batch_mask.sum(axis=1)
+
+        # shape: (B,)
+        score_normalized = score_ac_layout_gan / batch_mask
+        score_normalized[np.isnan(score_normalized)] = 0.0
+
+        return score_normalized
+
+    def _compute_layout_gan(
+        self, S: int, B: int, ai: npt.NDArray[np.float64]
+    ) -> npt.NDArray[np.float64]:
+
+        indices = np.arange(S)
+        ii, jj = np.meshgrid(indices, indices, indexing="ij")
+
+        # shape: ii (S, S) -> (1, S, S), jj (S, S) -> (1, S, S)
+        # shape: (1, S, S) -> (B, S, S)
+        ai[np.repeat((ii[None, :] >= jj[None, :]), axis=0, repeats=B)] = 0.0
+
+        # shape: (B, S, S) -> (B,)
+        score = ai.sum(axis=(1, 2))
+
+        return score
+
+    def _compute(
+        self,
+        *,
+        batch_bbox: Union[npt.NDArray[np.float64], List[List[int]]],
+        batch_mask: Union[npt.NDArray[np.bool_], List[List[bool]]],
+    ) -> Dict[str, npt.NDArray[np.float64]]:
+
+        # shape: (B, model_max_length, C)
+        batch_bbox = np.array(batch_bbox)
+        # shape: (B, model_max_length)
+        batch_mask = np.array(batch_mask)
+
+        assert batch_bbox.ndim == 3
+        assert batch_mask.ndim == 2
+
+        # S: model_max_length
+        B, S, C = batch_bbox.shape
+
+        # shape: batch_bbox (B, S, C), batch_mask (B, S) -> (B, S, 1) -> (B, S, C)
+        batch_bbox[np.repeat(~batch_mask[:, :, None], axis=2, repeats=C)] = 0.0
+        # shape: (C, B, S)
+        batch_bbox = batch_bbox.transpose(2, 0, 1)
+
+        A = self.__calculate_a1_ai(batch_bbox)
+
+        # shape: (B,)
+        score_ac_layout_gan = self._compute_ac_layout_gan(
+            S=S, batch_mask=batch_mask, **A
+        )
+        # shape: (B,)
+        score_layout_gan_pp = self._compute_layout_gan_pp(
+            score_ac_layout_gan=score_ac_layout_gan, batch_mask=batch_mask
+        )
+        # shape: (B,)
+        score_layout_gan = self._compute_layout_gan(B=B, S=S, ai=A["ai"])
+
+        return {
+            "overlap-ACLayoutGAN": score_ac_layout_gan,
+            "overlap-LayoutGAN++": score_layout_gan_pp,
+            "overlap-LayoutGAN": score_layout_gan,
+        }
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "huggingface-evaluate-layout-alignment"
+name = "huggingface-evaluate-layout-metrics"
 version = "0.1.0"
 description = ""
 authors = ["Shunsuke KITADA <shunsuke.kitada.0831@gmail.com>"]