Updates

.formatter.exs

@@ -1,8 +1,4 @@
+# Used by "mix format"
 [
-  inputs: [
-    "lib/**/*.{ex,exs}",
-    "test/**/*.{ex,exs}",
-    "mix/**/*.{ex,exs}",
-    "./mix.exs"
-  ]
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test,samples,mix}/**/*.{ex,exs}"]
 ]

.github/workflows/main.yml

@@ -0,0 +1,139 @@
+name: CI
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+on: [pull_request, push]
+
+jobs:
+  linux:
+    name: Test on Ubuntu (Elixir ${{ matrix.elixir_version }}, OTP ${{ matrix.otp_version }})
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        # Thoroughly check the most recent versions
+        elixir_version: ['1.13.4', '1.14.5', '1.15.7', '1.16.0-rc.0']
+        otp_version: ['23.3', '24.3', '25.3', '26.1']
+        # Spot check older versions
+        #
+        # Goal is to have old versions (elixir and erlang) have basically one test each.
+        # That should be enough to affirm that they are working ok.
+        include:
+          - elixir_version: '1.7.4'
+            otp_version: '20.3'
+          - elixir_version: '1.8.2'
+            otp_version: '20.3'
+          - elixir_version: '1.9.4'
+            otp_version: '21.3'
+          - elixir_version: '1.10.4'
+            otp_version: '21.3'
+          - elixir_version: '1.11.4'
+            otp_version: '22.3'
+          - elixir_version: '1.12.3'
+            otp_version: '23.3'
+          - elixir_version: '1.13.4'
+            otp_version: '22.3'
+        exclude:
+          - elixir_version: '1.15.7'
+            otp_version: '23.3'
+          - elixir_version: '1.16.0-rc.0'
+            otp_version: '23.3'
+          - elixir_version: '1.13.4'
+            otp_version: '26.1'
+          - elixir_version: '1.14.5'
+            otp_version: '26.1'
+
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: ${{ matrix.elixir_version }}
+          otp-version: ${{ matrix.otp_version }}
+      - name: Restore deps and _build
+        uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+          key: erlef-${{ runner.os }}-mix-${{ matrix.elixir_version }}-${{ matrix.otp_version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
+      - name: Restore plts
+        uses: actions/cache@v3
+        with:
+          path: tools/plts
+          key: erlef-${{ runner.os }}-dialyzer-${{ matrix.elixir_version }}-${{ matrix.otp_version }}-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
+        if: contains(matrix.elixir_version, '1.15')
+      - run: mix deps.get
+      - run: MIX_ENV=test mix compile --warnings-as-errors
+      - run: mix credo
+        if: contains(matrix.elixir_version, '1.15')
+      - name: Check if formatted
+        if: contains(matrix.elixir_version, '1.15')
+        run: mix format --check-formatted
+      - name: Actual Tests
+        # this will let warnings slip through but I don't wanna replicate all that magic
+        # right now
+        run: MIX_ENV=test mix coveralls.github || mix test --failed
+        # Apparently the one with `!` can't go without the fancy expression syntax
+        if: ${{ !contains(matrix.elixir_version, '1.15') }}
+      - name: Actual Tests WITH warnings as errors
+        run: MIX_ENV=test mix coveralls.github --warnings-as-errors || mix test --failed
+        if: contains(matrix.elixir_version, '1.15')
+      - name: Dialyzer
+        run: mix dialyzer --halt-exit-status
+        if: contains(matrix.elixir_version, '1.15')
+
+  macos:
+    name: Test on MacOS
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        # no versioning on brew but getting asdf or something was a bigger headache
+      - name: Install Elixir
+        run: brew install elixir
+      - name: Restore deps and _build
+        uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
+      - run: mix local.hex --force
+      - run: mix deps.get
+      - run: mix local.rebar --force
+      - run: MIX_ENV=test mix compile --warnings-as-errors
+      - run: mix test || mix test --failed
+
+  windows:
+    name: Test on Windows
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Restore chocolatey
+        uses: actions/cache@v3
+        with:
+          path: C:\Users\runneradmin\AppData\Local\Temp\chocolatey
+          key: ${{ runner.os }}-chocolatey-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-chocolatey-
+      - name: Install Elixir
+        run: |
+          choco install elixir --version=1.14.2 --no-progress
+          set MIX_ENV=test
+          echo "C:\ProgramData\chocolatey\lib\Elixir\bin;C:\ProgramData\chocolatey\bin;C:\ProgramData\chocolatey\lib\Elixir\tools\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - name: Build
+        run: |
+          echo "$PATH"
+          mix local.hex --force
+          mix deps.get
+          mix local.rebar --force
+          mix compile --warnings-as-errors
+      - name: Test
+        run: mix test || mix test --failed

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.8.1-otp-21
-erlang 21.3.2
+elixir 1.15.7-otp-26
+erlang 26.1.2

lib/benchee/formatters/json.ex

@@ -102,8 +102,53 @@ defmodule Benchee.Formatters.JSON do
       ...>        },
       ...>      ]
       ...>    }
-      iex> Benchee.Formatters.JSON.format(suite, %{file: "my_file.json"})
-      "[{\\"name\\":\\"My Job\\",\\"job_name\\":\\"My Job\\",\\"input_name\\":\\"Some Input\\",\\"run_time_data\\":{\\"statistics\\":{\\"absolute_difference\\":null,\\"average\\":500.0,\\"ips\\":2.0e3,\\"maximum\\":900,\\"median\\":450.0,\\"minimum\\":200,\\"mode\\":400,\\"percentiles\\":{\\"99\\":900},\\"relative_less\\":null,\\"relative_more\\":null,\\"sample_size\\":8,\\"std_dev\\":200.0,\\"std_dev_ips\\":800.0,\\"std_dev_ratio\\":0.4},\\"samples\\":[200,400,400,400,500,500,700,900]},\\"memory_usage_data\\":{\\"statistics\\":{\\"absolute_difference\\":null,\\"average\\":500.0,\\"ips\\":null,\\"maximum\\":900,\\"median\\":450.0,\\"minimum\\":200,\\"mode\\":null,\\"percentiles\\":{\\"99\\":900},\\"relative_less\\":null,\\"relative_more\\":null,\\"sample_size\\":8,\\"std_dev\\":200.0,\\"std_dev_ips\\":null,\\"std_dev_ratio\\":0.4},\\"samples\\":[200,400,400,400,500,500,700,900]},\\"tag\\":null}]"
+      iex> suite |> format(%{file: "my_file.json"}) |> Jason.decode!()
+      [
+        %{
+          "input_name" => "Some Input",
+          "name" => "My Job",
+          "job_name" => "My Job",
+          "tag" => nil,
+          "memory_usage_data" => %{
+            "samples" => [200, 400, 400, 400, 500, 500, 700, 900],
+            "statistics" => %{
+              "absolute_difference" => nil,
+              "average" => 500.0,
+              "ips" => nil,
+              "maximum" => 900,
+              "median" => 450.0,
+              "minimum" => 200,
+              "mode" => nil,
+              "percentiles" => %{"99" => 900},
+              "relative_less" => nil,
+              "relative_more" => nil,
+              "sample_size" => 8,
+              "std_dev" => 200.0,
+              "std_dev_ips" => nil,
+              "std_dev_ratio" => 0.4
+            }
+          },
+          "run_time_data" => %{
+            "samples" => [200, 400, 400, 400, 500, 500, 700, 900],
+            "statistics" => %{
+              "absolute_difference" => nil,
+              "average" => 500.0,
+              "ips" => 2000.0,
+              "maximum" => 900,
+              "median" => 450.0,
+              "minimum" => 200,
+              "mode" => 400,
+              "percentiles" => %{"99" => 900},
+              "relative_less" => nil,
+              "relative_more" => nil,
+              "sample_size" => 8,
+              "std_dev" => 200.0,
+              "std_dev_ips" => 800.0,
+              "std_dev_ratio" => 0.4
+            }
+          },
+        }
+      ]
 
   """
   @spec format(Suite.t(), map) :: String.t()

mix.exs

@@ -29,7 +29,7 @@ defmodule BencheeJSON.Mixfile do
   end
 
   def application do
-    [applications: [:logger, :benchee, :jason]]
+    []
   end
 
   defp deps do
@@ -40,7 +40,7 @@ defmodule BencheeJSON.Mixfile do
       {:credo, "~> 1.0", only: :dev},
       {:ex_doc, "~> 0.14", only: :dev},
       {:earmark, "~> 1.0", only: :dev},
-      {:dialyxir, "~> 0.5", only: :dev, runtime: false}
+      {:dialyxir, "~> 1.0", only: :dev, runtime: false}
     ]
   end
 

mix.lock

@@ -1,25 +1,19 @@
 %{
-  "benchee": {:hex, :benchee, "1.0.0", "2524b6c0e541309d969accf75f77e1d1653e46142afc68e9ea4ff35559a794bc", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
-  "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
-  "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
-  "credo": {:hex, :credo, "1.0.4", "d2214d4cc88c07f54004ffd5a2a27408208841be5eca9f5a72ce9e8e835f7ede", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
-  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
-  "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"},
-  "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
-  "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
-  "excoveralls": {:hex, :excoveralls, "0.10.6", "e2b9718c9d8e3ef90bc22278c3f76c850a9f9116faf4ebe9678063310742edc2", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
-  "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
-  "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
-  "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
-  "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
-  "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"},
-  "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
-  "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
-  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
-  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"},
-  "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"},
-  "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
-  "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], []},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
+  "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"},
+  "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
+  "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
+  "dialyxir": {:hex, :dialyxir, "1.4.2", "764a6e8e7a354f0ba95d58418178d486065ead1f69ad89782817c296d0d746a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "516603d8067b2fd585319e4b13d3674ad4f314a5902ba8130cd97dc902ce6bbd"},
+  "earmark": {:hex, :earmark, "1.4.46", "8c7287bd3137e99d26ae4643e5b7ef2129a260e3dcf41f251750cb4563c8fb81", [:mix], [], "hexpm", "798d86db3d79964e759ddc0c077d5eb254968ed426399fbf5a62de2b5ff8910a"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.37", "2ad73550e27c8946648b06905a57e4d454e4d7229c2dafa72a0348c99d8be5f7", [:mix], [], "hexpm", "6b19783f2802f039806f375610faa22da130b8edc21209d0bff47918bb48360e"},
+  "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
+  "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"},
+  "excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"},
+  "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
+  "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
+  "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
+  "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
+  "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
 }

samples/multiple_formatters.exs

@@ -1,10 +1,11 @@
 list = Enum.to_list(1..10_000)
-map_fun = fn(i) -> [i, i * i] end
+map_fun = fn i -> [i, i * i] end
 
-Benchee.run(%{
-  "flat_map"    => fn -> Enum.flat_map(list, map_fun) end,
-  "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
-},
+Benchee.run(
+  %{
+    "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
+    "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
+  },
   formatters: [
     {Benchee.Formatters.JSON, file: "my.json"},
     Benchee.Formatters.Console

test/benchee/formatters/json_integration_test.exs

@@ -56,7 +56,7 @@ defmodule Benchee.Formatters.JSONIntegrationTest do
                "name" => "List",
                "memory_usage_data" => %{
                  "statistics" => %{"average" => _, "ips" => _},
-                 "samples" => [376 | _]
+                 "samples" => [_ | _]
                }
              } = list
     end)

test/benchee/formatters/json_test.exs

@@ -1,6 +1,6 @@
 defmodule Benchee.Formatters.JSONTest do
   use ExUnit.Case, async: true
-  doctest Benchee.Formatters.JSON
+  doctest Benchee.Formatters.JSON, import: true
 
   alias Benchee.{Formatters.JSON, Suite}