diff --git a/script/build_results_site.rb b/script/build_results_site.rb index e98e55a..ddce20b 100644 --- a/script/build_results_site.rb +++ b/script/build_results_site.rb @@ -172,17 +172,19 @@ def entry(report, name) report["entries"].find { |e| e["name"] == name } end - # Per report: [i/s, error, relative to the reference's median]. Relative is nil for a suspect or zero result, or a report the reference does not have. + # Per report: [i/s, error, relative to the reference's median of the claimed report]. + # Every report of a benchmark shares that one baseline, so on the chart the higher line is the faster report. + # Relative is nil for a suspect or zero result, or when the reference has no usable pass of the claimed report. def cross_run(report, names, ref_ips) entries = names.each_with_index.map do |name, i| e = entry(report, name) or next nil - rel = ref_ips[i].empty? || !e["ips"].positive? || e["ips"] > SUSPECT_IPS ? nil : (e["ips"] / median(ref_ips[i])).round(4) + rel = ref_ips[0].empty? || !e["ips"].positive? || e["ips"] > SUSPECT_IPS ? nil : sig(e["ips"] / median(ref_ips[0])) [e["ips"].round(3), e["error"].to_f.round(1), rel] end { entries: entries, pass: report["pass"] } end - # The reference itself: its median pass, 1.0 relative to itself. + # The reference itself: its median pass, relative to its own claimed report (1.0 for the claimed report). def reference_run(passes, names, ref_ips) entries = names.each_with_index.map do |name, i| es = passes.filter_map { |r| entry(r, name) } @@ -190,7 +192,9 @@ def reference_run(passes, names, ref_ips) usable = es.select { |e| e["ips"].positive? && e["ips"] <= SUSPECT_IPS } pick = usable.empty? ? es : usable - [median(pick.map { |e| e["ips"] }).round(3), median(pick.map { |e| e["error"].to_f }).round(1), ref_ips[i].empty? ? nil : 1.0] + ips = median(pick.map { |e| e["ips"] }) + rel = ref_ips[0].empty? || ref_ips[i].empty? ? nil : sig(median(ref_ips[i]) / median(ref_ips[0])) + [ips.round(3), median(pick.map { |e| e["error"].to_f }).round(1), rel] end { entries: entries, passes: passes.map { |r| r["pass"] }.sort } end @@ -204,6 +208,11 @@ def variation(ref_ips) cvs.empty? ? nil : median(cvs) end + # Four significant digits: a report 27M times slower than the claimed one keeps its value instead of rounding to 0. + def sig(value) + Float(format("%.4g", value)) + end + def median(values) s = values.sort s.size.odd? ? s[s.size / 2] : (s[s.size / 2 - 1] + s[s.size / 2]) / 2.0 diff --git a/script/results_site.html b/script/results_site.html index be96fcd..634ca8a 100644 --- a/script/results_site.html +++ b/script/results_site.html @@ -200,10 +200,26 @@ /* Across Rubies: one card per benchmark, a chart with the Rubies on the x axis. */ .rcard { border-bottom: 1px solid var(--line); padding-block: 18px; display: grid; grid-template-columns: minmax(0, 1fr); gap: 8px; } -.rcard h3 { margin: 0; font-size: 15.5px; line-height: 1.35; color: var(--ink); text-wrap: balance; } -.rcard h3 code { font-weight: 400; } -.rcard .say { margin: 0; font-size: 14px; color: var(--text); max-width: 75ch; } -.rcard .say b { color: var(--ink); } +/* Card: the title says what is compared, the verdict what happened, the tiles the numbers behind it. */ +.rcard h3 { margin: 0; font-size: 20px; line-height: 1.3; color: var(--ink); overflow-wrap: anywhere; } +/* One line when it fits: fitTitles() shrinks a long title down to 14px first, and only lets it wrap past that. */ +.rcard h3.one { white-space: nowrap; overflow-wrap: normal; } +.rcard h3 code { font-weight: 700; font-size: .9em; } +.rcard .verdict { margin: 0; display: grid; grid-template-columns: 22px minmax(0, 1fr); gap: 8px; align-items: start; font-size: 15.5px; line-height: 1.45; color: var(--text); text-wrap: pretty; overflow-wrap: anywhere; } +.rcard .verdict code { font-weight: 700; color: var(--ink); } +.rcard .verdict b { color: var(--ink); } +.rcard .verdict .ico { width: 20px; height: 20px; font-size: 11px; margin-top: 1px; } +.ico.mix { background: #fff; color: var(--red); box-shadow: inset 0 0 0 3px var(--red); } +.stats { margin: 4px 0 2px; display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 8px; } +.stats > div { border: 1px solid var(--line); padding: 10px 12px; display: grid; align-content: start; gap: 2px; min-width: 0; } +.stats dt { font-size: 11.5px; font-weight: 700; letter-spacing: .06em; text-transform: uppercase; color: var(--muted); } +.stats dd { margin: 0; } +.stats .v { font: 700 22px/1.2 var(--mono); color: var(--ink); } +.stats .v.up { color: var(--green-dark); } +.stats .v.down { color: var(--red); } +.stats .n { font-size: 12.5px; line-height: 1.4; color: var(--muted); overflow-wrap: anywhere; } +.stats .n code { color: var(--text); } +.rcard td .lose { color: var(--red); } .rlegend { display: flex; flex-wrap: wrap; gap: 4px 16px; font-size: 13px; color: var(--text); } .rlegend span { display: inline-flex; align-items: center; gap: 6px; } .rlegend i { width: 14px; height: 3px; border-radius: 2px; display: inline-block; } @@ -211,6 +227,8 @@ .chart { position: relative; width: 100%; min-width: 0; } .chart svg { display: block; width: 100%; height: auto; overflow: visible; } .chart text { font-family: var(--mono); font-size: 11px; fill: var(--muted); } +.chart text.eng { font-family: var(--font); fill: var(--faint); } +.rlegend .sus svg { width: 10px; height: 8px; fill: var(--bg); stroke: var(--muted); stroke-width: 1.5; overflow: visible; } .chart .tick line { stroke: var(--grid); } .chart .base { stroke: var(--line); } .chart .cross { stroke: var(--faint); stroke-width: 1; opacity: 0; } @@ -221,8 +239,8 @@ .tipbox .r { display: flex; align-items: center; gap: 6px; font-family: var(--mono); font-size: 12px; } .tipbox .r i { width: 8px; height: 8px; border-radius: 50%; flex: none; box-shadow: 0 0 0 1.5px #fff; } .tipbox .n { color: #cfcfcf; font-size: 12px; margin-top: 3px; } -.rfoot { display: flex; flex-wrap: wrap; gap: 4px 16px; font-size: 13px; color: var(--muted); } -.rfoot .noisy { color: var(--red); font-weight: 700; } +.rfoot { margin: 8px 0 0; font-size: 13px; color: var(--muted); } +.rfoot.noisy { margin: 0; color: var(--red); font-weight: 700; } .rcard details summary { cursor: pointer; font-size: 13.5px; font-weight: 700; color: var(--ink); width: max-content; } .rcard .tb { overflow-x: auto; border: 1px solid var(--line); margin-top: 8px; } .rcard table { border-collapse: collapse; width: 100%; font-size: 13px; font-variant-numeric: tabular-nums; } @@ -234,6 +252,12 @@ .rcard td .sm { color: var(--yellow-ink); } .rcard td.off { color: var(--faint); } .rcard .links { font-size: 13px; } +/* Section headings: one per folder under code/, with how many benchmarks it shows. */ +.rsec { font-size: 24px; letter-spacing: -.01em; text-transform: none; color: var(--ink); margin: 40px 0 0; padding-top: 14px; border-top: 1px solid var(--line); display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 12px; scroll-margin-top: calc(var(--rpick, 0px) + 8px); } +.rsec small { font-size: 14px; font-weight: 400; color: var(--muted); letter-spacing: 0; } +.rsec .on { margin-left: auto; font: 500 13px var(--mono); color: var(--ink); float: none; } +.jump { font: 14px var(--font); color: var(--text); background: var(--bg); border: 1px solid var(--line); padding: 5px 9px; min-width: 0; max-width: 100%; } +.jump:focus-visible { outline: 2px solid var(--green); outline-offset: 0; border-color: var(--green); } .rhow { padding-block: 22px 0; display: grid; gap: 8px; font-size: 14px; color: var(--muted); } .rhow p { margin: 0; max-width: 75ch; } @@ -243,7 +267,7 @@ .picker { position: static; border-bottom: 1px solid var(--line); } .tip::after { position: fixed; left: 16px; right: 16px; top: auto; bottom: calc(16px + env(safe-area-inset-bottom, 0px)); transform: none; width: auto; max-width: none; font-size: 14px; padding: 12px 14px; } } -@media (max-width: 560px) { .detail { padding-left: 4px; } } +@media (max-width: 560px) { .detail { padding-left: 4px; } .stats { grid-template-columns: repeat(2, minmax(0, 1fr)); } .stats > div { padding: 8px 10px; } .stats .v { font-size: 16px; } } @media (max-width: 480px) { .services { grid-template-columns: minmax(0, 1fr); } } @@ -276,19 +300,23 @@

How fast is each idiom on each Ruby?

Every Ruby ran each benchmark on the same machine, one after the other, so here you can compare i/s across Rubies. A reference Ruby ran again and again between them, to show how steady the machine was during the run.

+

How to read these charts

Build
Show
+
+

How to read a card. The heading says what happened on the build you picked: ✓ the claimed idiom (the one the benchmark file says is fastest) wins on every Ruby, or wins on some and ties on the rest; = it ties on every Ruby; ~ it wins on some and another idiom wins on others; ✗ it never wins; – no Ruby could compare them. It compares the claimed idiom with the fastest of the others, and a gap within the measured error is a tie.

+

How to read a chart. The y axis is logarithmic: each gridline step is the same ratio, so a 100× gap between two idioms and a 10% change between two Rubies both show. Under "Relative", every line is divided by the claimed idiom on the reference Ruby, so 1× is that idiom there and the higher line is still the faster one.

How this is measured. Each shard of benchmark files ran on one GitHub runner: every build, in a random order, with benchmark-ips' default timing. The numbers are i/s as measured.

-

How steady the machine was. The reference Ruby ran first, again after every few builds, and last. It is the same code every time, so how much its i/s varies is the machine, not Ruby. Under each chart is that variation (standard deviation over mean). Over 5% the benchmark is marked noisy, and when the typical benchmark of a run is over 5%, the page warns at the top. Two Rubies whose i/s differ by less than that variation are shown as same-ish.

+

How steady the machine was. The reference Ruby ran first, again after every few builds, and last. It is the same code every time, so how much its i/s varies is the machine, not Ruby. "Numbers per build" shows that variation (standard deviation over mean). Over 5% the benchmark is marked noisy under its chart, and when the typical benchmark of a run is over 5%, the page warns at the top. Two Rubies whose i/s differ by less than that variation are shown as same-ish.

Only Rubies of the same run are compared. Weeks are not compared with each other: each week can land on a different CPU. Results over a billion iterations per second (under 1 ns each) are marked suspect and kept off the chart: the JIT most likely removed the work.

@@ -609,7 +637,7 @@

FastRuby.io keeps fast-ruby running

})();