From 3ca0a17d663ad1c28503438296b6f62f42e08574 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 28 Mar 2023 17:56:24 +0200 Subject: [PATCH 01/23] Add engine benchmark analysis tool (#5852) Add Engine benchmark analysis tool - a python script for downloading benchmark data, and Enso project for the analysis. I have also included benchmark data for 02/2022. Related issues and discussions: - https://github.com/enso-org/enso/issues/5714 - https://github.com/enso-org/enso/issues/5165 - https://github.com/enso-org/enso/discussions/5718 --- .../Engine_Benchs/package.yaml | 7 + .../Engine_Benchs/src/Main.enso | 39 + tools/performance/engine-benchmarks/README.md | 21 + .../engine-benchmarks/bench_download.py | 687 ++++++++++++++++++ .../engine-benchmarks/template_jinja.html | 184 +++++ 5 files changed, 938 insertions(+) create mode 100644 tools/performance/engine-benchmarks/Engine_Benchs/package.yaml create mode 100644 tools/performance/engine-benchmarks/Engine_Benchs/src/Main.enso create mode 100644 tools/performance/engine-benchmarks/README.md create mode 100755 tools/performance/engine-benchmarks/bench_download.py create mode 100644 tools/performance/engine-benchmarks/template_jinja.html diff --git a/tools/performance/engine-benchmarks/Engine_Benchs/package.yaml b/tools/performance/engine-benchmarks/Engine_Benchs/package.yaml new file mode 100644 index 000000000000..343dad8b2aaf --- /dev/null +++ b/tools/performance/engine-benchmarks/Engine_Benchs/package.yaml @@ -0,0 +1,7 @@ +name: Engine_Benchs +namespace: local +version: 0.0.1 +license: "" +authors: [] +maintainers: [] +prefer-local-libraries: true diff --git a/tools/performance/engine-benchmarks/Engine_Benchs/src/Main.enso b/tools/performance/engine-benchmarks/Engine_Benchs/src/Main.enso new file mode 100644 index 000000000000..b037163ed443 --- /dev/null +++ b/tools/performance/engine-benchmarks/Engine_Benchs/src/Main.enso @@ -0,0 +1,39 @@ +from Standard.Base import all +from Standard.Base.Data.Filter_Condition import Filter_Condition +from Standard.Base.Data.Map import Map +from Standard.Base.Data.Time.Date import Date +from Standard.Table import all +from Standard.Database import all +from Standard.Table.Data.Aggregate_Column import Aggregate_Column +import Standard.Visualization + +main = + operator4 = [Aggregate_Column.Maximum "commit_timestamp"] + operator11 = [Aggregate_Column.Minimum "commit_timestamp"] + operator13 = [Aggregate_Column.Group_By "label"] + number1 = 26 + text1 = "benchs.csv" + operator5 = enso_project.data / text1 + operator6 = operator5.read + operator7 = operator6.row_count + operator1 = operator6.column_names + operator9 = operator6.at 'commit_timestamp' + operator3 = operator6.aggregate operator4 + operator8 = operator6.aggregate operator11 + operator12 = operator6.aggregate operator13 + operator17 = operator12.at 'label' + operator18 = operator17.to_vector + operator15 = operator18.at number1 + operator19 = Filter_Condition.Equal operator15 + operator16 = operator6.filter 'label' operator19 + operator20 = operator16.select_columns ["commit_timestamp", "score"] True + operator14 = operator9.first + + + + + + +#### METADATA #### +[[{"index":{"value":5},"size":{"value":8}},"931526d6-4d50-425d-b042-fac34f13850b"],[{"index":{"value":13},"size":{"value":1}},"a2acfb78-817d-4240-8bd2-87a9e638e9af"],[{"index":{"value":14},"size":{"value":4}},"67f295f7-573f-4a89-a132-d86e617634fc"],[{"index":{"value":5},"size":{"value":13}},"d4c75e97-3159-4c8b-9e3c-2cfc6ed09624"],[{"index":{"value":0},"size":{"value":29}},"edc5f323-3cf2-435d-be75-39e8999ae3fd"],[{"index":{"value":35},"size":{"value":8}},"5a684d8b-565d-41bd-a21d-56208a06bdfd"],[{"index":{"value":43},"size":{"value":1}},"dc8abfc4-ad0f-4179-ba8d-c3fb1ba40d90"],[{"index":{"value":44},"size":{"value":4}},"ed341d54-fc9d-4806-93af-1e7757321c2f"],[{"index":{"value":35},"size":{"value":13}},"e27e2c44-3ce1-4a42-887a-fb6e6fb723d5"],[{"index":{"value":48},"size":{"value":1}},"619eb49f-e5a5-46bd-a12c-1923e59474a7"],[{"index":{"value":49},"size":{"value":4}},"1db50e60-ac84-487e-b93d-cca95b9dd7bd"],[{"index":{"value":35},"size":{"value":18}},"bc53fa2c-af79-42f4-80ba-146c709fcd3e"],[{"index":{"value":53},"size":{"value":1}},"a1c83316-b6a0-4d59-8ffe-90afe5bf88ba"],[{"index":{"value":54},"size":{"value":16}},"5ac9d69b-f16e-4d0e-8ec8-d817f4f898b7"],[{"index":{"value":35},"size":{"value":35}},"f31d3bc8-0636-4ad5-90f1-ca86d16ce48a"],[{"index":{"value":78},"size":{"value":16}},"67bcf3b4-504c-4a59-a27b-bf4a71981143"],[{"index":{"value":30},"size":{"value":64}},"86b180bd-3bd9-4c42-b35a-e1783d32f42c"],[{"index":{"value":100},"size":{"value":8}},"014d1d0a-99e9-4638-9129-267478534627"],[{"index":{"value":108},"size":{"value":1}},"d983d36a-fc38-4ed1-b7ab-333b0cfc8921"],[{"index":{"value":109},"size":{"value":4}},"0b64c01f-9476-4db0-a31b-c9601122b646"],[{"index":{"value":100},"size":{"value":13}},"4f289313-c6f9-4252-8d42-cc642eb45a59"],[{"index":{"value":113},"size":{"value":1}},"6167c113-c842-49bf-9f9a-1f3a70b6291d"],[{"index":{"value":114},"size":{"value":4}},"4b8acf66-55f6-4290-a171-d0863fd6dc21"],[{"index":{"value":100},"size":{"value":18}},"a6954222-c04f-482f-b0f3-470a66d86f41"],[{"index":{"value":118},"size":{"value":1}},"6810fd0d-8ad1-4f63-8204-fa433a373076"],[{"index":{"value":119},"size":{"value":3}},"eda718cd-5196-4334-82a9-f469e168f8a5"],[{"index":{"value":100},"size":{"value":22}},"33411327-7652-47d2-96ac-b7761dd276e3"],[{"index":{"value":130},"size":{"value":3}},"7c86a469-ba04-4ffa-a330-c190d79a0fed"],[{"index":{"value":95},"size":{"value":38}},"debd13b3-6543-445e-a2ec-affedd7fbe40"],[{"index":{"value":139},"size":{"value":8}},"2b5f6b91-84f2-4a79-a745-a3280cc4419d"],[{"index":{"value":147},"size":{"value":1}},"1d6fbfcb-e8fb-4c0c-bb1f-f8a7f1740126"],[{"index":{"value":148},"size":{"value":4}},"e8f8aff9-9fdd-46a0-9db4-a949d8b54b0e"],[{"index":{"value":139},"size":{"value":13}},"fe836d3a-1c71-4b88-8df6-2a97e5450314"],[{"index":{"value":152},"size":{"value":1}},"3e93815f-f434-405f-82e0-a058bb34fe97"],[{"index":{"value":153},"size":{"value":4}},"589fa082-e1e1-4511-bb40-8699900b589c"],[{"index":{"value":139},"size":{"value":18}},"633ebc7f-e3d2-4125-a0e1-392b458b8bdd"],[{"index":{"value":157},"size":{"value":1}},"f761c361-a465-4f75-8e27-ecc07e41ddb6"],[{"index":{"value":158},"size":{"value":4}},"d8cff0a6-f25b-4f2b-938e-cd02080e2f8f"],[{"index":{"value":139},"size":{"value":23}},"2528faba-a1dd-42ac-8ca6-8525bfce6692"],[{"index":{"value":162},"size":{"value":1}},"68d5ec5f-292c-4118-8ed5-62e9f8072803"],[{"index":{"value":163},"size":{"value":4}},"dead3a5d-4bc2-4673-b19f-bd41208b8b23"],[{"index":{"value":139},"size":{"value":28}},"594ee3a0-5825-4c1f-98e4-dd5ead9da7e0"],[{"index":{"value":175},"size":{"value":4}},"a535cddf-2028-45d8-aa61-783bae85d565"],[{"index":{"value":134},"size":{"value":45}},"b970390b-6d36-4abd-ac2b-e923e4a69cdb"],[{"index":{"value":185},"size":{"value":8}},"f072fe32-ec9c-4f18-a152-7902be2be04e"],[{"index":{"value":193},"size":{"value":1}},"d1982ba2-2ff3-4b56-8fc0-3d7a7eb466b0"],[{"index":{"value":194},"size":{"value":5}},"ee255d75-f684-4d46-8754-4be5051747b9"],[{"index":{"value":185},"size":{"value":14}},"52005190-14d8-407d-952b-aff5023986ae"],[{"index":{"value":180},"size":{"value":30}},"393df2a6-01e1-421c-a47f-e9c9ffb31852"],[{"index":{"value":216},"size":{"value":8}},"e3c60818-d4b4-49da-912e-ff90d8930f3c"],[{"index":{"value":224},"size":{"value":1}},"7d56841f-3c37-451c-905c-39dd7a11ebf9"],[{"index":{"value":225},"size":{"value":8}},"f2a0a254-ccae-4319-a3c7-c2733758c073"],[{"index":{"value":216},"size":{"value":17}},"e2ddc0bf-2054-41db-8519-307da6a5dd8c"],[{"index":{"value":211},"size":{"value":33}},"dc815cc4-508f-40d3-a13b-f8e02dedee63"],[{"index":{"value":250},"size":{"value":8}},"57f4cf9c-816f-47a6-9a9c-d00b71d398f3"],[{"index":{"value":258},"size":{"value":1}},"8c930e49-e5ee-4eb3-a03b-c354fa1f0e21"],[{"index":{"value":259},"size":{"value":5}},"743a9941-a512-4d92-9fbf-6ffc587d81bf"],[{"index":{"value":250},"size":{"value":14}},"c6b724a3-011a-4445-ba92-b2841965fc4e"],[{"index":{"value":264},"size":{"value":1}},"8b303800-53de-4bd9-b3e8-ff81eebb1de0"],[{"index":{"value":265},"size":{"value":4}},"ab06b298-5a8d-4199-bcf1-f0d978634223"],[{"index":{"value":250},"size":{"value":19}},"99c57192-1137-49ce-ba9a-99eeba9e87e2"],[{"index":{"value":269},"size":{"value":1}},"6861b630-60c6-4834-917c-478c8860f733"],[{"index":{"value":270},"size":{"value":16}},"8318c467-9f7b-496a-9182-e9a411bd340e"],[{"index":{"value":250},"size":{"value":36}},"f02677e6-3ede-4a03-ba83-4a35fba8a546"],[{"index":{"value":294},"size":{"value":16}},"faf63e01-a09c-4313-befc-d63752f755ea"],[{"index":{"value":245},"size":{"value":65}},"c6c0461b-6529-4c55-8754-368ee338051a"],[{"index":{"value":318},"size":{"value":8}},"07da081c-a3a9-4677-9dfb-e88e66372b0c"],[{"index":{"value":326},"size":{"value":1}},"45caa80d-2fe5-450c-9bdb-bd906b2985af"],[{"index":{"value":327},"size":{"value":13}},"12cd5136-9779-427a-b826-b7125e8fd254"],[{"index":{"value":318},"size":{"value":22}},"f005111c-dd05-44a1-8894-e42ca9fa930f"],[{"index":{"value":311},"size":{"value":29}},"b4f98a87-e4d9-4aa4-b81c-44266d98310f"],[{"index":{"value":342},"size":{"value":4}},"b10b8cf7-3376-4d46-a5e4-0d4705b5c838"],[{"index":{"value":347},"size":{"value":1}},"1cc99de4-74a4-4359-9656-617b9bb347ba"],[{"index":{"value":353},"size":{"value":9}},"51bb0373-c73d-4521-bc6f-fa3bef8ffd88"],[{"index":{"value":363},"size":{"value":1}},"86b05aef-394d-422c-b87f-1c44ced687d9"],[{"index":{"value":366},"size":{"value":16}},"49e581af-aecb-46b4-8012-4bb49b137de5"],[{"index":{"value":382},"size":{"value":1}},"84ce0dfe-0d78-4799-acf7-bbad725d9273"],[{"index":{"value":383},"size":{"value":7}},"6be4f5a3-f530-484a-9826-dbf14a0ee629"],[{"index":{"value":366},"size":{"value":24}},"621752d4-f7ea-491a-9c75-03db9d3db267"],[{"index":{"value":391},"size":{"value":18}},"765b8bd9-45c6-49c2-9afb-d417cc83f033"],[{"index":{"value":366},"size":{"value":43}},"981e00bd-183f-4dc0-b89f-12793ed8f88f"],[{"index":{"value":365},"size":{"value":45}},"18b4a06c-288e-47f6-8f1a-5cb2c54e3d1d"],[{"index":{"value":353},"size":{"value":57}},"0d9a4e95-c396-4e89-905c-76ca8107d509"],[{"index":{"value":415},"size":{"value":10}},"8a371a9a-eaf0-4e11-aac0-3162127be90a"],[{"index":{"value":426},"size":{"value":1}},"45fb5a5f-fdcd-4c94-8c6a-bc6d7f0df3d6"],[{"index":{"value":429},"size":{"value":16}},"b88fb5ab-32b3-4ccb-be4f-3c641d8ed4c0"],[{"index":{"value":445},"size":{"value":1}},"cc7a6e43-d9aa-4e15-8822-0f6e3fd7462d"],[{"index":{"value":446},"size":{"value":7}},"2e5fd261-3524-466d-b980-a6ce3e4f260d"],[{"index":{"value":429},"size":{"value":24}},"e1f3fe0f-fbc0-4f27-9db4-e9cb07106483"],[{"index":{"value":454},"size":{"value":18}},"0fdf2c74-2d62-4a6a-8cdd-c217a1537dc3"],[{"index":{"value":429},"size":{"value":43}},"f814aac2-68fd-409f-b978-bde0d8add4a4"],[{"index":{"value":428},"size":{"value":45}},"bfcefe18-7599-474b-82a5-74efc1f21b70"],[{"index":{"value":415},"size":{"value":58}},"842437e6-7cb0-4957-bae6-718b2aad3602"],[{"index":{"value":478},"size":{"value":10}},"c7265cfa-9ad3-42e0-a00c-51cd7dbca6ed"],[{"index":{"value":489},"size":{"value":1}},"2904a82a-3e5f-4603-94a0-8328e90c84b3"],[{"index":{"value":492},"size":{"value":16}},"4f3390cf-399a-4b08-9e26-e878cc7d1b58"],[{"index":{"value":508},"size":{"value":1}},"6ea52a87-d2ab-4fba-8d9d-cf9dc11d0a8d"],[{"index":{"value":509},"size":{"value":8}},"7f9ab74d-ab49-45ca-9ec1-7c508e44fc66"],[{"index":{"value":492},"size":{"value":25}},"b3b45e10-4eb2-402b-90a9-91d0b78582d4"],[{"index":{"value":518},"size":{"value":7}},"d73b497f-5960-4923-9182-b04f33af2d64"],[{"index":{"value":492},"size":{"value":33}},"ea1c0f2b-caad-42a2-8f6b-42b2707fc124"],[{"index":{"value":491},"size":{"value":35}},"8320c946-2b12-4abf-9914-4a73ddb8b072"],[{"index":{"value":478},"size":{"value":48}},"51883650-17dd-4fea-94d0-c27738d7964d"],[{"index":{"value":531},"size":{"value":7}},"ecd16e36-203c-486d-a74f-ef6b4f1074ce"],[{"index":{"value":539},"size":{"value":1}},"5295c797-8839-4ca4-95f9-889e47b7d262"],[{"index":{"value":541},"size":{"value":2}},"9e9c8fc3-0fdd-4d99-b273-ba9e7056ecd4"],[{"index":{"value":531},"size":{"value":12}},"b0f76972-3101-49b1-babf-53e067a918e9"],[{"index":{"value":548},"size":{"value":5}},"e91c9c50-7523-412f-a10e-32298d582964"],[{"index":{"value":554},"size":{"value":1}},"0ac1fbe2-8ad7-4bbe-bf48-d6e95cfa214d"],[{"index":{"value":556},"size":{"value":12}},"6ac781a3-6836-4f9c-8bcd-0ef47c10244d"],[{"index":{"value":548},"size":{"value":20}},"9c716434-9539-458a-8513-a4f06ac487e5"],[{"index":{"value":573},"size":{"value":9}},"ab52b70c-ffd6-487f-8d89-026da7215bd5"],[{"index":{"value":583},"size":{"value":1}},"fbf53e8f-8fdf-4260-a0b3-7e3ace6e65cd"],[{"index":{"value":585},"size":{"value":12}},"e43fce65-51b0-4c43-ba4f-133cab2a4def"],[{"index":{"value":597},"size":{"value":1}},"4a23824f-c6f3-4a37-b33c-0596335e7096"],[{"index":{"value":598},"size":{"value":4}},"4eb8e0f3-04b5-4cf2-a0f4-85f88faab3d9"],[{"index":{"value":585},"size":{"value":17}},"0c6adcd7-0781-4a33-ab47-42974eb5294f"],[{"index":{"value":603},"size":{"value":1}},"7a79a4ca-95db-4871-91f0-0b85684aa68a"],[{"index":{"value":605},"size":{"value":5}},"ac7100ab-cf52-4028-8192-7bdde99fa9af"],[{"index":{"value":585},"size":{"value":25}},"4787d28c-d96b-4153-ab01-0a34a5489797"],[{"index":{"value":573},"size":{"value":37}},"c9ff4726-de99-4d62-9e03-e952ff970b97"],[{"index":{"value":615},"size":{"value":9}},"8b39ebf7-3d77-476f-af67-e994f605e2c6"],[{"index":{"value":625},"size":{"value":1}},"277f6ea6-9aa8-44be-bca9-2595b7f4e4ba"],[{"index":{"value":627},"size":{"value":9}},"8fb231e2-ce2c-4b31-8d5e-71d8ab57df27"],[{"index":{"value":636},"size":{"value":1}},"0e7fb91c-57cb-4f2e-a3cb-2108c2924f1e"],[{"index":{"value":637},"size":{"value":4}},"3a486fd9-aeea-4502-b2e6-d9f0d336c17d"],[{"index":{"value":627},"size":{"value":14}},"87f557fb-ad89-4249-86f9-b766d1614db4"],[{"index":{"value":615},"size":{"value":26}},"8f4eca89-7749-42c8-8854-7ef536030521"],[{"index":{"value":646},"size":{"value":9}},"e12f21f1-45b9-4655-be78-0b2b00effab4"],[{"index":{"value":656},"size":{"value":1}},"aebaa580-0264-41a2-9f1e-b486ad74733b"],[{"index":{"value":658},"size":{"value":9}},"28e7e177-b88c-49e1-9696-6415e6d0866c"],[{"index":{"value":667},"size":{"value":1}},"dc3192ce-23ad-495d-8e1d-72b903273013"],[{"index":{"value":668},"size":{"value":9}},"4f064e55-820d-4dfa-a8fb-460bc1de2180"],[{"index":{"value":658},"size":{"value":19}},"a02c480e-fb38-43eb-a85f-31b5b28eef6f"],[{"index":{"value":646},"size":{"value":31}},"e8df2c0e-6761-43de-8afc-2bb96e6b93db"],[{"index":{"value":682},"size":{"value":9}},"ffa74a4f-53bf-4041-abe6-5984e3c37101"],[{"index":{"value":692},"size":{"value":1}},"8b402e3d-3f7b-4619-8e39-fb872a40a9a9"],[{"index":{"value":694},"size":{"value":9}},"2260dff5-e22d-4bfe-b957-669660a874cc"],[{"index":{"value":703},"size":{"value":1}},"34f8fa86-d2cf-4e78-85fe-dec18dff4fa3"],[{"index":{"value":704},"size":{"value":12}},"26cb247c-8686-43f5-bf8a-ba6578985060"],[{"index":{"value":694},"size":{"value":22}},"30002cb8-04c5-4491-8ef5-2e4f107c627b"],[{"index":{"value":682},"size":{"value":34}},"a3df19f1-7257-42d2-91b4-db23fb6f639a"],[{"index":{"value":721},"size":{"value":9}},"1985f399-f1e6-43a6-985b-81f129b295b9"],[{"index":{"value":731},"size":{"value":1}},"b2a2844c-70cf-497c-b85d-8e9cf7e6b257"],[{"index":{"value":733},"size":{"value":9}},"f5fb2978-0027-4fbd-a783-02af6923c2e2"],[{"index":{"value":742},"size":{"value":1}},"d3179d84-ccfd-4ab3-a1bb-5ed20eb2c363"],[{"index":{"value":743},"size":{"value":2}},"cb3f3503-acbd-4623-86e8-e4ea016403f8"],[{"index":{"value":733},"size":{"value":12}},"5aa03074-117b-4a01-9886-6c871e1b9c6b"],[{"index":{"value":746},"size":{"value":18}},"8529551e-b6ec-4be8-8f17-d87716af32e1"],[{"index":{"value":733},"size":{"value":31}},"f0c269ce-85b2-4631-8561-cf374ff38980"],[{"index":{"value":721},"size":{"value":43}},"c2be456b-7637-4c96-b3b5-6ee925b7ada5"],[{"index":{"value":769},"size":{"value":9}},"0a055351-24b4-4655-9d8e-0f90666250d9"],[{"index":{"value":779},"size":{"value":1}},"02c6d19c-b259-4d9d-af24-fde814dcb60a"],[{"index":{"value":781},"size":{"value":9}},"5ec284e6-8339-4cfc-a6cc-71598dd36aa0"],[{"index":{"value":790},"size":{"value":1}},"967b06c0-cfd0-407b-99d6-156b6bdea4ae"],[{"index":{"value":791},"size":{"value":9}},"7f993e68-a52d-426a-8ade-50291e7ef643"],[{"index":{"value":781},"size":{"value":19}},"e8724162-7c7c-4cdd-9301-79e92eb73bf2"],[{"index":{"value":801},"size":{"value":9}},"f5bf184c-6237-4144-9892-c201fc1bd636"],[{"index":{"value":781},"size":{"value":29}},"426caeea-df9e-4668-8739-9da02f6f55dc"],[{"index":{"value":769},"size":{"value":41}},"aa5e6a6b-1bc8-416f-8984-85b6046b432d"],[{"index":{"value":815},"size":{"value":9}},"1f2be60d-bbf9-405d-b3c9-e083bf3cae4c"],[{"index":{"value":825},"size":{"value":1}},"fa2a443a-5968-43ef-8ab5-95056fafb2f1"],[{"index":{"value":827},"size":{"value":9}},"f0a6acf7-275c-4633-97dd-4c859aa7641c"],[{"index":{"value":836},"size":{"value":1}},"407809ee-8b29-4f8d-aa97-8d675d622956"],[{"index":{"value":837},"size":{"value":9}},"df869251-c39f-43b8-a273-ebce156dfbe7"],[{"index":{"value":827},"size":{"value":19}},"c77aaddb-ced0-4c89-a869-14cc6615abed"],[{"index":{"value":847},"size":{"value":10}},"8890449b-5baf-42b4-9d5d-9b9f381b0ade"],[{"index":{"value":827},"size":{"value":30}},"d75251be-9370-470b-93dc-d6c9d5833124"],[{"index":{"value":815},"size":{"value":42}},"c9867a7b-b5a6-4736-a72b-b888f4780bc9"],[{"index":{"value":862},"size":{"value":10}},"957fdc1c-4e93-4c69-9da3-e742b5b10842"],[{"index":{"value":873},"size":{"value":1}},"4f292088-0f2b-44b9-8908-3c424d04427e"],[{"index":{"value":875},"size":{"value":9}},"32d87a4c-d936-4b97-8844-522b3a4c4627"],[{"index":{"value":884},"size":{"value":1}},"92de76ae-2a8e-4a27-9922-bc75a85ca7e0"],[{"index":{"value":885},"size":{"value":9}},"381ced9a-f543-4ddd-8a55-82d4ef665203"],[{"index":{"value":875},"size":{"value":19}},"30ac3755-aec4-434e-a397-616f6e79b4e4"],[{"index":{"value":895},"size":{"value":10}},"c8dabbaf-751c-4a9b-aadb-9f6fe84e53ef"],[{"index":{"value":875},"size":{"value":30}},"1c19e3d7-b750-43d6-ba48-d2118a11da94"],[{"index":{"value":862},"size":{"value":43}},"a52c8794-c37e-4fc6-bfcc-b3d427bbef3d"],[{"index":{"value":910},"size":{"value":10}},"a2790563-b1c5-4d4d-9282-3b656e3a1002"],[{"index":{"value":921},"size":{"value":1}},"ad4e9bd0-efe8-4d38-a977-28b20d2dd125"],[{"index":{"value":923},"size":{"value":10}},"e26074c9-03af-40a6-8417-5816b1bcde31"],[{"index":{"value":933},"size":{"value":1}},"340dcfb3-7c5f-41bc-bad7-c8983d536274"],[{"index":{"value":934},"size":{"value":2}},"47b1293a-6699-4c26-bf21-722eb6cf9e86"],[{"index":{"value":923},"size":{"value":13}},"c7f9d455-331c-430d-82e0-b9ada8f73c3e"],[{"index":{"value":937},"size":{"value":7}},"f2f58015-e828-498c-a049-910c7fa1fac2"],[{"index":{"value":923},"size":{"value":21}},"965001a6-eeaf-417c-8759-6c93dfdf9bdd"],[{"index":{"value":910},"size":{"value":34}},"b325605f-6061-4b59-ba7c-4e0f04f5533b"],[{"index":{"value":949},"size":{"value":10}},"7d5c9a00-fa2e-42a7-b8e0-e189f96a3403"],[{"index":{"value":960},"size":{"value":1}},"7d5a60e5-811f-4a9d-a723-7451f6bc319a"],[{"index":{"value":962},"size":{"value":10}},"b84e7fbb-9ca1-496c-b110-2e2bbf8d682b"],[{"index":{"value":972},"size":{"value":1}},"892037f6-a0d9-48de-a980-51ee7c39b316"],[{"index":{"value":973},"size":{"value":9}},"fe6d5bc4-53f6-4845-8382-0c8d9cffa313"],[{"index":{"value":962},"size":{"value":20}},"8278cc91-5e6c-4d1c-89a9-82e8536906fe"],[{"index":{"value":949},"size":{"value":33}},"54fcd516-9c3f-4d68-bb0a-ca49b98dc92d"],[{"index":{"value":987},"size":{"value":10}},"0c29fce9-55ac-4aa6-99a4-71689b37326e"],[{"index":{"value":998},"size":{"value":1}},"cc6ab154-28fd-4cba-b886-05dd08e3134d"],[{"index":{"value":1000},"size":{"value":10}},"e3151427-b6ff-4318-aec1-bd5c43cf00a2"],[{"index":{"value":1010},"size":{"value":1}},"aa8d8810-87d9-4445-b7f2-be5dc5b2b209"],[{"index":{"value":1011},"size":{"value":2}},"694262bb-1ca6-4e6c-9386-01601b9f3306"],[{"index":{"value":1000},"size":{"value":13}},"d72c77a8-9fb8-43cc-8294-fe879b8bc60e"],[{"index":{"value":1014},"size":{"value":7}},"7694d4fd-1b92-415e-8ba9-1ea3f73ade14"],[{"index":{"value":1000},"size":{"value":21}},"3f370f98-fc2a-4920-acd6-5caec6373afc"],[{"index":{"value":987},"size":{"value":34}},"2a588415-dac0-47cc-85c3-1eedbcb0a9dc"],[{"index":{"value":1026},"size":{"value":10}},"76ff0443-58de-4df4-a7c2-945f2971bc14"],[{"index":{"value":1037},"size":{"value":1}},"f116d4d7-d20e-4c1a-986b-631536a2d2a3"],[{"index":{"value":1039},"size":{"value":16}},"2e8a4a82-3623-460d-ba12-57ef0d223f8e"],[{"index":{"value":1055},"size":{"value":1}},"a652ddf5-221d-48c8-b05d-4327b30d5526"],[{"index":{"value":1056},"size":{"value":5}},"6317e213-c17b-4e44-965c-0c9e4bcd6b48"],[{"index":{"value":1039},"size":{"value":22}},"f1b77b5c-ec4b-4818-ae9e-c101d866e3cc"],[{"index":{"value":1062},"size":{"value":10}},"0707d055-777a-4e6d-9f50-44cc24bec776"],[{"index":{"value":1039},"size":{"value":33}},"345dd6b0-bf26-48ce-8378-aa45b589104c"],[{"index":{"value":1026},"size":{"value":46}},"e184da40-1ad3-41f0-aa55-779a01c9778f"],[{"index":{"value":1077},"size":{"value":10}},"1a8ebb32-e28a-45b0-84d0-2bf9218ad054"],[{"index":{"value":1088},"size":{"value":1}},"3bab8726-df25-4cef-84c6-bd510214fd33"],[{"index":{"value":1090},"size":{"value":9}},"79602d4f-0dab-42c4-b0f1-f06d47d869cc"],[{"index":{"value":1099},"size":{"value":1}},"ebcfcece-387a-45e3-a6f9-fb9ed1dbabe5"],[{"index":{"value":1100},"size":{"value":6}},"35e50326-1d63-4ac6-9eb9-c65a1354888c"],[{"index":{"value":1090},"size":{"value":16}},"f7abd8e0-9730-4e2c-a87f-62c33fbfe028"],[{"index":{"value":1107},"size":{"value":7}},"f78ae09a-7834-486c-9e65-dfa3e6d1e8fd"],[{"index":{"value":1090},"size":{"value":24}},"505e29bc-898c-4603-a194-65f7eebb942c"],[{"index":{"value":1115},"size":{"value":10}},"b5a97c2e-db9a-4417-8ba6-656bf0b0d808"],[{"index":{"value":1090},"size":{"value":35}},"1796308a-f800-4efb-9f01-38842c95187b"],[{"index":{"value":1077},"size":{"value":48}},"0da08d20-f303-42c8-b28a-54c03b0c49b9"],[{"index":{"value":1130},"size":{"value":10}},"943565bd-135d-4acd-9ef3-a0f881da3519"],[{"index":{"value":1141},"size":{"value":1}},"de14cc9c-cf5f-435b-ae4d-f9f1b7e52bfc"],[{"index":{"value":1143},"size":{"value":10}},"0392e653-d17c-4263-bb73-20bb017762ca"],[{"index":{"value":1153},"size":{"value":1}},"ab4954b1-ced5-4a29-bccc-11ebff47d18b"],[{"index":{"value":1154},"size":{"value":14}},"53a9778f-1e42-46f3-b838-ef38ccea106f"],[{"index":{"value":1143},"size":{"value":25}},"dffc846b-abd6-4b9d-9be3-71293a5146bb"],[{"index":{"value":1170},"size":{"value":18}},"3118e7b5-34a9-4c53-aaca-f5f7bcdff9f1"],[{"index":{"value":1190},"size":{"value":7}},"71bce570-85bd-435c-9dce-88e20cd0e4ab"],[{"index":{"value":1169},"size":{"value":29}},"2d2fc5a7-ae90-41b3-8df0-377c66a10d7a"],[{"index":{"value":1143},"size":{"value":55}},"934935e8-813b-4526-a68e-d2fce2594a20"],[{"index":{"value":1199},"size":{"value":4}},"6392a719-6213-45c7-963a-831ce7bcef82"],[{"index":{"value":1143},"size":{"value":60}},"e31710e3-0143-4cd8-ab27-7c20bafcc4f3"],[{"index":{"value":1130},"size":{"value":73}},"0dd27f3e-1e3d-4445-a7e6-208eb8de171a"],[{"index":{"value":1208},"size":{"value":10}},"a16d9b18-84ca-4317-855f-3d67a9c4950d"],[{"index":{"value":1219},"size":{"value":1}},"c741695e-3aec-421f-bee3-3183723e5f8b"],[{"index":{"value":1221},"size":{"value":9}},"90ecc894-68f5-482b-b617-d0e1385499f5"],[{"index":{"value":1230},"size":{"value":1}},"43eed7fc-4749-4acd-ad51-510e9fc4a280"],[{"index":{"value":1231},"size":{"value":5}},"74565039-4206-45c8-aea5-61df784c6a31"],[{"index":{"value":1221},"size":{"value":15}},"56697230-7a02-4246-9f28-35ae9347dbd8"],[{"index":{"value":1208},"size":{"value":28}},"6f90d07c-853e-4995-bfcd-ccb1cdc3294e"],[{"index":{"value":348},"size":{"value":888}},"f31efaae-2ff1-4648-abcc-4f9dd8bd7867"],[{"index":{"value":342},"size":{"value":894}},"2d9c7ee8-e889-4314-acef-be9271c34ea2"],[{"index":{"value":0},"size":{"value":1240}},"f1273bc9-c3ce-41f8-9336-6d6f81262403"]] +{"ide":{"node":{"426caeea-df9e-4668-8739-9da02f6f55dc":{"position":{"vector":[498.16125,-317.99982]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"aggregate"},"uploading_file":null,"selected":false,"visualization":{"name":{"content":{"content":"Table"}},"project":"Builtin"}},"345dd6b0-bf26-48ce-8378-aa45b589104c":{"position":{"vector":[158.61446,-983.7706]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"3f370f98-fc2a-4920-acd6-5caec6373afc":{"position":{"vector":[243.42368,-844.3346]},"intended_method":{"module":"Standard.Base.Data.Vector","defined_on_type":"Standard.Base.Data.Vector.Vector","name":"at"},"uploading_file":null,"selected":false,"visualization":{"name":{"content":{"content":"JSON"}},"project":"Builtin"}},"56697230-7a02-4246-9f28-35ae9347dbd8":{"position":{"vector":[1039.0,23.999529]},"intended_method":{"module":"Standard.Table.Data.Column","defined_on_type":"Standard.Table.Data.Column.Column","name":"first"},"uploading_file":null,"selected":false,"visualization":null},"8320c946-2b12-4abf-9914-4a73ddb8b072":{"position":{"vector":[216.27803,-421.55072]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"8278cc91-5e6c-4d1c-89a9-82e8536906fe":{"position":{"vector":[470.1698,-739.31165]},"intended_method":{"module":"Standard.Table.Data.Column","defined_on_type":"Standard.Table.Data.Column.Column","name":"to_vector"},"uploading_file":null,"selected":false,"visualization":null},"87f557fb-ad89-4249-86f9-b766d1614db4":{"position":{"vector":[111.56444,23.999529]},"intended_method":{"module":"Standard.Base.System.File","defined_on_type":"Standard.Base.System.File.File","name":"read"},"uploading_file":null,"selected":false,"visualization":null},"1c19e3d7-b750-43d6-ba48-d2118a11da94":{"position":{"vector":[83.583466,-525.0]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"aggregate"},"uploading_file":null,"selected":false,"visualization":null},"965001a6-eeaf-417c-8759-6c93dfdf9bdd":{"position":{"vector":[470.1698,-666.6788]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"at"},"uploading_file":null,"selected":false,"visualization":null},"d75251be-9370-470b-93dc-d6c9d5833124":{"position":{"vector":[-382.99994,-363.0]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"aggregate"},"uploading_file":null,"selected":false,"visualization":{"name":{"content":{"content":"Table"}},"project":"Builtin"}},"30002cb8-04c5-4491-8ef5-2e4f107c627b":{"position":{"vector":[-286.99994,-15.000187]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"column_names"},"uploading_file":null,"selected":false,"visualization":{"name":{"content":{"content":"Table"}},"project":"Builtin"}},"9e9c8fc3-0fdd-4d99-b273-ba9e7056ecd4":{"position":{"vector":[140.00784,-739.4624]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"a02c480e-fb38-43eb-a85f-31b5b28eef6f":{"position":{"vector":[803.0,-89.0]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"row_count"},"uploading_file":null,"selected":false,"visualization":null},"f0c269ce-85b2-4631-8561-cf374ff38980":{"position":{"vector":[307.00006,-55.0]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"at"},"uploading_file":null,"selected":false,"visualization":null},"bfcefe18-7599-474b-82a5-74efc1f21b70":{"position":{"vector":[-654.99994,-290.01126]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"6ac781a3-6836-4f9c-8bcd-0ef47c10244d":{"position":{"vector":[275.72974,217.98839]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"1796308a-f800-4efb-9f01-38842c95187b":{"position":{"vector":[-116.5526,-1095.6257]},"intended_method":{"module":"Standard.Table.Data.Table","defined_on_type":"Standard.Table.Data.Table.Table","name":"filter"},"uploading_file":null,"selected":false,"visualization":null},"e31710e3-0143-4cd8-ab27-7c20bafcc4f3":{"position":{"vector":[192.78017,-1166.8784]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":{"name":{"content":{"content":"Scatter Plot"}},"project":"Builtin"}},"4787d28c-d96b-4153-ab01-0a34a5489797":{"position":{"vector":[158.0,117.963745]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null},"18b4a06c-288e-47f6-8f1a-5cb2c54e3d1d":{"position":{"vector":[612.16125,-229.85553]},"intended_method":null,"uploading_file":null,"selected":false,"visualization":null}},"import":{"5604875829123660801":{},"6010822104071218392":{},"10907837657250088061":{},"2410219534913020646":{}},"project":null}} \ No newline at end of file diff --git a/tools/performance/engine-benchmarks/README.md b/tools/performance/engine-benchmarks/README.md new file mode 100644 index 000000000000..d6cb4c4d915a --- /dev/null +++ b/tools/performance/engine-benchmarks/README.md @@ -0,0 +1,21 @@ +# Engine benchmarks + +This directory contains a python script `bench_download.py` for downloading +Engine benchmark results from GitHub and `Engine_Benchs` Enso project for +analysing the downloaded data. + +Dependencies for `bench_download.py`: + +- python >= 3.7 +- `pandas` and `jinja2` + - Install, e.g., by `pip install pandas jinja2` +- GH CLI utility + - Install either from https://cli.github.com/ or with + `sudo apt-get install gh` + +Check `bench_download -h` for documentation and usage. Ensure that your +`/usr/bin/env python` links to Python version at least 3.7. + +One can also analyze the benchmarks in Enso IDE by running +`bench_download.py --create-csv` and then running `Engine_Benchs` project. The +created CSV is pasted into `Engine_Benchs/data` directory by default. diff --git a/tools/performance/engine-benchmarks/bench_download.py b/tools/performance/engine-benchmarks/bench_download.py new file mode 100755 index 000000000000..2b1abd69ba27 --- /dev/null +++ b/tools/performance/engine-benchmarks/bench_download.py @@ -0,0 +1,687 @@ +#!/usr/bin/env python + +""" +Script for downloading Engine benchmark results into a single static web page +that visualizes all the benchmarks. Without any options, downloads and +visualizes benchmark data for the last 14 days. + +It downloads the data synchronously and uses a cache directory by default. +It is advised to use `-v|--verbose` option all the time. + +It queries only successful benchmark runs. If there are no successful benchmarks +in a given period, no results will be written. + +The process of the script is roughly as follows: +- Gather all the benchmark results from GH API into job reports (JobReport dataclass) + - Use cache if possible to avoid unnecessary GH API queries +- Transform the gathered results into data for a particular benchmark sorted + by an appropriate commit timestamp. + - BenchmarkData class + +If you wish to inspect the data yourself, just use --create-csv option. + +Dependencies for the script: +- GH CLI utility + - https://cli.github.com/ + - Used for convenience to do the GH API queries. + - It needs to be installed, and you should also authenticate. +- Python version >= 3.7 +- Python 3rd party packages: + - pandas + - Used for convenience for a very simple data processing + - jinja2 + - Used as a template engine for the HTML. +""" + +import json +import logging +import logging.config +import os +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from csv import DictWriter +from datetime import datetime, timedelta +from os import path +from typing import List, Dict, Optional, Any, Union +from dataclasses import dataclass +if not (sys.version_info.major >= 3 and sys.version_info.minor >= 7): + print("ERROR: python version lower than 3.7") + exit(1) +try: + import pandas as pd + import numpy as np + import jinja2 +except ModuleNotFoundError as err: + print("ERROR: One of pandas, numpy, or jinja2 packages not installed") + exit(1) + + +BENCH_RUN_NAME = "Benchmark Engine" +DATE_FORMAT = "%Y-%m-%d" +# Workflod ID of engine benchmarks, got via `gh api '/repos/enso-org/enso/actions/workflows'` +BENCH_WORKFLOW_ID = 29450898 +GH_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +""" Date format as returned from responses in GH API""" +ENSO_COMMIT_BASE_URL = "https://github.com/enso-org/enso/commit/" +JINJA_TEMPLATE = "template_jinja.html" +""" Path to the Jinja HTML template """ + + +@dataclass +class Author: + name: str + + +@dataclass +class Commit: + """ Corresponds to the commit from GH API """ + id: str + author: Author + timestamp: str + message: str + + +@dataclass +class JobRun: + """ + Gathered via the GH API. Defines a single run of an Engine benchmark job. + """ + id: str + display_title: str + html_url: str + run_attempt: int + """ An event as defined by the GitHub API, for example 'push' or 'schedule' """ + event: str + head_commit: Commit + + +@dataclass +class JobReport: + """ + Gathered via the GH API - a report that is pushed as an aritfact to the job. + Contains a XML file with scores for all the benchmarks. + """ + label_score_dict: Dict[str, float] + """ A mapping of benchmark labels to their scores """ + bench_run: JobRun + + +@dataclass +class BenchmarkData: + """ + Data for a single benchmark compiled from all the job reports. + """ + @dataclass + class Entry: + score: float + commit_id: str + commit_msg: str + commit_url: str + commit_author: str + commit_timestamp: datetime + bench_run_url: str + bench_run_event: str + label: str + """ Label for the benchmark, as reported by org.enso.interpreter.bench.BenchmarksRunner """ + entries: List[Entry] + """ Entries sorted by timestamps """ + + +@dataclass +class ChartRow: + timestamp: datetime + score: float + tooltip: str + + +@dataclass +class TemplateBenchData: + id: str + rows: List[ChartRow] + score_diffs: List[str] + """ S string that is displayed in the selection info """ + commit_ids: List[str] + commit_msgs: List[str] + commit_authors: List[str] + commit_urls: List[str] + bench_run_urls: List[str] + """ URLs to Engine benchmark job """ + + +@dataclass +class JinjaData: + bench_datas: List[TemplateBenchData] + since: datetime + until: datetime + + +def _parse_bench_run_from_json(obj: Dict[Any, Any]) -> JobRun: + return JobRun( + id=str(obj["id"]), + html_url=obj["html_url"], + run_attempt=int(obj["run_attempt"]), + event=obj["event"], + display_title=obj["display_title"], + head_commit=Commit( + id=obj["head_commit"]["id"], + message=obj["head_commit"]["message"], + timestamp=obj["head_commit"]["timestamp"], + author=Author( + name=obj["head_commit"]["author"]["name"] + ) + ) + ) + + +def _parse_bench_report_from_json(obj: Dict[Any, Any]) -> JobReport: + return JobReport( + bench_run=_parse_bench_run_from_json(obj["bench_run"]), + label_score_dict=obj["label_score_dict"] + ) + + +def _bench_report_to_json(bench_report: JobReport) -> Dict[Any, Any]: + return { + "bench_run": { + "id": bench_report.bench_run.id, + "html_url": bench_report.bench_run.html_url, + "run_attempt": bench_report.bench_run.run_attempt, + "event": bench_report.bench_run.event, + "display_title": bench_report.bench_run.display_title, + "head_commit": { + "id": bench_report.bench_run.head_commit.id, + "message": bench_report.bench_run.head_commit.message, + "timestamp": bench_report.bench_run.head_commit.timestamp, + "author": { + "name": bench_report.bench_run.head_commit.author.name + } + } + }, + "label_score_dict": bench_report.label_score_dict + } + + +def _parse_bench_report_from_xml(bench_report_xml: str, bench_run: JobRun) -> "JobReport": + logging.debug(f"Parsing BenchReport from {bench_report_xml}") + with open(bench_report_xml, "r") as f: + lines = f.readlines() + label_pattern = re.compile("