diff --git a/.gitattributes b/.gitattributes index 36e06cd13ec8..e734aea1fd6b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,4 @@ poetry.lock linguist-generated=true requirements.txt linguist-generated=true docs/backends/app/requirements.txt linguist-generated=false docs/_freeze/**/html.json linguist-generated=true +docs/**/*.excalidraw linguist-generated=true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e048b599eff9..de7327471c24 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: args: ["--allow-missing-credentials"] - id: detect-private-key - id: end-of-file-fixer - exclude: (.+/snapshots/.+)|(.*docs/_freeze/.+\.json) + exclude: (.+/snapshots/.+)|(.*docs/_freeze/.+\.json)|(.+\.excalidraw) - id: fix-byte-order-marker - id: mixed-line-ending - id: trailing-whitespace diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 07ff2adfd3ef..245f1fd6b1e7 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -76,7 +76,6 @@ website: style: "docked" collapse-level: 2 contents: - - why.qmd - install.qmd - auto: tutorials/*.qmd - id: concepts @@ -84,7 +83,7 @@ website: style: "docked" collapse-level: 2 contents: - - what.qmd + - why.qmd - auto: concepts/*.qmd - id: backends title: "Backends" diff --git a/docs/concepts/frontend.qmd b/docs/concepts/frontend.qmd deleted file mode 100644 index e7c37042a75c..000000000000 --- a/docs/concepts/frontend.qmd +++ /dev/null @@ -1,3 +0,0 @@ -# Frontend - -The Ibis frontend is the user-facing API that you typically think about when you use the library. diff --git a/docs/concepts/internals.qmd b/docs/concepts/internals.qmd index e62be93f859d..44fd36964fb7 100644 --- a/docs/concepts/internals.qmd +++ b/docs/concepts/internals.qmd @@ -1,6 +1,6 @@ # Internals -The internals are designed to map the frontend to the backend. +The internals are designed to map the Ibis API to the backend. ## Primary Goals diff --git a/docs/portability.excalidraw b/docs/portability.excalidraw new file mode 100644 index 000000000000..dbc583c358e6 --- /dev/null +++ b/docs/portability.excalidraw @@ -0,0 +1,1369 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "rectangle", + "version": 897, + "versionNonce": 2035950368, + "isDeleted": false, + "id": "3satn-20N8u6a3ZJn1YFP", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 20, + "angle": 0, + "x": 1322.807558806446, + "y": 763.7138771151889, + "strokeColor": "#087f5b", + "backgroundColor": "#15aabf", + "width": 727.4732433894771, + "height": 1304.0415649311615, + "seed": 1249613600, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1693518742560, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 683, + "versionNonce": 1638401824, + "isDeleted": false, + "id": "QIEa-K6SpPTgNhgYp7s-K", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 20, + "angle": 0, + "x": 535.8952931180918, + "y": 749.3292280773187, + "strokeColor": "#087f5b", + "backgroundColor": "#15aabf", + "width": 727.4732433894771, + "height": 819.0817890700803, + "seed": 1249613600, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [], + "updated": 1693518738479, + "link": null, + "locked": false + }, + { + "id": "G8hyksok7shjI3OzrHncY", + "type": "rectangle", + "x": -5.008471682697632, + "y": 768.7727545529218, + "width": 482.80118976537767, + "height": 513.9678024911719, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "#15aabf", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 20, + "groupIds": [], + "roundness": { + "type": 3 + }, + "seed": 1249613600, + "version": 534, + "versionNonce": 1803016928, + "isDeleted": false, + "boundElements": null, + "updated": 1693518832580, + "link": null, + "locked": false + }, + { + "id": "DF5opBX7jj9OBEQX7Z5yZ", + "type": "diamond", + "x": 735.0227562311838, + "y": 103.29837145524095, + "width": 322, + "height": 326, + "angle": 0, + "strokeColor": "#0b7285", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 2 + }, + "seed": 1618143008, + "version": 663, + "versionNonce": 607832800, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "tmTumDZH-7xc93aZBEFea" + }, + { + "id": "FaatXgPWMNGMOpWebkL2c", + "type": "arrow" + }, + { + "id": "v5dEgMddZ6vGuDZjoq26p", + "type": "arrow" + } + ], + "updated": 1693518661794, + "link": null, + "locked": false + }, + { + "id": "tmTumDZH-7xc93aZBEFea", + "type": "text", + "x": 820.8367821100901, + "y": 243.79837145524095, + "width": 150.3719482421875, + "height": 45, + "angle": 0, + "strokeColor": "#0b7285", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 1435216608, + "version": 869, + "versionNonce": 1000816416, + "isDeleted": false, + "boundElements": null, + "updated": 1693518658845, + "link": null, + "locked": false, + "text": "Ibis API", + "fontSize": 36, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 32, + "containerId": "DF5opBX7jj9OBEQX7Z5yZ", + "originalText": "Ibis API", + "lineHeight": 1.25 + }, + { + "id": "Os56hCsbynRWnWTdNj59Q", + "type": "rectangle", + "x": 652.8558061273382, + "y": 652.0031007911264, + "width": 502.87491697201926, + "height": 85.98789831027557, + "angle": 0, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 3 + }, + "seed": 558132960, + "version": 424, + "versionNonce": 287397664, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "r7jyq4cmZztp3f0RI9Zj5" + }, + { + "id": "plv9OX5ALXbXXp5Ln3G1w", + "type": "arrow" + } + ], + "updated": 1693518640716, + "link": null, + "locked": false + }, + { + "id": "r7jyq4cmZztp3f0RI9Zj5", + "type": "text", + "x": 672.0573332168634, + "y": 672.4970499462642, + "width": 464.47186279296875, + "height": 45, + "angle": 0, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 341115616, + "version": 333, + "versionNonce": 540949280, + "isDeleted": false, + "boundElements": null, + "updated": 1693518623004, + "link": null, + "locked": false, + "text": "String-generating backends", + "fontSize": 36, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 32, + "containerId": "Os56hCsbynRWnWTdNj59Q", + "originalText": "String-generating backends", + "lineHeight": 1.25 + }, + { + "type": "rectangle", + "version": 566, + "versionNonce": 1780966112, + "isDeleted": false, + "id": "DvhrgeS3GmSVGEI4pIvRI", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1354.421489135725, + "y": 655.692561743038, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "width": 636, + "height": 88, + "seed": 558132960, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "type": "text", + "id": "V6t9Gshb1eWNi9PKWTXfd" + }, + { + "id": "FaatXgPWMNGMOpWebkL2c", + "type": "arrow" + } + ], + "updated": 1693518648700, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 551, + "versionNonce": 33328864, + "isDeleted": false, + "id": "V6t9Gshb1eWNi9PKWTXfd", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1399.8296007079907, + "y": 677.192561743038, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "width": 545.1837768554688, + "height": 45, + "seed": 341115616, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518549014, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Expression-generating backends", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "DvhrgeS3GmSVGEI4pIvRI", + "originalText": "Expression-generating backends", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "rectangle", + "version": 1620, + "versionNonce": 594728736, + "isDeleted": false, + "id": "gndz8aKpPILH6fDftjq96", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -8.169576085561971, + "y": 675.2191477491781, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "width": 489, + "height": 85, + "seed": 558132960, + "groupIds": [], + "roundness": { + "type": 3 + }, + "boundElements": [ + { + "type": "text", + "id": "2VCOnlHMRl-d_z9-abW0W" + }, + { + "id": "v5dEgMddZ6vGuDZjoq26p", + "type": "arrow" + } + ], + "updated": 1693518837419, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1655, + "versionNonce": 589408992, + "isDeleted": false, + "id": "2VCOnlHMRl-d_z9-abW0W", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 6.380503260141154, + "y": 695.2191477491781, + "strokeColor": "#5c940d", + "backgroundColor": "transparent", + "width": 459.89984130859375, + "height": 45, + "seed": 341115616, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518837419, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Direct-execution backends", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "gndz8aKpPILH6fDftjq96", + "originalText": "Direct-execution backends", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "id": "Ui-EyqSSge9Np4gyXRTX8", + "type": "ellipse", + "x": 573.6685106444917, + "y": 856.6214812932433, + "width": 313.93186541768836, + "height": 290.3279959388282, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 2 + }, + "seed": 292556576, + "version": 580, + "versionNonce": 1095449312, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "SfeNtnSGB0rPrzLkcAtm1" + } + ], + "updated": 1693518748037, + "link": null, + "locked": false + }, + { + "id": "SfeNtnSGB0rPrzLkcAtm1", + "type": "text", + "x": 661.1807928219846, + "y": 956.6390319143344, + "width": 138.9239501953125, + "height": 90, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": null, + "seed": 32046880, + "version": 281, + "versionNonce": 1075970784, + "isDeleted": false, + "boundElements": null, + "updated": 1693518750062, + "link": null, + "locked": false, + "text": "Apache \nImpala", + "fontSize": 36, + "fontFamily": 1, + "textAlign": "center", + "verticalAlign": "middle", + "baseline": 77, + "containerId": "Ui-EyqSSge9Np4gyXRTX8", + "originalText": "Apache Impala", + "lineHeight": 1.25 + }, + { + "type": "ellipse", + "version": 788, + "versionNonce": 1227508448, + "isDeleted": false, + "id": "ltqe3JfOW70vWV5ElNyIT", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 571.3917962014349, + "y": 1225.6662751192403, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "iScIpgg257viEBIxK8ShG" + } + ], + "updated": 1693518745854, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 511, + "versionNonce": 1838961440, + "isDeleted": false, + "id": "iScIpgg257viEBIxK8ShG", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 640.5620907690644, + "y": 1348.1838257403315, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 175.60792541503906, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518745854, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "ClickHouse", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "ltqe3JfOW70vWV5ElNyIT", + "originalText": "ClickHouse", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 842, + "versionNonce": 1531454176, + "isDeleted": false, + "id": "qEMoIdB24XqwgL_HyZzLa", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 923.7741041958541, + "y": 866.387347895011, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "-E5WxmwPMs7WqKSbKPs_V" + } + ], + "updated": 1693518681270, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 582, + "versionNonce": 1994914592, + "isDeleted": false, + "id": "-E5WxmwPMs7WqKSbKPs_V", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1005.7603930261789, + "y": 966.4048985161023, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 149.97593688964844, + "height": 90, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518681270, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Google \nBigQuery", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "qEMoIdB24XqwgL_HyZzLa", + "originalText": "Google BigQuery", + "lineHeight": 1.25, + "baseline": 77 + }, + { + "type": "ellipse", + "version": 1092, + "versionNonce": 615861024, + "isDeleted": false, + "id": "P34jIyXfcvNxSwoOH-kUk", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 931.0478489403114, + "y": 1231.9656723762253, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "NdeEfJQGffLMzAmPREUY9" + } + ], + "updated": 1693518679418, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 841, + "versionNonce": 611350240, + "isDeleted": false, + "id": "NdeEfJQGffLMzAmPREUY9", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1016.7781334371402, + "y": 1354.4832229973165, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 142.48794555664062, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518679418, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "HeavyAI", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "P34jIyXfcvNxSwoOH-kUk", + "originalText": "HeavyAI", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 1091, + "versionNonce": 262470432, + "isDeleted": false, + "id": "ErvGBEh6CzVJKf7T3Ko6p", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1361.0736775710388, + "y": 840.8960385983261, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "c-LKwSBsUUfWRQMUUUUgt" + } + ], + "updated": 1693518757062, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 860, + "versionNonce": 1704183520, + "isDeleted": false, + "id": "c-LKwSBsUUfWRQMUUUUgt", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1417.1759713452116, + "y": 918.4135892194173, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 201.74392700195312, + "height": 135, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518757062, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Apache \nArrow \nDataFusion", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "ErvGBEh6CzVJKf7T3Ko6p", + "originalText": "Apache Arrow DataFusion", + "lineHeight": 1.25, + "baseline": 122 + }, + { + "type": "ellipse", + "version": 1164, + "versionNonce": 1787795168, + "isDeleted": false, + "id": "tqUIpTcqmbQUYB8atReYf", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1698.5074439497573, + "y": 833.2306202977297, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "V6FRVugrJgZH-tHnkm7jO" + } + ], + "updated": 1693518758844, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 960, + "versionNonce": 1198811872, + "isDeleted": false, + "id": "V6FRVugrJgZH-tHnkm7jO", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1771.0977290569376, + "y": 955.7481709188208, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 168.7679443359375, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518777373, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Snowflake", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "tqUIpTcqmbQUYB8atReYf", + "originalText": "Snowflake", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 1512, + "versionNonce": 1214060320, + "isDeleted": false, + "id": "8fBoCFW4tatxGGwUk_HLp", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1352.6132698828128, + "y": 1204.7165601544466, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "eLwe6-4-q1Fwf_cdLMlsv" + } + ], + "updated": 1693518754603, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1312, + "versionNonce": 1224436448, + "isDeleted": false, + "id": "eLwe6-4-q1Fwf_cdLMlsv", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1440.1255520603056, + "y": 1304.7341107755378, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 138.9239501953125, + "height": 90, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518754603, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Apache \nSpark", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "8fBoCFW4tatxGGwUk_HLp", + "originalText": "Apache Spark", + "lineHeight": 1.25, + "baseline": 77 + }, + { + "type": "ellipse", + "version": 1589, + "versionNonce": 525970144, + "isDeleted": false, + "id": "Ewe5iyUvkvOL1dKiHnvCE", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1712.7854658183148, + "y": 1184.6332522884961, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "rpduLsJ31u0zwhP1UeMcv" + } + ], + "updated": 1693518755845, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1396, + "versionNonce": 1553134368, + "isDeleted": false, + "id": "rpduLsJ31u0zwhP1UeMcv", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1825.173739817097, + "y": 1307.1508029095874, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 89.17196655273438, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518755845, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Dask", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Ewe5iyUvkvOL1dKiHnvCE", + "originalText": "Dask", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 1771, + "versionNonce": 616451872, + "isDeleted": false, + "id": "jFSSEhu_n5skOGZN79miT", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1357.616782508544, + "y": 1546.776342656471, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "Fw-Wq44WIpU3J-DRS3Vt7" + } + ], + "updated": 1693518752963, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1586, + "versionNonce": 1214886624, + "isDeleted": false, + "id": "Fw-Wq44WIpU3J-DRS3Vt7", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1445.3990613291032, + "y": 1669.2938932775621, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 138.3839569091797, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518752963, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "DuckDB", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "jFSSEhu_n5skOGZN79miT", + "originalText": "DuckDB", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 1832, + "versionNonce": 302381792, + "isDeleted": false, + "id": "Qo3F_VgfqYPSEQZqJ02FS", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1703.4318891246614, + "y": 1544.777149384628, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "xmvO8cM6j-lWH_u2tmmtz" + } + ], + "updated": 1693518751044, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1671, + "versionNonce": 1868649248, + "isDeleted": false, + "id": "xmvO8cM6j-lWH_u2tmmtz", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1804.2821687386777, + "y": 1667.2947000057193, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 112.24795532226562, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518765226, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "Polars", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Qo3F_VgfqYPSEQZqJ02FS", + "originalText": "Polars", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 1385, + "versionNonce": 1042019040, + "isDeleted": false, + "id": "59j3zFDWZABQ4AhuikU7_", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 83.56917370673997, + "y": 881.3131290402264, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 313.93186541768836, + "height": 290.3279959388282, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [ + { + "type": "text", + "id": "qLojUJRaBTOGE8XM3UIrR" + } + ], + "updated": 1693518824704, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 1179, + "versionNonce": 939076384, + "isDeleted": false, + "id": "qLojUJRaBTOGE8XM3UIrR", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 179.23545521284612, + "y": 1003.8306796613165, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 122.61595153808594, + "height": 45, + "seed": 32046880, + "groupIds": [], + "roundness": null, + "boundElements": [], + "updated": 1693518824704, + "link": null, + "locked": false, + "fontSize": 36, + "fontFamily": 1, + "text": "pandas", + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "59j3zFDWZABQ4AhuikU7_", + "originalText": "pandas", + "lineHeight": 1.25, + "baseline": 32 + }, + { + "type": "ellipse", + "version": 2333, + "versionNonce": 658570976, + "isDeleted": false, + "id": "2FdeuKTGwdLi0E_M7uUeu", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1554.2813976338236, + "y": 1965.6826611676986, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 63.01973983174179, + "height": 60.85344526202827, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1693518595069, + "link": null, + "locked": false + }, + { + "type": "ellipse", + "version": 2446, + "versionNonce": 1218360032, + "isDeleted": false, + "id": "M6c9o4ZDQwxk_TDZVsSpb", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1773.8074524848673, + "y": 1964.8004941387303, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 58.82947285757625, + "height": 60.59360944126658, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1693518601328, + "link": null, + "locked": false + }, + { + "type": "ellipse", + "version": 2544, + "versionNonce": 970762976, + "isDeleted": false, + "id": "ac02It_wRuCbZe-VSqy2k", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1655.987024123536, + "y": 1965.5484186701453, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "width": 62.303681551346415, + "height": 61.6013537306726, + "seed": 292556576, + "groupIds": [], + "roundness": { + "type": 2 + }, + "boundElements": [], + "updated": 1693518593443, + "link": null, + "locked": false + }, + { + "id": "plv9OX5ALXbXXp5Ln3G1w", + "type": "arrow", + "x": 898.5656361500608, + "y": 425.6839784008932, + "width": 3.7084821955711504, + "height": 212.1252442299538, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 2 + }, + "seed": 1396884256, + "version": 165, + "versionNonce": 1929857824, + "isDeleted": false, + "boundElements": null, + "updated": 1693518666601, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 3.7084821955711504, + 212.1252442299538 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": { + "elementId": "Os56hCsbynRWnWTdNj59Q", + "focus": -0.004042043725041546, + "gap": 14.193878160279382 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "FaatXgPWMNGMOpWebkL2c", + "type": "arrow", + "x": 992.2686107730428, + "y": 339.06141737724175, + "width": 654.7042727422008, + "height": 306.1446611517617, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 2 + }, + "seed": 228272864, + "version": 207, + "versionNonce": 1051876128, + "isDeleted": false, + "boundElements": null, + "updated": 1693518658846, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 654.7042727422008, + 306.1446611517617 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "DF5opBX7jj9OBEQX7Z5yZ", + "focus": 0.17028130604106156, + "gap": 5.062808541682315 + }, + "endBinding": { + "elementId": "DvhrgeS3GmSVGEI4pIvRI", + "focus": 0.22099999823946306, + "gap": 10.486483214034592 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "id": "v5dEgMddZ6vGuDZjoq26p", + "type": "arrow", + "x": 810.7769665833746, + "y": 351.32655612235396, + "width": 593.1584159878962, + "height": 314.37432843397994, + "angle": 0, + "strokeColor": "#087f5b", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "roundness": { + "type": 2 + }, + "seed": 949795616, + "version": 535, + "versionNonce": 1802063584, + "isDeleted": false, + "boundElements": null, + "updated": 1693518837420, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -593.1584159878962, + 314.37432843397994 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "DF5opBX7jj9OBEQX7Z5yZ", + "focus": -0.2447790797199902, + "gap": 5.855786453443855 + }, + "endBinding": { + "elementId": "gndz8aKpPILH6fDftjq96", + "focus": -0.35991232861974126, + "gap": 9.518263192844188 + }, + "startArrowhead": null, + "endArrowhead": "arrow" + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/docs/portability.png b/docs/portability.png new file mode 100644 index 000000000000..b8f283e3567e Binary files /dev/null and b/docs/portability.png differ diff --git a/docs/what.qmd b/docs/what.qmd deleted file mode 100644 index fb20c94a02a3..000000000000 --- a/docs/what.qmd +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: What is Ibis? ---- - -Ibis is a versatile Python library for data analysis and manipulation. It simplifies data tasks, seamlessly integrates with backends, and provides efficient query execution. - -You can think of Ibis as a frontend for data that works with 15+ backends. diff --git a/docs/why.qmd b/docs/why.qmd index 8b3ea3f89cf6..28f132d4f1b6 100644 --- a/docs/why.qmd +++ b/docs/why.qmd @@ -1,35 +1,51 @@ --- -title: "Why use Ibis?" +title: "Why Ibis?" --- -Ibis is a powerful Python library designed for data analysis and manipulation. It offers a seamless integration with various backends, enabling users to effortlessly work with data stored in different formats and platforms. Whether you are a data scientist, analyst, or developer, Ibis provides numerous benefits that enhance your data-related workflows and empower you to make data-driven decisions. +Ibis is a portable Python dataframe library. If you've had issues with scaling data transformation code in Python, need to work with data in multiple data platforms, find yourself translating between other Python dataframe APIs, or just want a great Python dataframe experience, Ibis is for you. -## Simplified data manipulation +## Portability -One of the key advantages of Ibis is its intuitive and straightforward interface for data manipulation. With Ibis, you can effortlessly filter, transform, and aggregate your data using familiar Python syntax. This simplicity allows you to focus on the analysis rather than getting lost in complex code. +Ibis is portable -- that is, you can reuse the same code across multiple backends. Most Python dataframes are tightly coupled to their execution engine. And many databases only support SQL, with no Python API. Ibis solves this problem by providing a common API for data manipulation in Python, and compiling that API into the backend's native language. This means you can learn a single API and use it across any supported backend (execution engine). -## Seamless backend integration +![Portability](./portability.png) -Ibis seamlessly integrates with multiple backend systems, such as BigQuery, Snowflake, MySQL, PostgreSQL, and more. This flexibility allows you to leverage the power of Ibis regardless of where your data resides. Whether you prefer working with SQL or pandas-like syntax, Ibis enables you to harness the capabilities of your chosen backend effortlessly. +While portability with Ibis isn't perfect, commonalities across backends and SQL dialects combined with years of engineering effort produce a pretty good result. In the long-term, we hope for standard Intermediary Representation (IR) formats for transformation like [Substrait](https://substrait.io) to simplify this further. -## Efficient query execution +## Ecosystem -Ibis optimizes your queries for efficient execution and performance. It leverages the strengths of each backend to ensure that your data operations are executed as quickly as possible. This optimization translates into faster query results, reducing the time it takes to derive insights from your data. +Ibis is part of a larger ecosystem of Python data tools. Ibis is designed to work well with other tools in the ecosystem, and we hope to make it easier to use Ibis with other tools over time. -## Powerful analytics capabilities +### Local experience -Ibis empowers you to perform advanced analytics at scale. With built-in functions for statistical analysis, grouping, aggregation, and more, you can easily derive meaningful insights from your data. Whether you need to calculate descriptive statistics, perform regression analysis, or create custom analytics workflows, Ibis provides the tools you need. +By default, Ibis offers a great local experience for working with many file formats. DuckDB is the default backend, with Polars and DataFusion as two other great local options. Many of the backends can run locally but require more setup than a pip installation. -## Easy integration with data science ecosystem +### Scaling up and out -As an integral part of the Python data science ecosystem, Ibis seamlessly integrates with popular libraries such as pandas, PySpark, Polars, and DataFusion. This integration allows you to combine the power of Ibis with the extensive capabilities of these libraries, enabling you to build end-to-end data pipelines and perform complex data analyses with ease. +After prototyping on a local backend, directly scale in the cloud.You can prototype on DuckDB and deploy with MotherDuck. You can scale from any Python client with Ibis installed to whatever your backend supports. -## Robust input and output support +## Use cases -Ibis supports a wide range of input and output options, making it easy to work with different file formats and data platforms. Whether you need to read data from CSV files, write to Parquet files, or connect to a database, Ibis provides a consistent and reliable interface for handling your data. +You can use Ibis at any stage of your data workflow. Use the same framework for local exploration on a few files or production workloads on the most advanced data platforms. -## Active community and support +Ibis can help you with: -Ibis benefits from an active and thriving community of data professionals and open-source contributors. This community actively maintains and enhances the library, ensuring that it stays up-to-date and relevant to the evolving needs of data analysis. Additionally, comprehensive documentation and online resources make it easy to learn Ibis and find answers to your questions. +- data catalog exploration +- exploratory data analysis +- transforming data +- visualizing data +- data science and machine learning -In conclusion, Ibis is a valuable tool that simplifies data manipulation, integrates seamlessly with different backends, provides efficient query execution, offers powerful analytics capabilities, integrates with the data science ecosystem, supports various input and output options, and is backed by a vibrant community. By using Ibis, you can unlock the true potential of your data and accelerate your data analysis workflows. +## Supported backends + +{{< include /_tabsets/install.qmd >}} + +See the [backend support matrix](support_matrix.qmd) for details on operations supported. [Open a feature request](https://github.com/ibis-project/ibis/issues/new?assignees=&labels=feature&projects=&template=feature-request.yml&title=feat) if you'd like to see support for an operation in a given backend. If the backend supports it, we'll do our best to add it quickly! + +## Community + +Community discussions primarily take place on [GitHub](https://github.com/ibis-project/ibis/discussions). + +## Getting started + +If you're interested in trying Ibis we recommend the [getting started tutorial](./tutorials/getting_started.qmd).