From d14b2d215a5a0908c85135aab5e147314f523ebc Mon Sep 17 00:00:00 2001 From: Ben Beinke Date: Mon, 13 May 2024 11:27:45 +0200 Subject: [PATCH 1/2] Updated document_index notebook to use MultipleChunkRetrieverQa --- src/documentation/document_index.ipynb | 113 +++++++++++++++++++++---- 1 file changed, 97 insertions(+), 16 deletions(-) diff --git a/src/documentation/document_index.ipynb b/src/documentation/document_index.ipynb index e3ea71c9b..627ebd46a 100644 --- a/src/documentation/document_index.ipynb +++ b/src/documentation/document_index.ipynb @@ -2,9 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", @@ -22,7 +33,7 @@ " LimitedConcurrencyClient,\n", ")\n", "from intelligence_layer.core import InMemoryTracer\n", - "from intelligence_layer.examples import RetrieverBasedQa, RetrieverBasedQaInput\n", + "from intelligence_layer.examples import MultipleChunkRetrieverQa, RetrieverBasedQaInput\n", "\n", "load_dotenv()" ] @@ -64,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -74,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -98,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -119,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -198,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -220,9 +231,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='nelson_rockefeller'), created=datetime.datetime(2024, 4, 30, 15, 20, 59, 110906), version=6),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='jane_jacobs'), created=datetime.datetime(2024, 4, 30, 15, 20, 58, 999595), version=4),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), created=datetime.datetime(2024, 4, 30, 15, 20, 58, 893415), version=9),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1009'), created=datetime.datetime(2024, 4, 16, 18, 38, 25, 321925), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1008'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 984779), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1007'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 666370), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1006'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 351560), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1005'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 32960), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1004'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 721178), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1003'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 381143), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1002'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 101439), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1001'), created=datetime.datetime(2024, 4, 16, 18, 38, 22, 731025), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1000'), created=datetime.datetime(2024, 4, 16, 18, 38, 22, 304618), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1078'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 941525), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1076'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 606629), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1059'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 231763), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1057'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 844266), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1055'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 501809), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1054'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 221662), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1053'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 863937), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1052'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 505008), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1051'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 206234), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1050'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 882343), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1049'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 564074), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1048'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 263845), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1047'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 907021), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1046'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 563904), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1045'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 204945), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1044'), created=datetime.datetime(2024, 4, 16, 18, 11, 29, 135851), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1013'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 823863), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1012'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 506529), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1011'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 216216), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1010'), created=datetime.datetime(2024, 4, 16, 18, 11, 27, 890140), version=1),\n", + " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='demo'), created=datetime.datetime(2023, 10, 2, 10, 43, 40, 891681), version=1)]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "document_index.documents(collection_path)" ] @@ -240,9 +295,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.7355052, document_chunk=DocumentChunk(text='Robert Moses\\'\\'\\' (December 18, 1888 – July 29, 1981) was an American [[urban planner]] and public official who worked in the [[New York metropolitan area]] during the early to mid 20th century. Despite never being elected to any office, Moses is regarded as one of the most powerful and influential individuals in the history of New York City and New York State. The grand scale of his infrastructural projects and his philosophy of urban development influenced a generation of engineers, architects, and urban planners across the United States.\\n\\nMoses held various positions throughout his more than forty-year long career. He at times held up to 12 titles simultaneously, including [[New York City Parks Commissioner]] and chairman of the [[Long Island State Park Commission]].{{Cite web|url=https://www.pbs.org/wnet/need-to-know/environment/the-legacy-of-robert-moses/16018/|title=The legacy of Robert Moses|last=Sarachan|first=Sydney|date=January 17, 2013|website=Need to Know {{!}} PBS|language=en-US|access-date=December 3, 2019}} Having worked closely with New York governor [[Al Smith]] early in his career, Moses became expert in writing laws and navigating and manipulating the inner workings of state government. He created and led numerous semi-autonomous [[Public authority|public authorities]], through which he controlled millions of dollars in revenue and directly issued [[Bond (finance)|bonds]] to fund new ventures with little outside input or oversight.', start=0, end=1499, metadata=None)),\n", + " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.71257365, document_chunk=DocumentChunk(text=\"==Offices held==\\nThe many offices and professional titles that Moses held gave him unusually broad power to shape urban development in the New York metropolitan region. These include, according to the New York Preservation Archive Project:{{Cite web|url=http://www.nypap.org/preservation-history/robert-moses/|title=Robert Moses {{!}}|website=www.nypap.org|language=en-US|access-date=March 29, 2018}}\\n*[[Long Island State Park Commission]] (President, 1924–1963)\\n* New York State Council of Parks (Chairman, 1924–1963)\\n*[[Secretary of State of New York|New York Secretary of State]] (1927–1928)\\n* Bethpage State Park Authority (President, 1933–1963)\\n* Emergency Public Works Commission (Chairman, 1933–1934)\\n* Jones Beach Parkway Authority (President, 1933–1963)\\n*[[New York City Department of Parks and Recreation|New York City Department of Parks]] (Commissioner, 1934–1960)\\n* [[Triborough Bridge]] and Tunnel Authority (Chairman, 1934–1968)\\n* New York City Planning Commission (Commissioner, 1942–1960)\\n* New York State Power Authority (Chairman, 1954–1962)\\n* [[1964 New York World's Fair|New York's World Fair]] (President, 1960–1966)\\n* Office of the Governor of New York (Special Advisor on Housing, 1974–1975)\", start=9296, end=10521, metadata=None)),\n", + " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.7026457, document_chunk=DocumentChunk(text='Moses\\'s projects transformed the New York area and revolutionized the way cities in the U.S. were designed and built. As Long Island State Park Commissioner, Moses oversaw the construction of [[Jones Beach State Park]], the most visited public beach in the United States,{{cite news |url=http://www.longislandexchange.com/jones-beach.html |website=Long Island Exchange |title=Jones Beach |access-date=November 21, 2012 |archive-url=https://web.archive.org/web/20130121130008/http://www.longislandexchange.com/jones-beach.html |archive-date=January 21, 2013 |url-status=dead }} and was the primary architect of the [[Parkways in New York|New York State Parkway System]]. As head of the [[MTA Bridges and Tunnels|Triborough Bridge Authority]], Moses had near-complete control over bridges and tunnels in New York City as well as the tolls collected from them, and built, among others, the [[Robert F. Kennedy Bridge|Triborough Bridge]], the [[Brooklyn–Battery Tunnel]], and the [[Throgs Neck Bridge]], as well as several major highways. These roadways and bridges, alongside [[urban renewal]] efforts that saw the destruction of huge swaths of tenement housing and their replacement with large [[New York City Housing Authority|public housing projects]], transformed the physical fabric of New York and inspired other cities to undertake similar development endeavors.\\n\\nMoses\\'s reputation declined following the publication of [[Robert Caro]]\\'s [[Pulitzer Prize]]-winning biography \\'\\'[[The Power Broker]]\\'\\' (1974), which cast doubt on the purported benefits of many of Moses\\'s projects and further cast Moses as racist. In large part because of \\'\\'The Power Broker\\'\\', Moses is today considered a controversial figure in the history of New York City.', start=1502, end=3277, metadata=None)),\n", + " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.6688517, document_chunk=DocumentChunk(text=\"Robert Moses and his brother Paul attended several schools for their elementary and [[secondary education]], including the [[Ethical Culture Fieldston School|Ethical Culture School]], the [[Dwight School]] and the [[Mohegan Lake, New York#Historic places|Mohegan Lake School]], a military academy near [[Peekskill, New York|Peekskill]].{{sfn|Caro|1974|pp=35}}\\n\\nAfter graduating from [[Yale College]] (B.A., 1909) and [[Wadham College]], [[Oxford University|Oxford]] (B.A., Jurisprudence, 1911; M.A., 1913), and earning a Ph.D. in [[political science]] from [[Columbia University]] in 1914, Moses became attracted to New York City reform politics.{{Cite web|url=http://c250.columbia.edu/c250_celebrates/remarkable_columbians/robert_moses.html|title = Robert Moses}} A committed [[idealism|idealist]], he developed several plans to rid New York of [[Patronage#Politics|patronage hiring]] practices, including being the lead author of a 1919 proposal to reorganize the New York state government. None went very far, but Moses, due to his intelligence, caught the notice of [[Belle Moskowitz]], a friend and trusted advisor to Governor [[Al Smith]].{{sfn|Caro|1974}} When the state [[Secretary of State of New York|Secretary of State's]] position became appointive rather than elective, Smith named Moses. He served from 1927 to 1929.{{cite news |date=December 19, 1928 |title=Moses Resigns State Position |url=http://cdsun.library.cornell.edu/cgi-bin/cornell?a=d&d=CDS19281219.2.63.7# |newspaper=Cornell Daily Sun |location=Ithaca, NY |page=8}}\", start=4819, end=6382, metadata=None)),\n", + " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.6070359, document_chunk=DocumentChunk(text='==Influence==\\nDuring the 1920s, Moses sparred with [[Franklin D. Roosevelt]], then head of the Taconic State Park Commission, who favored the prompt construction of a [[parkway]] through the [[Hudson Valley]]. Moses succeeded in diverting funds to his Long Island parkway projects (the [[Northern State Parkway]], the [[Southern State Parkway]] and the [[Wantagh State Parkway]]), although the [[Taconic State Parkway]] was later completed as well.{{cite web|url=http://www.nycroads.com/roads/taconic/ |title=Taconic State Parkway |website=NYCRoads.com |access-date=May 25, 2006}} Moses helped build Long Island\\'s [[Meadowbrook State Parkway]]. It was the first fully divided limited access highway in the world.{{cite book|last=Leonard|first=Wallock|title=The Myth of The Master Builder|year=1991|publisher=Journal of Urban History|page=339}}\\n\\nMoses was a highly influential figure in the initiation of many of the reforms that restructured New York state\\'s government during the 1920s. A \\'Reconstruction Commission\\' headed by Moses produced a highly influential report that provided recommendations that would largely be adopted, including the consolidation of 187 existing agencies under 18 departments, a new executive budget system, and the four-year term limit for the governorship.{{sfn|Caro|1974|pp=106, 260}}', start=10524, end=11886, metadata=None))]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "document_index_retriever = DocumentIndexRetriever(\n", " document_index=document_index,\n", @@ -266,17 +336,28 @@ "\n", "Finally, since we have a ready-to use 'Retriever', we can employ it to do something more complicated than just search; it can serve as the basis for a question-answering system.\n", "\n", - "To do so, let's run a `RetrieverBasedQa` task:" + "To do so, let's run a `MultipleChunkRetrieverQa` task:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'The book about Robert Moses is called \"The Power Broker\" by Robert Caro.'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client = LimitedConcurrencyClient.from_env()\n", - "retriever_qa = RetrieverBasedQa(document_index_retriever)\n", + "retriever_qa = MultipleChunkRetrieverQa(document_index_retriever, insert_chunk_number=3)\n", "\n", "\n", "input = RetrieverBasedQaInput(\n", From d1a52c0f34ecb1650f4cebb36ca4dede9275d377 Mon Sep 17 00:00:00 2001 From: Ben Beinke Date: Mon, 13 May 2024 11:29:47 +0200 Subject: [PATCH 2/2] reset counts --- src/documentation/document_index.ipynb | 107 +++---------------------- 1 file changed, 13 insertions(+), 94 deletions(-) diff --git a/src/documentation/document_index.ipynb b/src/documentation/document_index.ipynb index 627ebd46a..cbe22b86a 100644 --- a/src/documentation/document_index.ipynb +++ b/src/documentation/document_index.ipynb @@ -2,20 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", @@ -75,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -85,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -130,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -209,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -231,53 +220,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='nelson_rockefeller'), created=datetime.datetime(2024, 4, 30, 15, 20, 59, 110906), version=6),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='jane_jacobs'), created=datetime.datetime(2024, 4, 30, 15, 20, 58, 999595), version=4),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), created=datetime.datetime(2024, 4, 30, 15, 20, 58, 893415), version=9),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1009'), created=datetime.datetime(2024, 4, 16, 18, 38, 25, 321925), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1008'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 984779), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1007'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 666370), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1006'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 351560), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1005'), created=datetime.datetime(2024, 4, 16, 18, 38, 24, 32960), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1004'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 721178), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1003'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 381143), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1002'), created=datetime.datetime(2024, 4, 16, 18, 38, 23, 101439), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1001'), created=datetime.datetime(2024, 4, 16, 18, 38, 22, 731025), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TKT1000'), created=datetime.datetime(2024, 4, 16, 18, 38, 22, 304618), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1078'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 941525), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1076'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 606629), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1059'), created=datetime.datetime(2024, 4, 16, 18, 38, 21, 231763), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1057'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 844266), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1055'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 501809), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1054'), created=datetime.datetime(2024, 4, 16, 18, 38, 20, 221662), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1053'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 863937), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1052'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 505008), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1051'), created=datetime.datetime(2024, 4, 16, 18, 38, 19, 206234), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1050'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 882343), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1049'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 564074), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1048'), created=datetime.datetime(2024, 4, 16, 18, 38, 18, 263845), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1047'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 907021), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1046'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 563904), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1045'), created=datetime.datetime(2024, 4, 16, 18, 38, 17, 204945), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1044'), created=datetime.datetime(2024, 4, 16, 18, 11, 29, 135851), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1013'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 823863), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1012'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 506529), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1011'), created=datetime.datetime(2024, 4, 16, 18, 11, 28, 216216), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='TCKT-1010'), created=datetime.datetime(2024, 4, 16, 18, 11, 27, 890140), version=1),\n", - " DocumentInfo(document_path=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='demo'), created=datetime.datetime(2023, 10, 2, 10, 43, 40, 891681), version=1)]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "document_index.documents(collection_path)" ] @@ -295,24 +240,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.7355052, document_chunk=DocumentChunk(text='Robert Moses\\'\\'\\' (December 18, 1888 – July 29, 1981) was an American [[urban planner]] and public official who worked in the [[New York metropolitan area]] during the early to mid 20th century. Despite never being elected to any office, Moses is regarded as one of the most powerful and influential individuals in the history of New York City and New York State. The grand scale of his infrastructural projects and his philosophy of urban development influenced a generation of engineers, architects, and urban planners across the United States.\\n\\nMoses held various positions throughout his more than forty-year long career. He at times held up to 12 titles simultaneously, including [[New York City Parks Commissioner]] and chairman of the [[Long Island State Park Commission]].{{Cite web|url=https://www.pbs.org/wnet/need-to-know/environment/the-legacy-of-robert-moses/16018/|title=The legacy of Robert Moses|last=Sarachan|first=Sydney|date=January 17, 2013|website=Need to Know {{!}} PBS|language=en-US|access-date=December 3, 2019}} Having worked closely with New York governor [[Al Smith]] early in his career, Moses became expert in writing laws and navigating and manipulating the inner workings of state government. He created and led numerous semi-autonomous [[Public authority|public authorities]], through which he controlled millions of dollars in revenue and directly issued [[Bond (finance)|bonds]] to fund new ventures with little outside input or oversight.', start=0, end=1499, metadata=None)),\n", - " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.71257365, document_chunk=DocumentChunk(text=\"==Offices held==\\nThe many offices and professional titles that Moses held gave him unusually broad power to shape urban development in the New York metropolitan region. These include, according to the New York Preservation Archive Project:{{Cite web|url=http://www.nypap.org/preservation-history/robert-moses/|title=Robert Moses {{!}}|website=www.nypap.org|language=en-US|access-date=March 29, 2018}}\\n*[[Long Island State Park Commission]] (President, 1924–1963)\\n* New York State Council of Parks (Chairman, 1924–1963)\\n*[[Secretary of State of New York|New York Secretary of State]] (1927–1928)\\n* Bethpage State Park Authority (President, 1933–1963)\\n* Emergency Public Works Commission (Chairman, 1933–1934)\\n* Jones Beach Parkway Authority (President, 1933–1963)\\n*[[New York City Department of Parks and Recreation|New York City Department of Parks]] (Commissioner, 1934–1960)\\n* [[Triborough Bridge]] and Tunnel Authority (Chairman, 1934–1968)\\n* New York City Planning Commission (Commissioner, 1942–1960)\\n* New York State Power Authority (Chairman, 1954–1962)\\n* [[1964 New York World's Fair|New York's World Fair]] (President, 1960–1966)\\n* Office of the Governor of New York (Special Advisor on Housing, 1974–1975)\", start=9296, end=10521, metadata=None)),\n", - " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.7026457, document_chunk=DocumentChunk(text='Moses\\'s projects transformed the New York area and revolutionized the way cities in the U.S. were designed and built. As Long Island State Park Commissioner, Moses oversaw the construction of [[Jones Beach State Park]], the most visited public beach in the United States,{{cite news |url=http://www.longislandexchange.com/jones-beach.html |website=Long Island Exchange |title=Jones Beach |access-date=November 21, 2012 |archive-url=https://web.archive.org/web/20130121130008/http://www.longislandexchange.com/jones-beach.html |archive-date=January 21, 2013 |url-status=dead }} and was the primary architect of the [[Parkways in New York|New York State Parkway System]]. As head of the [[MTA Bridges and Tunnels|Triborough Bridge Authority]], Moses had near-complete control over bridges and tunnels in New York City as well as the tolls collected from them, and built, among others, the [[Robert F. Kennedy Bridge|Triborough Bridge]], the [[Brooklyn–Battery Tunnel]], and the [[Throgs Neck Bridge]], as well as several major highways. These roadways and bridges, alongside [[urban renewal]] efforts that saw the destruction of huge swaths of tenement housing and their replacement with large [[New York City Housing Authority|public housing projects]], transformed the physical fabric of New York and inspired other cities to undertake similar development endeavors.\\n\\nMoses\\'s reputation declined following the publication of [[Robert Caro]]\\'s [[Pulitzer Prize]]-winning biography \\'\\'[[The Power Broker]]\\'\\' (1974), which cast doubt on the purported benefits of many of Moses\\'s projects and further cast Moses as racist. In large part because of \\'\\'The Power Broker\\'\\', Moses is today considered a controversial figure in the history of New York City.', start=1502, end=3277, metadata=None)),\n", - " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.6688517, document_chunk=DocumentChunk(text=\"Robert Moses and his brother Paul attended several schools for their elementary and [[secondary education]], including the [[Ethical Culture Fieldston School|Ethical Culture School]], the [[Dwight School]] and the [[Mohegan Lake, New York#Historic places|Mohegan Lake School]], a military academy near [[Peekskill, New York|Peekskill]].{{sfn|Caro|1974|pp=35}}\\n\\nAfter graduating from [[Yale College]] (B.A., 1909) and [[Wadham College]], [[Oxford University|Oxford]] (B.A., Jurisprudence, 1911; M.A., 1913), and earning a Ph.D. in [[political science]] from [[Columbia University]] in 1914, Moses became attracted to New York City reform politics.{{Cite web|url=http://c250.columbia.edu/c250_celebrates/remarkable_columbians/robert_moses.html|title = Robert Moses}} A committed [[idealism|idealist]], he developed several plans to rid New York of [[Patronage#Politics|patronage hiring]] practices, including being the lead author of a 1919 proposal to reorganize the New York state government. None went very far, but Moses, due to his intelligence, caught the notice of [[Belle Moskowitz]], a friend and trusted advisor to Governor [[Al Smith]].{{sfn|Caro|1974}} When the state [[Secretary of State of New York|Secretary of State's]] position became appointive rather than elective, Smith named Moses. He served from 1927 to 1929.{{cite news |date=December 19, 1928 |title=Moses Resigns State Position |url=http://cdsun.library.cornell.edu/cgi-bin/cornell?a=d&d=CDS19281219.2.63.7# |newspaper=Cornell Daily Sun |location=Ithaca, NY |page=8}}\", start=4819, end=6382, metadata=None)),\n", - " SearchResult(id=DocumentPath(collection_path=CollectionPath(namespace='aleph-alpha', collection='demo'), document_name='robert_moses'), score=0.6070359, document_chunk=DocumentChunk(text='==Influence==\\nDuring the 1920s, Moses sparred with [[Franklin D. Roosevelt]], then head of the Taconic State Park Commission, who favored the prompt construction of a [[parkway]] through the [[Hudson Valley]]. Moses succeeded in diverting funds to his Long Island parkway projects (the [[Northern State Parkway]], the [[Southern State Parkway]] and the [[Wantagh State Parkway]]), although the [[Taconic State Parkway]] was later completed as well.{{cite web|url=http://www.nycroads.com/roads/taconic/ |title=Taconic State Parkway |website=NYCRoads.com |access-date=May 25, 2006}} Moses helped build Long Island\\'s [[Meadowbrook State Parkway]]. It was the first fully divided limited access highway in the world.{{cite book|last=Leonard|first=Wallock|title=The Myth of The Master Builder|year=1991|publisher=Journal of Urban History|page=339}}\\n\\nMoses was a highly influential figure in the initiation of many of the reforms that restructured New York state\\'s government during the 1920s. A \\'Reconstruction Commission\\' headed by Moses produced a highly influential report that provided recommendations that would largely be adopted, including the consolidation of 187 existing agencies under 18 departments, a new executive budget system, and the four-year term limit for the governorship.{{sfn|Caro|1974|pp=106, 260}}', start=10524, end=11886, metadata=None))]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "document_index_retriever = DocumentIndexRetriever(\n", " document_index=document_index,\n", @@ -341,20 +271,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'The book about Robert Moses is called \"The Power Broker\" by Robert Caro.'" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "client = LimitedConcurrencyClient.from_env()\n", "retriever_qa = MultipleChunkRetrieverQa(document_index_retriever, insert_chunk_number=3)\n",