cli to automate some manual gst1 update work; do gst update

.github/workflows/pre-commit.yml

@@ -21,6 +21,10 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: Set git config to use personal access token in secrets
+        run: |
+          git config --global url."https://${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/climatepolicyradar/explorer.git".insteadOf "https://github.com/climatepolicyradar/explorer.git"
+
       - name: Install dependencies
         run: |
           python -m pip install "poetry==1.3.2" && poetry install && poetry run pre-commit install

.github/workflows/test.yml

@@ -21,12 +21,16 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: Set git config to use personal access token in secrets
+        run: |
+          git config --global url."https://${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/climatepolicyradar/explorer.git".insteadOf "https://github.com/climatepolicyradar/explorer.git"
+
       - name: Install dependencies
         run: |
           python -m pip install "poetry==1.2.2" && poetry install
+
       - name: Run test suite
         run: |
           poetry run python -m pytest \
-          --nbmake \
           --nbmake-find-import-errors \
           --nbmake-timeout=20 -vvv

classifiers/run_on_full_dataset.py

@@ -10,36 +10,15 @@
 from cpr_data_access.models import Span
 from dotenv import load_dotenv, find_dotenv
 
-from utils import load_text_block_sample, predict_from_text_blocks
+from classifiers.utils import load_text_block_sample, predict_from_text_blocks
 
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO
 )
 LOGGER = logging.getLogger(__name__)
 
 
-@click.command()
-@click.option(
-    "--wandb-artifact-name",
-    help="Weights and Biases artifact name. Should start with climatepolicyradar/. E.g. climatepolicyradar/sector-text-classifier/sector-text-classifier:v0",
-)
-@click.option(
-    "--output-dir",
-    type=click.Path(file_okay=False, path_type=Path),
-    help="Output directory for the spans.csv file.",
-)
-@click.option(
-    "--spans-csv-filename",
-    type=str,
-    default="spans.csv",
-    help="The filename to use for the spans CSV output file, including the .csv extension",
-)
-@click.option(
-    "--extra-output",
-    is_flag=True,
-    help="Whether to output an extra predictions.csv file. Output filename will use the same sufix as the spans CSV file.",
-)
-def cli(
+def run_on_full_dataset(
     wandb_artifact_name: str,
     output_dir: Path,
     spans_csv_filename: str,
@@ -117,5 +96,40 @@ def cli(
         )
 
 
+@click.command()
+@click.option(
+    "--wandb-artifact-name",
+    help="Weights and Biases artifact name. Should start with climatepolicyradar/. E.g. climatepolicyradar/sector-text-classifier/sector-text-classifier:v0",
+)
+@click.option(
+    "--output-dir",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Output directory for the spans.csv file.",
+)
+@click.option(
+    "--spans-csv-filename",
+    type=str,
+    default="spans.csv",
+    help="The filename to use for the spans CSV output file, including the .csv extension",
+)
+@click.option(
+    "--extra-output",
+    is_flag=True,
+    help="Whether to output an extra predictions.csv file. Output filename will use the same sufix as the spans CSV file.",
+)
+def cli(
+    wandb_artifact_name: str,
+    output_dir: Path,
+    spans_csv_filename: str,
+    extra_output: bool,
+) -> None:
+    return run_on_full_dataset(
+        wandb_artifact_name=wandb_artifact_name,
+        output_dir=output_dir,
+        spans_csv_filename=spans_csv_filename,
+        extra_output=extra_output,
+    )
+
+
 if __name__ == "__main__":
     cli()

concepts/response-measures/spans_cop28.csv

@@ -0,0 +1,34 @@
+document_id,text_block_text_hash,type,id,text,start_idx,end_idx,sentence,pred_probability,annotator,kb_ids
+UNFCCC.non-party.1863.0,0e41cf5d65ca9c7e4abbaa1aa34d512a__ec32f59370f7247403d882ff8d5b3560aa85314e843c5dd061c89792b13d7420,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,94,111,"- A clarification on the need to limit global warming temperatures to 1.5℃, the science-based response measures this implies and the actions to address already occurring and unavoidable climate impacts.",1.0,explorer,
+UNFCCC.party.1852.0,6639244aad9ce985d34327cf39a5b4ec__93dfd34cdd70c0637395c5a40c9027eea03a543370911ab9ae3ed429389805fe,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,24,41,"· Mitigation, including response measures",1.0,explorer,
+UNFCCC.party.1842.0,f27ec787bbc9b125af0b2ae156889fe8__269c85168ae9d0134f271b5cb28a56d9f3cb51cb7f763bcabe5350985684bb27,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,621,638,"Therefore, the assessment of the collective progress towards achieving the purpose of the Paris Agreement and its long-term goals, as mandated by its Article 14.1, means assessing the collective progress on ambition and gaps across mitigation; adaptation; means of implementation and support; response measures; and loss and damage.",1.0,explorer,
+UNFCCC.party.1837.0,6639244aad9ce985d34327cf39a5b4ec__93dfd34cdd70c0637395c5a40c9027eea03a543370911ab9ae3ed429389805fe,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,24,41,"· Mitigation, including response measures",1.0,explorer,
+UNFCCC.party.1837.0,c0e61564f9a891f9933bd7e1dc64afa2__3f2c95507cafd9b1f4bf5d94432bb2311228863e66889f7e80915d3a0052beaa,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,57,74,· The need to address the social and economic impacts of response measures and to address loss and damage in developing countries (paragraph 13 of the G77 and China's 25 May 2022 GST Submission),1.0,explorer,
+UNFCCC.party.1837.0,76cde74c37a8aac396397faa5c685c2a__5e6768eb79ce38b19423dcdeedb27d5b59fe0bfe74d5ef4122a513728e5a9c1e,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,331,348,"· Identifying the good practices, lessons learned, opportunities, challenges, needs, and gaps for enhancing implementation andambition , with respect to mitigation, adaptation, the provision of the means of implementation (finance, capacity building and technology transfer) to developing countries, addressing the consequences of response measures, and averting, minimizing, and addressing loss and damage as indicated in para 6(b) of decision 19/CMA.1 in a balanced manner (paragraph 15(iii) of the G77 and China's 25 May 2022 GST Submission).",1.0,explorer,
+UNFCCC.party.1848.0,7eb4b047e57b8145d8450160bed81dfc__e7a0f17e55ab3de00b643123d2c4f96717cf6fe93f5ef5d3e98a7d7feac1c5de,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,256,273,"The high-level events, convening representatives from Parties and non-Party stakeholders, could be organised according to relevant topics under mitigation, adaptation, and means of implementation, as well as the cross-cutting issues of loss and damage and response measures.",1.0,explorer,
+UNFCCC.party.1835.0,5bbdc8edcbf83d2457c7b0b87c45a96a__360803b11c5c956c8ec080b5986d2052f08751e373ff92389fad4c476b7f1969,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,140,157,"the Paris Agreement in a balanced and inclusive manner, considering mitigation, adaptation, means of implementation and support, as well as response measures and loss & damage.",1.0,explorer,
+UNFCCC.party.1841.0,8fce3492940bb916a80876dba50d71d8__d878033512769adde554e6cdb4ef1c10e03011d301b1302f19b2454c915e9d92,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,326,343,"Adaptation and Recognition of Adaptation Efforts: For the African Group it is particularly important to address the disparities in the treatment accorded to different thematic issues before the UNFCCC and restore trust, in a context where there is a bias in favour of mitigation, with adaptation, support, loss and damage and response measures being amongst the issues often relegated to process outcomes.",1.0,explorer,
+UNFCCC.party.1843.0,0d52574414db11917d60685866f3a097__98309e6ec4682ceddebeed7030697f0c52f13bf5428e75f67b2d02e045846771,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,64,81,(i) Address the social and economic consequences and impacts of response measures;,1.0,explorer,
+UNFCCC.party.1843.0,bb4e5b83df137ff00e62d88faefd1bde__e97a35db8b7b9601e0532fd5c30ed06973b5c75442df46f8c9777c46b3f552ce,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,83,100,- A paragraph on the manner how to address the social and economic consequences of response measures;,1.0,explorer,
+UNFCCC.non-party.1872.0,3f98713f2a0e69e04492831c4a71373c__9529b4675112a6732ad78e8f9e2ec897c65b0ceb44c8c7dacef078ec55870160,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,160,177,"o Outputs and GST events should retain a balanced, equity-rooted, approach to areas under the GST, including sufficient attention to loss and damage as well as response measures, and considering all work ""in the light of equity"" as mandated in 19/CMA.1.",1.0,explorer,
+UNFCCC.party.1879.0,0183cb638e45d8fe7163ed119e869821__daaf30e2b0256d1557599c97eb662bea21aa1afa1ef5d357b2e938d2f08c265b,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,28,45,"· There is recognition that response measures can have adverse, positive and co-beneficial social, economic, and environmental impacts.",1.0,explorer,
+UNFCCC.party.1879.0,0183cb638e45d8fe7163ed119e869821__daaf30e2b0256d1557599c97eb662bea21aa1afa1ef5d357b2e938d2f08c265b,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,496,513,"Gaps persist however, therefore work to assess the impacts of response measures remains pertinent to inform Parties' implementation of policies, including in the development of just transition strategies.",1.0,explorer,
+UNFCCC.party.1879.0,082b0a297b4f666b50a874e8d61a892d__f677285013b427044142294998a47236b71694e026bbe3b065f4da57cebfcce4,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,38,55,"· Encouraging continued assessment of response measures through the forum and the KCI, while ensuring a greater focus of co-benefits of response measures, including vis-a-vis health and biodiversity, with a view to better evaluate their impacts.",1.0,explorer,
+UNFCCC.party.1879.0,082b0a297b4f666b50a874e8d61a892d__f677285013b427044142294998a47236b71694e026bbe3b065f4da57cebfcce4,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,136,153,"· Encouraging continued assessment of response measures through the forum and the KCI, while ensuring a greater focus of co-benefits of response measures, including vis-a-vis health and biodiversity, with a view to better evaluate their impacts.",1.0,explorer,
+UNFCCC.party.1836.0,09bcf2c39b3b64f44819e35efe1b3637__de5074061be14c5dd781ab8b744c698750bba37bb82f2df403238ec84365b1ce,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,49,66,"· In addition, we have proposed key messages for response measures and loss and damage, noting that the GST ""may take into account, as appropriate, efforts related to"" these two areas.2",1.0,explorer,
+UNFCCC.party.1836.0,f5a733c787813f93d90c29591e13a31d__641ad6a6462783e7dcfc367a2e2410b34815b874537a7abb0d26806ae0b162bb,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,19,36,Efforts related to response measures (*also see U.S. submission on views on efforts related to addressing the social and economic impacts of response measures for additional details26);,1.0,explorer,
+UNFCCC.party.1836.0,f5a733c787813f93d90c29591e13a31d__641ad6a6462783e7dcfc367a2e2410b34815b874537a7abb0d26806ae0b162bb,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,141,158,Efforts related to response measures (*also see U.S. submission on views on efforts related to addressing the social and economic impacts of response measures for additional details26);,1.0,explorer,
+UNFCCC.party.1836.0,35f47f4d84692b4ff4ada54518eb9b4d__b514ea647ccf5a9aec00193003a3fbc0819a69788123f5c0198321b892fb7634,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,236,253,"Since the Paris Agreement was adopted, Parties and non-Parties have taken significant action to assess and address the positive and negative socioeconomic impacts of response measures domestically and under the 6-year workplan of the response measures forum and its Katowice Committee of Experts on the Impacts of the Implementation of Response Measures (KCI).",1.0,explorer,
+UNFCCC.party.1836.0,35f47f4d84692b4ff4ada54518eb9b4d__b514ea647ccf5a9aec00193003a3fbc0819a69788123f5c0198321b892fb7634,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,168,185,"Since the Paris Agreement was adopted, Parties and non-Parties have taken significant action to assess and address the positive and negative socioeconomic impacts of response measures domestically and under the 6-year workplan of the response measures forum and its Katowice Committee of Experts on the Impacts of the Implementation of Response Measures (KCI).",1.0,explorer,
+UNFCCC.party.1836.0,af7108a1b4079248fe8e90b475e050df__4482c44f1141764122937188c875d573591ad886e5d6d8e4b6ee7f4d4ac1a3eb,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,318,335,"Just transition of the workforce and economic diversification and transformation, supported by strong domestic policies and investments, can help to facilitate the transition to a net-zero GHG economy, in line with Article 2.1(a) of the Paris Agreement, and should be complementary policies to the implementation of response measures.",1.0,explorer,
+UNFCCC.party.1836.0,56d2ec6dbc9c24fcb649ef263f0157f2__6f9d6f7f8d0c353979b888f195a249f1aa54b9e4b3e8f011ddb6a3939f47c5d2,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,289,306,"The new work programme on just transition, established at COP27, should build on this work and further highlight ways in which Parties can simultaneously undertake response measures while supporting their impacted workers and communities.",1.0,explorer,
+UNFCCC.party.1836.0,56d2ec6dbc9c24fcb649ef263f0157f2__6f9d6f7f8d0c353979b888f195a249f1aa54b9e4b3e8f011ddb6a3939f47c5d2,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,6,23,o The response measures forum and its KCI have helped Parties build capacity to implement domestic just transition policies.,1.0,explorer,
+UNFCCC.party.1836.0,d67600d0ac36bb5e590861406a45a93d__390b1e6e090a1eebf814faec517e184ffe730011620a779716ef8a021dd42c24,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,298,315,Parties should therefore make efforts to maximize these opportunities by fostering domestic policy enabling conditions and maximizing such positive co-benefits from implementing response measures.,1.0,explorer,
+UNFCCC.party.1836.0,ca972589ce02f03b276384f0a3c7bde5__e718f79e7cc92a10e163176968643aecf7959a8fb2bc04d7c9127b37b8517494,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,28,45,"· Lack of implementation of response measures, especially by major emitters, and/or building new unabated fossil fuel infrastructure not only contributes to global GHG emissions, but also risks stranded assets and economic/job losses.",1.0,explorer,
+UNFCCC.party.1847.0,333f2d88d5dad9cb0673d6687fe492d3__96a74b230f9512a04e17fc4d3463786cef94d19d8a3e3a7fa1749654ad4d1daa,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,477,494,"The GST provides an important opportunity to shift the focus from negotiations to implementation and cooperation - in the three thematic areas (i) mitigation, including response measures; (ii) adaptation, including loss and damage; (iii) finance flows consistent with a pathway towards low greenhouse gas emissions and climate-resilient development, means of implementation and support.",1.0,explorer,
+UNFCCC.party.1847.0,50b842f47821013d474b3b24d0a130ee__c75c86e486e5135300b6b0718555b3f2ab8124dc95354b4eaab998e17f569751,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,322,339,"To assess and strengthen progress regarding the purpose and the long-term goals of the PA and to allow immediate uptake of the findings of TD1 and linkages to other relevant work streams, it seems pertinent to structure the outputs of the CO component along the three thematic areas of TD1, i.e. (i) mitigation, including response measures; (ii) adaptation, including loss and damage; (iii) finance flows consistent with a pathway towards low greenhouse gas emissions and climate-resilient development, means of implementation and support.",1.0,explorer,
+UNFCCC.party.1849.0,66955cb497603b0f47ddbbda0df82da6__11a2d7bd2009e3a57d9be60bec7305249d580a58c317a0bf915ef357007645e3,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,235,252,"Possible measures and good practices should be identified for the three thematic areas of the GST - mitigation, adaptation and means of implementation and support, as appropriate, including the efforts related to loss and damage and response measures.",1.0,explorer,
+UNFCCC.party.1880.0,5bc365d0fb593b6feb1c198673b40642__84b49f848e7b128f0ff8169ff8ebc341cd94944b5e12fde3ed63418749f6b69a,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,586,603,"However, gaps remain, and this is why work aimed at assessing the impacts of response measures remains relevant to inform the implementation of policies by Parties, including in the context of the development of just transition strategies.",1.0,explorer,
+UNFCCC.party.1880.0,5bc365d0fb593b6feb1c198673b40642__84b49f848e7b128f0ff8169ff8ebc341cd94944b5e12fde3ed63418749f6b69a,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,24,41,"· It is recognized that response measures can have both negative and positive or even co-beneficial repercussions on the social, economic and environmental levels.",1.0,explorer,
+UNFCCC.party.1880.0,5b83dcb15b7f30446ea689749084b3e9__9ac44aba6598f3ec573e0bc18a7f67d94e6410074be429670acb1c4f3d6559c2,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,46,63,"· Encouragement to continue the evaluation of response measures through the forum and the Katowice Expert Committee on the Impacts of Implemented Response Measures (CKI), while ensuring greater emphasis on co-benefits of response measures, particularly with regard to health and biodiversity, with a view to better assessing their impacts.",1.0,explorer,
+UNFCCC.party.1880.0,5b83dcb15b7f30446ea689749084b3e9__9ac44aba6598f3ec573e0bc18a7f67d94e6410074be429670acb1c4f3d6559c2,RESPONSE_MEASURES,RESPONSE_MEASURES,response measures,221,238,"· Encouragement to continue the evaluation of response measures through the forum and the Katowice Expert Committee on the Impacts of Implemented Response Measures (CKI), while ensuring greater emphasis on co-benefits of response measures, particularly with regard to health and biodiversity, with a view to better assessing their impacts.",1.0,explorer,