diff --git a/malariagen_data/anoph/to_plink.py b/malariagen_data/anoph/to_plink.py new file mode 100644 index 00000000..994ca656 --- /dev/null +++ b/malariagen_data/anoph/to_plink.py @@ -0,0 +1,167 @@ +from typing import Optional + +import allel # type: ignore +import numpy as np +import os +import bed_reader +from dask.diagnostics import ProgressBar + +from .snp_data import AnophelesSnpData +from . import base_params + + +class PlinkConverter( + AnophelesSnpData, +): + def __init__( + self, + **kwargs, + ): + # N.B., this class is designed to work cooperatively, and + # so it's important that any remaining parameters are passed + # to the superclass constructor. + super().__init__(**kwargs) + + def _create_plink_outfile( + self, + *, + results_dir, + region, + n_snps, + min_minor_ac, + thin_offset, + max_missing_an, + ): + return f"{results_dir}/{region}.{n_snps}.{min_minor_ac}.{thin_offset}.{max_missing_an}" + + def _biallelic_snps_to_plink( + self, + *, + results_dir, + region, + n_snps, + thin_offset, + sample_sets, + sample_query, + sample_indices, + site_mask, + min_minor_ac, + max_missing_an, + random_seed, + inline_array, + chunks, + ): + # Define output file + plink_file_path = self._create_plink_outfile( + results_dir=results_dir, + region=region, + n_snps=n_snps, + thin_offset=thin_offset, + min_minor_ac=min_minor_ac, + max_missing_an=max_missing_an, + ) + + bed_file_path = f"{plink_file_path}.bed" + if os.path.exists(bed_file_path): + return plink_file_path + + # Get snps + ds_snps = self.biallelic_snp_calls( + region=region, + sample_sets=sample_sets, + sample_query=sample_query, + sample_indices=sample_indices, + site_mask=site_mask, + min_minor_ac=min_minor_ac, + max_missing_an=max_missing_an, + n_snps=n_snps, + thin_offset=thin_offset, + random_seed=random_seed, + inline_array=inline_array, + chunks=chunks, + ) + + # Set up dataset with required vars for plink conversion + ds_snps_asc = ds_snps[ + [ + "variant_contig", + "variant_position", + "variant_allele", + "sample_id", + "call_genotype", + ] + ] + + # Compute gt ref counts + with self._spinner("Computing genotype ref counts"): + gt_asc = ds_snps_asc["call_genotype"].data.compute() + gn_ref = allel.GenotypeDaskArray(gt_asc).to_n_ref(fill=-127) + with ProgressBar(): + gn_ref = gn_ref.compute() + + # Ensure genotypes vary + loc_var = np.any(gn_ref != gn_ref[:, 0, np.newaxis], axis=1) + + # Load final data + with ProgressBar(): + ds_snps_final = ds_snps_asc[ + ["variant_contig", "variant_position", "variant_allele", "sample_id"] + ].isel(variants=loc_var) + + # Init vars for input to bed reader + gn_ref_final = gn_ref[loc_var] + val = gn_ref_final.T + alleles = ds_snps_final["variant_allele"].values + properties = { + "iid": ds_snps_final["sample_id"].values, + "chromosome": ds_snps_final["variant_contig"].values, + "bp_position": ds_snps_final["variant_position"].values, + "allele_1": alleles[:, 0], + "allele_2": alleles[:, 1], + } + + bed_reader.to_bed( + filepath=bed_file_path, + val=val, + properties=properties, + count_A1=True, + ) + + print(f"PLINK files written to to: {plink_file_path}") + + return plink_file_path + + def biallelic_snps_to_plink( + self, + results_dir, + region: base_params.regions, + n_snps: base_params.n_snps, + thin_offset: base_params.thin_offset = 0, + sample_sets: Optional[base_params.sample_sets] = None, + sample_query: Optional[base_params.sample_query] = None, + sample_indices: Optional[base_params.sample_indices] = None, + site_mask: Optional[base_params.site_mask] = None, + min_minor_ac: Optional[base_params.min_minor_ac] = 0, + max_missing_an: Optional[base_params.max_missing_an] = 0, + random_seed: base_params.random_seed = 42, + inline_array: base_params.inline_array = base_params.inline_array_default, + chunks: base_params.chunks = base_params.native_chunks, + ): + params = dict( + results_dir=results_dir, + region=region, + n_snps=n_snps, + thin_offset=thin_offset, + sample_sets=sample_sets, + sample_query=sample_query, + sample_indices=sample_indices, + site_mask=site_mask, + min_minor_ac=min_minor_ac, + max_missing_an=max_missing_an, + random_seed=random_seed, + ) + + filepath = self._biallelic_snps_to_plink( + inline_array=inline_array, chunks=chunks, **params + ) + return filepath diff --git a/malariagen_data/anopheles.py b/malariagen_data/anopheles.py index 0045492d..8886cd0c 100644 --- a/malariagen_data/anopheles.py +++ b/malariagen_data/anopheles.py @@ -49,6 +49,7 @@ from .anoph.pca import AnophelesPca from .anoph.sample_metadata import AnophelesSampleMetadata, locate_cohorts from .anoph.snp_data import AnophelesSnpData +from .anoph.to_plink import PlinkConverter from .anoph.g123 import AnophelesG123Analysis from .anoph.fst import AnophelesFstAnalysis from .anoph.h12 import AnophelesH12Analysis @@ -105,6 +106,7 @@ class AnophelesDataResource( AnophelesG123Analysis, AnophelesFstAnalysis, AnophelesSnpFrequencyAnalysis, + PlinkConverter, AnophelesPca, AnophelesIgv, AnophelesAimData, diff --git a/notebooks/plink_convert.ipynb b/notebooks/plink_convert.ipynb new file mode 100644 index 00000000..e4c75a52 --- /dev/null +++ b/notebooks/plink_convert.ipynb @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\nconst JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n // Clean up Bokeh references\n if (id != null && id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim();\n if (id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", + "application/vnd.bokehjs_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "_request non-retriable exception: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401\n", + "Traceback (most recent call last):\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py\", line 123, in retry_request\n", + " return await func(*args, **kwargs)\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py\", line 430, in _request\n", + " validate_response(status, contents, path, args)\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py\", line 110, in validate_response\n", + " raise HttpError(error)\n", + "gcsfs.retry.HttpError: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401\n" + ] + }, + { + "ename": "HttpError", + "evalue": "Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHttpError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/IPython/core/formatters.py:708\u001b[0m, in \u001b[0;36mPlainTextFormatter.__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 701\u001b[0m stream \u001b[38;5;241m=\u001b[39m StringIO()\n\u001b[1;32m 702\u001b[0m printer \u001b[38;5;241m=\u001b[39m pretty\u001b[38;5;241m.\u001b[39mRepresentationPrinter(stream, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbose,\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_width, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnewline,\n\u001b[1;32m 704\u001b[0m max_seq_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_seq_length,\n\u001b[1;32m 705\u001b[0m singleton_pprinters\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msingleton_printers,\n\u001b[1;32m 706\u001b[0m type_pprinters\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtype_printers,\n\u001b[1;32m 707\u001b[0m deferred_pprinters\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdeferred_printers)\n\u001b[0;32m--> 708\u001b[0m \u001b[43mprinter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpretty\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 709\u001b[0m printer\u001b[38;5;241m.\u001b[39mflush()\n\u001b[1;32m 710\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m stream\u001b[38;5;241m.\u001b[39mgetvalue()\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/IPython/lib/pretty.py:410\u001b[0m, in \u001b[0;36mRepresentationPrinter.pretty\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 407\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m meth(obj, \u001b[38;5;28mself\u001b[39m, cycle)\n\u001b[1;32m 408\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mobject\u001b[39m \\\n\u001b[1;32m 409\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mcallable\u001b[39m(\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__repr__\u001b[39m\u001b[38;5;124m'\u001b[39m)):\n\u001b[0;32m--> 410\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_repr_pprint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcycle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 412\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _default_pprint(obj, \u001b[38;5;28mself\u001b[39m, cycle)\n\u001b[1;32m 413\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/IPython/lib/pretty.py:778\u001b[0m, in \u001b[0;36m_repr_pprint\u001b[0;34m(obj, p, cycle)\u001b[0m\n\u001b[1;32m 776\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"A pprint that just redirects to the normal repr function.\"\"\"\u001b[39;00m\n\u001b[1;32m 777\u001b[0m \u001b[38;5;66;03m# Find newlines and replace them with p.break_()\u001b[39;00m\n\u001b[0;32m--> 778\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mrepr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 779\u001b[0m lines \u001b[38;5;241m=\u001b[39m output\u001b[38;5;241m.\u001b[39msplitlines()\n\u001b[1;32m 780\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m p\u001b[38;5;241m.\u001b[39mgroup():\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/ag3.py:206\u001b[0m, in \u001b[0;36mAg3.__repr__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__repr__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 205\u001b[0m text \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 207\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStorage URL : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mData releases available : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreleases)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 209\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mResults cache : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_results_cache\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCohorts analysis : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cohorts_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAIM analysis : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_aim_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSite filters analysis : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_site_filters_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 213\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSoftware version : malariagen_data \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmalariagen_data\u001b[38;5;241m.\u001b[39m__version__\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mClient location : \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient_location\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m---\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease note that data are subject to terms of use,\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 217\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfor more information see https://www.malariagen.net/data\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mor contact data@malariagen.net. For API documentation see \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://malariagen.github.io/malariagen-data-python/v\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmalariagen_data\u001b[38;5;241m.\u001b[39m__version__\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/Ag3.html\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 220\u001b[0m )\n\u001b[1;32m 221\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m text\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/anoph/base.py:272\u001b[0m, in \u001b[0;36mAnophelesBase.releases\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 270\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 271\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pre:\n\u001b[0;32m--> 272\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_discover_releases\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 274\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_public_releases()\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/anoph/base.py:253\u001b[0m, in \u001b[0;36mAnophelesBase._discover_releases\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;129m@doc\u001b[39m(\n\u001b[1;32m 246\u001b[0m summary\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;124m Here we discover which releases are available, by listing the storage\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 251\u001b[0m )\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_discover_releases\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m]:\n\u001b[0;32m--> 253\u001b[0m sub_dirs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m([p\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mls\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_base_path\u001b[49m\u001b[43m)\u001b[49m])\n\u001b[1;32m 254\u001b[0m discovered_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28msorted\u001b[39m(\n\u001b[1;32m 256\u001b[0m [\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m )\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m discovered_releases\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:118\u001b[0m, in \u001b[0;36msync_wrapper..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m obj \u001b[38;5;129;01mor\u001b[39;00m args[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:103\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m FSTimeoutError \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreturn_result\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 103\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 104\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m return_result\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:56\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(event, coro, result, timeout)\u001b[0m\n\u001b[1;32m 54\u001b[0m coro \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mwait_for(coro, timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 56\u001b[0m result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n\u001b[1;32m 58\u001b[0m result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m ex\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:995\u001b[0m, in \u001b[0;36mGCSFileSystem._ls\u001b[0;34m(self, path, detail, prefix, versions, refresh, **kwargs)\u001b[0m\n\u001b[1;32m 993\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 994\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 995\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m entry \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_list_objects(\n\u001b[1;32m 996\u001b[0m path, prefix\u001b[38;5;241m=\u001b[39mprefix, versions\u001b[38;5;241m=\u001b[39mversions, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 997\u001b[0m ):\n\u001b[1;32m 998\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m versions \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeneration\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m entry:\n\u001b[1;32m 999\u001b[0m entry \u001b[38;5;241m=\u001b[39m entry\u001b[38;5;241m.\u001b[39mcopy()\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:564\u001b[0m, in \u001b[0;36mGCSFileSystem._list_objects\u001b[0;34m(self, path, prefix, versions, **kwargs)\u001b[0m\n\u001b[1;32m 561\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m key:\n\u001b[1;32m 562\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[0;32m--> 564\u001b[0m items, prefixes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_do_list_objects(\n\u001b[1;32m 565\u001b[0m path, prefix\u001b[38;5;241m=\u001b[39mprefix, versions\u001b[38;5;241m=\u001b[39mversions, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 566\u001b[0m )\n\u001b[1;32m 568\u001b[0m pseudodirs \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 569\u001b[0m {\n\u001b[1;32m 570\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbucket\u001b[39m\u001b[38;5;124m\"\u001b[39m: bucket,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 576\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m prefix \u001b[38;5;129;01min\u001b[39;00m prefixes\n\u001b[1;32m 577\u001b[0m ]\n\u001b[1;32m 578\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (items \u001b[38;5;241m+\u001b[39m pseudodirs):\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:638\u001b[0m, in \u001b[0;36mGCSFileSystem._do_list_objects\u001b[0;34m(self, path, max_results, delimiter, prefix, versions, **kwargs)\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_concurrent_list_objects_helper(\n\u001b[1;32m 627\u001b[0m items\u001b[38;5;241m=\u001b[39mitems,\n\u001b[1;32m 628\u001b[0m bucket\u001b[38;5;241m=\u001b[39mbucket,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 632\u001b[0m page_size\u001b[38;5;241m=\u001b[39mdefault_page_size,\n\u001b[1;32m 633\u001b[0m )\n\u001b[1;32m 635\u001b[0m \u001b[38;5;66;03m# If the user has not configured inventory report, proceed to use\u001b[39;00m\n\u001b[1;32m 636\u001b[0m \u001b[38;5;66;03m# sequential listing.\u001b[39;00m\n\u001b[1;32m 637\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 638\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sequential_list_objects_helper(\n\u001b[1;32m 639\u001b[0m bucket\u001b[38;5;241m=\u001b[39mbucket,\n\u001b[1;32m 640\u001b[0m delimiter\u001b[38;5;241m=\u001b[39mdelimiter,\n\u001b[1;32m 641\u001b[0m start_offset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 642\u001b[0m end_offset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 643\u001b[0m prefix\u001b[38;5;241m=\u001b[39mprefix,\n\u001b[1;32m 644\u001b[0m versions\u001b[38;5;241m=\u001b[39mversions,\n\u001b[1;32m 645\u001b[0m page_size\u001b[38;5;241m=\u001b[39mdefault_page_size,\n\u001b[1;32m 646\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:732\u001b[0m, in \u001b[0;36mGCSFileSystem._sequential_list_objects_helper\u001b[0;34m(self, bucket, delimiter, start_offset, end_offset, prefix, versions, page_size)\u001b[0m\n\u001b[1;32m 729\u001b[0m prefixes \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 730\u001b[0m items \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 732\u001b[0m page \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(\n\u001b[1;32m 733\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGET\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 734\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb/\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m/o\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 735\u001b[0m bucket,\n\u001b[1;32m 736\u001b[0m delimiter\u001b[38;5;241m=\u001b[39mdelimiter,\n\u001b[1;32m 737\u001b[0m prefix\u001b[38;5;241m=\u001b[39mprefix,\n\u001b[1;32m 738\u001b[0m startOffset\u001b[38;5;241m=\u001b[39mstart_offset,\n\u001b[1;32m 739\u001b[0m endOffset\u001b[38;5;241m=\u001b[39mend_offset,\n\u001b[1;32m 740\u001b[0m maxResults\u001b[38;5;241m=\u001b[39mpage_size,\n\u001b[1;32m 741\u001b[0m json_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 742\u001b[0m versions\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrue\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m versions \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 743\u001b[0m )\n\u001b[1;32m 745\u001b[0m prefixes\u001b[38;5;241m.\u001b[39mextend(page\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprefixes\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n\u001b[1;32m 746\u001b[0m items\u001b[38;5;241m.\u001b[39mextend(page\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mitems\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:437\u001b[0m, in \u001b[0;36mGCSFileSystem._call\u001b[0;34m(self, method, path, json_out, info_out, *args, **kwargs)\u001b[0m\n\u001b[1;32m 433\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_call\u001b[39m(\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28mself\u001b[39m, method, path, \u001b[38;5;241m*\u001b[39margs, json_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, info_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 435\u001b[0m ):\n\u001b[1;32m 436\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmethod\u001b[38;5;241m.\u001b[39mupper()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00margs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mheaders\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 437\u001b[0m status, headers, info, contents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request(\n\u001b[1;32m 438\u001b[0m method, path, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 439\u001b[0m )\n\u001b[1;32m 440\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m json_out:\n\u001b[1;32m 441\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mloads(contents)\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/decorator.py:221\u001b[0m, in \u001b[0;36mdecorate..fun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kwsyntax:\n\u001b[1;32m 220\u001b[0m args, kw \u001b[38;5;241m=\u001b[39m fix(args, kw, sig)\n\u001b[0;32m--> 221\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m caller(func, \u001b[38;5;241m*\u001b[39m(extras \u001b[38;5;241m+\u001b[39m args), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:158\u001b[0m, in \u001b[0;36mretry_request\u001b[0;34m(func, retries, *args, **kwargs)\u001b[0m\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 157\u001b[0m logger\u001b[38;5;241m.\u001b[39mexception(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m non-retriable exception: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 158\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:123\u001b[0m, in \u001b[0;36mretry_request\u001b[0;34m(func, retries, *args, **kwargs)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retry \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 122\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;28mmin\u001b[39m(random\u001b[38;5;241m.\u001b[39mrandom() \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m (retry \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m), \u001b[38;5;241m32\u001b[39m))\n\u001b[0;32m--> 123\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\n\u001b[1;32m 125\u001b[0m HttpError,\n\u001b[1;32m 126\u001b[0m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 129\u001b[0m aiohttp\u001b[38;5;241m.\u001b[39mclient_exceptions\u001b[38;5;241m.\u001b[39mClientError,\n\u001b[1;32m 130\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 131\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28misinstance\u001b[39m(e, HttpError)\n\u001b[1;32m 133\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m e\u001b[38;5;241m.\u001b[39mcode \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m400\u001b[39m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequester pays\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m e\u001b[38;5;241m.\u001b[39mmessage\n\u001b[1;32m 135\u001b[0m ):\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:430\u001b[0m, in \u001b[0;36mGCSFileSystem._request\u001b[0;34m(self, method, path, headers, json, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 427\u001b[0m info \u001b[38;5;241m=\u001b[39m r\u001b[38;5;241m.\u001b[39mrequest_info \u001b[38;5;66;03m# for debug only\u001b[39;00m\n\u001b[1;32m 428\u001b[0m contents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m r\u001b[38;5;241m.\u001b[39mread()\n\u001b[0;32m--> 430\u001b[0m \u001b[43mvalidate_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstatus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 431\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m status, headers, info, contents\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:110\u001b[0m, in \u001b[0;36mvalidate_response\u001b[0;34m(status, content, path, args)\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBad Request: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mmsg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m error:\n\u001b[0;32m--> 110\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError(error)\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m status:\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m: status, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessage\u001b[39m\u001b[38;5;124m\"\u001b[39m: msg}) \u001b[38;5;66;03m# text-like\u001b[39;00m\n", + "\u001b[0;31mHttpError\u001b[0m: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "_request non-retriable exception: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401\n", + "Traceback (most recent call last):\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py\", line 123, in retry_request\n", + " return await func(*args, **kwargs)\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py\", line 430, in _request\n", + " validate_response(status, contents, path, args)\n", + " File \"/Users/dennistpw/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py\", line 110, in validate_response\n", + " raise HttpError(error)\n", + "gcsfs.retry.HttpError: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401\n" + ] + }, + { + "ename": "HttpError", + "evalue": "Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHttpError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/IPython/core/formatters.py:344\u001b[0m, in \u001b[0;36mBaseFormatter.__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 342\u001b[0m method \u001b[38;5;241m=\u001b[39m get_real_method(obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprint_method)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m method \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/ag3.py:248\u001b[0m, in \u001b[0;36mAg3._repr_html_\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_repr_html_\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 224\u001b[0m html \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 225\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 228\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 229\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 230\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 236\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 237\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 238\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 239\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 242\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 243\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 244\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[0;32m--> 248\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 249\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 250\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 251\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 255\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 266\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 267\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 268\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 269\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 272\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 274\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 275\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 278\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 279\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 280\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 281\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 284\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 285\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 286\u001b[0m \u001b[38;5;124m \u001b[39m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;124m
MalariaGEN Ag3 API client
\u001b[39m\n\u001b[1;32m 231\u001b[0m \u001b[38;5;124m Please note that data are subject to terms of use,\u001b[39m\n\u001b[1;32m 232\u001b[0m \u001b[38;5;124m for more information see \u001b[39m\n\u001b[1;32m 233\u001b[0m \u001b[38;5;124m the MalariaGEN website or contact data@malariagen.net.\u001b[39m\n\u001b[1;32m 234\u001b[0m \u001b[38;5;124m See also the Ag3 API docs.\u001b[39m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;124m Storage URL\u001b[39m\n\u001b[1;32m 241\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 246\u001b[0m \u001b[38;5;124m Data releases available\u001b[39m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreleases)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;124m Results cache\u001b[39m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_results_cache\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;124m Cohorts analysis\u001b[39m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cohorts_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 264\u001b[0m \u001b[38;5;124m AIM analysis\u001b[39m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_aim_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 270\u001b[0m \u001b[38;5;124m Site filters analysis\u001b[39m\n\u001b[1;32m 271\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_site_filters_analysis\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 276\u001b[0m \u001b[38;5;124m Software version\u001b[39m\n\u001b[1;32m 277\u001b[0m \u001b[38;5;124m malariagen_data \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmalariagen_data\u001b[38;5;241m.\u001b[39m__version__\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 282\u001b[0m \u001b[38;5;124m Client location\u001b[39m\n\u001b[1;32m 283\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient_location\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m
\u001b[39m\n\u001b[1;32m 288\u001b[0m \u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 289\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m html\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/anoph/base.py:272\u001b[0m, in \u001b[0;36mAnophelesBase.releases\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 270\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 271\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pre:\n\u001b[0;32m--> 272\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_discover_releases\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 274\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_public_releases()\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/malariagen_data/anoph/base.py:253\u001b[0m, in \u001b[0;36mAnophelesBase._discover_releases\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;129m@doc\u001b[39m(\n\u001b[1;32m 246\u001b[0m summary\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;124m Here we discover which releases are available, by listing the storage\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 251\u001b[0m )\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_discover_releases\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m]:\n\u001b[0;32m--> 253\u001b[0m sub_dirs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msorted\u001b[39m([p\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mls\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_base_path\u001b[49m\u001b[43m)\u001b[49m])\n\u001b[1;32m 254\u001b[0m discovered_releases \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28msorted\u001b[39m(\n\u001b[1;32m 256\u001b[0m [\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m )\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m discovered_releases\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:118\u001b[0m, in \u001b[0;36msync_wrapper..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m obj \u001b[38;5;129;01mor\u001b[39;00m args[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:103\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m FSTimeoutError \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreturn_result\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 103\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 104\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m return_result\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/fsspec/asyn.py:56\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(event, coro, result, timeout)\u001b[0m\n\u001b[1;32m 54\u001b[0m coro \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mwait_for(coro, timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 56\u001b[0m result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n\u001b[1;32m 58\u001b[0m result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m ex\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:995\u001b[0m, in \u001b[0;36mGCSFileSystem._ls\u001b[0;34m(self, path, detail, prefix, versions, refresh, **kwargs)\u001b[0m\n\u001b[1;32m 993\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 994\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 995\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m entry \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_list_objects(\n\u001b[1;32m 996\u001b[0m path, prefix\u001b[38;5;241m=\u001b[39mprefix, versions\u001b[38;5;241m=\u001b[39mversions, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 997\u001b[0m ):\n\u001b[1;32m 998\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m versions \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeneration\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m entry:\n\u001b[1;32m 999\u001b[0m entry \u001b[38;5;241m=\u001b[39m entry\u001b[38;5;241m.\u001b[39mcopy()\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:564\u001b[0m, in \u001b[0;36mGCSFileSystem._list_objects\u001b[0;34m(self, path, prefix, versions, **kwargs)\u001b[0m\n\u001b[1;32m 561\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m key:\n\u001b[1;32m 562\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[0;32m--> 564\u001b[0m items, prefixes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_do_list_objects(\n\u001b[1;32m 565\u001b[0m path, prefix\u001b[38;5;241m=\u001b[39mprefix, versions\u001b[38;5;241m=\u001b[39mversions, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 566\u001b[0m )\n\u001b[1;32m 568\u001b[0m pseudodirs \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 569\u001b[0m {\n\u001b[1;32m 570\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbucket\u001b[39m\u001b[38;5;124m\"\u001b[39m: bucket,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 576\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m prefix \u001b[38;5;129;01min\u001b[39;00m prefixes\n\u001b[1;32m 577\u001b[0m ]\n\u001b[1;32m 578\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (items \u001b[38;5;241m+\u001b[39m pseudodirs):\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:638\u001b[0m, in \u001b[0;36mGCSFileSystem._do_list_objects\u001b[0;34m(self, path, max_results, delimiter, prefix, versions, **kwargs)\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_concurrent_list_objects_helper(\n\u001b[1;32m 627\u001b[0m items\u001b[38;5;241m=\u001b[39mitems,\n\u001b[1;32m 628\u001b[0m bucket\u001b[38;5;241m=\u001b[39mbucket,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 632\u001b[0m page_size\u001b[38;5;241m=\u001b[39mdefault_page_size,\n\u001b[1;32m 633\u001b[0m )\n\u001b[1;32m 635\u001b[0m \u001b[38;5;66;03m# If the user has not configured inventory report, proceed to use\u001b[39;00m\n\u001b[1;32m 636\u001b[0m \u001b[38;5;66;03m# sequential listing.\u001b[39;00m\n\u001b[1;32m 637\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 638\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sequential_list_objects_helper(\n\u001b[1;32m 639\u001b[0m bucket\u001b[38;5;241m=\u001b[39mbucket,\n\u001b[1;32m 640\u001b[0m delimiter\u001b[38;5;241m=\u001b[39mdelimiter,\n\u001b[1;32m 641\u001b[0m start_offset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 642\u001b[0m end_offset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 643\u001b[0m prefix\u001b[38;5;241m=\u001b[39mprefix,\n\u001b[1;32m 644\u001b[0m versions\u001b[38;5;241m=\u001b[39mversions,\n\u001b[1;32m 645\u001b[0m page_size\u001b[38;5;241m=\u001b[39mdefault_page_size,\n\u001b[1;32m 646\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:732\u001b[0m, in \u001b[0;36mGCSFileSystem._sequential_list_objects_helper\u001b[0;34m(self, bucket, delimiter, start_offset, end_offset, prefix, versions, page_size)\u001b[0m\n\u001b[1;32m 729\u001b[0m prefixes \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 730\u001b[0m items \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 732\u001b[0m page \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(\n\u001b[1;32m 733\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGET\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 734\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb/\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m/o\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 735\u001b[0m bucket,\n\u001b[1;32m 736\u001b[0m delimiter\u001b[38;5;241m=\u001b[39mdelimiter,\n\u001b[1;32m 737\u001b[0m prefix\u001b[38;5;241m=\u001b[39mprefix,\n\u001b[1;32m 738\u001b[0m startOffset\u001b[38;5;241m=\u001b[39mstart_offset,\n\u001b[1;32m 739\u001b[0m endOffset\u001b[38;5;241m=\u001b[39mend_offset,\n\u001b[1;32m 740\u001b[0m maxResults\u001b[38;5;241m=\u001b[39mpage_size,\n\u001b[1;32m 741\u001b[0m json_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 742\u001b[0m versions\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrue\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m versions \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 743\u001b[0m )\n\u001b[1;32m 745\u001b[0m prefixes\u001b[38;5;241m.\u001b[39mextend(page\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprefixes\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n\u001b[1;32m 746\u001b[0m items\u001b[38;5;241m.\u001b[39mextend(page\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mitems\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:437\u001b[0m, in \u001b[0;36mGCSFileSystem._call\u001b[0;34m(self, method, path, json_out, info_out, *args, **kwargs)\u001b[0m\n\u001b[1;32m 433\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_call\u001b[39m(\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28mself\u001b[39m, method, path, \u001b[38;5;241m*\u001b[39margs, json_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, info_out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 435\u001b[0m ):\n\u001b[1;32m 436\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmethod\u001b[38;5;241m.\u001b[39mupper()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00margs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mheaders\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 437\u001b[0m status, headers, info, contents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request(\n\u001b[1;32m 438\u001b[0m method, path, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 439\u001b[0m )\n\u001b[1;32m 440\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m json_out:\n\u001b[1;32m 441\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mloads(contents)\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/decorator.py:221\u001b[0m, in \u001b[0;36mdecorate..fun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kwsyntax:\n\u001b[1;32m 220\u001b[0m args, kw \u001b[38;5;241m=\u001b[39m fix(args, kw, sig)\n\u001b[0;32m--> 221\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m caller(func, \u001b[38;5;241m*\u001b[39m(extras \u001b[38;5;241m+\u001b[39m args), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:158\u001b[0m, in \u001b[0;36mretry_request\u001b[0;34m(func, retries, *args, **kwargs)\u001b[0m\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 157\u001b[0m logger\u001b[38;5;241m.\u001b[39mexception(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m non-retriable exception: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 158\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:123\u001b[0m, in \u001b[0;36mretry_request\u001b[0;34m(func, retries, *args, **kwargs)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m retry \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 122\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;28mmin\u001b[39m(random\u001b[38;5;241m.\u001b[39mrandom() \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m (retry \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m), \u001b[38;5;241m32\u001b[39m))\n\u001b[0;32m--> 123\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\n\u001b[1;32m 125\u001b[0m HttpError,\n\u001b[1;32m 126\u001b[0m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 129\u001b[0m aiohttp\u001b[38;5;241m.\u001b[39mclient_exceptions\u001b[38;5;241m.\u001b[39mClientError,\n\u001b[1;32m 130\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 131\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28misinstance\u001b[39m(e, HttpError)\n\u001b[1;32m 133\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m e\u001b[38;5;241m.\u001b[39mcode \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m400\u001b[39m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequester pays\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m e\u001b[38;5;241m.\u001b[39mmessage\n\u001b[1;32m 135\u001b[0m ):\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/core.py:430\u001b[0m, in \u001b[0;36mGCSFileSystem._request\u001b[0;34m(self, method, path, headers, json, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 427\u001b[0m info \u001b[38;5;241m=\u001b[39m r\u001b[38;5;241m.\u001b[39mrequest_info \u001b[38;5;66;03m# for debug only\u001b[39;00m\n\u001b[1;32m 428\u001b[0m contents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m r\u001b[38;5;241m.\u001b[39mread()\n\u001b[0;32m--> 430\u001b[0m \u001b[43mvalidate_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstatus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 431\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m status, headers, info, contents\n", + "File \u001b[0;32m~/miniconda3/envs/malariagen_plink/lib/python3.8/site-packages/gcsfs/retry.py:110\u001b[0m, in \u001b[0;36mvalidate_response\u001b[0;34m(status, content, path, args)\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBad Request: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mmsg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m error:\n\u001b[0;32m--> 110\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError(error)\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m status:\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m: status, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessage\u001b[39m\u001b[38;5;124m\"\u001b[39m: msg}) \u001b[38;5;66;03m# text-like\u001b[39;00m\n", + "\u001b[0;31mHttpError\u001b[0m: Anonymous caller does not have storage.objects.list access to the Google Cloud Storage bucket. Permission 'storage.objects.list' denied on resource (or it may not exist)., 401" + ] + } + ], + "source": [ + "import malariagen_data\n", + "\n", + "ag3 = malariagen_data.Ag3(pre=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'Ag3' object has no attribute 'biallelic_snps_to_plink'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mag3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbiallelic_snps_to_plink\u001b[49m(results_dir\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/dennistpw/Projects/malariagen-data-python/\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 2\u001b[0m region\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m2L:100000-2000000\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 3\u001b[0m n_snps\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2000\u001b[39m,\n\u001b[1;32m 4\u001b[0m sample_sets\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mAG1000G-AO\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 5\u001b[0m )\n", + "\u001b[0;31mAttributeError\u001b[0m: 'Ag3' object has no attribute 'biallelic_snps_to_plink'" + ] + } + ], + "source": [ + "ag3.biallelic_snps_to_plink(results_dir='/Users/dennistpw/Projects/malariagen-data-python/',\n", + " region='2L:100000-2000000',\n", + " n_snps=2000,\n", + " sample_sets='AG1000G-AO',\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "malariagen_plink", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index 39a48d44..d5235190 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,8 @@ orjson = "*" yaspin = "*" # provides efficient neighbour-joining tree implementation biotite = "*" +bed_reader = "*" + # dev extras pytest = "*" diff --git a/tests/anoph/test_plink_converter.py b/tests/anoph/test_plink_converter.py new file mode 100644 index 00000000..5425b04c --- /dev/null +++ b/tests/anoph/test_plink_converter.py @@ -0,0 +1,136 @@ +import random +import pytest +from pytest_cases import parametrize_with_cases + +from malariagen_data import af1 as _af1 +from malariagen_data import ag3 as _ag3 +from malariagen_data.anoph.to_plink import PlinkConverter + +import os +import bed_reader + + +@pytest.fixture +def ag3_sim_api(ag3_sim_fixture): + return PlinkConverter( + url=ag3_sim_fixture.url, + config_path=_ag3.CONFIG_PATH, + major_version_number=_ag3.MAJOR_VERSION_NUMBER, + major_version_path=_ag3.MAJOR_VERSION_PATH, + pre=True, + aim_metadata_dtype={ + "aim_species_fraction_arab": "float64", + "aim_species_fraction_colu": "float64", + "aim_species_fraction_colu_no2l": "float64", + "aim_species_gambcolu_arabiensis": object, + "aim_species_gambiae_coluzzii": object, + "aim_species": object, + }, + gff_gene_type="gene", + gff_gene_name_attribute="Name", + gff_default_attributes=("ID", "Parent", "Name", "description"), + default_site_mask="gamb_colu_arab", + results_cache=ag3_sim_fixture.results_cache_path.as_posix(), + taxon_colors=_ag3.TAXON_COLORS, + virtual_contigs=_ag3.VIRTUAL_CONTIGS, + ) + + +@pytest.fixture +def af1_sim_api(af1_sim_fixture): + return PlinkConverter( + url=af1_sim_fixture.url, + config_path=_af1.CONFIG_PATH, + major_version_number=_af1.MAJOR_VERSION_NUMBER, + major_version_path=_af1.MAJOR_VERSION_PATH, + pre=False, + gff_gene_type="protein_coding_gene", + gff_gene_name_attribute="Note", + gff_default_attributes=("ID", "Parent", "Note", "description"), + default_site_mask="funestus", + results_cache=af1_sim_fixture.results_cache_path.as_posix(), + taxon_colors=_af1.TAXON_COLORS, + ) + + +# N.B., here we use pytest_cases to parametrize tests. Each +# function whose name begins with "case_" defines a set of +# inputs to the test functions. See the documentation for +# pytest_cases for more information, e.g.: +# +# https://smarie.github.io/python-pytest-cases/#basic-usage +# +# We use this approach here because we want to use fixtures +# as test parameters, which is otherwise hard to do with +# pytest alone. + + +def case_ag3_sim(ag3_sim_fixture, ag3_sim_api): + return ag3_sim_fixture, ag3_sim_api + + +def case_af1_sim(af1_sim_fixture, af1_sim_api): + return af1_sim_fixture, af1_sim_api + + +@parametrize_with_cases("fixture,api", cases=".") +def test_plink_converter(fixture, api: PlinkConverter, tmp_path): + # Parameters for selecting input data, filtering, and converting. + all_sample_sets = api.sample_sets()["sample_set"].to_list() + + data_params = dict( + region=random.choice(api.contigs), + sample_sets=random.sample(all_sample_sets, 2), + site_mask=random.choice((None,) + api.site_mask_ids), + min_minor_ac=1, + max_missing_an=1, + thin_offset=1, + random_seed=random.randint(1, 2000), + ) + + # Load a ds containing the randomly generated samples and regions to get the number of available snps to subset from. + ds = api.biallelic_snp_calls( + **data_params, + ) + + n_snps_available = ds.sizes["variants"] + n_snps = random.randint(1, n_snps_available) + + # Define plink params. + plink_params = dict(results_dir=tmp_path, n_snps=n_snps, **data_params) + + # Make the plink files. + api.biallelic_snps_to_plink(**plink_params) + + # Test to see if bed, bim, fam output files exist. + file_path = f"{tmp_path}/{plink_params['region']}.{plink_params['n_snps']}.{plink_params['min_minor_ac']}.{plink_params['thin_offset']}.{plink_params['max_missing_an']}" + + assert os.path.exists(f"{file_path}.bed") + assert os.path.exists(f"{file_path}.bim") + assert os.path.exists(f"{file_path}.fam") + + # Read bed, bim, and fam files (bed_reader searches for the .bim and.fam files matching the prefix of the .bed file). + bed = bed_reader.open_bed(f"{file_path}.bed") + + # Load a ds containing the same data exported to PLINK to test against. + ds_test = api.biallelic_snp_calls( + **data_params, + n_snps=n_snps, + ) + + # Test to make sure that the rows and columns (no. variants and no. samples) of the .bed file match. + assert bed.shape[1] == ds_test.variant_position.shape[0] + assert bed.shape[0] == ds_test.samples.shape[0] + + # Test to see if sample_id is exported correctly (stored in the .fam file). + assert set(bed.iid) == set(ds_test.sample_id.values) + + # Test to see if variant position is exported to the.bim correctly. + assert set(bed.bp_position) == set(ds_test.variant_position.values) + + # Test to make sure chromosome ID is exported to the .bim file correctly (coerce to str to match types). + assert set(bed.chromosome) == set(ds_test.variant_contig.values.astype(str)) + + # Test to make sure that the major and minor allele are exported to the .bim file as expected (coerce to str to match types). + assert set(bed.allele_1) == set(ds_test.variant_allele.values[:, 0].astype(str)) + assert set(bed.allele_2) == set(ds_test.variant_allele.values[:, 1].astype(str))