Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[python/ci] Restore pre-merge lint checks #2247

Merged
merged 2 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/python-ci-minimal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ jobs:
python_version: ${{ matrix.python-version }}
cc: ${{ matrix.cc }}
cxx: ${{ matrix.cxx }}
report_codecov: ${{ matrix.python-version == '3.10' }}
run_lint: ${{ matrix.python-version == '3.10' }}
report_codecov: ${{ matrix.python-version == '3.11' }}
run_lint: ${{ matrix.python-version == '3.11' }}
secrets: inherit
6 changes: 4 additions & 2 deletions apis/python/tests/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,10 @@ def test_add_matrices(tmp_path):
See https://github.com/single-cell-data/TileDB-SOMA/issues/1565."""
# Create a soma object from an anndata object
soma_path = tmp_path.as_posix()
h5ad_path = HERE.parent / 'testdata/pbmc-small.h5ad'
soma_uri = soma.io.from_h5ad(soma_path, input_path=h5ad_path, measurement_name="RNA")
h5ad_path = HERE.parent / "testdata/pbmc-small.h5ad"
soma_uri = soma.io.from_h5ad(
soma_path, input_path=h5ad_path, measurement_name="RNA"
)

# Synthesize some new data to be written into two matrices within the soma object (ensuring it's different from the
# original data, so that writes must be performed)
Expand Down
40 changes: 24 additions & 16 deletions profiler/src/profiler/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
import hashlib
import json
import os
import uuid
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
from botocore.client import ClientError
import tempfile

import attr
import boto3
import uuid
from botocore.client import ClientError


@attr.define
class ProfileData:
Expand Down Expand Up @@ -134,6 +135,7 @@ def add(self, data: ProfileData) -> str:
def close(self):
pass


class S3ProfileDB(ProfileDB):
"""Represents a S3-based implementation of a ProfileDB database.
Each run is stored as a separate S3 object under a key with the structure `<bucket>/<command>/<timestamp>`.
Expand All @@ -143,21 +145,20 @@ def read_object_keys(self, prefix: str, suffix: str) -> List[str]:
# return all the objects kets starting with prefix and ending with suffix
result = self.s3.list_objects(Bucket=self.bucket_name, Prefix=prefix)
keys: List[str] = []
for o in result.get('Contents'):
object_key = o.get('Key')
for o in result.get("Contents"):
object_key = o.get("Key")
if object_key.endswith(suffix):
keys.append(object_key)
return keys

def read_s3_text(self, key: str) -> str:
# Assume the key is associated with one object. Otherwise, return the first object
result = self.s3.list_objects(Bucket=self.bucket_name, Prefix=key)
for o in result.get('Contents'):
data = self.s3.get_object(Bucket=self.bucket_name, Key=o.get('Key'))
contents = data['Body'].read().decode("utf-8")
for o in result.get("Contents"):
data = self.s3.get_object(Bucket=self.bucket_name, Key=o.get("Key"))
contents = data["Body"].read().decode("utf-8")
return contents


def bucket_exist_and_accessible(self):
try:
self.s3.head_bucket(Bucket=self.bucket.name)
Expand All @@ -167,13 +168,16 @@ def bucket_exist_and_accessible(self):

def __init__(self, bucket_name: str):
# Initialize bucket's info
self.s3 = boto3.client('s3')
self.s3 = boto3.client("s3")
self.bucket_name = bucket_name
self.bucket = boto3.resource('s3').Bucket(self.bucket_name)
self.bucket = boto3.resource("s3").Bucket(self.bucket_name)
# Check if the bucket exists
if not self.bucket_exist_and_accessible():
raise(
Exception(f"Bucket {self.bucket_name} does not exist or access is not granted."))
raise (
Exception(
f"Bucket {self.bucket_name} does not exist or access is not granted."
)
)

def __str__(self):
result = ""
Expand All @@ -186,9 +190,11 @@ def __str__(self):
command = self.read_s3_text(command_file_key)
# Get the number of runs.
n_runs = len(self.read_object_keys(runs_prefix, ".json"))
result =+ f"[{command_file_key}] \"{command}\": {n_runs} runs\n"
result = +f'[{command_file_key}] "{command}": {n_runs} runs\n'
return result
return Exception(f"Bucket {self.bucket_name} does not exist or access is not granted.")
return Exception(
f"Bucket {self.bucket_name} does not exist or access is not granted."
)

def find(self, command) -> List[ProfileData]:
key = _command_key(command)
Expand All @@ -208,7 +214,9 @@ def add(self, data: ProfileData) -> str:

with open(filename, "w") as fp:
fp.write(data.command.strip())
self.s3.upload_file(os.path.abspath(str(fp.name)), self.bucket_name, s3_command_key)
self.s3.upload_file(
os.path.abspath(str(fp.name)), self.bucket_name, s3_command_key
)
os.unlink(filename)

key2 = data.timestamp
Expand Down
2 changes: 1 addition & 1 deletion profiler/src/profiler/profiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from .context_generator import host_context

# import context_generator
from .data import FileBasedProfileDB, ProfileData, ProfileDB, S3ProfileDB
from .data import ProfileData, ProfileDB, S3ProfileDB

GNU_TIME_FORMAT = (
'Command being timed: "%C"\n'
Expand Down
Loading