Merge branch 'main' of github.com:astrophpeter/blast

.github/workflows/docker_image_workflow.yml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: ./app

.pre-commit-config.yaml

@@ -1,4 +1,8 @@
 exclude: '(^data/|^app/entrypoints/)'
+# See https://pre-commit.ci/ for more config options
+ci:
+    # autoupdate_schedule: 'quarterly'
+    autoupdate_branch: 'pre-commit'
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
@@ -8,7 +12,7 @@ repos:
   - id: trailing-whitespace
   - id: check-byte-order-marker
 - repo: https://github.com/psf/black
-  rev: 24.3.0
+  rev: 23.12.1
   hooks:
   - id: black
 - repo: https://github.com/asottile/reorder-python-imports

app/Dockerfile

@@ -1,4 +1,4 @@
-FROM hub.ncsa.illinois.edu/blast/deps@sha256:0fe60eb8fee7c93664ee67e2b82fab048311f2bdf574011a402ca656f9e1408d as deps
+FROM hub.ncsa.illinois.edu/blast/deps@sha256:c402391fa2490e6c56518084c4469f9dc23bb4a6691645bd0a1cc491e05d2f94 as deps
 
 FROM python:3.11
 ENV PYTHONUNBUFFERED 1

app/api/views.py

@@ -19,7 +19,6 @@
 
 ### Filter Sets ###
 class TransientFilter(django_filters.FilterSet):
-
     redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte")
     redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte")
 
@@ -30,7 +29,6 @@ class Meta:
 
 ### Filter Sets ###
 class HostFilter(django_filters.FilterSet):
-
     redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte")
     redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte")
     photometric_redshift_lte = django_filters.NumberFilter(
@@ -54,7 +52,6 @@ class Meta:
 
 
 class TaskRegisterFilter(django_filters.FilterSet):
-
     transient = django_filters.Filter(field_name="transient__name")
     status = django_filters.Filter(field_name="status__message")
     task = django_filters.Filter(field_name="task__name")
@@ -71,7 +68,6 @@ class Meta:
 
 
 class CutoutFilter(django_filters.FilterSet):
-
     filter = django_filters.Filter(field_name="filter__name")
     transient = django_filters.Filter(field_name="transient__name")
 
@@ -81,7 +77,6 @@ class Meta:
 
 
 class AperturePhotometryFilter(django_filters.FilterSet):
-
     filter = django_filters.Filter(field_name="filter__name")
     transient = django_filters.Filter(field_name="transient__name")
 
@@ -91,7 +86,6 @@ class Meta:
 
 
 class SEDFittingResultFilter(django_filters.FilterSet):
-
     transient = django_filters.Filter(field_name="transient__name")
     aperture_type = django_filters.Filter(field_name="aperture__type")
 

app/app/settings.py

@@ -29,6 +29,7 @@
 SILKY_PYTHON_PROFILER = (
     os.environ.get("SILKY_PYTHON_PROFILER", "false").lower() == "true"
 )
+SILKY_INTERCEPT_PERCENT = int(os.environ.get("SILKY_INTERCEPT_PERCENT", "0"))
 
 
 HOSTNAMES = os.environ.get("DJANGO_HOSTNAMES", "localhost").split(",")

app/entrypoints/initialize_dustmaps.py

@@ -9,8 +9,8 @@
 
 # Download data if is missing
 for data_file in [
-    f'''{config["data_dir"]}/sfd/SFD_dust_4096_ngp.fits''',
-    f'''{config["data_dir"]}/sfd/SFD_dust_4096_sgp.fits''',
+    f"""{config["data_dir"]}/sfd/SFD_dust_4096_ngp.fits""",
+    f"""{config["data_dir"]}/sfd/SFD_dust_4096_sgp.fits""",
 ]:
     if not os.path.exists(data_file):
         dustmaps.sfd.fetch()
@@ -19,5 +19,5 @@
 # Create dustmap config file if it is missing
 dustmap_config = f"{os.environ['HOME']}/.dustmapsrc"
 if not os.path.exists(dustmap_config):
-    with open(dustmap_config, 'w') as fout:
-        print(f'''{{"data_dir": "{config["data_dir"]}"}}''', file=fout)
+    with open(dustmap_config, "w") as fout:
+        print(f"""{{"data_dir": "{config["data_dir"]}"}}""", file=fout)

app/entrypoints/setup_superuser.py

@@ -3,9 +3,9 @@
 
 User = get_user_model()  # get the currently active user model,
 
-username = os.environ.get('DJANGO_SUPERUSER_USERNAME', '')
-email = os.environ.get('DJANGO_SUPERUSER_EMAIL', '')
-password = os.environ.get('DJANGO_SUPERUSER_PASSWORD', '')
+username = os.environ.get("DJANGO_SUPERUSER_USERNAME", "")
+email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "")
+password = os.environ.get("DJANGO_SUPERUSER_PASSWORD", "")
 
 if not User.objects.filter(username=username).exists():
     User.objects.create_superuser(

app/host/SBI/run_sbi.py

@@ -77,7 +77,6 @@
 
 
 def main():
-
     # training set
     data = h5py.File(sbi_params["train_fname"], "r")
     x_train = np.array(data["theta"])  # physical parameters

app/host/SBI/run_sbi_blast.py

@@ -253,12 +253,12 @@ def fit_sbi_pp(observations, n_filt_cuts=True, fit_type="global"):
         wavelengths = np.append(wavelengths, f.transmission_curve().wave_effective)
 
     obs = {}
-    obs["mags"] = (
-        mags  ##np.array([maggies_to_asinh(p) for p in observations['maggies']])
-    )
-    obs["mags_unc"] = (
-        mags_unc  ##2.5/np.log(10)*observations['maggies_unc']/observations['maggies']
-    )
+    obs[
+        "mags"
+    ] = mags  ##np.array([maggies_to_asinh(p) for p in observations['maggies']])
+    obs[
+        "mags_unc"
+    ] = mags_unc  ##2.5/np.log(10)*observations['maggies_unc']/observations['maggies']
     obs["redshift"] = observations["redshift"]
     obs["wavelengths"] = wavelengths
     obs["filternames"] = filternames

app/host/SBI/sbi_pp.py

@@ -600,7 +600,6 @@ def sbi_missing_and_noisy(obs, run_params, sbi_params):
     cnt_timeout = 0
     timeout_flag = False
     while cnt < run_params["nmc"]:
-
         samp_y_guess = np.copy(observed)
 
         # first, fill in the missing bands

app/host/SBI/sbi_pp_fordebugging.py

@@ -77,7 +77,6 @@ def build_obs(**extras):  ##transient, aperture_type):
     """
     filters = []
     for filter in all_filters:
-
         filters.append(filter.transmission_curve())
 
     obs_data = dict(
@@ -1079,7 +1078,6 @@ def sbi_missing_and_noisy(obs, run_params, sbi_params):
     cnt_timeout = 0
     timeout_flag = False
     while cnt < run_params["nmc"]:
-
         samp_y_guess = np.copy(observed)
 
         # first, fill in the missing bands

app/host/SBI/snr_vs_mag.py

@@ -11,7 +11,6 @@
 
 
 def main():
-
     for f in Filter.objects.all():
         phot = AperturePhotometry.objects.filter(filter=f, magnitude__isnull=False)
         mag = np.array(phot.values_list("magnitude", flat=True))

app/host/SBI/train_sbi.py

@@ -67,7 +67,6 @@ def build_obs(**extras):  ##transient, aperture_type):
     """
     filters = []
     for filter in all_filters:
-
         filters.append(filter.transmission_curve())
 
     obs_data = dict(
@@ -373,7 +372,6 @@ def loc(mass):
 
 
 def draw_thetas(flat=False):
-
     if flat:
         zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample()
         logmass = priors.FastUniform(a=7.0, b=12.5).sample()
@@ -463,7 +461,6 @@ def draw_thetas(flat=False):
 
 
 class TrainSBI(CronJobBase):
-
     RUN_EVERY_MINS = 3
 
     schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
@@ -478,7 +475,6 @@ def build_all(self, **kwargs):
         )
 
     def do(self, do_phot=True, do_train=False):
-
         # parameters
         needed_size = 150000
         run_params = {"ichunk": 0, "needed_size": needed_size}

app/host/SBI/train_sbi_local.py

@@ -67,7 +67,6 @@ def build_obs(**extras):  ##transient, aperture_type):
     """
     filters = []
     for filter in all_filters:
-
         filters.append(filter.transmission_curve())
 
     obs_data = dict(
@@ -373,7 +372,6 @@ def loc(mass):
 
 
 def draw_thetas(flat=False):
-
     if flat:
         zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample()
         ### go 1.5 dex less for local
@@ -464,7 +462,6 @@ def draw_thetas(flat=False):
 
 
 class TrainSBI(CronJobBase):
-
     RUN_EVERY_MINS = 3
 
     schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
@@ -479,7 +476,6 @@ def build_all(self, **kwargs):
         )
 
     def do(self, do_phot=True, do_train=False):
-
         # parameters
         needed_size = 150000
         run_params = {"ichunk": 0, "needed_size": needed_size}

app/host/cutouts.py

@@ -457,7 +457,6 @@ def TWOMASS_cutout(position, image_size=None, filter=None):
                 break
 
     if fits_image is not None:
-
         cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs)
         fits_image[0].data = cutout.data
         fits_image[0].header.update(cutout.wcs.to_header())

app/host/debug_utils.py

@@ -24,7 +24,6 @@ def _overwrite_or_create_object(model, unique_object_query, object_data):
 
 
 def get_failed_tasks(transient_name=None):
-
     if transient_name is None:
         failed_task_register = TaskRegister.objects.filter(status__message="failed")
     else:
@@ -37,11 +36,9 @@ def get_failed_tasks(transient_name=None):
 
 
 def rerun_failed_task(task_register):
-
     task = task_register.task
     for ptask in tasks.periodic_tasks:
         if ptask.task_name == task.name:
-
             print(f"Running {task.name}")
             status = ptask._run_process(task_register.transient)
             print(f"Status: {status}")
@@ -52,7 +49,6 @@ def rerun_failed_task(task_register):
 
 
 def set_tasks_unprocessed(transient_name):
-
     transient = Transient.objects.get(name=transient_name)
     all_tasks = TaskRegister.objects.filter(transient=transient)
     for t in all_tasks:

app/host/host_utils.py

@@ -336,7 +336,6 @@ def check_global_contamination(global_aperture_phot, aperture_primary):
         global_aperture_phot.aperture.cutout.fits.name,
         aperture_primary.cutout.fits.name,
     ]:
-
         # UV photons are too sparse, segmentation map
         # builder cannot easily handle these
         if "/GALEX/" in cutout_name:
@@ -402,7 +401,6 @@ def select_cutout_aperture(cutouts, choice=0):
 
 
 def select_aperture(transient):
-
     cutouts = Cutout.objects.filter(transient=transient)
     if len(cutouts):
         cutout_for_aperture = select_cutout_aperture(cutouts)

app/host/migrations/0001_initial.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []

app/host/migrations/0002_auto_20220601_1914.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0001_initial"),
     ]

app/host/migrations/0003_acknowledgement.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0002_auto_20220601_1914"),
     ]

app/host/migrations/0004_filter_kcorrect_name.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0003_acknowledgement"),
     ]

app/host/migrations/0005_prospectorresult_host.py

@@ -5,7 +5,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0004_filter_kcorrect_name"),
     ]

app/host/migrations/0006_host_milkyway_dust_reddening.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0005_prospectorresult_host"),
     ]

app/host/migrations/0007_transient_processing_status.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0006_host_milkyway_dust_reddening"),
     ]

app/host/migrations/0008_auto_20220822_2335.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("host", "0007_transient_processing_status"),
     ]