diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml deleted file mode 100644 index d74bb806..00000000 --- a/.github/workflows/continuous-integration.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Continuous Integration - -on: - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - push: - branches: [main] - paths-ignore: - - 'docs/**' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - # - name: Run tests - # run: | - # docker compose \ - # --project-name blast \ - # -f docker/docker-compose.yml \ - # --env-file env/.env.default \ - # --env-file env/.env.ci \ - # --profile ci \ - # up --build --exit-code-from app_ci - - - uses: codecov/codecov-action@v4 - with: - directory: app/ diff --git a/.github/workflows/docker_image_workflow.yml b/.github/workflows/docker_image_workflow.yml index ce524ba5..a2ba30f4 100644 --- a/.github/workflows/docker_image_workflow.yml +++ b/.github/workflows/docker_image_workflow.yml @@ -51,7 +51,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: ./app diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90db7884..9907abcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,8 @@ exclude: '(^data/|^app/entrypoints/)' +# See https://pre-commit.ci/ for more config options +ci: + # autoupdate_schedule: 'quarterly' + autoupdate_branch: 'pre-commit' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -8,7 +12,7 @@ repos: - id: trailing-whitespace - id: check-byte-order-marker - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/asottile/reorder-python-imports diff --git a/app/Dockerfile b/app/Dockerfile index 2fc3bf37..978490a3 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,4 +1,4 @@ -FROM hub.ncsa.illinois.edu/blast/deps@sha256:0fe60eb8fee7c93664ee67e2b82fab048311f2bdf574011a402ca656f9e1408d as deps +FROM hub.ncsa.illinois.edu/blast/deps@sha256:c402391fa2490e6c56518084c4469f9dc23bb4a6691645bd0a1cc491e05d2f94 as deps FROM python:3.11 ENV PYTHONUNBUFFERED 1 diff --git a/app/api/views.py b/app/api/views.py index 9e53a9aa..3ea7ca7f 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -19,7 +19,6 @@ ### Filter Sets ### class TransientFilter(django_filters.FilterSet): - redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte") redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte") @@ -30,7 +29,6 @@ class Meta: ### Filter Sets ### class HostFilter(django_filters.FilterSet): - redshift_lte = django_filters.NumberFilter(field_name="redshift", lookup_expr="lte") redshift_gte = django_filters.NumberFilter(field_name="redshift", lookup_expr="gte") photometric_redshift_lte = django_filters.NumberFilter( @@ -54,7 +52,6 @@ class Meta: class TaskRegisterFilter(django_filters.FilterSet): - transient = django_filters.Filter(field_name="transient__name") status = django_filters.Filter(field_name="status__message") task = django_filters.Filter(field_name="task__name") @@ -71,7 +68,6 @@ class Meta: class CutoutFilter(django_filters.FilterSet): - filter = django_filters.Filter(field_name="filter__name") transient = django_filters.Filter(field_name="transient__name") @@ -81,7 +77,6 @@ class Meta: class AperturePhotometryFilter(django_filters.FilterSet): - filter = django_filters.Filter(field_name="filter__name") transient = django_filters.Filter(field_name="transient__name") @@ -91,7 +86,6 @@ class Meta: class SEDFittingResultFilter(django_filters.FilterSet): - transient = django_filters.Filter(field_name="transient__name") aperture_type = django_filters.Filter(field_name="aperture__type") diff --git a/app/app/settings.py b/app/app/settings.py index 98130450..dce45389 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -29,6 +29,7 @@ SILKY_PYTHON_PROFILER = ( os.environ.get("SILKY_PYTHON_PROFILER", "false").lower() == "true" ) +SILKY_INTERCEPT_PERCENT = int(os.environ.get("SILKY_INTERCEPT_PERCENT", "0")) HOSTNAMES = os.environ.get("DJANGO_HOSTNAMES", "localhost").split(",") diff --git a/app/entrypoints/initialize_dustmaps.py b/app/entrypoints/initialize_dustmaps.py index 972a1b24..55d1a164 100644 --- a/app/entrypoints/initialize_dustmaps.py +++ b/app/entrypoints/initialize_dustmaps.py @@ -9,8 +9,8 @@ # Download data if is missing for data_file in [ - f'''{config["data_dir"]}/sfd/SFD_dust_4096_ngp.fits''', - f'''{config["data_dir"]}/sfd/SFD_dust_4096_sgp.fits''', + f"""{config["data_dir"]}/sfd/SFD_dust_4096_ngp.fits""", + f"""{config["data_dir"]}/sfd/SFD_dust_4096_sgp.fits""", ]: if not os.path.exists(data_file): dustmaps.sfd.fetch() @@ -19,5 +19,5 @@ # Create dustmap config file if it is missing dustmap_config = f"{os.environ['HOME']}/.dustmapsrc" if not os.path.exists(dustmap_config): - with open(dustmap_config, 'w') as fout: - print(f'''{{"data_dir": "{config["data_dir"]}"}}''', file=fout) + with open(dustmap_config, "w") as fout: + print(f"""{{"data_dir": "{config["data_dir"]}"}}""", file=fout) diff --git a/app/entrypoints/setup_superuser.py b/app/entrypoints/setup_superuser.py index 7627cfb7..640555cc 100644 --- a/app/entrypoints/setup_superuser.py +++ b/app/entrypoints/setup_superuser.py @@ -3,9 +3,9 @@ User = get_user_model() # get the currently active user model, -username = os.environ.get('DJANGO_SUPERUSER_USERNAME', '') -email = os.environ.get('DJANGO_SUPERUSER_EMAIL', '') -password = os.environ.get('DJANGO_SUPERUSER_PASSWORD', '') +username = os.environ.get("DJANGO_SUPERUSER_USERNAME", "") +email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "") +password = os.environ.get("DJANGO_SUPERUSER_PASSWORD", "") if not User.objects.filter(username=username).exists(): User.objects.create_superuser( diff --git a/app/host/SBI/run_sbi.py b/app/host/SBI/run_sbi.py index a9c78d2e..977ac210 100644 --- a/app/host/SBI/run_sbi.py +++ b/app/host/SBI/run_sbi.py @@ -77,7 +77,6 @@ def main(): - # training set data = h5py.File(sbi_params["train_fname"], "r") x_train = np.array(data["theta"]) # physical parameters diff --git a/app/host/SBI/run_sbi_blast.py b/app/host/SBI/run_sbi_blast.py index c266efce..fdbd3af9 100644 --- a/app/host/SBI/run_sbi_blast.py +++ b/app/host/SBI/run_sbi_blast.py @@ -253,12 +253,12 @@ def fit_sbi_pp(observations, n_filt_cuts=True, fit_type="global"): wavelengths = np.append(wavelengths, f.transmission_curve().wave_effective) obs = {} - obs["mags"] = ( - mags ##np.array([maggies_to_asinh(p) for p in observations['maggies']]) - ) - obs["mags_unc"] = ( - mags_unc ##2.5/np.log(10)*observations['maggies_unc']/observations['maggies'] - ) + obs[ + "mags" + ] = mags ##np.array([maggies_to_asinh(p) for p in observations['maggies']]) + obs[ + "mags_unc" + ] = mags_unc ##2.5/np.log(10)*observations['maggies_unc']/observations['maggies'] obs["redshift"] = observations["redshift"] obs["wavelengths"] = wavelengths obs["filternames"] = filternames diff --git a/app/host/SBI/sbi_pp.py b/app/host/SBI/sbi_pp.py index 6749d991..4a9b5174 100644 --- a/app/host/SBI/sbi_pp.py +++ b/app/host/SBI/sbi_pp.py @@ -600,7 +600,6 @@ def sbi_missing_and_noisy(obs, run_params, sbi_params): cnt_timeout = 0 timeout_flag = False while cnt < run_params["nmc"]: - samp_y_guess = np.copy(observed) # first, fill in the missing bands diff --git a/app/host/SBI/sbi_pp_fordebugging.py b/app/host/SBI/sbi_pp_fordebugging.py index 42ae1219..3967bc4a 100644 --- a/app/host/SBI/sbi_pp_fordebugging.py +++ b/app/host/SBI/sbi_pp_fordebugging.py @@ -77,7 +77,6 @@ def build_obs(**extras): ##transient, aperture_type): """ filters = [] for filter in all_filters: - filters.append(filter.transmission_curve()) obs_data = dict( @@ -1079,7 +1078,6 @@ def sbi_missing_and_noisy(obs, run_params, sbi_params): cnt_timeout = 0 timeout_flag = False while cnt < run_params["nmc"]: - samp_y_guess = np.copy(observed) # first, fill in the missing bands diff --git a/app/host/SBI/snr_vs_mag.py b/app/host/SBI/snr_vs_mag.py index 3ea6ac0d..354d48e0 100644 --- a/app/host/SBI/snr_vs_mag.py +++ b/app/host/SBI/snr_vs_mag.py @@ -11,7 +11,6 @@ def main(): - for f in Filter.objects.all(): phot = AperturePhotometry.objects.filter(filter=f, magnitude__isnull=False) mag = np.array(phot.values_list("magnitude", flat=True)) diff --git a/app/host/SBI/train_sbi.py b/app/host/SBI/train_sbi.py index 591f9ef2..801d7ed6 100644 --- a/app/host/SBI/train_sbi.py +++ b/app/host/SBI/train_sbi.py @@ -67,7 +67,6 @@ def build_obs(**extras): ##transient, aperture_type): """ filters = [] for filter in all_filters: - filters.append(filter.transmission_curve()) obs_data = dict( @@ -373,7 +372,6 @@ def loc(mass): def draw_thetas(flat=False): - if flat: zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() logmass = priors.FastUniform(a=7.0, b=12.5).sample() @@ -463,7 +461,6 @@ def draw_thetas(flat=False): class TrainSBI(CronJobBase): - RUN_EVERY_MINS = 3 schedule = Schedule(run_every_mins=RUN_EVERY_MINS) @@ -478,7 +475,6 @@ def build_all(self, **kwargs): ) def do(self, do_phot=True, do_train=False): - # parameters needed_size = 150000 run_params = {"ichunk": 0, "needed_size": needed_size} diff --git a/app/host/SBI/train_sbi_local.py b/app/host/SBI/train_sbi_local.py index db3bdea9..2ceac91f 100644 --- a/app/host/SBI/train_sbi_local.py +++ b/app/host/SBI/train_sbi_local.py @@ -67,7 +67,6 @@ def build_obs(**extras): ##transient, aperture_type): """ filters = [] for filter in all_filters: - filters.append(filter.transmission_curve()) obs_data = dict( @@ -373,7 +372,6 @@ def loc(mass): def draw_thetas(flat=False): - if flat: zred = priors.FastUniform(a=0.0, b=0.2 + 1e-3).sample() ### go 1.5 dex less for local @@ -464,7 +462,6 @@ def draw_thetas(flat=False): class TrainSBI(CronJobBase): - RUN_EVERY_MINS = 3 schedule = Schedule(run_every_mins=RUN_EVERY_MINS) @@ -479,7 +476,6 @@ def build_all(self, **kwargs): ) def do(self, do_phot=True, do_train=False): - # parameters needed_size = 150000 run_params = {"ichunk": 0, "needed_size": needed_size} diff --git a/app/host/cutouts.py b/app/host/cutouts.py index fa0c3430..4042627b 100644 --- a/app/host/cutouts.py +++ b/app/host/cutouts.py @@ -457,7 +457,6 @@ def TWOMASS_cutout(position, image_size=None, filter=None): break if fits_image is not None: - cutout = Cutout2D(fits_image[0].data, position, image_size, wcs=wcs) fits_image[0].data = cutout.data fits_image[0].header.update(cutout.wcs.to_header()) diff --git a/app/host/debug_utils.py b/app/host/debug_utils.py index 420c9504..009e6688 100644 --- a/app/host/debug_utils.py +++ b/app/host/debug_utils.py @@ -24,7 +24,6 @@ def _overwrite_or_create_object(model, unique_object_query, object_data): def get_failed_tasks(transient_name=None): - if transient_name is None: failed_task_register = TaskRegister.objects.filter(status__message="failed") else: @@ -37,11 +36,9 @@ def get_failed_tasks(transient_name=None): def rerun_failed_task(task_register): - task = task_register.task for ptask in tasks.periodic_tasks: if ptask.task_name == task.name: - print(f"Running {task.name}") status = ptask._run_process(task_register.transient) print(f"Status: {status}") @@ -52,7 +49,6 @@ def rerun_failed_task(task_register): def set_tasks_unprocessed(transient_name): - transient = Transient.objects.get(name=transient_name) all_tasks = TaskRegister.objects.filter(transient=transient) for t in all_tasks: diff --git a/app/host/host_utils.py b/app/host/host_utils.py index 696dc45b..ef74f086 100644 --- a/app/host/host_utils.py +++ b/app/host/host_utils.py @@ -336,7 +336,6 @@ def check_global_contamination(global_aperture_phot, aperture_primary): global_aperture_phot.aperture.cutout.fits.name, aperture_primary.cutout.fits.name, ]: - # UV photons are too sparse, segmentation map # builder cannot easily handle these if "/GALEX/" in cutout_name: @@ -402,7 +401,6 @@ def select_cutout_aperture(cutouts, choice=0): def select_aperture(transient): - cutouts = Cutout.objects.filter(transient=transient) if len(cutouts): cutout_for_aperture = select_cutout_aperture(cutouts) diff --git a/app/host/migrations/0001_initial.py b/app/host/migrations/0001_initial.py index 673ce396..6d97d8f1 100644 --- a/app/host/migrations/0001_initial.py +++ b/app/host/migrations/0001_initial.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [] diff --git a/app/host/migrations/0002_auto_20220601_1914.py b/app/host/migrations/0002_auto_20220601_1914.py index 61f93808..6672ba97 100644 --- a/app/host/migrations/0002_auto_20220601_1914.py +++ b/app/host/migrations/0002_auto_20220601_1914.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0001_initial"), ] diff --git a/app/host/migrations/0003_acknowledgement.py b/app/host/migrations/0003_acknowledgement.py index 8037d47b..161c1d6b 100644 --- a/app/host/migrations/0003_acknowledgement.py +++ b/app/host/migrations/0003_acknowledgement.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0002_auto_20220601_1914"), ] diff --git a/app/host/migrations/0004_filter_kcorrect_name.py b/app/host/migrations/0004_filter_kcorrect_name.py index 38f171ff..3915bca1 100644 --- a/app/host/migrations/0004_filter_kcorrect_name.py +++ b/app/host/migrations/0004_filter_kcorrect_name.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0003_acknowledgement"), ] diff --git a/app/host/migrations/0005_prospectorresult_host.py b/app/host/migrations/0005_prospectorresult_host.py index a8066e0a..d5266bfd 100644 --- a/app/host/migrations/0005_prospectorresult_host.py +++ b/app/host/migrations/0005_prospectorresult_host.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0004_filter_kcorrect_name"), ] diff --git a/app/host/migrations/0006_host_milkyway_dust_reddening.py b/app/host/migrations/0006_host_milkyway_dust_reddening.py index d7aa93d3..154b8b07 100644 --- a/app/host/migrations/0006_host_milkyway_dust_reddening.py +++ b/app/host/migrations/0006_host_milkyway_dust_reddening.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0005_prospectorresult_host"), ] diff --git a/app/host/migrations/0007_transient_processing_status.py b/app/host/migrations/0007_transient_processing_status.py index aa8bc2ad..066c7c64 100644 --- a/app/host/migrations/0007_transient_processing_status.py +++ b/app/host/migrations/0007_transient_processing_status.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0006_host_milkyway_dust_reddening"), ] diff --git a/app/host/migrations/0008_auto_20220822_2335.py b/app/host/migrations/0008_auto_20220822_2335.py index 55a3cf72..27d6128d 100644 --- a/app/host/migrations/0008_auto_20220822_2335.py +++ b/app/host/migrations/0008_auto_20220822_2335.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0007_transient_processing_status"), ] diff --git a/app/host/migrations/0009_alter_aperturephotometry_flux.py b/app/host/migrations/0009_alter_aperturephotometry_flux.py index ac507fcb..41bf0898 100644 --- a/app/host/migrations/0009_alter_aperturephotometry_flux.py +++ b/app/host/migrations/0009_alter_aperturephotometry_flux.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0008_auto_20220822_2335"), ] diff --git a/app/host/migrations/0010_auto_20221030_1810.py b/app/host/migrations/0010_auto_20221030_1810.py index 28976364..730f8059 100644 --- a/app/host/migrations/0010_auto_20221030_1810.py +++ b/app/host/migrations/0010_auto_20221030_1810.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0009_alter_aperturephotometry_flux"), ] diff --git a/app/host/migrations/0011_filter_magnitude_zero_point_keyword.py b/app/host/migrations/0011_filter_magnitude_zero_point_keyword.py index 512f2b2f..5e42777c 100644 --- a/app/host/migrations/0011_filter_magnitude_zero_point_keyword.py +++ b/app/host/migrations/0011_filter_magnitude_zero_point_keyword.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0010_auto_20221030_1810"), ] diff --git a/app/host/migrations/0012_filter_ab_offset.py b/app/host/migrations/0012_filter_ab_offset.py index 7ddc228d..86728087 100644 --- a/app/host/migrations/0012_filter_ab_offset.py +++ b/app/host/migrations/0012_filter_ab_offset.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0011_filter_magnitude_zero_point_keyword"), ] diff --git a/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py b/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py index 94792993..fb0fd800 100644 --- a/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py +++ b/app/host/migrations/0013_sedfittingresult_mass_surviving_ratio.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0012_filter_ab_offset"), ] diff --git a/app/host/migrations/0014_taskregister_user_warning.py b/app/host/migrations/0014_taskregister_user_warning.py index 436e9ec5..52e729e5 100644 --- a/app/host/migrations/0014_taskregister_user_warning.py +++ b/app/host/migrations/0014_taskregister_user_warning.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0013_sedfittingresult_mass_surviving_ratio"), ] diff --git a/app/host/migrations/0015_alter_taskregister_user_warning.py b/app/host/migrations/0015_alter_taskregister_user_warning.py index 0ca92ab5..3b936056 100644 --- a/app/host/migrations/0015_alter_taskregister_user_warning.py +++ b/app/host/migrations/0015_alter_taskregister_user_warning.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0014_taskregister_user_warning"), ] diff --git a/app/host/migrations/0016_transient_added_by.py b/app/host/migrations/0016_transient_added_by.py index 1cc9ed7c..b4ae5fb7 100644 --- a/app/host/migrations/0016_transient_added_by.py +++ b/app/host/migrations/0016_transient_added_by.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("host", "0015_alter_taskregister_user_warning"), diff --git a/app/host/migrations/0017_alter_aperturephotometry_is_validated.py b/app/host/migrations/0017_alter_aperturephotometry_is_validated.py index bd7bd4b7..1953b16f 100644 --- a/app/host/migrations/0017_alter_aperturephotometry_is_validated.py +++ b/app/host/migrations/0017_alter_aperturephotometry_is_validated.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0016_transient_added_by"), ] diff --git a/app/host/migrations/0018_auto_20240205_0240.py b/app/host/migrations/0018_auto_20240205_0240.py index 6081612f..21245e8d 100644 --- a/app/host/migrations/0018_auto_20240205_0240.py +++ b/app/host/migrations/0018_auto_20240205_0240.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0017_alter_aperturephotometry_is_validated"), ] diff --git a/app/host/migrations/0019_cutout_message.py b/app/host/migrations/0019_cutout_message.py index 14b8b01c..819f31b0 100644 --- a/app/host/migrations/0019_cutout_message.py +++ b/app/host/migrations/0019_cutout_message.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0018_auto_20240205_0240"), ] diff --git a/app/host/migrations/0020_transient_progress.py b/app/host/migrations/0020_transient_progress.py index ed5f3408..f701490b 100644 --- a/app/host/migrations/0020_transient_progress.py +++ b/app/host/migrations/0020_transient_progress.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0019_cutout_message"), ] diff --git a/app/host/migrations/0021_alter_transient_progress.py b/app/host/migrations/0021_alter_transient_progress.py index c972f0d5..e8e3bde8 100644 --- a/app/host/migrations/0021_alter_transient_progress.py +++ b/app/host/migrations/0021_alter_transient_progress.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("host", "0020_transient_progress"), ] diff --git a/app/host/plotting_utils.py b/app/host/plotting_utils.py index 46ed3729..28290cb9 100644 --- a/app/host/plotting_utils.py +++ b/app/host/plotting_utils.py @@ -2,7 +2,6 @@ import os from math import pi -import extinction import numpy as np import pandas as pd import prospect.io.read_results as reader @@ -14,37 +13,38 @@ from astropy.wcs import WCS from bokeh.embed import components from bokeh.layouts import gridplot -from bokeh.models import Circle from bokeh.models import ColumnDataSource -from bokeh.models import Cross -from bokeh.models import Ellipse -from bokeh.models import Grid from bokeh.models import HoverTool from bokeh.models import LabelSet -from bokeh.models import Legend -from bokeh.models import LinearAxis -from bokeh.models import LogColorMapper -from bokeh.models import Plot from bokeh.models import Range1d -from bokeh.models import Scatter from bokeh.palettes import Category20 from bokeh.plotting import ColumnDataSource from bokeh.plotting import figure -from bokeh.plotting import show from bokeh.transform import cumsum -from host.catalog_photometry import filter_information -from host.host_utils import survey_list from host.models import Filter from host.photometric_calibration import maggies_to_mJy -from host.photometric_calibration import mJy_to_maggies -from host.prospector import build_model from host.prospector import build_obs -from .models import Aperture +# import extinction +# from bokeh.models import Circle +# from bokeh.models import Cross +# from bokeh.models import Ellipse +# from bokeh.models import Grid +# from bokeh.models import Legend +# from bokeh.models import LinearAxis +# from bokeh.models import LogColorMapper +# from bokeh.models import Plot +# from bokeh.models import Scatter +# from bokeh.plotting import show +# from host.catalog_photometry import filter_information +# from host.host_utils import survey_list +# from host.photometric_calibration import mJy_to_maggies +# from host.prospector import build_model + +# from .models import Aperture def scale_image(image_data): - transform = AsinhStretch() + PercentileInterval(99.5) scaled_data = transform(image_data) @@ -52,7 +52,6 @@ def scale_image(image_data): def plot_image(image_data, figure): - # sometimes low image mins mess up the plotting perc01 = np.nanpercentile(image_data, 1) @@ -103,15 +102,15 @@ def plot_aperture(figure, aperture, wcs, plotting_kwargs=None): def plot_image_grid(image_dict, apertures=None): - figures = [] for survey, image in image_dict.items(): fig = figure( title=survey, x_axis_label="", y_axis_label="", - plot_width=1000, - plot_height=1000, + # plot_width=1000, + # plot_height=1000, + sizing_mode="scale_both", ) fig = plot_image(fig, image) if apertures is not None: @@ -130,7 +129,6 @@ def plot_image_grid(image_dict, apertures=None): def plot_cutout_image( cutout=None, transient=None, global_aperture=None, local_aperture=None ): - title = cutout.filter if cutout is not None else "No cutout selected" if cutout is not None: @@ -142,8 +140,9 @@ def plot_cutout_image( title=f"{title}", x_axis_label="", y_axis_label="", - plot_width=700, - plot_height=int(np.shape(image_data)[0] / np.shape(image_data)[1] * 700), + # plot_width=500, + # plot_height=int(np.shape(image_data)[0] / np.shape(image_data)[1] * 700), + sizing_mode="scale_both", ) fig.axis.visible = False fig.xgrid.visible = False @@ -153,9 +152,10 @@ def plot_cutout_image( "legend_label": f"{transient.name}", "size": 30, "line_width": 2, + "marker": "cross", } plot_position( - transient, wcs, plotting_kwargs=transient_kwargs, plotting_func=fig.cross + transient, wcs, plotting_kwargs=transient_kwargs, plotting_func=fig.scatter ) if transient.host is not None: @@ -164,9 +164,13 @@ def plot_cutout_image( "size": 25, "line_width": 2, "line_color": "red", + "marker": "x", } plot_position( - transient.host, wcs, plotting_kwargs=host_kwargs, plotting_func=fig.x + transient.host, + wcs, + plotting_kwargs=host_kwargs, + plotting_func=fig.scatter, ) if global_aperture.exists(): filter_name = global_aperture[0].cutout.filter.name @@ -200,8 +204,9 @@ def plot_cutout_image( title=f"{title}", x_axis_label="", y_axis_label="", - plot_width=700, - plot_height=700, + # plot_width=700, + # plot_height=700, + sizing_mode="scale_both", ) fig.axis.visible = False fig.xgrid.visible = False @@ -246,8 +251,9 @@ def asinh_to_maggies(x): fig = figure( title="", - width=700, - height=400, + # max_width=600, + sizing_mode="stretch_width", + max_height=400, min_border=0, # toolbar_location=None, x_axis_type="log", @@ -288,7 +294,7 @@ def asinh_to_maggies(x): ("mag (AB)", "@mag"), ("mag error (AB)", "@mag_error"), ] - hover = HoverTool(renderers=[p], tooltips=TOOLTIPS, toggleable=False) + hover = HoverTool(renderers=[p], tooltips=TOOLTIPS, visible=False) fig.add_tools(hover) # second check on SED file @@ -356,15 +362,14 @@ def asinh_to_maggies(x): pwave = [f.wave_effective for f in obs["filters"]] if transient.best_redshift < 0.015: - fig.circle( + fig.scatter( pwave, maggies_to_mJy(model_data["phot"]) * 10 ** (0.4 * mag_off), size=10, ) else: - fig.circle(pwave, maggies_to_mJy(model_data["phot"]), size=10) + fig.scatter(pwave, maggies_to_mJy(model_data["phot"]), size=10) - fig.width = 600 fig.legend.location = "top_left" script, div = components(fig) return {f"bokeh_sed_{type}_script": script, f"bokeh_sed_{type}_div": div} @@ -387,9 +392,9 @@ def plot_errorbar( # to do the mouse-over if source is not None: - p = figure.circle("x", "y", color=color, source=source, **point_kwargs) + p = figure.scatter("x", "y", color=color, source=source, **point_kwargs) else: - p = figure.circle(x, y, color=color, source=source, **point_kwargs) + p = figure.scatter(x, y, color=color, source=source, **point_kwargs) if xerr: x_err_x = [] @@ -410,7 +415,6 @@ def plot_errorbar( def plot_bar_chart(data_dict): - x_label = "" y_label = "Transients" transient_numbers = list(data_dict.values()) @@ -430,6 +434,7 @@ def plot_bar_chart(data_dict): y_axis_label=y_label, x_range=vals["processing"], y_range=Range1d(start=0, end=max(transient_numbers) * 1.1), + sizing_mode="scale_both", ) labels = LabelSet( @@ -490,7 +495,6 @@ def plot_pie_chart(data_dict): def plot_timeseries(): - fig = figure( title="", width=700, diff --git a/app/host/postprocess_prosp.py b/app/host/postprocess_prosp.py index 61bea886..d2f91995 100644 --- a/app/host/postprocess_prosp.py +++ b/app/host/postprocess_prosp.py @@ -287,7 +287,6 @@ def run_all( obs=None, **extra ): - # XXX read in prospector outputs if obs is None: res, obs, _ = reader.results_from(fname, dangerous=False) diff --git a/app/host/prospector.py b/app/host/prospector.py index c4a3b3bc..42ca8c65 100644 --- a/app/host/prospector.py +++ b/app/host/prospector.py @@ -767,7 +767,6 @@ def prospector_result_to_blast( parametric_sfh=False, sbipp=False, ): - # write the results hdf5_file_name = ( f"{sed_output_root}/{transient.name}/{transient.name}_{aperture.type}.h5" @@ -781,7 +780,6 @@ def prospector_result_to_blast( os.remove(hdf5_file) if sbipp: - hf = h5py.File(hdf5_file_name, "w") sdat = hf.create_group("sampling") diff --git a/app/host/slurm/run_single_transient.py b/app/host/slurm/run_single_transient.py index a7ab9003..5945e768 100644 --- a/app/host/slurm/run_single_transient.py +++ b/app/host/slurm/run_single_transient.py @@ -65,18 +65,15 @@ class run_single(CronJobBase): - RUN_EVERY_MINS = 3 schedule = Schedule(run_every_mins=RUN_EVERY_MINS) code = "host.slurm.run_single_transient.run_single" def do(self): - self.run() def run(self): - transients = Transient.objects.filter(name=transient_name) if not len(transients): transient = Transient.objects.create( @@ -123,18 +120,15 @@ def run(self): class run_single_sed(CronJobBase): - RUN_EVERY_MINS = 3 schedule = Schedule(run_every_mins=RUN_EVERY_MINS) code = "host.slurm.run_single_transient.run_single" def do(self): - self.run() def run(self): - transients = Transient.objects.filter(name=transient_name) if not len(transients): transient = Transient.objects.create( diff --git a/app/host/tables.py b/app/host/tables.py index a6155161..7ff0e6f4 100644 --- a/app/host/tables.py +++ b/app/host/tables.py @@ -6,7 +6,6 @@ class TransientTable(tables.Table): - name = tables.TemplateColumn( "{{ record.name }}", verbose_name="Name", @@ -89,7 +88,6 @@ class Meta: } def order_best_spec_redshift(self, queryset, is_descending): - queryset = queryset.annotate( best_spec_redshift=Coalesce("redshift", "host__redshift"), ).order_by(("-" if is_descending else "") + "best_spec_redshift") diff --git a/app/host/templates/host/base.html b/app/host/templates/host/base.html index a106fd46..9650edf9 100644 --- a/app/host/templates/host/base.html +++ b/app/host/templates/host/base.html @@ -3,8 +3,6 @@ {% block title %}{% endblock %} - - @@ -94,7 +92,7 @@ -
+
{% block body %} {% endblock %}
@@ -113,11 +111,13 @@
+ - {% block script %}{% endblock %} diff --git a/app/host/templates/host/photometry_card.html b/app/host/templates/host/photometry_card.html index 7eaad677..2e85385b 100644 --- a/app/host/templates/host/photometry_card.html +++ b/app/host/templates/host/photometry_card.html @@ -19,7 +19,7 @@

Aperture Photometry

-
+

Aperture details

RA, DEC (J2000) = {{local_aperture.ra}}, {{local_aperture.dec}}
@@ -54,7 +54,7 @@

Aperture details


-
+

Aperture details

RA, DEC (J2000) = {{global_aperture.ra}}, {{global_aperture.dec}}
semi major axis, semi minor axis (arcsec) = {{global_aperture.semi_major_axis}}, {{global_aperture.semi_minor_axis}}
diff --git a/app/host/templates/host/results.html b/app/host/templates/host/results.html index 14d4ee0a..27590fd9 100644 --- a/app/host/templates/host/results.html +++ b/app/host/templates/host/results.html @@ -16,11 +16,9 @@

{{transient.name}}

{% if is_auth %} Reprocess {% endif %} -
-
- +
-
+
{% include "host/information_card.html" %} @@ -32,7 +30,7 @@

{{transient.name}}

-
+
{% include "host/cutout_card.html" %} {% include "host/sed_card.html" %} diff --git a/app/host/templates/host/sed_card.html b/app/host/templates/host/sed_card.html index a473ecef..9d06fdba 100644 --- a/app/host/templates/host/sed_card.html +++ b/app/host/templates/host/sed_card.html @@ -21,7 +21,7 @@

Spectral Energy Distribution

 
-
+
@@ -33,7 +33,7 @@

Spectral Energy Distribution

{% if contam_warning %} Warning: possible contaminating objects in the global aperture
{% endif %} -
+
diff --git a/app/host/templates/host/sed_inference_card.html b/app/host/templates/host/sed_inference_card.html index 8a9ba361..54c0499d 100644 --- a/app/host/templates/host/sed_inference_card.html +++ b/app/host/templates/host/sed_inference_card.html @@ -21,7 +21,7 @@

Host SED inference

-
+

Local parameter details

{% if local_sed_results %} @@ -61,7 +61,7 @@

Local parameter details

-
+

Global parameter details

{% if global_sed_results %} diff --git a/app/host/tests/test_aperture_construction.py b/app/host/tests/test_aperture_construction.py index aa10bda5..8111a11b 100644 --- a/app/host/tests/test_aperture_construction.py +++ b/app/host/tests/test_aperture_construction.py @@ -12,7 +12,6 @@ class TestApertureConstruction(TestCase): - fixtures = [ "../fixtures/initial/setup_survey_data.yaml", "../fixtures/initial/setup_filter_data.yaml", @@ -33,7 +32,6 @@ def test_aperture_construction(self): assert status_message == "processed" def test_aperture_failures(self): - data = np.zeros((500, 5000), dtype=np.float64) hdu = fits.PrimaryHDU(data=data) hdulist = fits.HDUList(hdus=[hdu]) diff --git a/app/host/tests/test_ebv.py b/app/host/tests/test_ebv.py index b2ee352e..5ce9afcf 100644 --- a/app/host/tests/test_ebv.py +++ b/app/host/tests/test_ebv.py @@ -19,7 +19,6 @@ class EBVTest(TestCase): ] def test_mwebv_transient(self): - transient = Transient.objects.get(name="2010H") mwebv_cls = MWEBV_Transient() status_message = mwebv_cls._run_process(transient) diff --git a/app/host/tests/test_host_match.py b/app/host/tests/test_host_match.py index e091069c..b4602980 100644 --- a/app/host/tests/test_host_match.py +++ b/app/host/tests/test_host_match.py @@ -8,7 +8,6 @@ class TestHostMatch(TestCase): - fixtures = [ "../fixtures/initial/setup_survey_data.yaml", "../fixtures/initial/setup_filter_data.yaml", diff --git a/app/host/tests/test_photometry.py b/app/host/tests/test_photometry.py index ed055715..e0d22d8a 100644 --- a/app/host/tests/test_photometry.py +++ b/app/host/tests/test_photometry.py @@ -21,7 +21,6 @@ class TestValidatePhotometry(TestCase): ] def test_validate_local_photometry(self): - transient = Transient.objects.get(name="2010H") vlp_cls = ValidateLocalPhotometry() @@ -41,7 +40,6 @@ def test_validate_local_photometry(self): assert status_message == "processed" def test_validate_global_photometry(self): - transient = Transient.objects.get(name="2010H") vgp_cls = ValidateGlobalPhotometry() diff --git a/app/host/tests/test_processing.py b/app/host/tests/test_processing.py index c6598878..50605aa4 100644 --- a/app/host/tests/test_processing.py +++ b/app/host/tests/test_processing.py @@ -110,7 +110,6 @@ def _failed_status_message(self): self.two_prereqs_suc_runner = TestRunnerTwoPrereqsSuc() def test_run_process(self): - self.processed_runner.run_process() # 2022testone is the oldest transient so should be selected and @@ -126,7 +125,6 @@ def test_run_process(self): self.assertTrue(task_register.status.message == "processed") def test_run_failed(self): - try: self.failed_runner.run_process() except ValueError: diff --git a/app/host/tests/test_sedfitting.py b/app/host/tests/test_sedfitting.py index 017f55a0..9ca47b00 100644 --- a/app/host/tests/test_sedfitting.py +++ b/app/host/tests/test_sedfitting.py @@ -61,7 +61,6 @@ def test_build_obs(self): class SEDFittingFullTest(TestCase): - fixtures = [ "../fixtures/initial/setup_survey_data.yaml", "../fixtures/initial/setup_filter_data.yaml", diff --git a/app/host/transient_tasks.py b/app/host/transient_tasks.py index aa2635a7..f241f58e 100644 --- a/app/host/transient_tasks.py +++ b/app/host/transient_tasks.py @@ -400,11 +400,9 @@ def _run_process(self, transient): # make new aperture # adjust semi-major/minor axes for size if f"{cutout.name}_global" != aperture.name: - if not len( Aperture.objects.filter(cutout__name=f"{cutout.name}_global") ): - semi_major_axis = ( aperture.semi_major_axis_arcsec - aperture.cutout.filter.image_fwhm_arcsec # / 2.354 @@ -514,7 +512,6 @@ def _run_process(self, transient): return "phot valid failed" for local_aperture_phot in local_aperture_photometry: - is_validated = check_local_radius( redshift, local_aperture_phot.filter.image_fwhm_arcsec, diff --git a/app/host/views.py b/app/host/views.py index c96d44d1..966ed11d 100644 --- a/app/host/views.py +++ b/app/host/views.py @@ -37,7 +37,6 @@ def filter_transient_categories(qs, value): - if value == "Transients with Basic Information": qs = qs.filter( pk__in=TaskRegister.objects.filter( @@ -93,7 +92,6 @@ def filter_transient_categories(qs, value): class TransientFilter(django_filters.FilterSet): - hostmatch = django_filters.ChoiceFilter( choices=[ ("All Transients", "All Transients"), @@ -116,7 +114,6 @@ class Meta: fields = ["hostmatch", "ex"] def filter_transients(self, qs, name, value): - qs = filter_transient_categories(qs, value) return qs @@ -124,7 +121,6 @@ def filter_transients(self, qs, name, value): @silk_profile(name="List transients") def transient_list(request): - transients = Transient.objects.order_by("-public_timestamp") transientfilter = TransientFilter(request.GET, queryset=transients) @@ -137,7 +133,6 @@ def transient_list(request): @login_required def transient_uploads(request): - errors = [] uploaded_transient_names = [] @@ -197,11 +192,9 @@ def transient_uploads(request): def analytics(request): - analytics_results = {} for aggregate in ["total", "not completed", "completed", "waiting"]: - transients = TaskRegisterSnapshot.objects.filter( aggregate_type__exact=aggregate ) @@ -212,9 +205,9 @@ def analytics(request): else: transients_current = None - analytics_results[f"{aggregate}_transients_current".replace(" ", "_")] = ( - transients_current - ) + analytics_results[ + f"{aggregate}_transients_current".replace(" ", "_") + ] = transients_current bokeh_processing_context = plot_timeseries() return render( @@ -370,7 +363,6 @@ def results(request, slug): def reprocess_transient(request, slug): - tasks = TaskRegister.objects.filter(transient__name=slug) for t in tasks: t.status = Status.objects.get(message="not processed") @@ -380,7 +372,6 @@ def reprocess_transient(request, slug): def download_chains(request, slug, aperture_type): - sed_result = get_object_or_404( SEDFittingResult, transient__name=slug, aperture__type=aperture_type ) @@ -393,7 +384,6 @@ def download_chains(request, slug, aperture_type): def download_modelfit(request, slug, aperture_type): - sed_result = get_object_or_404( SEDFittingResult, transient__name=slug, aperture__type=aperture_type ) @@ -406,7 +396,6 @@ def download_modelfit(request, slug, aperture_type): def download_percentiles(request, slug, aperture_type): - sed_result = get_object_or_404( SEDFittingResult, transient__name=slug, aperture__type=aperture_type ) @@ -424,7 +413,6 @@ def acknowledgements(request): def home(request): - analytics_results = {} for aggregate, qs_value in zip( @@ -441,7 +429,6 @@ def home(request): "Transients with SED Fitting", ], ): - analytics_results[aggregate] = len( filter_transient_categories(Transient.objects.all(), qs_value) ) diff --git a/app/requirements.txt b/app/requirements.txt index 4b228e38..98d93b77 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -16,7 +16,7 @@ dustmaps==1.0.13 astro-sedpy==0.3.2 extinction==0.4.6 astro-prospector==1.3.0 -bokeh==2.4.0 +bokeh==3.4.1 dynesty==2.1.3 torch==2.2.2 sbi==0.22.0 diff --git a/env/.env.default b/env/.env.default index 64954cc8..e7f90e58 100644 --- a/env/.env.default +++ b/env/.env.default @@ -16,6 +16,8 @@ DJANGO_SUPERUSER_PASSWORD = password DJANGO_SUPERUSER_USERNAME = admin DJANGO_SUPERUSER_EMAIL = test@example.com SILKY_PYTHON_PROFILER = false +FLOWER_UNAUTHENTICATED_API = false +SILKY_INTERCEPT_PERCENT = 0 # RabbitMQ RABBITMQ_USERNAME = guest