Skip to content

Commit

Permalink
Merge pull request #19 from LSSTDESC/u/rknop/dev
Browse files Browse the repository at this point in the history
Some refactoring, add full-cycle alert tests for fastdb_dev
  • Loading branch information
buckleygeer authored Nov 4, 2024
2 parents eecb1e8 + a862a59 commit 7593b17
Show file tree
Hide file tree
Showing 30 changed files with 1,956 additions and 1,406 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ tom_desc/elasticc2/static/elasticc2/alertsendfinish
tom_desc/elasticc2/static/elasticc2/alertsend

# Log directory
logs/
logs/**
!logs/.placeholder

# FastDB files
tom_desc/fastdb_dev/management/commands/insert_last_time.sql
Expand Down
1 change: 1 addition & 0 deletions docker_mongodb/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ RUN set -eux; \
jq \
numactl \
procps \
netcat-openbsd \
; \
rm -rf /var/lib/apt/lists/*

Expand Down
675 changes: 0 additions & 675 deletions fastdb_get_alerts/process_alerts.py

This file was deleted.

1 change: 1 addition & 0 deletions logs/.placeholder
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This is here so that the logs directory will be created on git checkout.
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: tom-rknop-dev-app
namespace: desc-tom
name: tom-app
namespace: desc-tom-rknop-dev
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
workload.user.cattle.io/workloadselector: deployment-desc-tom-tom-rknop-dev-app
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-app
strategy:
rollingUpdate:
maxSurge: 1
Expand All @@ -19,31 +19,46 @@ spec:
template:
metadata:
annotations:
cattle.io/timestamp: "2023-02-06T16:44:40Z"
cattle.io/timestamp: "2023-02-27T21:51:32Z"
field.cattle.io/ports: '[[]]'
nersc.gov/collab_uids: 70268,99896
nersc.gov/gid: "95089"
nersc.gov/gids: 45703,60152,57177,58102,59318,60070,63477,64483,79186,70268,92576,94721,95089,96414,99650
nersc.gov/gids: 95089,57177,60152,96414
nersc.gov/roles: user
nersc.gov/uid: "95089"
nersc.gov/username: raknop
creationTimestamp: null
labels:
workload.user.cattle.io/workloadselector: deployment-desc-tom-tom-rknop-dev-app
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-app
spec:
affinity: {}
containers:
- env:
- name: DB_HOST
value: tom-rknop-dev-postgres
value: tom-postgres
- name: DB_NAME
value: tom_desc
- name: DB_PASS
value: fragile
- name: DB_USER
value: postgres
image: registry.nersc.gov/m1727/raknop/tom_desc_bindmount:chimaera
- name: ALERCE_KAFKA_SERVER
value: b-3-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196,b-2-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196,b-1-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196
- name: FINK_GROUP_ID
value: lsstfr-johann
- name: FINK_SERVER
value: 134.158.74.95:24499,
- name: FINK_TOPIC
value: fink_early_sn_candidates_ztf
- name: FINK_USERNAME
value: johann
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /secrets/pitt_google_auth_key.json
- name: GOOGLE_CLOUD_PROJECT
value: elasticc-challenge
image: registry.nersc.gov/m1727/raknop/tom_server_bindmount
imagePullPolicy: Always
name: tom-rknop-dev-app
name: tom-app
resources: {}
securityContext:
allowPrivilegeEscalation: false
Expand All @@ -62,11 +77,15 @@ spec:
tty: true
volumeMounts:
- mountPath: /secrets
name: tom-rknop-dev-secrets
name: tom-secrets
- mountPath: /tom_desc
name: tom-rknop-dev-deployment
name: tom-deployment
- mountPath: /code
name: tom-checkout
- mountPath: /query_results
name: tom-rknop-dev-query-results
name: tom-query-results
- mountPath: /sample
name: tom-sample
dnsConfig: {}
dnsPolicy: ClusterFirst
restartPolicy: Always
Expand All @@ -77,33 +96,52 @@ spec:
imagePullSecrets:
- name: registry-nersc
volumes:
- name: tom-rknop-dev-secrets
- name: tom-secrets
secret:
defaultMode: 256
optional: false
secretName: tom-rknop-dev-secrets
secretName: tom-secrets
- hostPath:
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/dev/tom_desc/tom_desc
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/rknop_dev/tom_desc
type: Directory
name: tom-rknop-dev-deployment
name: tom-checkout
- hostPath:
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/dev/query_results
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/rknop_dev/tom_desc/tom_desc
type: Directory
name: tom-rknop-dev-query-results
name: tom-deployment
- hostPath:
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/rknop_dev/query_results
type: Directory
name: tom-query-results
- hostPath:
path: /global/cfs/cdirs/desc-td/ELASTICC2
type: Directory
name: tom-sample
---
apiVersion: v1
kind: Service
metadata:
name: tom-rknop-dev-app
namespace: desc-tom
annotations:
field.cattle.io/targetWorkloadIds: '["deployment:desc-tom-rknop-dev:tom-app"]'
name: tom-app
namespace: desc-tom-rknop-dev
ownerReferences:
- apiVersion: apps/v1beta2
controller: true
kind: deployment
name: tom-app
uid: e8f8e9fa-9bb1-475a-810d-483dbc7f6bc8
spec:
clusterIP: None
clusterIPs:
- None
ports:
- name: default
port: 42
protocol: TCP
targetPort: 42
selector:
workload.user.cattle.io/workloadselector: deployment-desc-tom-tom-rknop-dev-app
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-app
sessionAffinity: None
type: ClusterIP
status:
Expand All @@ -112,15 +150,23 @@ status:
apiVersion: v1
kind: Service
metadata:
annotations:
field.cattle.io/targetWorkloadIds: '["deployment:desc-tom-rknop-dev:tom-app"]'
name: tom-ingress-service
namespace: desc-tom
namespace: desc-tom-rknop-dev
ownerReferences:
- apiVersion: v1beta1/extensions
controller: true
kind: Ingress
name: tom-app
uid: e8f8e9fa-9bb1-475a-810d-483dbc7f6bc8
spec:
ports:
- port: 8080
protocol: TCP
targetPort: 8080
selector:
workload.user.cattle.io/workloadselector: deployment-desc-tom-tom-rknop-dev-app
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-app
sessionAffinity: None
type: ClusterIP
status:
Expand All @@ -130,18 +176,20 @@ apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nersc.gov/clustername: development
nersc.gov/clustername: production
nginx.ingress.kubernetes.io/preserve-trailing-slash: "true"
nginx.ingress.kubernetes.io/proxy-body-size: 500m
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
generation: 3
generation: 1
labels:
cattle.io/creator: norman
name: tom-rknop-dev-app-ingress
namespace: desc-tom
name: tom-desc-app
namespace: desc-tom-rknop-dev
selfLink: /apis/networking.k8s.io/v1/namespaces/desc-tom-rknop-dev/ingresses/tom-app
spec:
ingressClassName: nginx
rules:
- host: tom-rknop-dev-app-ingress.desc-tom.development.svc.spin.nersc.org
- host: tom-desc-app.desc-tom-rknop-dev.production.svc.spin.nersc.org
http:
paths:
- backend:
Expand All @@ -162,5 +210,5 @@ spec:
tls:
- hosts:
- desc-tom-rknop-dev.lbl.gov
secretName: desc-tom-rknop-dev-cert
secretName: tom-cert
---
125 changes: 125 additions & 0 deletions spin_admin/rknop-dev/tom-brokerpoll.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: tom-brokerpoll
namespace: desc-tom-rknop-dev
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-brokerpoll
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
cattle.io/timestamp: "2023-02-06T16:44:40Z"
field.cattle.io/ports: '[[]]'
nersc.gov/collab_uids: 70268,99896
nersc.gov/gid: "95089"
nersc.gov/gids: 45703,60152,57177,58102,59318,60070,63477,64483,79186,70268,92576,94721,95089,96414,99650
nersc.gov/roles: user
nersc.gov/uid: "95089"
nersc.gov/username: raknop
creationTimestamp: null
labels:
workload.user.cattle.io/workloadselector: deployment-desc-tom-rknop-dev-tom-brokerpoll
spec:
containers:
- env:
- name: DB_HOST
value: tom-postgres
- name: DB_NAME
value: tom_desc
- name: DB_PASS
value: fragile
- name: DB_USER
value: postgres
- name: CASSANDRA_HOST
value: tom-cassandra
- name: CASSANDRA_DB
value: tom_desc
- name: CASSANDRA_USER
value: cassandra
- name: CASSANDRA_PASSWORD
value: cassandra
- name: ALERCE_TOPIC_RELDATEOFFSET
value: '-36'
- name: ALERCE_KAFKA_SERVER
value: b-3-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196,b-2-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196,b-1-public.publicproduction.o8ncxm.c18.kafka.us-east-1.amazonaws.com:9196
- name: FINK_GROUP_ID
value: lsstfr-johann
- name: FINK_SERVER
value: 134.158.74.95:24499,
- name: FINK_TOPIC
value: fink_early_sn_candidates_ztf
- name: FINK_USERNAME
value: johann
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /secrets/pitt_google_auth_key.json
- name: GOOGLE_CLOUD_PROJECT
value: elasticc-challenge
image: registry.nersc.gov/m1727/raknop/tom_server_bindmount
imagePullPolicy: Always
name: tom-brokerpoll
resources: {}
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
privileged: false
readOnlyRootFilesystem: false
runAsNonRoot: true
runAsUser: 95089
stdin: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
tty: true
volumeMounts:
- mountPath: /secrets
name: tom-secrets
- mountPath: /tom_desc
name: tom-deployment
command: [ "dumb-init" ]
args:
- -r
- 15:10
- python
- manage.py
- brokerpoll2
- --do-antares
- --antares-topic
- elasticc2-2
- --do-pitt
- --pitt-project
- elasticc-challenge
- --pitt-topic
- elasticc-SuperNNova
- --do-alerce
dnsConfig: {}
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 57177
terminationGracePeriodSeconds: 30
imagePullSecrets:
- name: registry-nersc
volumes:
- name: tom-secrets
secret:
defaultMode: 256
optional: false
secretName: tom-secrets
- hostPath:
path: /global/cfs/cdirs/lsst/groups/TD/SOFTWARE/tom_deployment/rknop_dev/tom_desc/tom_desc
type: Directory
name: tom-deployment
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ data:
tls.key: PUT THE RIGHT THING HERE
kind: Secret
metadata:
name: desc-tom-rknop-dev-cert
namespace: desc-tom
name: tom-cert
namespace: desc-tom-rknop-dev
type: kubernetes.io/tls
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tom-rknop-dev-postgres-pvc
namespace: desc-tom
name: rknop-dev-mongodb-pvc-20241030
namespace: desc-tom-rknop-dev
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200Gi
storage: 2048Gi
storageClassName: nfs-client
volumeMode: Filesystem
Loading

0 comments on commit 7593b17

Please sign in to comment.