diff --git a/Makefile b/Makefile index 9af878d7..adb291d6 100644 --- a/Makefile +++ b/Makefile @@ -8,13 +8,19 @@ RPMBUILD_DIR=${HOME}/rpmbuild specfile := $(ROOTDIR)/config/spec/$(NAME).spec versionfile := ${ROOTDIR}/lib/version.py tarball_dirs = bin lib etc man templates config # Dirs we need for the tarball +OSR=$(shell /bin/sh -c ' . /etc/os-release; echo $$VERSION_ID | sed -e s/\..*//') + + -tracing-name := $(NAME)_tracing_deps -tracing-specfile := $(ROOTDIR)/config/spec/$(tracing-name).spec -all: tarball set-version tracing-rpm rpm clean .PHONY: all clean tarball set-version rpm clean-all +ifeq($OSR,7) +# we only need the tracing-rpm on SL7/EL7 + +all: tarball set-version tracing-rpm rpm clean +tracing-name := $(NAME)_tracing_deps +tracing-specfile := $(ROOTDIR)/config/spec/$(tracing-name).spec tracing-rpm: NOWFILE := $(shell mktemp) tracing-rpm: rpmSpecsDir := $(RPMBUILD_DIR)/SPECS tracing-rpm: @@ -24,6 +30,11 @@ tracing-rpm: find $(RPMBUILD_DIR)/RPMS -type f -name "$(tracing-name)*.rpm" -newer $(NOWFILE) -exec cp {} $(ROOTDIR) \; echo "Created RPM and copied it to current working directory" ((test -e $(NOWFILE)) && (rm $(NOWFILE)) && echo "Cleaned up tempfile") || echo "$(NOWFILE) does not exist. Continuing anyway" +else + +all: tarball set-version rpm clean + +endif rpm: rpmSourcesDir := $(RPMBUILD_DIR)/SOURCES rpm: rpmSpecsDir := $(RPMBUILD_DIR)/SPECS diff --git a/bin/jobsub_cmd b/bin/jobsub_cmd index f5b602db..75ade559 100755 --- a/bin/jobsub_cmd +++ b/bin/jobsub_cmd @@ -176,10 +176,9 @@ def main() -> None: if cmd == "jobsub_q": cmd_jobsub_q = True - # add -global -schedd-constraint IsJobsubLite==True + # add -schedd-constraint IsJobsubLite==True execargs.insert(0, "IsJobsubLite==True") execargs.insert(0, "-schedd-constraint") - execargs.insert(0, "-global") if default_constraint: execargs.extend( diff --git a/lib/condor.py b/lib/condor.py index b8e2f95d..13c2af4d 100644 --- a/lib/condor.py +++ b/lib/condor.py @@ -305,6 +305,7 @@ def submit( check=False, ) sys.stdout.write(output.stdout) + sys.stderr.write(output.stderr) hl = f"\n{'=-'*30}\n\n" # highlight line to make result stand out diff --git a/lib/dagnabbit.py b/lib/dagnabbit.py index 48efa70a..8625dff3 100644 --- a/lib/dagnabbit.py +++ b/lib/dagnabbit.py @@ -174,6 +174,16 @@ def parse_dagnabbit( raise # print(f"vars(res): {repr(vars(res))}") # handle -f drobpox: etc. in dag stages + + if res.need_storage_modify: + sys.stderr.write(f"Warning at file {dagfile} line {linenum}\n") + sys.stderr.write(f"flag --need-storage-modify ignored in DAG\n") + sys.stderr.flush() + if res.need_scope: + sys.stderr.write(f"Warning at file {dagfile} line {linenum}\n") + sys.stderr.write(f"flag --need-scope ignored in DAG\n") + sys.stderr.flush() + do_tarballs(res) thesevalues = values.copy() thesevalues["mail"] = "never" diff --git a/lib/tarfiles.py b/lib/tarfiles.py index b8c85d96..468c8bbe 100644 --- a/lib/tarfiles.py +++ b/lib/tarfiles.py @@ -356,6 +356,7 @@ def tarfile_in_dropbox(args: argparse.Namespace, origtfn: str) -> Optional[str]: publisher.update_cid() elif args.use_dropbox == "pnfs": + cred_set = get_creds(vars(args)) location = dcache_persistent_path(args.group, tfn) existing = fake_ifdh.ls(location) if existing: @@ -400,8 +401,8 @@ def __init__( self.cid = cid self.cid_url = _quote(cid, safe="") # Encode CID for passing to URL self.verbose = verbose - self.__auth_kwargs: Dict[str, Any] + print(f"Publishing hash {cid} to RCDS:") if cred_set.token is not None: print( f"Using bearer token located at {cred_set.token} to authenticate to RCDS" diff --git a/templates/simple/simple.cmd b/templates/simple/simple.cmd index cf1cb150..57a9fcfc 100644 --- a/templates/simple/simple.cmd +++ b/templates/simple/simple.cmd @@ -15,7 +15,7 @@ JOBSUBJOBSECTION=$(Process) TRACEPARENT="{{traceparent}}" +TraceParent=$(TRACEPARENT) -environment = CM1=$(CM1);CM2=$(CM2);CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};{% if token is defined and token %}BEARER_TOKEN_FILE=.condor_creds/{% if role is defined and role and role != 'Analysis' %}{{group}}_{{role | lower}}_{{oauth_handle}}.use{%else%}{{group}}_{{oauth_handle}}.use{%endif%}{% endif %};CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};TRACEPARENT=$(TRACEPARENT);{{environment|join(';')}} +environment = CM1=$(CM1);CM2=$(CM2);CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};{% if token is defined and token %}BEARER_TOKEN_FILE=.condor_creds/{% if role is defined and role and role != 'Analysis' %}{{group}}_{{role | lower}}_{{oauth_handle}}.use{%else%}{{group}}_{{oauth_handle}}.use{%endif%}{% endif %};CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};TRACEPARENT={{traceparent}};{{environment|join(';')}} rank = Mips / 2 + Memory job_lease_duration = 3600 transfer_output = True diff --git a/tests/test_submit_wait_int.py b/tests/test_submit_wait_int.py index c6e595be..7f650ba6 100644 --- a/tests/test_submit_wait_int.py +++ b/tests/test_submit_wait_int.py @@ -209,7 +209,7 @@ def run_launch(cmd, expected_out=1, get_dir=False): def lookaround_launch(extra, verify_files=""): """Simple submit of our lookaround script""" assert run_launch( - f"jobsub_submit --mail-never --verbose=1 -e SAM_EXPERIMENT {extra} file://`pwd`/job_scripts/lookaround.sh {verify_files}" + f"jobsub_submit --mail-never --verbose=2 -e SAM_EXPERIMENT {extra} file://`pwd`/job_scripts/lookaround.sh {verify_files}" ) @@ -374,7 +374,7 @@ def fife_launch(extra): """ jobsub_submit \ --mail-never \ - --verbose=1 \ + --verbose=2 \ -e EXPERIMENT \ -e IFDH_DEBUG \ -e IFDH_FORCE \