Bug fixes

Makefile

@@ -8,13 +8,19 @@ RPMBUILD_DIR=${HOME}/rpmbuild
 specfile := $(ROOTDIR)/config/spec/$(NAME).spec
 versionfile := ${ROOTDIR}/lib/version.py
 tarball_dirs = bin lib etc man templates config # Dirs we need for the tarball
+OSR=$(shell /bin/sh -c ' . /etc/os-release; echo $$VERSION_ID | sed -e s/\..*//')
+
+
 
-tracing-name := $(NAME)_tracing_deps
-tracing-specfile := $(ROOTDIR)/config/spec/$(tracing-name).spec
 
-all: tarball set-version tracing-rpm rpm clean
 .PHONY: all clean tarball set-version rpm clean-all
 
+ifeq($OSR,7)
+# we only need the tracing-rpm on SL7/EL7
+
+all: tarball set-version tracing-rpm rpm clean
+tracing-name := $(NAME)_tracing_deps
+tracing-specfile := $(ROOTDIR)/config/spec/$(tracing-name).spec
 tracing-rpm: NOWFILE := $(shell mktemp)
 tracing-rpm: rpmSpecsDir := $(RPMBUILD_DIR)/SPECS
 tracing-rpm:
@@ -24,6 +30,11 @@ tracing-rpm:
 	find $(RPMBUILD_DIR)/RPMS -type f -name "$(tracing-name)*.rpm" -newer $(NOWFILE) -exec cp {} $(ROOTDIR) \;
 	echo "Created RPM and copied it to current working directory"
 	((test -e $(NOWFILE)) && (rm $(NOWFILE)) && echo "Cleaned up tempfile") || echo "$(NOWFILE) does not exist.  Continuing anyway"
+else
+
+all: tarball set-version rpm clean
+
+endif
 
 rpm: rpmSourcesDir := $(RPMBUILD_DIR)/SOURCES
 rpm: rpmSpecsDir := $(RPMBUILD_DIR)/SPECS

bin/jobsub_cmd

@@ -176,10 +176,9 @@ def main() -> None:
     if cmd == "jobsub_q":
         cmd_jobsub_q = True
 
-        # add -global -schedd-constraint IsJobsubLite==True
+        # add -schedd-constraint IsJobsubLite==True
         execargs.insert(0, "IsJobsubLite==True")
         execargs.insert(0, "-schedd-constraint")
-        execargs.insert(0, "-global")
 
         if default_constraint:
             execargs.extend(

lib/condor.py

@@ -305,6 +305,7 @@ def submit(
                 check=False,
             )
             sys.stdout.write(output.stdout)
+            sys.stderr.write(output.stderr)
 
         hl = f"\n{'=-'*30}\n\n"  # highlight line to make result stand out
 

lib/dagnabbit.py

@@ -174,6 +174,16 @@ def parse_dagnabbit(
                         raise
                     # print(f"vars(res): {repr(vars(res))}")
                     # handle -f drobpox: etc. in dag stages
+
+                    if res.need_storage_modify:
+                        sys.stderr.write(f"Warning at file {dagfile} line {linenum}\n")
+                        sys.stderr.write(f"flag --need-storage-modify ignored in DAG\n")
+                        sys.stderr.flush()
+                    if res.need_scope:
+                        sys.stderr.write(f"Warning at file {dagfile} line {linenum}\n")
+                        sys.stderr.write(f"flag --need-scope ignored in DAG\n")
+                        sys.stderr.flush()
+
                     do_tarballs(res)
                     thesevalues = values.copy()
                     thesevalues["mail"] = "never"

lib/tarfiles.py

@@ -356,6 +356,7 @@ def tarfile_in_dropbox(args: argparse.Namespace, origtfn: str) -> Optional[str]:
             publisher.update_cid()
 
     elif args.use_dropbox == "pnfs":
+        cred_set = get_creds(vars(args))
         location = dcache_persistent_path(args.group, tfn)
         existing = fake_ifdh.ls(location)
         if existing:
@@ -400,8 +401,8 @@ def __init__(
         self.cid = cid
         self.cid_url = _quote(cid, safe="")  # Encode CID for passing to URL
         self.verbose = verbose
-
         self.__auth_kwargs: Dict[str, Any]
+        print(f"Publishing hash {cid} to RCDS:")
         if cred_set.token is not None:
             print(
                 f"Using bearer token located at {cred_set.token} to authenticate to RCDS"

templates/simple/simple.cmd

@@ -15,7 +15,7 @@ JOBSUBJOBSECTION=$(Process)
 TRACEPARENT="{{traceparent}}"
 +TraceParent=$(TRACEPARENT)
 
-environment        = CM1=$(CM1);CM2=$(CM2);CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};{% if token is defined and token %}BEARER_TOKEN_FILE=.condor_creds/{% if role is defined and role and role != 'Analysis' %}{{group}}_{{role | lower}}_{{oauth_handle}}.use{%else%}{{group}}_{{oauth_handle}}.use{%endif%}{% endif %};CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};TRACEPARENT=$(TRACEPARENT);{{environment|join(';')}}
+environment        = CM1=$(CM1);CM2=$(CM2);CLUSTER=$(Cluster);PROCESS=$(Process);JOBSUBJOBSECTION=$(JOBSUBJOBSECTION);CONDOR_TMP={{outdir}};{% if token is defined and token %}BEARER_TOKEN_FILE=.condor_creds/{% if role is defined and role and role != 'Analysis' %}{{group}}_{{role | lower}}_{{oauth_handle}}.use{%else%}{{group}}_{{oauth_handle}}.use{%endif%}{% endif %};CONDOR_EXEC=/tmp;DAGMANJOBID=$(DAGManJobId);GRID_USER={{user}};JOBSUBJOBID=$(CLUSTER).$(PROCESS)@{{schedd}};EXPERIMENT={{group}};TRACEPARENT={{traceparent}};{{environment|join(';')}}
 rank               = Mips / 2 + Memory
 job_lease_duration = 3600
 transfer_output    = True

tests/test_submit_wait_int.py

@@ -209,7 +209,7 @@ def run_launch(cmd, expected_out=1, get_dir=False):
 def lookaround_launch(extra, verify_files=""):
     """Simple submit of our lookaround script"""
     assert run_launch(
-        f"jobsub_submit --mail-never --verbose=1 -e SAM_EXPERIMENT {extra} file://`pwd`/job_scripts/lookaround.sh {verify_files}"
+        f"jobsub_submit --mail-never --verbose=2 -e SAM_EXPERIMENT {extra} file://`pwd`/job_scripts/lookaround.sh {verify_files}"
     )
 
 
@@ -374,7 +374,7 @@ def fife_launch(extra):
         """
         jobsub_submit \
           --mail-never \
-          --verbose=1 \
+          --verbose=2 \
           -e EXPERIMENT \
           -e IFDH_DEBUG \
           -e IFDH_FORCE \