Merge pull request #37 from germanfgv/master

Migrates to Python3 and add RepackStats

setup.py

@@ -29,7 +29,7 @@ def define_the_build(self, dist, system_name, patch_x = ''):
   # as we have a makefile to build all the rest above anyway. Any c++
   # products we built go into data_files as python just needs to copy them.
   # Headers get special handling, we map them automatically to directories.
-  py_version = (string.split(sys.version))[0]
+  py_version = sys.version.split()[0]
   pylibdir = '%slib/python%s/site-packages' % (patch_x, py_version[0:3])
   dist.py_modules = ['T0WmaDataSvc.__init__']
   dist.packages = system['python']
@@ -59,7 +59,7 @@ def initialize_options(self):
 
   def finalize_options(self):
     if self.system not in systems:
-      print "System %s unrecognised, please use '-s T0WmaDataSvc'" % self.system
+      print("System %s unrecognised, please use '-s T0WmaDataSvc'" % self.system)
       sys.exit(1)
 
     # Expand various sources and maybe do the c++ build.
@@ -103,10 +103,10 @@ def initialize_options(self):
   def finalize_options(self):
     # Check options.
     if self.system not in systems:
-      print "System %s unrecognised, please use '-s T0WmaDataSvc'" % self.system
+      print("System %s unrecognised, please use '-s T0WmaDataSvc'" % self.system)
       sys.exit(1)
     if self.patch and not os.path.isdir("%s/xbin" % self.prefix):
-      print "Patch destination %s does not look like a valid location." % self.prefix
+      print("Patch destination %s does not look like a valid location." % self.prefix)
       sys.exit(1)
 
     # Expand various sources, but don't build anything from c++ now.

src/python/Data.py

@@ -9,6 +9,7 @@
 from T0WmaDataSvc.DataRunDatasetDone import *
 from T0WmaDataSvc.DataPromptRecoStatus import *
 from T0WmaDataSvc.DataDatasetLocked import *
+from T0WmaDataSvc.DataRepackStats import *
 
 class Data(DatabaseRESTApi):
   """Server object for REST data access API."""
@@ -29,5 +30,6 @@ def __init__(self, app, config, mount):
                 "run_stream_done": RunStreamDone(app, self, config, mount),
                 "run_dataset_done": RunDatasetDone(app,self, config, mount),
                 "dataset_locked": DatasetLocked(app, self, config, mount),
-                "promptreco_status": PromptRecoStatus(app, self, config, mount)
+                "promptreco_status": PromptRecoStatus(app, self, config, mount),
+                "repack_stats": RepackStats(app, self, config, mount)
                 })

src/python/DataRepackStats.py

@@ -0,0 +1,38 @@
+from WMCore.REST.Server import RESTEntity, restcall, rows
+from WMCore.REST.Tools import tools
+from WMCore.REST.Validation import *
+from WMCore.REST.Format import JSONFormat, PrettyJSONFormat
+from T0WmaDataSvc.Regexps import *
+from operator import itemgetter
+
+class RepackStats(RESTEntity):
+  """REST entity for retrieving run/stream processing status."""
+  def validate(self, apiobj, method, api, param, safe):
+    """Validate request input data."""
+    validate_str('run', param, safe, RX_RUN, optional = False)
+
+  @restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())])
+  @tools.expires(secs=300)
+  def get(self,run):
+    """Check run mean repacking time
+
+    :arg int run: the run number
+    :returns: median and mean repacking time in hours"""
+
+    sql = """select MEDIAN(time) as median, AVG(time) as mean 
+                  from (SELECT (cast(t0_repacked_time as date) - cast(t0_checked_time as date))*24 as time
+                        FROM file_transfer_status_offline
+                        WHERE filename like '%'||:run||'%')
+          """
+
+    c, _ = self.api.execute(sql, run = run)
+
+    runs=[]
+    for result in c.fetchall():
+      if result[0]:
+        stats={'median' : result[0],
+               'mean' : result[1]
+               }
+        runs.append(stats)
+
+    return runs

test/python/T0WmaDataSvc_t/RESTSites_t.py

@@ -10,6 +10,7 @@ class RESTSites_t(unittest.TestCase):
 
     def setUp(self):
         self.goodurls = [('/hello/', {'rows': ['world']})]
+        self.meantime = [('/repack_mean_time/?run=341169', {'rows': ['world']})]
         self.badurls = [('/wronghello/', {})]
 
     def testGoodGetJSON(self):
@@ -30,6 +31,10 @@ def testBaadGetDefault(self):
         req = Requests('localhost:8308')
         self.runReq(self.badurls, req, 400)
 
+    def testMeanTime(self):
+        req = Requests('localhost:8308')
+        self.runReq(self.meantime, req, 400)
+
     def runReq(self, urls, req, code):
         #TODO: check keys returned are good
         for u in urls: