【chunkserver & ut】split read-write thread and add localut.sh

conf/chunkserver.conf.example

@@ -141,11 +141,14 @@ storeng.sync_write=false
 
 #
 # Concurrent apply module
-#
-# 并发模块的并发度，一般是10
-concurrentapply.size=10
-# 并发模块线程的队列深度
-concurrentapply.queuedepth=1
+# 并发模块写线程的并发度，一般是10
+wconcurrentapply.size=10
+# 并发模块写线程的队列深度
+wconcurrentapply.queuedepth=1
+# 并发模块读线程的并发度，一般是5
+rconcurrentapply.size=5
+# 并发模块读线程的队列深度
+rconcurrentapply.queuedepth=1
 
 #
 # Chunkfile pool

coverage/check_coverage.sh

@@ -0,0 +1,118 @@
+#!/bin/bash 
+
+line_cover_base=75
+mds_branch_base=70
+snapshot_branch_base=70
+client_branch_base=78
+other_branch_base=65
+line_cover_all=`cat coverage/index.html | grep -A 5  "Lines"  | grep % | awk -F "%" '{print $1}' | awk -F '>' '{print $2}' | awk -F '.' '{print $1}'`
+
+if(("$line_cover_all" < "$line_cover_base"))
+then
+    echo "line cover not ok!.";
+    echo $line_cover_all;
+    exit -1
+else
+    echo "line cover ok!.";
+    echo $line_cover_all;    
+fi
+
+for i in `find coverage -type d | grep mds`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > mds.all
+if [ -s mds.all ]; then    
+    mds_branch=`cat mds.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    mds_branch=0
+fi
+if(("$mds_branch" < "$mds_branch_base"))
+then
+    echo "mds_branch cover not ok!.";
+    echo $mds_branch;
+    exit -1
+else
+    echo "mds_branch cover ok!.";
+    echo $mds_branch;    
+fi
+
+for i in `find coverage -type d | grep tools`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > tools.all
+if [ -s tools.all ]; then
+    tools_branch=`cat tools.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    tools_branch=0
+fi
+
+if(("$tools_branch" < "$other_branch_base"))
+then
+    echo "tools_branch cover not ok!.";
+    echo $tools_branch;
+    #exit -1
+else
+    echo "tools_branch cover ok!.";
+    echo $tools_branch;    
+fi
+
+for i in `find coverage -type d | grep common`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > common.all
+if [ -s common.all ]; then
+    common_branch=`cat common.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    common_branch=0
+fi	
+if(("$common_branch" < "$other_branch_base"))
+then
+    echo "common_branch cover not ok!.";
+    echo $common_branch;
+    exit -1
+else
+    echo "common_branch cover ok!.";
+    echo $common_branch;    
+fi
+
+
+for i in `find coverage -type d | grep chunkserver`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > chunkserver.all
+if [ -s chunkserver.all ]; then
+    chunkserver_branch=`cat chunkserver.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    chunkserver_branch=0
+fi
+if(("$chunkserver_branch" < "$other_branch_base"))
+then
+    echo "chunkserver_branch cover not ok!.";
+    echo $chunkserver_branch;
+    exit -1
+else
+    echo "chunkserver_branch cover ok!.";
+    echo $chunkserver_branch;    
+fi
+
+
+for i in `find coverage/client -type d | grep client`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > client.all
+if [ -s chunkserver.all ]; then
+    client_branch=`cat client.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    client_branch=0
+fi
+if(("$client_branch" < "$client_branch_base"))
+then
+    echo "client_branch cover not ok!.";
+    echo $client_branch;
+    exit -1
+else
+    echo "client_branch cover ok!.";
+    echo $client_branch;    
+fi
+
+for i in `find coverage -type d | grep fs`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > sfs.all
+if [ -s sfs.all ]; then
+    sfs_branch=`cat sfs.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    sfs_branch=0
+fi
+if(("$sfs_branch" < "$other_branch_base"))
+then
+    echo "sfs_branch cover not ok!.";
+    echo $sfs_branch;
+    #exit -1
+else
+    echo "sfs_branch cover ok!.";
+    echo $sfs_branch;    
+fi
+

coverage/filterbr.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <[email protected]>, GPLv3
+
+import sys
+
+def skip_comments(lines):
+  state = 0
+  for line in lines:
+    n = len(line)
+    l = ''
+    p = 0
+    while p < n:
+      if state == 0:
+        a = line.find('//', p)
+        b = line.find('/*', p)
+        if a > -1 and (a < b or b == -1):
+          l += line[p:a]
+          p = n
+        elif b > -1 and (b < a or a == -1):
+          l += line[p:b]
+          p = b+2
+          state = 1
+        else:
+          l += line[p:]
+          p = n
+      elif state == 1:
+        a = line.rfind('*/', p)
+        if a == -1:
+          p = n
+        else:
+          p = a + 2
+          state = 0
+    yield l
+
+def cond_lines(lines):
+  state = 0
+  pcnt = 0
+  for nr, line in enumerate(lines, 1):
+    if not line:
+      continue
+    n = len(line)
+    p = 0
+    do_yield = False
+    while p < n:
+      if state == 0:
+#        p = line.strip().startswith('if', p)
+        p = line.find('if', p)
+        if p == -1:
+          p = n
+          continue
+        if (p == 0 or not line[p-1].isalpha()) \
+            and (p+2 == len(line) or not line[p+2].isalpha()):
+          do_yield = True
+          state = 1
+        p += 2
+      elif state == 1:
+        do_yield = True
+        p = line.find('(', p)
+        if p == -1:
+          p = n
+        else:
+          p += 1
+          state = 2
+          pcnt = 1
+      elif state == 2:
+        do_yield = True
+        for p in range(p, n):
+          if line[p] == '(':
+            pcnt += 1
+          elif line[p] == ')':
+            pcnt -= 1
+          if not pcnt:
+            state = 0
+            break
+        p += 1
+    if do_yield:
+      yield nr
+
+def cond_lines_from_file(filename):
+  with open(filename) as f:
+    yield from cond_lines(skip_comments(f))
+
+def filter_lcov_trace(lines):
+  nrs = set()
+  for line in lines:
+    if line.startswith('SF:'):
+      nrs = set(cond_lines_from_file(line[3:-1]))
+    elif line.startswith('BRDA:'):
+      xs = line[5:].split(',')
+      nr = int(xs[0]) if xs else 0
+      if nr not in nrs:
+        continue
+    yield line
+
+def filter_lcov_trace_file(s_filename, d_file):
+  with open(s_filename) as f:
+    for l in filter_lcov_trace(f):
+      print(l, end='', file=d_file)
+
+if __name__ == '__main__':
+  #for l in cond_lines_from_file(sys.argv[1]):
+  #  print(l)
+
+  filter_lcov_trace_file(sys.argv[1], sys.stdout)
+
+  #with open(sys.argv[1]) as f:
+  #  for l in skip_comments(f):
+  #    print(l)
+
+

coverage/gen-coverage-nebd.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <[email protected]>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' , '*/client_proto/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+