From b528fd385102aa91e7fc2f6074748db5aa484830 Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 6 Jan 2016 15:19:25 +0800 Subject: [PATCH 1/6] add io dump tool --- optools/io_dump.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 optools/io_dump.py diff --git a/optools/io_dump.py b/optools/io_dump.py new file mode 100644 index 00000000..c32f88be --- /dev/null +++ b/optools/io_dump.py @@ -0,0 +1,84 @@ +# -*- coding:utf-8 -*- +import os +import time +import datetime +import sys +# dump process_name io usage on device +class IoDump(object): + def __init__(self, process_name=[], top_size = 10): + self.process_data = {} + self.process_name = process_name + + def readable(self, num): + for unit in ['','K','M','G','T','P','E','Z']: + if abs(num) < 1024.0: + return "%3.1f%s"%(num, unit) + num /= 1024.0 + return "%.1f%s" % (num, 'Y') + + def collect(self): + entries = os.listdir("/proc") + statistics = {} + for entry in entries: + try: + pid = int(entry) + exe_path = os.path.realpath("/proc/%d/exe"%pid) + offset = exe_path.rfind("/") + bin = exe_path[offset+1:] + if bin in self.process_name: + status, read_bytes_ps, write_bytes_ps = self.calc_process(pid) + if not status: + continue + stat = statistics.get(bin,{"read_bytes_ps":0, "write_bytes_ps":0}) + stat["read_bytes_ps"] += read_bytes_ps + stat["write_bytes_ps"] += write_bytes_ps + statistics[bin] = stat + except Exception,e: + pass + stat_array = [] + for key in statistics: + item = {"name":key} + item.update(statistics[key]) + stat_array.append(item) + sorted_array = sorted(stat_array, key=lambda item : item["read_bytes_ps"], reverse = True) + output = [] + now = time.time() + output.append("-------------------------------------------------") + output.append(datetime.datetime.fromtimestamp(now).strftime('%Y-%m-%d %H:%M:%S')) + for item in sorted_array: + output.append("%s %s/s %s/s"%(item["name"], self.readable(item["read_bytes_ps"]), + self.readable(item["write_bytes_ps"]))) + print "\n".join(output) + + def calc_process(self, pid): + data = {'read':0, + 'write':0, + 'cancell':0} + fd = open("/proc/%d/io"%pid, "r") + for line in fd.readlines(): + parts = line.replace("\n","").split(":") + if parts[0] == "read_bytes": + data["read"] = int(parts[1][1:]) + elif parts[0] == "write_bytes": + data["write"] = int(parts[1][1:]) + elif parts[0] == "cancelled_write_bytes": + data["cancell"] = int(parts[1][1:]) + fd.close() + if pid in self.process_data: + old_data = self.process_data[pid] + read_bytes_ps = data["read"] - old_data["read"] + write_bytes_ps = (data["write"] - data["cancell"]) - (old_data["write"] - old_data["cancell"]) + self.process_data[pid] = data + return True, read_bytes_ps, write_bytes_ps + else: + self.process_data[pid] = data + return False, 0, 0 + +def main(): + dump = IoDump(sys.argv) + while True: + dump.collect() + time.sleep(1) + +if __name__ == "__main__": + main() From f623c1422d8a8be834999838851453b26470507e Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 6 Jan 2016 15:24:48 +0800 Subject: [PATCH 2/6] add label --- optools/io_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optools/io_dump.py b/optools/io_dump.py index c32f88be..d2affc99 100644 --- a/optools/io_dump.py +++ b/optools/io_dump.py @@ -46,7 +46,7 @@ def collect(self): output.append("-------------------------------------------------") output.append(datetime.datetime.fromtimestamp(now).strftime('%Y-%m-%d %H:%M:%S')) for item in sorted_array: - output.append("%s %s/s %s/s"%(item["name"], self.readable(item["read_bytes_ps"]), + output.append("%s read:%s/s write:%s/s"%(item["name"], self.readable(item["read_bytes_ps"]), self.readable(item["write_bytes_ps"]))) print "\n".join(output) From ba092ddd5dd98c0b3ba407e7f5638d4ab6686ac1 Mon Sep 17 00:00:00 2001 From: imotai Date: Tue, 19 Jan 2016 14:01:58 +0800 Subject: [PATCH 3/6] add preempt tool --- optools/preempt.py | 102 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 optools/preempt.py diff --git a/optools/preempt.py b/optools/preempt.py new file mode 100644 index 00000000..72880f24 --- /dev/null +++ b/optools/preempt.py @@ -0,0 +1,102 @@ +import subprocess +import sys +import os +import json +def get_jobs(): + p = subprocess.Popen(["./galaxy","jobs"], stdout=subprocess.PIPE) + out, err = p.communicate() + if err: + print "fail to list jobs" + sys.exit(-1) + lines = out.splitlines() + # job id and job name pair + jobs = {} + for line in lines[2:]: + parts = line.split(" ") + if len(parts) < 3: + continue + jobs[parts[2].strip()] = parts[3].strip() + return jobs + +def get_all_pods(jobs): + pods = {} + for jobid in jobs: + print "get pods from job %s"%jobid + p = subprocess.Popen(["./galaxy", "pods", "--j=%s"%jobid], stdout=subprocess.PIPE) + out, err = p.communicate() + lines = out.splitlines() + for line in lines[2:]: + parts = line.split(" ") + if len(parts) < 3: + continue + pods[parts[2].strip()] = {"jobid": jobid, "name":jobs[jobid], "state":parts[3].strip()} + return pods + +def preempt(pods, jobid, podid, endpoint): + pods_on_agent = [] + p = subprocess.Popen(["./galaxy", "pods", "--e=%s"%endpoint], stdout=subprocess.PIPE) + out, err = p.communicate() + if err: + print "fail to ./galaxy pods -e %s for err %s"%(endpoint, err) + sys.exit(-1) + lines = out.splitlines() + print " %s will preempt the following pods on %s:"%(pods[podid]["name"], endpoint) + for line in lines[2:]: + parts = line.split(" ") + if len(parts) < 2: + continue + ipodid = parts[2].strip() + if ipodid not in pods: + continue + name = pods[ipodid]["name"] + if name.find("nfs") != -1: + continue + pods_on_agent.append(ipodid) + print name + yes = raw_input('sure to preempt (y/n):') + if yes != "y": + return False + preempt_json = { + "addr":endpoint, + "pending_pod":{ + "jobid":jobid, + "podid":podid + }, + "preempted_pods":[] + } + for pod in pods_on_agent: + preempt_json["preempted_pods"].append({"jobid": pods[pod]["jobid"], "podid":pod}) + filename = endpoint.replace(".", "_").replace(":", "_")+ ".json" + with open(filename, "wb+") as fd: + content = json.dumps(preempt_json) + fd.write(content) + p = subprocess.Popen(["./galaxy", "preempt", "--f=%s"%filename], stdout=subprocess.PIPE) + out, err = p.communicate() + if p.returncode == 0: + return True + return False + +def real_main(jobid, podid, endpoint): + pods = {} + if os.path.isfile("pods_cache"): + print "use pods_cache to load pods" + with open("pods_cache", "rb") as fd: + pods = json.load(fd) + else: + print "get pods from galaxy" + jobs = get_jobs() + pods = get_all_pods(jobs) + with open("pods_cache", "wb") as fd: + fd.write(json.dumps(pods)) + ok = preempt(pods, jobid, podid, endpoint) + if ok : + print "preempt for pod %s successfully"%podid + else: + print "fail to preempt for pod %s "%podid + +if __name__ == "__main__": + if len(sys.argv) < 4: + print "python preempt.py jobid podid endpoint" + sys.exit(-1) + real_main(sys.argv[1], sys.argv[2], sys.argv[3]) + From 39f797af48d7e6ee3bb0c4e1f8163486bf5bd53b Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 20 Jan 2016 11:30:27 +0800 Subject: [PATCH 4/6] update galaxy --- optools/galaxy | 83 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/optools/galaxy b/optools/galaxy index 6e014608..a09ca9b4 100755 --- a/optools/galaxy +++ b/optools/galaxy @@ -8,11 +8,90 @@ # # processname: galaxy -PROD_ADDR=ftp://xxxxxx/tmp/galaxy.tar.gz +PROD_ADDR=ftp://yq01-ps-rtg0000.yq01.baidu.com/home/galaxy/prod/galaxy.tar.gz . /etc/rc.d/init.d/functions AGENT_IP=`hostname -i` AGENT_HOME=/home/galaxy/agent RETVAL=0 + +netCards=$(/sbin/ifconfig | awk '/Ethernet/{a=$0; getline; b=$0; print a,b}' | grep "inet addr" | awk '{print $1}') + +readd_mod() { + local DEV=$1 + local IP=$(cat /etc/sysconfig/network-scripts/ifcfg-${DEV} | grep IPADDR | cut -d "=" -f 2) + local GATEWAY=$(ip route | grep ${DEV} | grep via | awk '{print $3}' | head -1) + if ! grep "GATEWAY" /etc/sysconfig/network-scripts/ifcfg-${DEV} > /dev/null; then + echo "GATEWAY=${GATEWAY}" >> /etc/sysconfig/network-scripts/ifcfg-${DEV} + fi + local ETH=${DEV} + local driver=`ethtool -i ${ETH} | grep driver | awk '{print $2}'` + + if [[ ${driver} == "igb" ]] && ! grep "RSS=8,8" /etc/modprobe.conf | grep "igb" &>/dev/null ;then + sed -i '/options igb.*/d' /etc/modprobe.conf + sed -i '/^$/d' /etc/modprobe.conf + echo "options igb RSS=8,8" >> /etc/modprobe.conf + + rmmod igb && modprobe igb && /etc/init.d/network start && /etc/rc.local + fi +} + +set_affinity() { + MASK=$((1<<$VEC)) + printf "%s mask=%X for /proc/irq/%d/smp_affinity\n" $DEV $MASK $IRQ 2>/dev/null + printf "%X" $MASK > /proc/irq/$IRQ/smp_affinity 2>/dev/null + #echo $DEV mask=$MASK for /proc/irq/$IRQ/smp_affinity + #echo $MASK > /proc/irq/$IRQ/smp_affinity +} + +if [[ ${#netCards} -eq 0 ]]; then + exit 1 +fi +# check for irqbalance running +IRQBALANCE_ON=`ps ax | grep -v grep | grep -q irqbalance; echo $?` +if [ "$IRQBALANCE_ON" == "0" ] ; then + echo " WARNING: irqbalance is running and will" + echo " likely override this script's affinitization." + echo " Please stop the irqbalance service and/or execute" + echo " 'killall irqbalance'" +fi + +for DEV in $(echo ${netCards}); do + # + # Set up the desired devices. + # + eth_interrputs=$(cat /proc/interrupts | grep "${DEV}" | wc -l ) + if [[ ${eth_interrputs} -le 2 ]]; then + readd_mod ${DEV} + fi + for DIR in rx tx TxRx fp ''; do + MAX=`grep $DEV-$DIR /proc/interrupts | wc -l` + if [ "$MAX" == "0" ] ; then + MAX=`egrep -i "$DEV:.*$DIR" /proc/interrupts | wc -l` + fi + if [ "$MAX" == "0" ] ; then + echo no $DIR vectors found on $DEV + continue + #exit 1 + fi + for VEC in `seq 0 1 $MAX`; do + if [ "x$DIR" = "x" ]; then + IRQ=`cat /proc/interrupts | grep -i $DEV-$DIR$VEC"$" | cut -d: -f1 | sed "s/ //g"` + else + IRQ=`cat /proc/interrupts | grep -i $DEV-$DIR-$VEC"$" | cut -d: -f1 | sed "s/ //g"` + fi + + if [ -n "$IRQ" ]; then + set_affinity + else + IRQ=`cat /proc/interrupts | egrep -i $DEV:v$VEC-$DIR"$" | cut -d: -f1 | sed "s/ //g"` + if [ -n "$IRQ" ]; then + set_affinity + fi + fi + done + done +done + init() { // config core dump dir mkdir -p /home/disk2/coresave @@ -34,6 +113,7 @@ init() { mkdir -p $AGENT_HOME/gc_dir mkdir -p $AGENT_HOME/log echo 0 > /proc/sys/kernel/printk + ulimit -n 65536 } download_pkg() { @@ -48,6 +128,7 @@ download_pkg() { echo "--agent_ip=$AGENT_IP" >> conf/galaxy.flag df -h | grep ssd | awk '{print $6}' >> conf/mount_bind.template df -h | grep disk | awk '{print $6}' >> conf/mount_bind.template + echo "/home/opt" >> conf/mount_bind.template } start() { From ea7a0c7b6c2fe466cea9d7024b38d0f7e1b60424 Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 20 Jan 2016 11:33:54 +0800 Subject: [PATCH 5/6] update galaxy --- optools/galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optools/galaxy b/optools/galaxy index a09ca9b4..6e7dcdd3 100755 --- a/optools/galaxy +++ b/optools/galaxy @@ -8,7 +8,7 @@ # # processname: galaxy -PROD_ADDR=ftp://yq01-ps-rtg0000.yq01.baidu.com/home/galaxy/prod/galaxy.tar.gz +PROD_ADDR=xxxxx . /etc/rc.d/init.d/functions AGENT_IP=`hostname -i` AGENT_HOME=/home/galaxy/agent From b5baba2c6de791d1915fbcc4189a933582affd62 Mon Sep 17 00:00:00 2001 From: imotai Date: Wed, 20 Jan 2016 19:06:41 +0800 Subject: [PATCH 6/6] remove tera depd --- build.sh | 152 ++++++++++++++----------------------------------------- 1 file changed, 39 insertions(+), 113 deletions(-) mode change 100755 => 100644 build.sh diff --git a/build.sh b/build.sh old mode 100755 new mode 100644 index 28d1653f..1e902cc7 --- a/build.sh +++ b/build.sh @@ -29,6 +29,33 @@ else echo "install boost done" fi +if [ -f "Python-2.7.11.tgz" ] +then + echo "python exist" +else + echo "start download python" + wget https://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz > /dev/null + tar zxf Python-2.7.11.tgz >/dev/null + cd Python-2.7.11 + ./configure --prefix=${DEPS_PREFIX} --disable-shared >/dev/null + make -j4 && make install>/dev/null + echo "install python done" + cd - +fi + +if [ -f "setuptools-19.2.tar.gz" ] +then + echo "setuptools exist" +else + echo "start download setuptools" + wget https://pypi.python.org/packages/source/s/setuptools/setuptools-19.2.tar.gz + tar -zxvf setuptools-19.2.tar.gz >/dev/null + cd setuptools-19.2 + python setup.py install >/dev/null + echo "install setuptools done" + cd - +fi + if [ -d "rapidjson" ] then echo "rapid json exist" @@ -56,6 +83,9 @@ else make -j4 >/dev/null make install cd - + cd protobuf-2.6.1/python + python setup.py build && python setup.py install + cd - echo "install protobuf done" fi @@ -92,28 +122,17 @@ else echo "PROTOBUF_DIR=${DEPS_PREFIX}" >> depends.mk echo "SNAPPY_DIR=${DEPS_PREFIX}" >> depends.mk echo "PREFIX=${DEPS_PREFIX}" >> depends.mk - cd src + cd - + cd sofa-pbrpc-1.0.0/src PROTOBUF_DIR=${DEPS_PREFIX} sh compile_proto.sh - cd .. - make -j4 >/dev/null - make install - cd .. -fi - -if [ -f "zookeeper-3.4.6.tar.gz" ] -then - echo "zookeeper-3.4.6.tar.gz exist" -else - # zookeeper - wget http://apache.arvixe.com/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz - tar zxf zookeeper-3.4.6.tar.gz - cd zookeeper-3.4.6/src/c - ./configure ${DEPS_CONFIG} >/dev/null + cd - + cd sofa-pbrpc-1.0.0 make -j4 >/dev/null make install cd - fi + if [ -f "CMake-3.2.1.tar.gz" ] then echo "CMake-3.2.1.tar.gz exist" @@ -142,69 +161,6 @@ else cd - fi -if [ -f "glog-0.3.3.tar.gz" ] -then - echo "glog-0.3.3.tar.gz exist" -else - # glog - wget --no-check-certificate -O glog-0.3.3.tar.gz https://github.com/google/glog/archive/v0.3.3.tar.gz - tar zxf glog-0.3.3.tar.gz - cd glog-0.3.3 - ./configure ${DEPS_CONFIG} CPPFLAGS=-I${DEPS_PREFIX}/include LDFLAGS=-L${DEPS_PREFIX}/lib >/dev/null - make -j4 >/dev/null - make install - cd - -fi - -if [ -d "gtest_archive" ] -then - echo "gtest_archive exist" -else - - # gtest - # wget --no-check-certificate https://googletest.googlecode.com/files/gtest-1.7.0.zip - git clone --depth=1 https://github.com/xupeilin/gtest_archive - mv gtest_archive/gtest-1.7.0.zip . - unzip gtest-1.7.0.zip - cd gtest-1.7.0 - ./configure ${DEPS_CONFIG} >/dev/null - make -j8 >/dev/null - cp -a lib/.libs/* ${DEPS_PREFIX}/lib - cp -a include/gtest ${DEPS_PREFIX}/include - cd - -fi - -if [ -f "libunwind-0.99-beta.tar.gz" ] -then - echo "libunwind-0.99-beta.tar.gz exist" -else - # libunwind for gperftools - wget http://download.savannah.gnu.org/releases/libunwind/libunwind-0.99-beta.tar.gz - tar zxf libunwind-0.99-beta.tar.gz - cd libunwind-0.99-beta - ./configure ${DEPS_CONFIG} >/dev/null - make CFLAGS=-fPIC -j4 >/dev/null - make CFLAGS=-fPIC install - cd - -fi - -if [ -d "gperftools" ] -then - echo "gperftools exist" -else - - # gperftools (tcmalloc) - # wget --no-check-certificate https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.2.1.tar.gz - git clone --depth=1 https://github.com/00k/gperftools - mv gperftools/gperftools-2.2.1.tar.gz . - tar zxf gperftools-2.2.1.tar.gz - cd gperftools-2.2.1 - ./configure ${DEPS_CONFIG} CPPFLAGS=-I${DEPS_PREFIX}/include LDFLAGS=-L${DEPS_PREFIX}/lib >/dev/null - make -j4 >/dev/null - make install - cd - -fi - if [ -d "leveldb" ] then echo "leveldb exist" @@ -239,37 +195,12 @@ else export PATH=${DEPS_PREFIX}/bin:$PATH export BOOST_PATH=${DEPS_PREFIX}/boost_1_57_0 export PBRPC_PATH=${DEPS_PREFIX}/ - make -j4 ins >/dev/null && make -j4 install_sdk + make -j4 ins >/dev/null && make -j4 install_sdk >/dev/null && make python >/dev/null mkdir -p output/bin && cp ins output/bin + cp -rf output/python/* ../../optools/ cd - fi -if [ -d "tera" ] -then - echo "tera exist" -else - - # tera - git clone https://github.com/baidu/tera - depends_file=depends.mk.template - cd tera - sed -i 's#^SOFA_PBRPC_PREFIX=.*#SOFA_PBRPC_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^PROTOBUF_PREFIX=.*#PROTOBUF_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^SNAPPY_PREFIX=.*#SNAPPY_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^ZOOKEEPER_PREFIX=.*#ZOOKEEPER_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^GFLAGS_PREFIX=.*#GFLAGS_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^GLOG_PREFIX=.*#GLOG_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^GTEST_PREFIX=.*#GTEST_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^GPERFTOOLS_PREFIX=.*#GPERFTOOLS_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -i 's#^BOOST_INCDIR=.*#BOOST_INCDIR='${DEPS_PREFIX}'\/boost_1_57_0#' ${depends_file} - sed -i 's#^INS_PREFIX=.*#INS_PREFIX='${DEPS_PREFIX}'#' ${depends_file} - sed -e '$ c -lgtest_main -lgtest -lglog -lgflags -ltcmalloc_minimal -lunwind' ${depends_file} > depends.mk.new - mv depends.mk.new depends.mk - make -j8 >/dev/null - cp -a build/lib/*.a ${DEPS_PREFIX}/lib - cp -a build/include/*.h ${DEPS_PREFIX}/include - cd - -fi if [ -d "mdt" ] then @@ -282,16 +213,11 @@ else sed -i 's/^SOFA_PBRPC_PREFIX=.*/SOFA_PBRPC_PREFIX=..\/..\/thirdparty/' depends.mk sed -i 's/^PROTOBUF_PREFIX=.*/PROTOBUF_PREFIX=..\/..\/thirdparty/' depends.mk sed -i 's/^SNAPPY_PREFIX=.*/SNAPPY_PREFIX=..\/..\/thirdparty/' depends.mk - sed -i 's/^ZOOKEEPER_PREFIX=.*/ZOOKEEPER_PREFIX=..\/..\/thirdparty/' depends.mk sed -i 's/^GFLAGS_PREFIX=.*/GFLAGS_PREFIX=..\/..\/thirdparty/' depends.mk sed -i 's/^GLOG_PREFIX=.*/GLOG_PREFIX=..\/..\/thirdparty/' depends.mk - sed -i 's/^GTEST_PREFIX=.*/GTEST_PREFIX=..\/..\/thirdparty/' depends.mk - sed -i 's/^GPERFTOOLS_PREFIX=.*/GPERFTOOLS_PREFIX=..\/..\/thirdparty/' depends.mk sed -i 's/^BOOST_INCDIR=.*/BOOST_INCDIR=..\/..\/thirdparty\/boost_1_57_0/' depends.mk - sed -i 's/^INS_PREFIX=.*/INS_PREFIX=..\/..\/thirdparty/' depends.mk - sed -i 's/^TERA_PREFIX=.*/TERA_PREFIX=..\/..\/thirdparty/' depends.mk - sed -i '/-lgtest_main/c -lgtest_main -lgtest -lglog -lgflags -ltcmalloc_minimal -lunwind' depends.mk - make -j8 >/dev/null + sed -i '/-lgtest_main/c -lglog -lgflags' depends.mk + make -j8 libftrace.a >/dev/null cp src/ftrace/collector/logger.h ${DEPS_PREFIX}/include cp libftrace.a ${DEPS_PREFIX}/lib cd -