From 90c008834b25ec9ded8f3da8264c285e3d2ba4ec Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Wed, 6 Jun 2018 11:46:25 -0600
Subject: [PATCH 01/50] Increased number of rings between Flow Functions based
 on RSS

Now all flows have flexible number of intermediate rings which
depends on RSS queues number. Packets for same sessions
follow exact same routes.

Scheduler now has one more notion - flow function instance (ffi),
main concepts of new scheduler are:
1) Flow Function - covering structure, includes name,
working function, parameters, user contexts and other general information.
Different flow functions are completely different and can't be compared to each other.
2) Flow function instance - structure for working at different RSS flows.
Contains determination of current RSS flow.
Instances of one flow function are absolutely independent:
different cores, mempools, input/output rings and handled sessions.
3) Instance clone - structure for one process that is done useful work.
Contains determination of core and start/stop/report flags/channels.
Clones of one instance work at different cores. However they are not independent:
They share input/output rings, high level protocol sessions and lines in various tables.

Added new user options:
RestrictedCloning - forbids scheduler to add more that one clone per instance which can result to reordering
MaxInIndex - limits instance number per flow function. Can various from 1 to RSS restriction

Changed constants:
As now we have one mempool per each instance we limit default size:
mempool - 8191 instead of 4*8191
ring - 64 * burstSize instead of 256 * burstSize

Details:
Each flow has constant number of parallel rings.
Flows which started with receive have MaxInIndex parallel rings
Other flows like KNI, generate, PCAP read have 1 parallel ring
Merge will merge parallel rings by their numbers:
flows with 1 and 16 rings will merge as first to first ring
Basicaly all functions have one instance and one clone. This instance
handles all input parallel rings. If some function can't handle input flow
scheduler will add additional instance and divide input parallel rings
like half and half. If there is only on input ring and instance still
can't handle it scheduler will add clones (if this is permitted by user).

Pluses:
No reordering, flow locality
No locks for the same lines per session in any intermediate tables
Can reassemble high level protocols like TCP or HTML
Faster scheduler (working independently with each instance)

Minuses:
Memory usage for additional mempools and rings.
---
 flow/flow.go      | 429 +++++++++++++++++++++++--------------------
 flow/scheduler.go | 456 ++++++++++++++++++++++++++++++----------------
 low/low.go        |  58 +++---
 low/low.h         | 149 +++++++--------
 4 files changed, 635 insertions(+), 457 deletions(-)

diff --git a/flow/flow.go b/flow/flow.go
index ce801906..4eedbc85 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -48,7 +48,7 @@ var schedState *scheduler
 var vEach [10][burstSize]uint8
 
 type processSegment struct {
-	out      []*low.Ring
+	out      []low.Rings
 	contexts []UserContext
 	stype    uint8
 }
@@ -56,9 +56,10 @@ type processSegment struct {
 // Flow is an abstraction for connecting flow functions with each other.
 // Flow shouldn't be understood in any way beyond this.
 type Flow struct {
-	current  *low.Ring
+	current  low.Rings
 	segment  *processSegment
 	previous **Func
+	inIndexNumber int32
 }
 
 type partitionCtx struct {
@@ -138,41 +139,41 @@ type Kni struct {
 }
 
 type receiveParameters struct {
-	out  *low.Ring
+	out  low.Rings
 	port *low.Port
 	kni  bool
 }
 
-func addReceiver(portId uint16, kni bool, out *low.Ring) {
+func addReceiver(portId uint16, kni bool, out low.Rings, inIndexNumber int32) {
 	par := new(receiveParameters)
 	par.port = low.GetPort(portId)
 	par.out = out
 	par.kni = kni
 	if kni {
-		schedState.addFF("KNI receiver", nil, recvKNI, nil, par, nil, sendReceiveKNI)
+		schedState.addFF("KNI receiver", nil, recvKNI, nil, par, nil, sendReceiveKNI, 0)
 	} else {
-		schedState.addFF("receiver", nil, recvRSS, nil, par, nil, receiveRSS)
+		schedState.addFF("receiver", nil, recvRSS, nil, par, nil, receiveRSS, inIndexNumber)
 	}
 }
 
 type generateParameters struct {
-	out                    *low.Ring
+	out                    low.Rings
 	generateFunction       GenerateFunction
 	vectorGenerateFunction VectorGenerateFunction
 	mempool                *low.Mempool
 	targetSpeed            float64
 }
 
-func addGenerator(out *low.Ring, generateFunction GenerateFunction, context UserContext) {
+func addGenerator(out low.Rings, generateFunction GenerateFunction, context UserContext) {
 	par := new(generateParameters)
 	par.out = out
 	par.generateFunction = generateFunction
 	ctx := make([]UserContext, 1, 1)
 	ctx[0] = context
-	schedState.addFF("generator", nil, nil, pGenerate, par, &ctx, generate)
+	schedState.addFF("generator", nil, nil, pGenerate, par, &ctx, generate, 0)
 }
 
-func addFastGenerator(out *low.Ring, generateFunction GenerateFunction,
+func addFastGenerator(out low.Rings, generateFunction GenerateFunction,
 	vectorGenerateFunction VectorGenerateFunction, targetSpeed uint64, context UserContext) {
 	par := new(generateParameters)
 	par.out = out
@@ -182,41 +183,41 @@ func addFastGenerator(out *low.Ring, generateFunction GenerateFunction,
 	par.targetSpeed = float64(targetSpeed)
 	ctx := make([]UserContext, 1, 1)
 	ctx[0] = context
-	schedState.addFF("fast generator", nil, nil, pFastGenerate, par, &ctx, fastGenerate)
+	schedState.addFF("fast generator", nil, nil, pFastGenerate, par, &ctx, fastGenerate, 0)
 }
 
 type sendParameters struct {
-	in    *low.Ring
+	in    low.Rings
 	queue int16
 	port  uint16
 }
 
-func addSender(port uint16, queue int16, in *low.Ring) {
+func addSender(port uint16, queue int16, in low.Rings, inIndexNumber int32) {
 	par := new(sendParameters)
 	par.port = port
 	par.queue = queue
 	par.in = in
 	if queue != -1 {
-		schedState.addFF("sender", nil, send, nil, par, nil, sendReceiveKNI)
+		schedState.addFF("sender", nil, send, nil, par, nil, sendReceiveKNI, inIndexNumber)
 	} else {
-		schedState.addFF("KNI sender", nil, send, nil, par, nil, sendReceiveKNI)
+		schedState.addFF("KNI sender", nil, send, nil, par, nil, sendReceiveKNI, inIndexNumber)
 	}
 }
 
 type copyParameters struct {
-	in      *low.Ring
-	out     *low.Ring
-	outCopy *low.Ring
+	in      low.Rings
+	out     low.Rings
+	outCopy low.Rings
 	mempool *low.Mempool
 }
 
-func addCopier(in *low.Ring, out *low.Ring, outCopy *low.Ring) {
+func addCopier(in low.Rings, out low.Rings, outCopy low.Rings, inIndexNumber int32) {
 	par := new(copyParameters)
 	par.in = in
 	par.out = out
 	par.outCopy = outCopy
 	par.mempool = low.CreateMempool("copy")
-	schedState.addFF("copy", nil, nil, pcopy, par, nil, segmentCopy)
+	schedState.addFF("copy", nil, nil, pcopy, par, nil, segmentCopy, inIndexNumber)
 }
 
 func makePartitioner(N uint64, M uint64) *Func {
@@ -262,32 +263,32 @@ func makeHandler(handleFunction HandleFunction, vectorHandleFunction VectorHandl
 }
 
 type writeParameters struct {
-	in       *low.Ring
+	in       low.Rings
 	filename string
 }
 
-func addWriter(filename string, in *low.Ring) {
+func addWriter(filename string, in low.Rings, inIndexNumber int32) {
 	par := new(writeParameters)
 	par.in = in
 	par.filename = filename
-	schedState.addFF("writer", write, nil, nil, par, nil, readWrite)
+	schedState.addFF("writer", write, nil, nil, par, nil, readWrite, inIndexNumber)
 }
 
 type readParameters struct {
-	out      *low.Ring
+	out      low.Rings
 	filename string
 	repcount int32
 }
 
-func addReader(filename string, out *low.Ring, repcount int32) {
+func addReader(filename string, out low.Rings, repcount int32) {
 	par := new(readParameters)
 	par.out = out
 	par.filename = filename
 	par.repcount = repcount
-	schedState.addFF("reader", read, nil, nil, par, nil, readWrite)
+	schedState.addFF("reader", read, nil, nil, par, nil, readWrite, 0)
 }
 
-func makeSlice(out *low.Ring, segment *processSegment) *Func {
+func makeSlice(out low.Rings, segment *processSegment) *Func {
 	f := new(Func)
 	f.sFunc = constructSlice
 	f.vFunc = vConstructSlice
@@ -298,22 +299,22 @@ func makeSlice(out *low.Ring, segment *processSegment) *Func {
 }
 
 type segmentParameters struct {
-	in        *low.Ring
-	out       *([](*low.Ring))
+	in        low.Rings
+	out       *([]low.Rings)
 	firstFunc *Func
 	stype     *uint8
 }
 
-func addSegment(in *low.Ring, first *Func) *processSegment {
+func addSegment(in low.Rings, first *Func, inIndexNumber int32) *processSegment {
 	par := new(segmentParameters)
 	par.in = in
 	par.firstFunc = first
 	segment := new(processSegment)
-	segment.out = make([](*low.Ring), 0, 0)
+	segment.out = make([]low.Rings, 0, 0)
 	segment.contexts = make([](UserContext), 0, 0)
 	par.out = &segment.out
 	par.stype = &segment.stype
-	schedState.addFF("segment", nil, nil, segmentProcess, par, &segment.contexts, segmentCopy)
+	schedState.addFF("segment", nil, nil, segmentProcess, par, &segment.contexts, segmentCopy, inIndexNumber)
 	return segment
 }
 
@@ -360,6 +361,7 @@ type port struct {
 	willKNI        bool // will this port has assigned KNI device
 	port           uint16
 	MAC            [common.EtherAddrLen]uint8
+	InIndex     int32
 }
 
 // Config is a struct with all parameters, which user can pass to NFF-GO library
@@ -410,6 +412,11 @@ type Config struct {
 	// Maximum simultaneous receives that should handle all
 	// input at your network card
 	MaxRecv int
+	// Limits parallel instances. 1 for one instance, 1000 for RSS count determine instances
+	MaxInIndex int32
+	// Scheduler should clone functions even if ti can lead to reordering.
+	// This option should be switch off for all high level reassembling like TCP or HTTP
+	RestrictedCloning bool
 }
 
 // SystemInit is initialization of system. This function should be always called before graph construction.
@@ -432,8 +439,9 @@ func SystemInit(args *Config) error {
 	schedulerOffRemove := args.PersistentClones
 	stopDedicatedCore := args.StopOnDedicatedCore
 	hwtxchecksum = args.HWTXChecksum
+	anyway := !args.RestrictedCloning
 
-	mbufNumber := uint(4 * 8191)
+	mbufNumber := uint(8191)
 	if args.MbufNumber != 0 {
 		mbufNumber = args.MbufNumber
 	}
@@ -443,7 +451,7 @@ func SystemInit(args *Config) error {
 		mbufCacheSize = args.MbufCacheSize
 	}
 
-	sizeMultiplier = 256
+	sizeMultiplier = 64
 	if args.RingSize != 0 {
 		sizeMultiplier = args.RingSize
 	}
@@ -478,11 +486,19 @@ func SystemInit(args *Config) error {
 	}
 	common.SetLogType(logType)
 
-	maxRecv := 3
+	maxRecv := 2
 	if args.MaxRecv != 0 {
 		needKNI = args.MaxRecv
 	}
 
+	maxInIndex := int32(16)
+	if schedulerOff == true {
+		maxInIndex = 1
+	}
+	if args.MaxInIndex != 0 {
+		maxInIndex = args.MaxInIndex
+	}
+
 	argc, argv := low.InitDPDKArguments(args.DPDKArgs)
 	// We want to add new clone if input ring is approximately 80% full
 	maxPacketsToClone := uint32(sizeMultiplier * burstSize / 5 * 4)
@@ -494,13 +510,18 @@ func SystemInit(args *Config) error {
 	createdPorts = make([]port, low.GetPortsNumber(), low.GetPortsNumber())
 	for i := range createdPorts {
 		createdPorts[i].port = uint16(i)
+		if maxInIndex > low.CheckPortRSS(createdPorts[i].port) {
+			createdPorts[i].InIndex = low.CheckPortRSS(createdPorts[i].port)
+		} else {
+			createdPorts[i].InIndex = maxInIndex
+		}
 	}
 	portPair = make(map[uint32](*port))
 	// Init scheduler
 	common.LogTitle(common.Initialization, "------------***------ Initializing scheduler -----***------------")
-	StopRing := low.CreateRing(burstSize * sizeMultiplier)
+	StopRing := low.CreateRings(burstSize * sizeMultiplier, 50 /*Maximum possible rings*/)
 	common.LogDebug(common.Initialization, "Scheduler can use cores:", cpus)
-	schedState = newScheduler(cpus, schedulerOff, schedulerOffRemove, stopDedicatedCore, StopRing, checkTime, debugTime, maxPacketsToClone, maxRecv)
+	schedState = newScheduler(cpus, schedulerOff, schedulerOffRemove, stopDedicatedCore, StopRing, checkTime, debugTime, maxPacketsToClone, maxRecv, anyway)
 	// Init packet processing
 	packet.SetHWTXChecksumFlag(hwtxchecksum)
 	for i := 0; i < 10; i++ {
@@ -522,7 +543,7 @@ func SystemStart() error {
 	for i := range createdPorts {
 		if createdPorts[i].wasRequested {
 			if err := low.CreatePort(createdPorts[i].port, createdPorts[i].willReceive,
-				uint16(createdPorts[i].txQueuesNumber), true, hwtxchecksum); err != nil {
+				uint16(createdPorts[i].txQueuesNumber), true, hwtxchecksum, createdPorts[i].InIndex); err != nil {
 				return err
 			}
 		}
@@ -572,7 +593,7 @@ func SetSenderFile(IN *Flow, filename string) error {
 	if err := checkFlow(IN); err != nil {
 		return err
 	}
-	addWriter(filename, finishFlow(IN))
+	addWriter(filename, finishFlow(IN), IN.inIndexNumber)
 	return nil
 }
 
@@ -581,9 +602,9 @@ func SetSenderFile(IN *Flow, filename string) error {
 // file is read infinitely in circle.
 // Returns new opened flow with read packets.
 func SetReceiverFile(filename string, repcount int32) (OUT *Flow) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
-	addReader(filename, ring, repcount)
-	return newFlow(ring)
+	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	addReader(filename, rings, repcount)
+	return newFlow(rings, 1)
 }
 
 // SetReceiver adds receive function to flow graph.
@@ -599,9 +620,9 @@ func SetReceiver(portId uint16) (OUT *Flow, err error) {
 	}
 	createdPorts[portId].wasRequested = true
 	createdPorts[portId].willReceive = true
-	ring := low.CreateRing(burstSize * sizeMultiplier)
-	addReceiver(portId, false, ring)
-	return newFlow(ring), nil
+	rings := low.CreateRings(burstSize * sizeMultiplier, createdPorts[portId].InIndex)
+	addReceiver(portId, false, rings, createdPorts[portId].InIndex)
+	return newFlow(rings, createdPorts[portId].InIndex), nil
 }
 
 // SetReceiverKNI adds function receive from KNI to flow graph.
@@ -609,9 +630,9 @@ func SetReceiver(portId uint16) (OUT *Flow, err error) {
 // Receive queue will be added to port automatically.
 // Returns new opened flow with received packets
 func SetReceiverKNI(kni *Kni) (OUT *Flow) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
-	addReceiver(kni.portId, true, ring)
-	return newFlow(ring)
+	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	addReceiver(kni.portId, true, rings, 1)
+	return newFlow(rings, 1)
 }
 
 // SetFastGenerator adds clonable generate function to flow graph.
@@ -619,13 +640,13 @@ func SetReceiverKNI(kni *Kni) (OUT *Flow) {
 // Returns new open flow with generated packets.
 // Function tries to achieve target speed by cloning.
 func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
+	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
 	if targetSpeed > 0 {
-		addFastGenerator(ring, f, nil, targetSpeed, context)
+		addFastGenerator(rings, f, nil, targetSpeed, context)
 	} else {
 		return nil, common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
 	}
-	return newFlow(ring), nil
+	return newFlow(rings, 1), nil
 }
 
 // SetVectorFastGenerator adds clonable vector generate function to flow graph.
@@ -633,13 +654,13 @@ func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContex
 // Returns new open flow with generated packets.
 // Function tries to achieve target speed by cloning.
 func SetVectorFastGenerator(f VectorGenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
+	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
 	if targetSpeed > 0 {
-		addFastGenerator(ring, nil, f, targetSpeed, context)
+		addFastGenerator(rings, nil, f, targetSpeed, context)
 	} else {
 		return nil, common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
 	}
-	return newFlow(ring), nil
+	return newFlow(rings, 1), nil
 }
 
 // SetGenerator adds non-clonable generate flow function to flow graph.
@@ -648,9 +669,9 @@ func SetVectorFastGenerator(f VectorGenerateFunction, targetSpeed uint64, contex
 // Single packet non-clonable flow function will be added. It can be used for waiting of
 // input user packets.
 func SetGenerator(f GenerateFunction, context UserContext) (OUT *Flow) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
-	addGenerator(ring, f, context)
-	return newFlow(ring)
+	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	addGenerator(rings, f, context)
+	return newFlow(rings, 1)
 }
 
 // SetSender adds send function to flow graph.
@@ -664,7 +685,7 @@ func SetSender(IN *Flow, portId uint16) error {
 		return common.WrapWithNFError(nil, "Requested send port exceeds number of ports which can be used by DPDK (bind to DPDK).", common.ReqTooManyPorts)
 	}
 	createdPorts[portId].wasRequested = true
-	addSender(portId, createdPorts[portId].txQueuesNumber, finishFlow(IN))
+	addSender(portId, createdPorts[portId].txQueuesNumber, finishFlow(IN), IN.inIndexNumber)
 	createdPorts[portId].txQueuesNumber++
 	return nil
 }
@@ -676,7 +697,7 @@ func SetSenderKNI(IN *Flow, kni *Kni) error {
 	if err := checkFlow(IN); err != nil {
 		return err
 	}
-	addSender(kni.portId, -1, finishFlow(IN))
+	addSender(kni.portId, -1, finishFlow(IN), IN.inIndexNumber)
 	return nil
 }
 
@@ -686,19 +707,19 @@ func SetCopier(IN *Flow) (OUT *Flow, err error) {
 	if err := checkFlow(IN); err != nil {
 		return nil, err
 	}
-	ringFirst := low.CreateRing(burstSize * sizeMultiplier)
-	ringSecond := low.CreateRing(burstSize * sizeMultiplier)
+	ringFirst := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
+	ringSecond := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
 	if IN.segment == nil {
-		addCopier(IN.current, ringFirst, ringSecond)
+		addCopier(IN.current, ringFirst, ringSecond, IN.inIndexNumber)
 	} else {
-		tRing := low.CreateRing(burstSize * sizeMultiplier)
+		tRing := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
 		ms := makeSlice(tRing, IN.segment)
 		segmentInsert(IN, ms, false, nil, 0, 0)
-		addCopier(tRing, ringFirst, ringSecond)
+		addCopier(tRing, ringFirst, ringSecond, IN.inIndexNumber)
 		IN.segment = nil
 	}
 	IN.current = ringFirst
-	return newFlow(ringSecond), nil
+	return newFlow(ringSecond, IN.inIndexNumber), nil
 }
 
 // SetPartitioner adds partition function to flow graph.
@@ -719,7 +740,7 @@ func SetPartitioner(IN *Flow, N uint64, M uint64) (OUT *Flow, err error) {
 	if err := segmentInsert(IN, partition, false, *ctx, 0, 0); err != nil {
 		return nil, err
 	}
-	return newFlowSegment(IN.segment, &partition.next[1]), nil
+	return newFlowSegment(IN.segment, &partition.next[1], IN.inIndexNumber), nil
 }
 
 // SetSeparator adds separate function to flow graph.
@@ -731,7 +752,7 @@ func SetSeparator(IN *Flow, separateFunction SeparateFunction, context UserConte
 	if err := segmentInsert(IN, separate, false, context, 1, 1); err != nil {
 		return nil, err
 	}
-	return newFlowSegment(IN.segment, &separate.next[0]), nil
+	return newFlowSegment(IN.segment, &separate.next[0], IN.inIndexNumber), nil
 }
 
 // SetVectorSeparator adds vector separate function to flow graph.
@@ -743,7 +764,7 @@ func SetVectorSeparator(IN *Flow, vectorSeparateFunction VectorSeparateFunction,
 	if err := segmentInsert(IN, separate, false, context, 2, 1); err != nil {
 		return nil, err
 	}
-	return newFlowSegment(IN.segment, &separate.next[0]), nil
+	return newFlowSegment(IN.segment, &separate.next[0], IN.inIndexNumber), nil
 }
 
 // SetSplitter adds split function to flow graph.
@@ -759,7 +780,7 @@ func SetSplitter(IN *Flow, splitFunction SplitFunction, flowNumber uint, context
 	segmentInsert(IN, split, true, context, 1, 0)
 	OutArray = make([](*Flow), flowNumber, flowNumber)
 	for i := range OutArray {
-		OutArray[i] = newFlowSegment(IN.segment, &split.next[i])
+		OutArray[i] = newFlowSegment(IN.segment, &split.next[i], IN.inIndexNumber)
 	}
 	return OutArray, nil
 }
@@ -777,7 +798,7 @@ func SetVectorSplitter(IN *Flow, vectorSplitFunction VectorSplitFunction, flowNu
 	segmentInsert(IN, split, true, context, 2, 0)
 	OutArray = make([](*Flow), flowNumber, flowNumber)
 	for i := range OutArray {
-		OutArray[i] = newFlowSegment(IN.segment, &split.next[i])
+		OutArray[i] = newFlowSegment(IN.segment, &split.next[i], IN.inIndexNumber)
 	}
 	return OutArray, nil
 }
@@ -833,7 +854,7 @@ func SetHandlerDrop(IN *Flow, separateFunction SeparateFunction, context UserCon
 	if err := segmentInsert(IN, separate, false, context, 1, 1); err != nil {
 		return err
 	}
-	return SetStopper(newFlowSegment(IN.segment, &separate.next[0]))
+	return SetStopper(newFlowSegment(IN.segment, &separate.next[0], IN.inIndexNumber))
 }
 
 // SetVectorHandlerDrop adds vector handle function to flow graph.
@@ -845,7 +866,7 @@ func SetVectorHandlerDrop(IN *Flow, vectorSeparateFunction VectorSeparateFunctio
 	if err := segmentInsert(IN, separate, false, context, 2, 1); err != nil {
 		return err
 	}
-	return SetStopper(newFlowSegment(IN.segment, &separate.next[0]))
+	return SetStopper(newFlowSegment(IN.segment, &separate.next[0], IN.inIndexNumber))
 }
 
 // SetMerger adds merge function to flow graph.
@@ -853,21 +874,27 @@ func SetVectorHandlerDrop(IN *Flow, vectorSeparateFunction VectorSeparateFunctio
 // All input flows will be closed. All packets from all these flows will be sent to new flow.
 // This function isn't use any cores. It changes output flows of other functions at initialization stage.
 func SetMerger(InArray ...*Flow) (OUT *Flow, err error) {
-	ring := low.CreateRing(burstSize * sizeMultiplier)
+	max := int32(0)
+	for i := range InArray {
+		if InArray[i].inIndexNumber > max {
+			max = InArray[i].inIndexNumber
+		}
+	}
+	rings := low.CreateRings(burstSize * sizeMultiplier, max)
 	for i := range InArray {
 		if err := checkFlow(InArray[i]); err != nil {
 			return nil, err
 		}
 		if InArray[i].segment == nil {
-			merge(InArray[i].current, ring)
+			merge(InArray[i].current, rings)
 			closeFlow(InArray[i])
 		} else {
 			// TODO merge finishes segment even if this is merge inside it. Need to optimize.
-			ms := makeSlice(ring, InArray[i].segment)
+			ms := makeSlice(rings, InArray[i].segment)
 			segmentInsert(InArray[i], ms, true, nil, 0, 0)
 		}
 	}
-	return newFlow(ring), nil
+	return newFlow(rings, max), nil
 }
 
 // GetPortMACAddress returns default MAC address of an Ethernet port.
@@ -888,27 +915,28 @@ func SetIPForPort(port uint16, ip uint32) error {
 }
 
 // Service functions for Flow
-func newFlow(ring *low.Ring) *Flow {
+func newFlow(rings low.Rings, inIndexNumber int32) *Flow {
 	OUT := new(Flow)
-	OUT.current = ring
+	OUT.current = rings
+	OUT.inIndexNumber = inIndexNumber
 	openFlowsNumber++
 	return OUT
 }
 
-func newFlowSegment(segment *processSegment, previous **Func) *Flow {
-	OUT := newFlow(nil)
+func newFlowSegment(segment *processSegment, previous **Func, inIndexNumber int32) *Flow {
+	OUT := newFlow(nil, inIndexNumber)
 	OUT.segment = segment
 	OUT.previous = previous
 	return OUT
 }
 
-func finishFlow(IN *Flow) *low.Ring {
-	var ring *low.Ring
+func finishFlow(IN *Flow) low.Rings {
+	var ring low.Rings
 	if IN.segment == nil {
 		ring = IN.current
 		closeFlow(IN)
 	} else {
-		ring = low.CreateRing(burstSize * sizeMultiplier)
+		ring = low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
 		ms := makeSlice(ring, IN.segment)
 		segmentInsert(IN, ms, true, nil, 0, 0)
 	}
@@ -926,11 +954,12 @@ func segmentInsert(IN *Flow, f *Func, willClose bool, context UserContext, setTy
 		return err
 	}
 	if IN.segment == nil {
-		IN.segment = addSegment(IN.current, f)
+		IN.segment = addSegment(IN.current, f, IN.inIndexNumber)
 		IN.segment.stype = setType
 	} else {
 		if setType > 0 && IN.segment.stype > 0 && setType != IN.segment.stype {
-			ring := low.CreateRing(burstSize * sizeMultiplier)
+			// Try to combine scalar and vector code. Start new segment
+			ring := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
 			ms := makeSlice(ring, IN.segment)
 			segmentInsert(IN, ms, false, nil, 0, 0)
 			IN.segment = nil
@@ -939,6 +968,7 @@ func segmentInsert(IN *Flow, f *Func, willClose bool, context UserContext, setTy
 			return nil
 		}
 		if setType > 0 && IN.segment.stype == 0 {
+			// Current segment is universal. Set new scalar/vector type to it
 			IN.segment.stype = setType
 		}
 		*IN.previous = f
@@ -953,7 +983,7 @@ func segmentInsert(IN *Flow, f *Func, willClose bool, context UserContext, setTy
 	return nil
 }
 
-func segmentProcess(parameters interface{}, stopper [2]chan int, report chan reportPair, context []UserContext) {
+func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int, report chan reportPair, context []UserContext) {
 	// For scalar and vector parts
 	lp := parameters.(*segmentParameters)
 	IN := lp.in
@@ -1000,88 +1030,91 @@ func segmentProcess(parameters interface{}, stopper [2]chan int, report chan rep
 			report <- currentSpeed
 			currentSpeed = reportPair{}
 		default:
-			n := IN.DequeueBurst(InputMbufs, burstSize)
-			if n == 0 {
-				if pause != 0 {
-					time.Sleep(time.Duration(pause) * time.Nanosecond)
+			for q := int32(1); q < inIndex[0]+1; q++ {
+				n := IN[inIndex[q]].DequeueBurst(InputMbufs, burstSize)
+				if n == 0 {
+					if pause != 0 {
+						// pause should be non 0 only if function works with ONE inIndex
+						time.Sleep(time.Duration(pause) * time.Nanosecond)
+					}
+					continue
 				}
-				continue
-			}
-			if scalar { // Scalar code
-				for i := uint(0); i < n; i++ {
-					currentFunc := firstFunc
-					tempPacket = packet.ExtractPacket(InputMbufs[i])
-					for {
-						nextIndex := currentFunc.sFunc(tempPacket, currentFunc, context[currentFunc.contextIndex])
-						if currentFunc.followingNumber == 0 {
-							// We have constructSlice -> put packets to output slices
-							OutputMbufs[nextIndex][countOfPackets[nextIndex]] = InputMbufs[i]
-							countOfPackets[nextIndex]++
-							if reportMbits {
-								currentSpeed.Bytes += uint64(tempPacket.GetPacketLen())
+				if scalar { // Scalar code
+					for i := uint(0); i < n; i++ {
+						currentFunc := firstFunc
+						tempPacket = packet.ExtractPacket(InputMbufs[i])
+						for {
+							nextIndex := currentFunc.sFunc(tempPacket, currentFunc, context[currentFunc.contextIndex])
+							if currentFunc.followingNumber == 0 {
+								// We have constructSlice -> put packets to output slices
+								OutputMbufs[nextIndex][countOfPackets[nextIndex]] = InputMbufs[i]
+								countOfPackets[nextIndex]++
+								if reportMbits {
+									currentSpeed.Bytes += uint64(tempPacket.GetPacketLen())
+								}
+								break
 							}
-							break
+							currentFunc = currentFunc.next[nextIndex]
 						}
-						currentFunc = currentFunc.next[nextIndex]
 					}
-				}
-				for index := 0; index < outNumber; index++ {
-					if countOfPackets[index] == 0 {
-						continue
+					for index := 0; index < outNumber; index++ {
+						if countOfPackets[index] == 0 {
+							continue
+						}
+						safeEnqueue(OUT[index][inIndex[q]], OutputMbufs[index], uint(countOfPackets[index]))
+						currentSpeed.Packets += uint64(countOfPackets[index])
+						countOfPackets[index] = 0
 					}
-					safeEnqueue(OUT[index], OutputMbufs[index], uint(countOfPackets[index]))
-					currentSpeed.Packets += uint64(countOfPackets[index])
-					countOfPackets[index] = 0
-				}
-			} else { // Vector code
-				packet.ExtractPackets(tempPackets, InputMbufs, n)
-				def[0].f = firstFunc
-				for i := uint(0); i < burstSize; i++ {
-					def[0].mask[i] = (i < n)
-				}
-				st := 0
-				for st != -1 {
-					cur := def[st].f
-					cur.vFunc(tempPackets, &def[st].mask, &answers, cur, context[cur.contextIndex])
-					if cur.followingNumber == 0 {
-						// We have constructSlice -> put packets inside ring, it is an end of segment
-						count := FillSliceFromMask(InputMbufs, &def[st].mask, OutputMbufs[0])
-						safeEnqueue(OUT[answers[0]], OutputMbufs[0], uint(count))
-						currentSpeed.Packets += uint64(count)
-					} else if cur.followingNumber == 1 {
-						// We have simple handle. Mask will remain the same, current function will be changed
-						def[st].f = cur.next[0]
-						st++
-					} else {
-						step := 0
-						currentMask = def[st].mask
-						for i := uint8(0); i < cur.followingNumber; i++ {
-							cont := asm.GenerateMask(&answers, &(vEach[i]), &currentMask, &def[st+step].mask)
-							if !cont {
-								def[st+step].f = cur.next[i]
-								step++
+				} else { // Vector code
+					packet.ExtractPackets(tempPackets, InputMbufs, n)
+					def[0].f = firstFunc
+					for i := uint(0); i < burstSize; i++ {
+						def[0].mask[i] = (i < n)
+					}
+					st := 0
+					for st != -1 {
+						cur := def[st].f
+						cur.vFunc(tempPackets, &def[st].mask, &answers, cur, context[cur.contextIndex])
+						if cur.followingNumber == 0 {
+							// We have constructSlice -> put packets inside ring, it is an end of segment
+							count := FillSliceFromMask(InputMbufs, &def[st].mask, OutputMbufs[0])
+							safeEnqueue(OUT[answers[0]][inIndex[q]], OutputMbufs[0], uint(count))
+							currentSpeed.Packets += uint64(count)
+						} else if cur.followingNumber == 1 {
+							// We have simple handle. Mask will remain the same, current function will be changed
+							def[st].f = cur.next[0]
+							st++
+						} else {
+							step := 0
+							currentMask = def[st].mask
+							for i := uint8(0); i < cur.followingNumber; i++ {
+								cont := asm.GenerateMask(&answers, &(vEach[i]), &currentMask, &def[st+step].mask)
+								if !cont {
+									def[st+step].f = cur.next[i]
+									step++
+								}
 							}
+							st += step
 						}
-						st += step
+						st--
 					}
-					st--
 				}
 			}
 		}
 	}
 }
 
-func recvRSS(parameters interface{}, flag *int32, coreID int) {
+func recvRSS(parameters interface{}, inIndex []int32, flag *int32, coreID int) {
 	srp := parameters.(*receiveParameters)
-	low.ReceiveRSS(uint16(srp.port.PortId), int16(srp.port.QueuesNumber-1), srp.out, flag, coreID)
+	low.ReceiveRSS(uint16(srp.port.PortId), inIndex, srp.out, flag, coreID)
 }
 
-func recvKNI(parameters interface{}, flag *int32, coreID int) {
+func recvKNI(parameters interface{}, inIndex []int32, flag *int32, coreID int) {
 	srp := parameters.(*receiveParameters)
-	low.ReceiveKNI(uint16(srp.port.PortId), srp.out, flag, coreID)
+	low.ReceiveKNI(uint16(srp.port.PortId), srp.out[0], flag, coreID)
 }
 
-func pGenerate(parameters interface{}, stopper [2]chan int, report chan reportPair, context []UserContext) {
+func pGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int, report chan reportPair, context []UserContext) {
 	// Function is unclonable, report is always nil
 	gp := parameters.(*generateParameters)
 	OUT := gp.out
@@ -1101,12 +1134,12 @@ func pGenerate(parameters interface{}, stopper [2]chan int, report chan reportPa
 				common.LogFatal(common.Debug, err)
 			}
 			generateFunction(tempPacket, context[0])
-			safeEnqueueOne(OUT, tempPacket.ToUintptr())
+			safeEnqueueOne(OUT[0], tempPacket.ToUintptr())
 		}
 	}
 }
 
-func pFastGenerate(parameters interface{}, stopper [2]chan int, report chan reportPair, context []UserContext) {
+func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int, report chan reportPair, context []UserContext) {
 	gp := parameters.(*generateParameters)
 	OUT := gp.out
 	generateFunction := gp.generateFunction
@@ -1137,6 +1170,7 @@ func pFastGenerate(parameters interface{}, stopper [2]chan int, report chan repo
 		default:
 			err := low.AllocateMbufs(bufs, mempool, burstSize)
 			if err != nil {
+				low.ReportMempoolsState()
 				common.LogFatal(common.Debug, err)
 			}
 			if vector == false {
@@ -1152,7 +1186,7 @@ func pFastGenerate(parameters interface{}, stopper [2]chan int, report chan repo
 				packet.ExtractPackets(tempPackets, bufs, burstSize)
 				vectorGenerateFunction(tempPackets, context[0])
 			}
-			safeEnqueue(OUT, bufs, burstSize)
+			safeEnqueue(OUT[0], bufs, burstSize)
 			currentSpeed.Packets += uint64(burstSize)
 			// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
 			// we just want to slow goroutine down without parking, so loop is OK for this.
@@ -1167,7 +1201,7 @@ func pFastGenerate(parameters interface{}, stopper [2]chan int, report chan repo
 }
 
 // TODO reassembled packets are not supported
-func pcopy(parameters interface{}, stopper [2]chan int, report chan reportPair, context []UserContext) {
+func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report chan reportPair, context []UserContext) {
 	cp := parameters.(*copyParameters)
 	IN := cp.in
 	OUT := cp.out
@@ -1194,42 +1228,45 @@ func pcopy(parameters interface{}, stopper [2]chan int, report chan reportPair,
 			report <- currentSpeed
 			currentSpeed = reportPair{}
 		default:
-			n := IN.DequeueBurst(bufs1, burstSize)
-			if n != 0 {
-				if err := low.AllocateMbufs(bufs2, mempool, n); err != nil {
-					common.LogFatal(common.Debug, err)
-				}
-				for i := uint(0); i < n; i++ {
-					// TODO Maybe we need to prefetcht here?
-					tempPacket1 = packet.ExtractPacket(bufs1[i])
-					tempPacket2 = packet.ExtractPacket(bufs2[i])
-					packet.GeneratePacketFromByte(tempPacket2, tempPacket1.GetRawPacketBytes())
-					if reportMbits {
-						currentSpeed.Bytes += uint64(tempPacket1.GetPacketLen())
+			for q := int32(1); q < inIndex[0]+1; q++ {
+				n := IN[inIndex[q]].DequeueBurst(bufs1, burstSize)
+				if n != 0 {
+					if err := low.AllocateMbufs(bufs2, mempool, n); err != nil {
+						common.LogFatal(common.Debug, err)
+					}
+					for i := uint(0); i < n; i++ {
+						// TODO Maybe we need to prefetcht here?
+						tempPacket1 = packet.ExtractPacket(bufs1[i])
+						tempPacket2 = packet.ExtractPacket(bufs2[i])
+						packet.GeneratePacketFromByte(tempPacket2, tempPacket1.GetRawPacketBytes())
+						if reportMbits {
+							currentSpeed.Bytes += uint64(tempPacket1.GetPacketLen())
+						}
 					}
+					safeEnqueue(OUT[inIndex[q]], bufs1, uint(n))
+					safeEnqueue(OUTCopy[inIndex[q]], bufs2, uint(n))
+					currentSpeed.Packets += uint64(n)
 				}
-				safeEnqueue(OUT, bufs1, uint(n))
-				safeEnqueue(OUTCopy, bufs2, uint(n))
-				currentSpeed.Packets += uint64(n)
-			}
-			// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
-			// we just want to slow goroutine down without parking, so loop is OK for this.
-			// time.Now lasts approximately 70ns and this satisfies us
-			if pause != 0 {
-				a := time.Now()
-				for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
+				// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
+				// we just want to slow goroutine down without parking, so loop is OK for this.
+				// time.Now lasts approximately 70ns and this satisfies us
+				if pause != 0 {
+					// pause should be 0 only if function works with ONE inIndex
+					a := time.Now()
+					for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
+					}
 				}
 			}
 		}
 	}
 }
 
-func send(parameters interface{}, flag *int32, coreID int) {
+func send(parameters interface{}, inIndex []int32, flag *int32, coreID int) {
 	srp := parameters.(*sendParameters)
-	low.Send(srp.port, srp.queue, srp.in, flag, coreID)
+	low.Send(srp.port, srp.queue, srp.in, inIndex[0], flag, coreID)
 }
 
-func merge(from, to *low.Ring) {
+func merge(from low.Rings, to low.Rings) {
 	// We should change out rings in all flow functions which we added before
 	// and change them to one "after merge" ring.
 	// We don't proceed stop and send functions here because they don't have
@@ -1239,22 +1276,22 @@ func merge(from, to *low.Ring) {
 	for i := range schedState.ff {
 		switch parameters := schedState.ff[i].Parameters.(type) {
 		case *receiveParameters:
-			if parameters.out == from {
+			if parameters.out[0] == from[0] {
 				parameters.out = to
 			}
 		case *generateParameters:
-			if parameters.out == from {
+			if parameters.out[0] == from[0] {
 				parameters.out = to
 			}
 		case *readParameters:
-			if parameters.out == from {
+			if parameters.out[0] == from[0] {
 				parameters.out = to
 			}
 		case *copyParameters:
-			if parameters.out == from {
+			if parameters.out[0] == from[0] {
 				parameters.out = to
 			}
-			if parameters.outCopy == from {
+			if parameters.outCopy[0] == from[0] {
 				parameters.outCopy = to
 			}
 		}
@@ -1322,7 +1359,7 @@ func vConstructSlice(packets []*packet.Packet, mask *[burstSize]bool, answers *[
 	answers[0] = uint8(ve.bufIndex)
 }
 
-func write(parameters interface{}, stopper [2]chan int) {
+func write(parameters interface{}, inIndex []int32, stopper [2]chan int) {
 	wp := parameters.(*writeParameters)
 	IN := wp.in
 	filename := wp.filename
@@ -1347,21 +1384,23 @@ func write(parameters interface{}, stopper [2]chan int) {
 			stopper[1] <- 1
 			return
 		default:
-			n := IN.DequeueBurst(bufIn, 1)
-			if n == 0 {
-				continue
-			}
-			tempPacket = packet.ExtractPacket(bufIn[0])
-			err := tempPacket.WritePcapOnePacket(f)
-			if err != nil {
-				common.LogFatal(common.Debug, err)
+			for q := int32(0); q < inIndex[0]; q++ {
+				n := IN[q].DequeueBurst(bufIn, 1)
+				if n == 0 {
+					continue
+				}
+				tempPacket = packet.ExtractPacket(bufIn[0])
+				err := tempPacket.WritePcapOnePacket(f)
+				if err != nil {
+					common.LogFatal(common.Debug, err)
+				}
+				low.DirectStop(1, bufIn)
 			}
-			low.DirectStop(1, bufIn)
 		}
 	}
 }
 
-func read(parameters interface{}, stopper [2]chan int) {
+func read(parameters interface{}, inIndex []int32, stopper [2]chan int) {
 	rp := parameters.(*readParameters)
 	OUT := rp.out
 	filename := rp.filename
@@ -1412,7 +1451,7 @@ func read(parameters interface{}, stopper [2]chan int) {
 			}
 			// TODO we need packet reassembly here. However we don't
 			// use mbuf packet_type here, so it is impossible.
-			safeEnqueueOne(OUT, tempPacket.ToUintptr())
+			safeEnqueueOne(OUT[0], tempPacket.ToUintptr())
 		}
 	}
 }
@@ -1424,7 +1463,7 @@ func safeEnqueue(place *low.Ring, data []uintptr, number uint) {
 	done := place.EnqueueBurst(data, number)
 	if done < number {
 		schedState.Dropped += number - uint(done)
-		done2 := schedState.StopRing.EnqueueBurst(data[done:number], number-uint(done))
+		done2 := schedState.StopRing[0].EnqueueBurst(data[done:number], number-uint(done))
 		// If stop ring is crowded a function will call C stop directly without
 		// moving forward. It prevents constant crowd stop and increases
 		// performance on "long accelerating" topologies in 1.5x times.
diff --git a/flow/scheduler.go b/flow/scheduler.go
index 7fe50bed..88d3837d 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -55,6 +55,22 @@ type clonePair struct {
 	flag    int32
 }
 
+type instance struct {
+	inIndex []int32
+	// Clones of this instance
+	clone []*clonePair
+	// Channel for reporting current speed.
+	report       chan reportPair
+	currentSpeed reportPair
+	// Current saved speed when making a clone.
+	// It will be compared with immediate current speed to stop a clone.
+	previousSpeed []reportPair
+	// Number of clones of this instance
+	cloneNumber int
+	pause       int
+	ff          *flowFunction
+}
+
 // UserContext is used inside flow packet and is going for user via it
 type UserContext interface {
 	Copy() interface{}
@@ -62,9 +78,9 @@ type UserContext interface {
 }
 
 // Function types which are used inside flow functions
-type uncloneFlowFunction func(interface{}, [2]chan int)
-type cloneFlowFunction func(interface{}, [2]chan int, chan reportPair, []UserContext)
-type cFlowFunction func(interface{}, *int32, int)
+type uncloneFlowFunction func(interface{}, []int32, [2]chan int)
+type cloneFlowFunction func(interface{}, []int32, [2]chan int, chan reportPair, []UserContext)
+type cFlowFunction func(interface{}, []int32, *int32, int)
 
 type ffType int
 
@@ -87,29 +103,21 @@ type flowFunction struct {
 	cFunction cFlowFunction
 	// Main body of clonable flow function
 	cloneFunction cloneFlowFunction
-	// Number of clones of this function
-	cloneNumber int
-	// Clones of this function. They are determined
-	// by their core and their stopper channel
-	clone []*clonePair
-	// Channel for reporting current speed.
-	// It is one for all clones
-	report chan reportPair
-	// Current saved speed when making a clone.
-	// It will be compared with immediate current speed to stop a clone.
-	previousSpeed []reportPair
-	// For debug purposes
-	currentSpeed reportPair
+	// Number of instances of this function
+	instanceNumber int
+	// Instances of this function
+	instance     []*instance
+	oneCoreSpeed reportPair
 	// Name of flow function
-	name    string
-	context *[]UserContext
-	pause   int
-	fType   ffType
+	name          string
+	context       *[]UserContext
+	fType         ffType
+	inIndexNumber int32
 }
 
 // Adding every flow function to scheduler list
 func (scheduler *scheduler) addFF(name string, ucfn uncloneFlowFunction, Cfn cFlowFunction, cfn cloneFlowFunction,
-	par interface{}, context *[]UserContext, fType ffType) {
+	par interface{}, context *[]UserContext, fType ffType, inIndexNumber int32) {
 	ff := new(flowFunction)
 	nameC := 1
 	tName := name
@@ -124,10 +132,9 @@ func (scheduler *scheduler) addFF(name string, ucfn uncloneFlowFunction, Cfn cFl
 	ff.cFunction = Cfn
 	ff.cloneFunction = cfn
 	ff.Parameters = par
-	ff.report = make(chan reportPair, 50)
 	ff.context = context
-	ff.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
 	ff.fType = fType
+	ff.inIndexNumber = inIndexNumber
 	scheduler.ff = append(scheduler.ff, ff)
 }
 
@@ -136,8 +143,9 @@ type scheduler struct {
 	cores             []core
 	off               bool
 	offRemove         bool
+	anyway            bool
 	stopDedicatedCore bool
-	StopRing          *low.Ring
+	StopRing          low.Rings
 	usedCores         uint8
 	checkTime         uint
 	debugTime         uint
@@ -152,8 +160,8 @@ type core struct {
 	isfree bool
 }
 
-func newScheduler(cpus []int, schedulerOff bool, schedulerOffRemove bool,
-	stopDedicatedCore bool, stopRing *low.Ring, checkTime uint, debugTime uint, maxPacketsToClone uint32, maxRecv int) *scheduler {
+func newScheduler(cpus []int, schedulerOff bool, schedulerOffRemove bool, stopDedicatedCore bool,
+	stopRing low.Rings, checkTime uint, debugTime uint, maxPacketsToClone uint32, maxRecv int, anyway bool) *scheduler {
 	coresNumber := len(cpus)
 	// Init scheduler
 	scheduler := new(scheduler)
@@ -169,6 +177,7 @@ func newScheduler(cpus []int, schedulerOff bool, schedulerOffRemove bool,
 	scheduler.debugTime = debugTime
 	scheduler.maxPacketsToClone = maxPacketsToClone
 	scheduler.maxRecv = maxRecv
+	scheduler.anyway = anyway
 
 	return scheduler
 }
@@ -195,7 +204,7 @@ func (scheduler *scheduler) systemStart() (err error) {
 		low.Stop(scheduler.StopRing, &scheduler.stopFlag, core)
 	}()
 	for i := range scheduler.ff {
-		if err = scheduler.startFF(scheduler.ff[i]); err != nil {
+		if err = scheduler.ff[i].startNewInstance(constructNewIndex(scheduler.ff[i].inIndexNumber), scheduler); err != nil {
 			return err
 		}
 	}
@@ -204,17 +213,34 @@ func (scheduler *scheduler) systemStart() (err error) {
 	return nil
 }
 
-func (scheduler *scheduler) startFF(ff *flowFunction) (err error) {
+func (ff *flowFunction) startNewInstance(inIndex []int32, scheduler *scheduler) (err error) {
+	ffi := new(instance)
+	common.LogDebug(common.Initialization, "Start new instance for", ff.name)
+	ffi.inIndex = inIndex
+	ffi.report = make(chan reportPair, 50)
+	ffi.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
+	ffi.ff = ff
+	err = ffi.startNewClone(scheduler, ff.instanceNumber)
+	if err == nil {
+		ff.instance = append(ff.instance, ffi)
+		ff.instanceNumber++
+	}
+	return err
+}
+
+func (ffi *instance) startNewClone(scheduler *scheduler, n int) (err error) {
+	ff := ffi.ff
 	core, index, err := scheduler.getCore()
 	if err != nil {
+		common.LogWarning(common.Debug, "Can't start new clone for", ff.name, "instance", n)
 		return err
 	}
-	common.LogDebug(common.Initialization, "Start new FlowFunction or clone for", ff.name, "at", core, "core")
-	ff.clone = append(ff.clone, &clonePair{index, [2]chan int{nil, nil}, process})
-	ff.cloneNumber++
+	common.LogDebug(common.Initialization, "Start new clone for", ff.name, "instance", n, "at", core, "core")
+	ffi.clone = append(ffi.clone, &clonePair{index, [2]chan int{nil, nil}, process})
+	ffi.cloneNumber++
 	if ff.fType != receiveRSS && ff.fType != sendReceiveKNI {
-		ff.clone[ff.cloneNumber-1].channel[0] = make(chan int)
-		ff.clone[ff.cloneNumber-1].channel[1] = make(chan int)
+		ffi.clone[ffi.cloneNumber-1].channel[0] = make(chan int)
+		ffi.clone[ffi.cloneNumber-1].channel[1] = make(chan int)
 	}
 	go func() {
 		if ff.fType != receiveRSS && ff.fType != sendReceiveKNI {
@@ -222,32 +248,52 @@ func (scheduler *scheduler) startFF(ff *flowFunction) (err error) {
 				common.LogFatal(common.Initialization, "Failed to set affinity to", core, "core: ", err)
 			}
 			if ff.fType == segmentCopy || ff.fType == fastGenerate || ff.fType == generate {
-				ff.cloneFunction(ff.Parameters, ff.clone[ff.cloneNumber-1].channel, ff.report, cloneContext(ff.context))
+				ff.cloneFunction(ff.Parameters, ffi.inIndex, ffi.clone[ffi.cloneNumber-1].channel, ffi.report, cloneContext(ff.context))
 			} else {
-				ff.uncloneFunction(ff.Parameters, ff.clone[ff.cloneNumber-1].channel)
+				ff.uncloneFunction(ff.Parameters, ffi.inIndex, ffi.clone[ffi.cloneNumber-1].channel)
 			}
 		} else {
-			ff.cFunction(ff.Parameters, &ff.clone[ff.cloneNumber-1].flag, core)
+			ff.cFunction(ff.Parameters, ffi.inIndex, &ffi.clone[ffi.cloneNumber-1].flag, core)
 		}
 	}()
 	return nil
 }
 
-func (scheduler *scheduler) stopFF(ff *flowFunction) {
-	if ff.clone[ff.cloneNumber-1].channel[0] != nil {
-		ff.clone[ff.cloneNumber-1].channel[0] <- -1
-		<-ff.clone[ff.cloneNumber-1].channel[1]
-		close(ff.clone[ff.cloneNumber-1].channel[0])
-		close(ff.clone[ff.cloneNumber-1].channel[1])
+func (ff *flowFunction) stopClone(ffi *instance, scheduler *scheduler) {
+	if ffi.clone[ffi.cloneNumber-1].channel[0] != nil {
+		ffi.clone[ffi.cloneNumber-1].channel[0] <- -1
+		<-ffi.clone[ffi.cloneNumber-1].channel[1]
+		close(ffi.clone[ffi.cloneNumber-1].channel[0])
+		close(ffi.clone[ffi.cloneNumber-1].channel[1])
 	} else {
-		atomic.StoreInt32(&ff.clone[ff.cloneNumber-1].flag, stopRequest)
-		for atomic.LoadInt32(&ff.clone[ff.cloneNumber-1].flag) != wasStopped {
+		atomic.StoreInt32(&ffi.clone[ffi.cloneNumber-1].flag, stopRequest)
+		for atomic.LoadInt32(&ffi.clone[ffi.cloneNumber-1].flag) != wasStopped {
 			runtime.Gosched()
 		}
 	}
-	scheduler.setCoreByIndex(ff.clone[ff.cloneNumber-1].index)
-	ff.clone = ff.clone[:len(ff.clone)-1]
-	ff.cloneNumber--
+	scheduler.setCoreByIndex(ffi.clone[ffi.cloneNumber-1].index)
+	ffi.clone = ffi.clone[:len(ffi.clone)-1]
+	ffi.cloneNumber--
+}
+
+func (ff *flowFunction) stopInstance(from int, to int, scheduler *scheduler) {
+	common.LogDebug(common.Initialization, "Stop instance for", ff.name)
+	fromInstance := ff.instance[from]
+	for fromInstance.cloneNumber != 0 {
+		ff.stopClone(fromInstance, scheduler)
+	}
+	if fromInstance.report != nil {
+		close(fromInstance.report)
+	}
+	if to != -1 {
+		toInstance := ff.instance[to]
+		for q := int32(0); q < fromInstance.inIndex[0]; q++ {
+			toInstance.inIndex[toInstance.inIndex[0]+q+1] = fromInstance.inIndex[q+1]
+		}
+		toInstance.inIndex[0] += fromInstance.inIndex[0]
+		ff.instance = append(ff.instance[:from], ff.instance[from+1:]...)
+	}
+	ff.instanceNumber--
 }
 
 func (scheduler *scheduler) systemStop() {
@@ -257,11 +303,8 @@ func (scheduler *scheduler) systemStop() {
 		runtime.Gosched()
 	}
 	for i := range scheduler.ff {
-		for scheduler.ff[i].cloneNumber != 0 {
-			scheduler.stopFF(scheduler.ff[i])
-		}
-		if scheduler.ff[i].report != nil {
-			close(scheduler.ff[i].report)
+		for scheduler.ff[i].instanceNumber != 0 {
+			scheduler.ff[i].stopInstance(0, -1, scheduler)
 		}
 	}
 	scheduler.setCoreByIndex(0) // scheduler
@@ -307,46 +350,85 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 			// Firstly we check removing clones. We can remove one clone if:
 			// 1. flow function has clones or it is fastGenerate
 			// 2. scheduler removing is switched on
-			if (ff.cloneNumber > 1 || ff.fType == fastGenerate) && (scheduler.offRemove == false) {
+			if scheduler.offRemove == false {
 				switch ff.fType {
-				case segmentCopy:
-					// 3. current speed of flow function is lower, than saved speed with less number of clones
-					if ff.currentSpeed.Packets < ff.previousSpeed[ff.cloneNumber-1].Packets {
-						// Save current speed as speed of flow function with this number of clones before removing
-						ff.previousSpeed[ff.cloneNumber] = ff.currentSpeed
-						scheduler.stopFF(ff)
-						ff.updatePause(ff.cloneNumber)
-						// After removing a clone we don't want to try to add clone immediately
-						continue
+				case segmentCopy: // Both instances and clones
+					for q := 0; q < ff.instanceNumber; q++ {
+						ffi := ff.instance[q]
+						if ffi.cloneNumber > 1 {
+							// 3. current speed of flow function is lower, than saved speed with less number of clones
+							if ffi.currentSpeed.Packets < ffi.previousSpeed[ffi.cloneNumber-1].Packets {
+								// Save current speed as speed of flow function with this number of clones before removing
+								ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
+								ff.stopClone(ffi, scheduler)
+								ffi.updatePause(ffi.cloneNumber)
+							}
+						}
+					}
+					if ff.instanceNumber > 1 {
+						min0 := 0
+						min1 := 1
+						if ff.instance[0].currentSpeed.Packets > ff.instance[1].currentSpeed.Packets {
+							min0 = 1
+							min1 = 0
+						}
+						for q := 2; q < ff.instanceNumber; q++ {
+							if ff.instance[q].currentSpeed.Packets < ff.instance[min0].currentSpeed.Packets {
+								min1 = min0
+								min0 = q
+								continue
+							}
+							if ff.instance[q].currentSpeed.Packets < ff.instance[min1].currentSpeed.Packets {
+								min1 = q
+							}
+						}
+						if ff.instance[min0].currentSpeed.Packets+ff.instance[min1].currentSpeed.Packets < ff.oneCoreSpeed.Packets {
+							ff.stopInstance(min0, min1, scheduler)
+						}
 					}
-				case fastGenerate:
+				case fastGenerate: // Only clones, no instances
 					targetSpeed := (ff.Parameters.(*generateParameters)).targetSpeed
-					speedPKTS := float64(ff.currentSpeed.normalize(schedTime).Packets)
+					speedPKTS := float64(ff.instance[0].currentSpeed.normalize(schedTime).Packets)
 					// 3. Current speed is much bigger than target speed
 					if speedPKTS > 1.1*targetSpeed {
+						ffi := ff.instance[0]
 						// 4. TODO strange heuristic, it is required to check this
-						if ff.cloneNumber > 1 && targetSpeed/float64(ff.cloneNumber+1)*float64(ff.cloneNumber) > speedPKTS {
-							scheduler.stopFF(ff)
-							ff.updatePause(0)
+						if ffi.cloneNumber > 1 && targetSpeed/float64(ffi.cloneNumber+1)*float64(ffi.cloneNumber) > speedPKTS {
+							ff.stopClone(ffi, scheduler)
+							ffi.updatePause(0)
 							continue
 						} else {
-							if ff.pause == 0 {
+							if ffi.pause == 0 {
 								// This is proportion between required and current speeds.
 								// 1000000000 is converting to nanoseconds
-								ff.updatePause(int((speedPKTS - targetSpeed) / speedPKTS / targetSpeed * 1000000000))
-							} else if float64(ff.pause)*generatePauseStep < 2 {
+								ffi.updatePause(int((speedPKTS - targetSpeed) / speedPKTS / targetSpeed * 1000000000))
+							} else if float64(ffi.pause)*generatePauseStep < 2 {
 								// Multiplying can't be used here due to integer truncation
-								ff.updatePause(ff.pause + 3)
+								ffi.updatePause(ffi.pause + 3)
 							} else {
-								ff.updatePause(int((1 + generatePauseStep) * float64(ff.pause)))
+								ffi.updatePause(int((1 + generatePauseStep) * float64(ffi.pause)))
 							}
 						}
 					}
-				case receiveRSS:
-					if low.CheckRSSPacketCount(ff.Parameters.(*receiveParameters).port) < RSSCloneMin {
-						scheduler.stopFF(ff)
-						low.DecreaseRSS((ff.Parameters.(*receiveParameters)).port)
-						continue
+				case receiveRSS: // Only instances, no clones
+					if ff.instanceNumber > 1 {
+						first := -1
+						for q := 0; q < ff.instanceNumber; q++ {
+							var q1 int32
+							for q1 = 1; q1 < ff.instance[q].inIndex[0]+1; q1++ {
+								if low.CheckRSSPacketCount(ff.Parameters.(*receiveParameters).port, int16(ff.instance[q].inIndex[q1])) > RSSCloneMin {
+									break
+								}
+							}
+							if q1 == ff.instance[q].inIndex[0]+1 {
+								if first == -1 {
+									first = q
+								} else {
+									ff.stopInstance(first, q, scheduler)
+									break
+								}
+							}
+						}
 					}
 				case readWrite, generate, sendReceiveKNI:
 				}
@@ -354,55 +436,74 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 			// Secondly we check adding clones. We can add one clone if:
 			// 1. scheduler is switched on
 			// 2. ouput ring is quite empty
-			if scheduler.off == false && ff.checkOutputRingClonable(scheduler.maxPacketsToClone) {
+			if scheduler.off == false {
 				switch ff.fType {
-				case segmentCopy:
-					// 3. check function signals that we need to clone
-					// 4. we don't know flow function speed with more clones, or we know it and it is bigger than current speed
-					if ff.checkInputRingClonable(scheduler.maxPacketsToClone) &&
-						(ff.previousSpeed[ff.cloneNumber+1].Packets == 0 || ff.previousSpeed[ff.cloneNumber+1].Packets > ff.currentSpeed.Packets) {
-						// Save current speed as speed of flow function with this number of clones before adding
-						ff.previousSpeed[ff.cloneNumber] = ff.currentSpeed
-						if scheduler.startFF(ff) == nil {
-							// Add a pause to all clones. This pause depends on the number of clones.
-							ff.updatePause(ff.cloneNumber)
-						} else {
-							common.LogWarning(common.Debug, "Can't start new clone for", ff.name)
+				case segmentCopy: // Both instances and clones
+					l := ff.instanceNumber
+					a := true
+					for q := 0; q < l; q++ {
+						ffi := ff.instance[q]
+						if ffi.checkInputRingClonable(scheduler.maxPacketsToClone) {
+							if ffi.inIndex[0] > 1 {
+								ff.oneCoreSpeed = ffi.currentSpeed
+								if ff.startNewInstance(constructZeroIndex(ffi.inIndex), scheduler) == nil {
+									ff.instance[ff.instanceNumber-1].inIndex = constructDuplicatedIndex(ffi.inIndex)
+									a = false
+								}
+							}
 						}
-						continue
 					}
-					// Scheduler can't add new clones if saved flow function speed with more
-					// clones is slower than current. However this speed can be changed due to
-					// external reasons. So flow function should check and update this speed
-					// regular time.
-					if checkRequired == true {
-						// Regular flow function speed check after each checkTime milliseconds,
-						// it is done by erasing previously measured speed data. After this scheduler can
-						// add new clone. If performance decreases with added clone, it will be stopped
-						// with setting speed data.
-						ff.previousSpeed[ff.cloneNumber+1].Packets = 0
+					if a == true {
+						for q := 0; q < l; q++ {
+							ffi := ff.instance[q]
+							if ffi.checkInputRingClonable(scheduler.maxPacketsToClone) {
+								if ffi.inIndex[0] == 1 && scheduler.anyway && ffi.checkOutputRingClonable(scheduler.maxPacketsToClone) &&
+									(ffi.previousSpeed[ffi.cloneNumber+1].Packets == 0 ||
+										ffi.previousSpeed[ffi.cloneNumber+1].Packets > ffi.currentSpeed.Packets) {
+									// Save current speed as speed of flow function with this number of clones before adding
+									ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
+									if ffi.startNewClone(scheduler, q) == nil {
+										// Add a pause to all clones. This pause depends on the number of clones.
+										ffi.updatePause(ffi.cloneNumber)
+									}
+									continue
+								}
+							}
+							// Scheduler can't add new clones if saved flow function speed with more
+							// clones is slower than current. However this speed can be changed due to
+							// external reasons. So flow function should check and update this speed
+							// regular time.
+							if checkRequired == true {
+								// Regular flow function speed check after each checkTime milliseconds,
+								// it is done by erasing previously measured speed data. After this scheduler can
+								// add new clone. If performance decreases with added clone, it will be stopped
+								// with setting speed data.
+								ffi.previousSpeed[ffi.cloneNumber+1].Packets = 0
+							}
+						}
 					}
-				case fastGenerate:
+				case fastGenerate: // Only clones, no instances
 					// 3. speed is not enough
-					if float64(ff.currentSpeed.normalize(schedTime).Packets) < (ff.Parameters.(*generateParameters)).targetSpeed {
-						if ff.pause != 0 {
-							ff.updatePause(int((1 - generatePauseStep) * float64(ff.pause)))
-						} else {
+					if float64(ff.instance[0].currentSpeed.normalize(schedTime).Packets) < (ff.Parameters.(*generateParameters)).targetSpeed {
+						if ff.instance[0].pause != 0 {
+							ff.instance[0].updatePause(int((1 - generatePauseStep) * float64(ff.instance[0].pause)))
+						} else if ff.instance[0].checkOutputRingClonable(scheduler.maxPacketsToClone) {
 							// 3. there is no pause
-							if scheduler.startFF(ff) == nil {
-								ff.updatePause(0)
-							} else {
-								common.LogWarning(common.Debug, "Can't start new clone for", ff.name)
+							if ff.instance[0].startNewClone(scheduler, 0) == nil {
+								ff.instance[0].updatePause(0)
 							}
 							continue
 						}
 					}
-				case receiveRSS:
+				case receiveRSS: // Only instances, no clones
 					// 3. Number of packets in RSS is big enogh to cause overwrite (drop) problems
-					if ff.checkInputRingClonable(RSSCloneMax) && ff.cloneNumber < scheduler.maxRecv {
-						if low.IncreaseRSS((ff.Parameters.(*receiveParameters)).port) {
-							if scheduler.startFF(ff) != nil {
-								common.LogWarning(common.Debug, "Can't start new clone for", ff.name)
+					if ff.instanceNumber < scheduler.maxRecv && ff.inIndexNumber > 1 {
+						for q := 0; q < ff.instanceNumber; q++ {
+							if ff.instance[q].checkInputRingClonable(RSSCloneMax) && ff.instance[q].checkOutputRingClonable(scheduler.maxPacketsToClone) {
+								if ff.startNewInstance(constructZeroIndex(ff.instance[q].inIndex), scheduler) == nil {
+									ff.instance[ff.instanceNumber-1].inIndex = constructDuplicatedIndex(ff.instance[q].inIndex)
+								}
+								break
 							}
 						}
 					}
@@ -429,25 +530,27 @@ func cloneContext(ctx *[]UserContext) []UserContext {
 	return ret
 }
 
-func (ff *flowFunction) updatePause(pause int) {
-	ff.pause = pause
-	for j := 0; j < ff.cloneNumber; j++ {
-		ff.clone[j].channel[0] <- ff.pause
+func (ffi *instance) updatePause(pause int) {
+	ffi.pause = pause
+	for j := 0; j < ffi.cloneNumber; j++ {
+		ffi.clone[j].channel[0] <- ffi.pause
 	}
 }
 
 func (ff *flowFunction) printDebug(schedTime uint) {
 	switch ff.fType {
 	case segmentCopy:
-		out := ff.currentSpeed.normalize(schedTime)
-		if reportMbits {
-			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s")
-		} else {
-			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S")
+		for q := 0; q < ff.instanceNumber; q++ {
+			out := ff.instance[q].currentSpeed.normalize(schedTime)
+			if reportMbits {
+				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s")
+			} else {
+				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S, cloneNumber:", ff.instance[q].cloneNumber)
+			}
 		}
 	case fastGenerate:
 		targetSpeed := (ff.Parameters.(*generateParameters)).targetSpeed
-		out := ff.currentSpeed.normalize(schedTime)
+		out := ff.instance[0].currentSpeed.normalize(schedTime)
 		if reportMbits {
 			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s, target speed is", int64(targetSpeed), "PKT/S")
 		} else {
@@ -468,15 +571,18 @@ func (current reportPair) normalize(schedTime uint) reportPair {
 func (ff *flowFunction) updateCurrentSpeed() {
 	// Gather and sum current speeds from all clones of current flow function
 	// Flow function itself and all clones put their speeds in one channel
-	ff.currentSpeed = reportPair{}
-	t := len(ff.report) - ff.cloneNumber
-	for k := 0; k < t; k++ {
-		<-ff.report
-	}
-	for k := 0; k < ff.cloneNumber; k++ {
-		temp := <-ff.report
-		ff.currentSpeed.Packets += temp.Packets
-		ff.currentSpeed.Bytes += temp.Bytes
+	for q := 0; q < ff.instanceNumber; q++ {
+		ffi := ff.instance[q]
+		ffi.currentSpeed = reportPair{}
+		t := len(ffi.report) - ffi.cloneNumber
+		for k := 0; k < t; k++ {
+			<-ffi.report
+		}
+		for k := 0; k < ffi.cloneNumber; k++ {
+			temp := <-ffi.report
+			ffi.currentSpeed.Packets += temp.Packets
+			ffi.currentSpeed.Bytes += temp.Bytes
+		}
 	}
 }
 
@@ -496,46 +602,86 @@ func (scheduler *scheduler) getCore() (int, int, error) {
 	return 0, 0, common.WrapWithNFError(nil, "Requested number of cores isn't enough.", common.NotEnoughCores)
 }
 
-func (ff *flowFunction) checkInputRingClonable(min uint32) bool {
-	switch ff.Parameters.(type) {
+func (ffi *instance) checkInputRingClonable(min uint32) bool {
+	switch ffi.ff.Parameters.(type) {
 	case *segmentParameters:
-		if ff.Parameters.(*segmentParameters).in.GetRingCount() > min {
-			return true
+		for q := int32(0); q < ffi.inIndex[0]; q++ {
+			if ffi.ff.Parameters.(*segmentParameters).in[ffi.inIndex[q+1]].GetRingCount() > min {
+				return true
+			}
 		}
 	case *copyParameters:
-		if ff.Parameters.(*copyParameters).in.GetRingCount() > min {
-			return true
+		for q := int32(0); q < ffi.inIndex[0]; q++ {
+			if ffi.ff.Parameters.(*copyParameters).in[ffi.inIndex[q+1]].GetRingCount() > min {
+				return true
+			}
 		}
 	case *receiveParameters:
-		if low.CheckRSSPacketCount(ff.Parameters.(*receiveParameters).port) > min {
-			return true
+		for q := int32(0); q < ffi.inIndex[0]; q++ {
+			if low.CheckRSSPacketCount(ffi.ff.Parameters.(*receiveParameters).port, int16(ffi.inIndex[q+1])) > min {
+				return true
+			}
 		}
 	}
 	return false
 }
 
-func (ff *flowFunction) checkOutputRingClonable(max uint32) bool {
-	switch ff.Parameters.(type) {
-	case *receiveParameters:
-		if ff.Parameters.(*receiveParameters).out.GetRingCount() <= max {
-			return true
-		}
+func (ffi *instance) checkOutputRingClonable(max uint32) bool {
+	switch ffi.ff.Parameters.(type) {
 	case *generateParameters:
-		if ff.Parameters.(*generateParameters).out.GetRingCount() <= max {
+		if ffi.ff.Parameters.(*generateParameters).out[0].GetRingCount() <= max {
 			return true
 		}
+	case *receiveParameters:
+		for q := int32(0); q < ffi.inIndex[0]; q++ {
+			if ffi.ff.Parameters.(*receiveParameters).out[ffi.inIndex[q+1]].GetRingCount() <= max {
+				return true
+			}
+		}
 	case *copyParameters:
-		if ff.Parameters.(*copyParameters).out.GetRingCount() <= max ||
-			ff.Parameters.(*copyParameters).outCopy.GetRingCount() <= max {
-			return true
+		for q := int32(0); q < ffi.inIndex[0]; q++ {
+			if ffi.ff.Parameters.(*copyParameters).out[ffi.inIndex[q+1]].GetRingCount() <= max ||
+				ffi.ff.Parameters.(*copyParameters).outCopy[ffi.inIndex[q+1]].GetRingCount() <= max {
+				return true
+			}
 		}
 	case *segmentParameters:
-		p := *(ff.Parameters.(*segmentParameters).out)
+		p := *(ffi.ff.Parameters.(*segmentParameters).out)
 		for i := range p {
-			if p[i].GetRingCount() <= max {
-				return true
+			for q := int32(0); q < ffi.inIndex[0]; q++ {
+				if p[i][ffi.inIndex[q+1]].GetRingCount() <= max {
+					return true
+				}
 			}
 		}
 	}
 	return false
 }
+
+func constructZeroIndex(old []int32) []int32 {
+	return make([]int32, len(old), len(old))
+}
+
+func constructDuplicatedIndex(old []int32) []int32 {
+	newIndex := make([]int32, len(old), len(old))
+	oldLen := old[0]/2 + old[0]%2
+	newLen := old[0] / 2
+	for q := int32(0); q < newLen; q++ {
+		newIndex[q+1] = old[q+1+oldLen]
+	}
+	newIndex[0] = newLen
+	old[0] = oldLen
+	return newIndex
+}
+
+func constructNewIndex(inIndexNumber int32) []int32 {
+	if inIndexNumber == 0 {
+		return nil
+	}
+	newIndex := make([]int32, inIndexNumber+1, inIndexNumber+1)
+	for q := int32(0); q < inIndexNumber; q++ {
+		newIndex[q+1] = q
+	}
+	newIndex[0] = inIndexNumber
+	return newIndex
+}
diff --git a/low/low.go b/low/low.go
index 68b4a6b8..fdaf1c01 100644
--- a/low/low.go
+++ b/low/low.go
@@ -44,6 +44,7 @@ func DirectSend(m *Mbuf, port uint16) bool {
 
 // Ring is a ring buffer for pointers
 type Ring C.struct_nff_go_ring
+type Rings []*Ring
 
 // Mbuf is a message buffer.
 type Mbuf C.struct_rte_mbuf
@@ -70,16 +71,8 @@ func GetPort(n uint16) *Port {
 	return p
 }
 
-func CheckRSSPacketCount(p *Port) uint32 {
-	return uint32(C.checkRSSPacketCount((*C.struct_cPort)(p), C.int16_t(p.QueuesNumber-1)))
-}
-
-func DecreaseRSS(p *Port) {
-	C.changeRSSReta((*C.struct_cPort)(p), false)
-}
-
-func IncreaseRSS(p *Port) bool {
-	return bool(C.changeRSSReta((*C.struct_cPort)(p), true))
+func CheckRSSPacketCount(p *Port, queue int16) uint32 {
+	return uint32(C.checkRSSPacketCount((*C.struct_cPort)(p), (C.int16_t(queue))))
 }
 
 // GetPortMACAddress gets MAC address of given port.
@@ -242,6 +235,15 @@ func CreateRing(count uint) *Ring {
 	return (*Ring)(unsafe.Pointer(C.nff_go_ring_create(C.CString(name), C.uint(count), C.SOCKET_ID_ANY, 0x0000)))
 }
 
+// CreateRings creates ring with given name and count.
+func CreateRings(count uint, inIndexNumber int32) Rings {
+	rings := make(Rings, inIndexNumber, inIndexNumber)
+	for i := int32(0); i < inIndexNumber; i++ {
+		rings[i] = CreateRing(count)
+	}
+	return rings
+}
+
 // EnqueueBurst enqueues data to ring buffer.
 func (ring *Ring) EnqueueBurst(buffer []uintptr, count uint) uint {
 	return nffgoRingMpEnqueueBurst((*C.struct_nff_go_ring)(ring), buffer, count)
@@ -445,11 +447,11 @@ func (ring *Ring) GetRingCount() uint32 {
 }
 
 // ReceiveRSS - get packets from port and enqueue on a Ring.
-func ReceiveRSS(port uint16, queue int16, OUT *Ring, flag *int32, coreID int) {
+func ReceiveRSS(port uint16, inIndex []int32, OUT Rings, flag *int32, coreID int) {
 	if C.rte_eth_dev_socket_id(C.uint16_t(port)) != C.int(C.rte_lcore_to_socket_id(C.uint(coreID))) {
 		common.LogWarning(common.Initialization, "Receive port", port, "is on remote NUMA node to polling thread - not optimal performance.")
 	}
-	C.receiveRSS(C.uint16_t(port), C.int16_t(queue), OUT.DPDK_ring, (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
+	C.receiveRSS(C.uint16_t(port), (*C.int32_t)(unsafe.Pointer(&(inIndex[0]))), C.extractDPDKRings((**C.struct_nff_go_ring)(unsafe.Pointer(&(OUT[0]))), C.int32_t(len(OUT))), (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
 }
 
 // ReceiveKNI - get packets from Linux core and enqueue on a Ring.
@@ -458,17 +460,17 @@ func ReceiveKNI(port uint16, OUT *Ring, flag *int32, coreID int) {
 }
 
 // Send - dequeue packets and send.
-func Send(port uint16, queue int16, IN *Ring, flag *int32, coreID int) {
+func Send(port uint16, queue int16, IN Rings, inIndexNumber int32, flag *int32, coreID int) {
 	t := C.rte_eth_dev_socket_id(C.uint16_t(port))
 	if queue != -1 && t != C.int(C.rte_lcore_to_socket_id(C.uint(coreID))) {
 		common.LogWarning(common.Initialization, "Send port", port, "is on remote NUMA node to polling thread - not optimal performance.")
 	}
-	C.nff_go_send(C.uint16_t(port), C.int16_t(queue), IN.DPDK_ring, (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
+	C.nff_go_send(C.uint16_t(port), C.int16_t(queue), C.extractDPDKRings((**C.struct_nff_go_ring)(unsafe.Pointer(&(IN[0]))), C.int32_t(len(IN))), C.int32_t(len(IN)), (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
 }
 
 // Stop - dequeue and free packets.
-func Stop(IN *Ring, flag *int32, coreID int) {
-	C.nff_go_stop(IN.DPDK_ring, (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
+func Stop(IN Rings, flag *int32, coreID int) {
+	C.nff_go_stop(C.extractDPDKRings((**C.struct_nff_go_ring)(unsafe.Pointer(&(IN[0]))), C.int32_t(len(IN))), (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
 }
 
 // InitDPDKArguments allocates and initializes arguments for dpdk.
@@ -512,16 +514,21 @@ func GetPortsNumber() int {
 	return int(C.rte_eth_dev_count())
 }
 
+func CheckPortRSS(port uint16) int32 {
+	return int32(C.check_port_rss(C.uint16_t(port)))
+}
+
 // CreatePort initializes a new port using global settings and parameters.
-func CreatePort(port uint16, willReceive bool, sendQueuesNumber uint16, promiscuous bool, hwtxchecksum bool) error {
-	var mempool *C.struct_rte_mempool
+func CreatePort(port uint16, willReceive bool, sendQueuesNumber uint16, promiscuous bool, hwtxchecksum bool, inIndex int32) error {
+	var mempools **C.struct_rte_mempool
 	if willReceive {
-		mempool = (*C.struct_rte_mempool)(CreateMempool("receive"))
+		m := CreateMempools("receive", inIndex)
+		mempools = (**C.struct_rte_mempool)(unsafe.Pointer(&(m[0])))
 	} else {
-		mempool = nil
+		mempools = nil
 	}
 	if C.port_init(C.uint16_t(port), C.bool(willReceive), C.uint16_t(sendQueuesNumber),
-		mempool, C._Bool(promiscuous), C._Bool(hwtxchecksum)) != 0 {
+		mempools, C._Bool(promiscuous), C._Bool(hwtxchecksum), C.int32_t(inIndex)) != 0 {
 		msg := common.LogError(common.Initialization, "Cannot init port ", port, "!")
 		return common.WrapWithNFError(nil, msg, common.FailToInitPort)
 	}
@@ -543,6 +550,15 @@ func CreateMempool(name string) *Mempool {
 	return (*Mempool)(mempool)
 }
 
+func CreateMempools(name string, inIndex int32) []*Mempool {
+	m := make([]*Mempool, inIndex, inIndex)
+	for i := int32(0); i < inIndex; i++ {
+		m[i] = CreateMempool(name)
+	}
+	return m
+}
+
+
 // SetAffinity sets cpu affinity mask.
 func SetAffinity(coreID int) error {
 	// go tool trace shows that each proc executes different goroutine. However it is expected behavior
diff --git a/low/low.h b/low/low.h
index 4c6c41cb..7fa49f01 100644
--- a/low/low.h
+++ b/low/low.h
@@ -146,44 +146,16 @@ int checkRSSPacketCount(struct cPort *port, int16_t queue) {
 	return rte_eth_rx_queue_count(port->PortId, queue);
 }
 
-bool changeRSSReta(struct cPort *port, bool increment) {
+int check_port_rss(uint16_t port) {
 	struct rte_eth_dev_info dev_info;
 	memset(&dev_info, 0, sizeof(dev_info));
-	rte_eth_dev_info_get(port->PortId, &dev_info);
-
-	struct rte_eth_rss_reta_entry64 reta_conf[APP_RETA_SIZE_MAX];
-	memset(reta_conf, 0, sizeof(reta_conf));
-
-	// This is required, hanging without this
-	for (int i = 0; i < dev_info.reta_size / RTE_RETA_GROUP_SIZE; i++) {
-		reta_conf[i].mask = UINT64_MAX;
-	}
-
-	if (increment) {
-		if (port->QueuesNumber == dev_info.max_rx_queues) {
-			return false;
-		}
-		port->QueuesNumber++;
-	} else {
-		port->QueuesNumber--;
-	}
-
-	rte_eth_dev_rss_reta_query(port->PortId, reta_conf, dev_info.reta_size);
-
-	// http://dpdk.org/doc/api/examples_2ip_pipeline_2init_8c-example.html#a27
-	for (int i = 0; i < dev_info.reta_size / RTE_RETA_GROUP_SIZE; i++) {
-		for (int j = 0; j < RTE_RETA_GROUP_SIZE; j++) {
-			reta_conf[i].reta[j] = j % port->QueuesNumber;
-		}
-	}
-	rte_eth_dev_rss_reta_update(port->PortId, reta_conf, dev_info.reta_size);
-	return true;
-	// TODO we don't start or stop queues here. Is it right?
+	rte_eth_dev_info_get(port, &dev_info);
+	return dev_info.max_rx_queues;
 }
 
 // Initializes a given port using global settings and with the RX buffers
 // coming from the mbuf_pool passed as a parameter.
-int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct rte_mempool *mbuf_pool, bool promiscuous, bool hwtxchecksum) {
+int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct rte_mempool **mbuf_pools, bool promiscuous, bool hwtxchecksum, int32_t inIndex) {
 	uint16_t rx_rings, tx_rings = sendQueuesNumber;
 
 	struct rte_eth_dev_info dev_info;
@@ -191,12 +163,7 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 	rte_eth_dev_info_get(port, &dev_info);
 
 	if (willReceive) {
-		// TODO Investigate this
-		if (dev_info.max_rx_queues > 16) {
-			rx_rings = 16;
-		} else {
-			rx_rings = dev_info.max_rx_queues;
-		}
+		rx_rings = inIndex;
 		if (tx_rings == 0) {
 			// All receive ports should have at least one send queue to handle ARP
 			tx_rings = 1;
@@ -226,7 +193,7 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 	/* Allocate and set up RX queues per Ethernet port. */
 	for (q = 0; q < rx_rings; q++) {
 		retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE,
-				rte_eth_dev_socket_id(port), NULL, mbuf_pool);
+				rte_eth_dev_socket_id(port), NULL, mbuf_pools[q]);
 		if (retval < 0)
 			return retval;
 	}
@@ -260,10 +227,6 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 		.PortId  = port,
 		.QueuesNumber = rx_rings
 	};
-	// Stop all queues except the last one. Queues will be added later if needed
-	for (q = 0; q < rx_rings - 1; q++) {
-		changeRSSReta(&newPort, false);
-	}
 
 	return 0;
 }
@@ -356,27 +319,29 @@ static inline struct rte_mbuf* reassemble(struct rte_ip_frag_tbl* tbl, struct rt
 	return buf;
 }
 
-void receiveRSS(uint16_t port, int16_t queue, struct rte_ring *out_ring, volatile int *flag, int coreId) {
+void receiveRSS(uint16_t port, volatile int32_t *inIndex, struct rte_ring **out_rings, volatile int *flag, int coreId) {
 	setAffinity(coreId);
 	struct rte_mbuf *bufs[BURST_SIZE];
 	REASSEMBLY_INIT
-
 	while (*flag == process) {
-		// Get packets from port
-		uint16_t rx_pkts_number = rte_eth_rx_burst(port, queue, bufs, BURST_SIZE);
-		if (unlikely(rx_pkts_number == 0)) {
-			continue;
-		}
-		rx_pkts_number = handleReceived(bufs, rx_pkts_number, tbl, pdeath_row);
-
-		uint16_t pushed_pkts_number = rte_ring_enqueue_burst(out_ring, (void*)bufs, rx_pkts_number, NULL);
-		// Free any packets which can't be pushed to the ring. The ring is probably full.
-		handleUnpushed((void*)bufs, pushed_pkts_number, rx_pkts_number);
+		for (int q = 0; q < inIndex[0]; q++) {
+			// Get packets from port
+			uint16_t rx_pkts_number = rte_eth_rx_burst(port, inIndex[q+1], bufs, BURST_SIZE);
+			if (unlikely(rx_pkts_number == 0)) {
+				continue;
+			}
+			rx_pkts_number = handleReceived(bufs, rx_pkts_number, tbl, pdeath_row);
+
+			uint16_t pushed_pkts_number = rte_ring_enqueue_burst(out_rings[inIndex[q+1]], (void*)bufs, rx_pkts_number, NULL);
+			// Free any packets which can't be pushed to the ring. The ring is probably full.
+			handleUnpushed((void*)bufs, pushed_pkts_number, rx_pkts_number);
 #ifdef DEBUG
-		receive_received += rx_pkts_number;
-		receive_pushed += pushed_pkts_number;
+			receive_received += rx_pkts_number;
+			receive_pushed += pushed_pkts_number;
 #endif
+		}
 	}
+	free(out_rings);
 	*flag = wasStopped;
 }
 
@@ -405,58 +370,62 @@ void receiveKNI(uint16_t port, struct rte_ring *out_ring, volatile int *flag, in
 	*flag = wasStopped;
 }
 
-void nff_go_send(uint16_t port, int16_t queue, struct rte_ring *in_ring, volatile int *flag, int coreId) {
+void nff_go_send(uint16_t port, int16_t queue, struct rte_ring **in_rings, int32_t inIndexNumber, volatile int *flag, int coreId) {
 	setAffinity(coreId);
 
 	struct rte_mbuf *bufs[BURST_SIZE];
 	uint16_t buf;
 	uint16_t tx_pkts_number;
 	while (*flag == process) {
-		// Get packets for TX from ring
-		uint16_t pkts_for_tx_number = rte_ring_mc_dequeue_burst(in_ring, (void*)bufs, BURST_SIZE, NULL);
-
-		if (unlikely(pkts_for_tx_number == 0))
-			continue;
-
-		if (queue != -1) {
-			tx_pkts_number = rte_eth_tx_burst(port, queue, bufs, pkts_for_tx_number);
-		} else {
-			// if queue == "-1" this means that this send is to KNI device
-			tx_pkts_number = rte_kni_tx_burst(kni[port], bufs, pkts_for_tx_number);
-		}
-		// Free any unsent packets
-		handleUnpushed(bufs, tx_pkts_number, pkts_for_tx_number);
+		for (int q = 0; q < inIndexNumber; q++) {
+			// Get packets for TX from ring
+			uint16_t pkts_for_tx_number = rte_ring_mc_dequeue_burst(in_rings[q], (void*)bufs, BURST_SIZE, NULL);
+
+			if (unlikely(pkts_for_tx_number == 0))
+				continue;
+
+			if (queue != -1) {
+				tx_pkts_number = rte_eth_tx_burst(port, queue, bufs, pkts_for_tx_number);
+			} else {
+				// if queue == "-1" this means that this send is to KNI device
+				tx_pkts_number = rte_kni_tx_burst(kni[port], bufs, pkts_for_tx_number);
+			}
+			// Free any unsent packets
+			handleUnpushed(bufs, tx_pkts_number, pkts_for_tx_number);
 #ifdef DEBUG
-		send_required += pkts_for_tx_number;
-		send_sent += tx_pkts_number;
+			send_required += pkts_for_tx_number;
+			send_sent += tx_pkts_number;
 #endif
+		}
 	}
+	free(in_rings);
 	*flag = wasStopped;
 }
 
-void nff_go_stop(struct rte_ring *in_ring, volatile int *flag, int coreId) {
+void nff_go_stop(struct rte_ring **in_rings, volatile int *flag, int coreId) {
 	setAffinity(coreId);
-
 	struct rte_mbuf *bufs[BURST_SIZE];
 	uint16_t buf;
 	// Flag is used for both scheduler and stop.
 	// stopRequest will stop scheduler and this loop will stop with stopRequest+1
 	while (*flag == process || *flag == stopRequest) {
-		// Get packets for freeing from ring
-		uint16_t pkts_for_free_number = rte_ring_mc_dequeue_burst(in_ring, (void*)bufs, BURST_SIZE, NULL);
+		for (int q = 0; q < 50 /*Maximum possible rings*/; q++) {
+			// Get packets for freeing from ring
+			uint16_t pkts_for_free_number = rte_ring_mc_dequeue_burst(in_rings[q], (void*)bufs, BURST_SIZE, NULL);
 
-		if (unlikely(pkts_for_free_number == 0))
-			continue;
-
-		// Free all these packets
-		for (buf = 0; buf < pkts_for_free_number; buf++) {
-			rte_pktmbuf_free(bufs[buf]);
-		}
+			if (unlikely(pkts_for_free_number == 0))
+				continue;
 
+			// Free all these packets
+			for (buf = 0; buf < pkts_for_free_number; buf++) {
+				rte_pktmbuf_free(bufs[buf]);
+			}
 #ifdef DEBUG
-		stop_freed += pkts_for_free_number;
+			stop_freed += pkts_for_free_number;
 #endif
+		}
 	}
+	free(in_rings);
 	*flag = wasStopped;
 }
 
@@ -581,6 +550,14 @@ struct nff_go_ring {
 	uint32_t offset;
 };
 
+struct rte_ring** extractDPDKRings(struct nff_go_ring** r, int32_t inIndexNumber) {
+	struct rte_ring **output = malloc(inIndexNumber*sizeof(struct rte_ring *));
+	for (int i = 0; i < inIndexNumber; i++) {
+		output[i] = r[i]->DPDK_ring;
+	}
+	return output;
+}
+
 struct nff_go_ring *
 nff_go_ring_create(const char *name, unsigned count, int socket_id, unsigned flags) {
 	struct nff_go_ring* r = malloc(sizeof(struct nff_go_ring));

From aed8501e0d01162079035dfd0959927b25725d50 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Thu, 21 Jun 2018 10:29:28 -0500
Subject: [PATCH 02/50] Switched to virtio network cards in VMs because they
 are the most stable

---
 vagrant/Vagrantfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 2a78b8de..453ad063 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -125,7 +125,7 @@ SHELL
                             auto_config: false,
                             virtualbox__intnet: "#{vm_name}-link-#{user}-#{i}-#{j}",
                             :mac => '3cfdfea4ddf0',
-                            :model_type => 'e1000',
+                            :model_type => 'virtio',
                             :libvirt__forward_mode => 'none',
                             :libvirt__tunnel_type => 'udp',
                             :libvirt__tunnel_local_port => vm_second_port_base + i * vm_links_number + j,
@@ -134,7 +134,7 @@ SHELL
             node.vm.network "private_network",
                             auto_config: false,
                             virtualbox__intnet: "#{vm_name}-link-#{user}-#{i}-#{j}",
-                            :model_type => 'e1000',
+                            :model_type => 'virtio',
                             :libvirt__forward_mode => 'none',
                             :libvirt__tunnel_type => 'udp',
                             :libvirt__tunnel_local_port => vm_second_port_base + i * vm_links_number + j,
@@ -149,7 +149,7 @@ SHELL
           node.vm.network "private_network",
                           auto_config: false,
                           virtualbox__intnet: "#{vm_name}-link-#{user}-#{i + 1}-#{j}",
-                          :model_type => 'e1000',
+                          :model_type => 'virtio',
                           :libvirt__forward_mode => 'none',
                           :libvirt__tunnel_type => 'udp',
                           :libvirt__tunnel_local_port => vm_port_base + (i + 1) * vm_links_number + j,

From 833361a3a4f938ac04eaab5905779142dc170dc2 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Mon, 25 Jun 2018 09:32:43 -0500
Subject: [PATCH 03/50] Fixed pktgen URL and directory after dpdk site redesign

---
 mk/include.mk | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mk/include.mk b/mk/include.mk
index d657e2ae..1d702142 100644
--- a/mk/include.mk
+++ b/mk/include.mk
@@ -12,9 +12,9 @@ ifndef DPDK_URL
 DPDK_URL=http://fast.dpdk.org/rel/dpdk-$(DPDK_VERSION).tar.xz
 endif
 PKTGEN_VERSION=3.4.9
-PKTGEN_DIR=pktgen-$(PKTGEN_VERSION)
+PKTGEN_DIR=pktgen-dpdk-pktgen-$(PKTGEN_VERSION)
 ifndef PKTGEN_URL
-PKTGEN_URL=http://dpdk.org/browse/apps/pktgen-dpdk/snapshot/pktgen-$(PKTGEN_VERSION).tar.xz
+PKTGEN_URL=http://git.dpdk.org/apps/pktgen-dpdk/snapshot/pktgen-dpdk-pktgen-$(PKTGEN_VERSION).tar.xz
 endif
 export RTE_SDK = $(PROJECT_ROOT)/dpdk/$(DPDK_DIR)
 export RTE_TARGET = x86_64-native-linuxapp-gcc

From c908936a5d90aa6f8ffc8e2b6a50175dfb10f130 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Fri, 8 Jun 2018 19:36:06 +0300
Subject: [PATCH 04/50] Added speed regulation to generator, ability to
 generate imix, fixed errors

---
 test/localTesting/pktgen/README.md            |  79 +++-
 test/localTesting/pktgen/generate.go          | 390 ++++++++++++------
 .../pktgen/parseConfig/parseConfig.go         | 129 ++++--
 test/localTesting/pktgen/testing/mix.json     | 110 +++++
 test/localTesting/pktgen/testing/run.sh       |  26 +-
 5 files changed, 564 insertions(+), 170 deletions(-)
 create mode 100644 test/localTesting/pktgen/testing/mix.json

diff --git a/test/localTesting/pktgen/README.md b/test/localTesting/pktgen/README.md
index fb888de8..c5b0ca3c 100644
--- a/test/localTesting/pktgen/README.md
+++ b/test/localTesting/pktgen/README.md
@@ -6,17 +6,32 @@ Pktgen parses config in json format and generates packets according to it in pca
 ### Command-line options:
 * --totalPackets sets the number of packets to generate, default value is 10000000
 * --infile sets the name of the file with packet configurations, default value is "config.json"
-* --outfile sets the name of the file to write output to, default value is "pkts_generated.pcap"
+* --outfile sets the name of the file to write output to, default value is "pkts_generated.pcap". Can be set with port (the output is copied).
+* --outport sets the port number to send output to. Can be used alone, with or instead of outfile.
+* --speed sets the speed of generator
+* --cycle sets cycle execution to generate infinite number of packets
+* --portConfig specifies config per port portNum: 'path', portNum2: 'path2'. For example: 1: 'ip4.json', 0: 'mix.json'
 
 ### Configuration syntax:
-File should be a structure containing structure with ethernet configuration:
+File should be a structure containing structure with ethernet header or mix configuration:
 ```json
 {
     "ether": {
     }
 }
 ```
-Inside each header can be either data or next level header.
+or mix following regexp "mix[0-9]*"
+```json
+{
+    "mix1": {
+    },
+    "mix2": {
+    },
+    "mix3": {
+    }
+}
+```
+Inside each packet header can be either data or next level header, inside mix is packet header and quantity.
 #### packet data configuration:
 possible options are:
 * "raw":
@@ -325,4 +340,60 @@ Ethernet source is set to sha by default, destination is broadcast.
     ]
 }
 ```
-
+#### mix config:
+Mix should contain packet configuration "ether" and "quantity".
+```json
+"mix1": {
+    "ether": {
+                "saddr": {
+                    "range": {
+                        "min": "00:25:96:FF:FE:12",
+                        "start": "00:30:00:FF:FE:12",
+                        "max": "00:FF:96:FF:FE:12",
+                        "incr": 10
+                    }
+                },
+                "daddr": "00:FF:96:FF:FE:12",
+                "randbytes":    {
+                    "size": 40,
+                    "deviation": 0
+                }   
+             },
+    "quantity": 6
+},
+"mix2": {
+    "ether": {
+                "saddr": "00:FF:96:FF:FE:12",
+                "daddr": "00:FF:96:FF:FE:12",
+                "randbytes":    {
+                    "size": 500,
+                    "deviation": 0
+                }   
+             },
+    "quantity": 3
+},
+"mix3": {
+            "ether": {
+                        "saddr": "00:25:96:FF:FE:12",
+                        "daddr": "00:00:96:FF:00:00",
+                        "ip": {
+                            "version": 4,
+                            "saddr": "1.1.127.1",
+                            "daddr": "1.1.1.3",
+                            "tcp": {
+                                "sport": 8080,
+                                "dport": 2000,
+                                "seq": "increasing",
+                                "flags": ["ack", "fin", "syn"],
+                                "randbytes":    {
+                                    "size": 1466,
+                                    "deviation": 0
+                                }
+                            }
+                        }
+                    },
+            "quantity": 1
+        }
+```
+The following sequence of packets will be generated for config above:
+6 packets "mix1" configuration, 3 packets "mix2" configuration and 1 packet "mix3" and this chain will be repeated until a needed number is generated.
diff --git a/test/localTesting/pktgen/generate.go b/test/localTesting/pktgen/generate.go
index 374fb6d5..710f5b25 100644
--- a/test/localTesting/pktgen/generate.go
+++ b/test/localTesting/pktgen/generate.go
@@ -14,6 +14,8 @@ import (
 	"math/rand"
 	"net"
 	"os"
+	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -25,43 +27,120 @@ import (
 )
 
 var (
-	count uint64
-
+	count         uint64
 	testDoneEvent *sync.Cond
-	configuration *parseConfig.PacketConfig
-
-	outFile      string
-	inFile       string
-	totalPackets uint64
+	number        uint64
+	isCycled      bool
 )
 
+type mapFlags struct {
+	value map[int]string
+	set   bool
+}
+
+func (m *mapFlags) String() string {
+	s := ""
+	for key, value := range (*m).value {
+		if s != "" {
+			s += ","
+		}
+		s += fmt.Sprintf("%d: '%s'", key, value)
+	}
+	return s
+}
+
+func (m *mapFlags) Set(s string) error {
+	ss := strings.Split(s, ",")
+	(*m).value = make(map[int]string)
+	for _, val := range ss {
+		val = strings.TrimSpace(val)
+		pair := strings.Split(val, ":")
+		key, err := strconv.Atoi(pair[0])
+		if err != nil {
+			return err
+		}
+		value := strings.Trim(pair[1], "' ")
+		(*m).value[key] = value
+	}
+	(*m).set = true
+	return nil
+}
+
 func main() {
-	flag.StringVar(&outFile, "outfile", "pkts_generated.pcap", "file to write output to")
+	var (
+		portflag           uint
+		outFile, inFile    string
+		outPort            uint16
+		usePort, useFile   bool
+		speed              uint64
+		m                  sync.Mutex
+		fileFlow, portFlow *flow.Flow
+		eachPortConfig     mapFlags
+	)
+	flag.StringVar(&outFile, "outfile", "", "file to write output to")
+	flag.UintVar(&portflag, "outport", 70000, "port to send output to")
 	flag.StringVar(&inFile, "infile", "config.json", "file with configurations for generator")
-	flag.Uint64Var(&totalPackets, "totalPackets", 10000000, "stop after generation totalPackets number")
+	flag.Uint64Var(&number, "number", 10000000, "stop after generation number number")
+	flag.BoolVar(&isCycled, "cycle", false, "cycle execution and generate infinite number of packets")
+	flag.Uint64Var(&speed, "speed", 6000000, "speed of fast generator, Pkts/s")
+	flag.Var(&eachPortConfig, "portConfig", "specify config per port portNum: 'path', portNum2: 'path2'. For example: 1: 'ip4.json', 0: 'mix.json'")
 	flag.Parse()
 
-	var err error
-	configuration, err = ReadConfig(inFile)
-	if err != nil {
-		panic(fmt.Sprintf("config reading failed: %v", err))
-	}
-
 	// Init NFF-GO system at 16 available cores
-	config := flow.Config{
-		CPUList: "0-15",
-	}
+	config := flow.Config{}
 	flow.CheckFatal(flow.SystemInit(&config))
 
-	var m sync.Mutex
 	testDoneEvent = sync.NewCond(&m)
 
-	generator, err := getGenerator()
-	flow.CheckFatal(err)
-	// Create packet flow
-	outputFlow := flow.SetGenerator(generator, nil)
+	if eachPortConfig.set {
+		for key, value := range eachPortConfig.value {
+			configuration, err := ReadConfig(value)
+			if err != nil {
+				panic(fmt.Sprintf("%s config reading failed: %v", value, err))
+			}
+			context, err := getGeneratorContext(configuration)
+			flow.CheckFatal(err)
+			outFlow, err := flow.SetFastGenerator(generate, speed, &context)
+			flow.CheckFatal(err)
+			flow.CheckFatal(flow.SetSender(outFlow, uint16(key)))
+		}
+	} else {
+		if portflag != 70000 {
+			usePort = true
+			outPort = uint16(portflag)
+		}
+		if outFile != "" {
+			useFile = true
+		}
+		if !usePort && !useFile {
+			useFile = true
+			outFile = "pkts_generated.pcap"
+		}
+		configuration, err := ReadConfig(inFile)
+		if err != nil {
+			panic(fmt.Sprintf("config reading failed: %v", err))
+		}
+
+		context, err := getGeneratorContext(configuration)
+		flow.CheckFatal(err)
+		fileFlow, err = flow.SetFastGenerator(generate, speed, &context)
+		flow.CheckFatal(err)
+		if useFile && usePort {
+			portFlow, err = flow.SetCopier(fileFlow)
+			flow.CheckFatal(err)
+		} else {
+			if usePort {
+				portFlow = fileFlow
+			}
+		}
+		if useFile {
+			flow.CheckFatal(flow.SetSenderFile(fileFlow, outFile))
+		}
+		if usePort {
+			flow.CheckFatal(flow.SetSender(portFlow, outPort))
+		}
+	}
 
-	flow.CheckFatal(flow.SetSenderFile(outputFlow, outFile))
 	// Start pipeline
 	go func() {
 		flow.CheckFatal(flow.SystemStart())
@@ -72,49 +151,45 @@ func main() {
 	testDoneEvent.Wait()
 	testDoneEvent.L.Unlock()
 
-	sent := atomic.LoadUint64(&count)
-
 	// Print report
-	println("Sent", sent, "packets")
+	println("Sent", atomic.LoadUint64(&count), "packets")
 }
 
 // ReadConfig function reads and parses config file.
-func ReadConfig(fileName string) (*parseConfig.PacketConfig, error) {
+func ReadConfig(fileName string) ([]*parseConfig.MixConfig, error) {
 	f, err := os.Open(fileName)
 	if err != nil {
 		return nil, fmt.Errorf("opening file failed with: %v ", err)
 	}
-	cfg, err := parseConfig.ParsePacketConfig(f)
+	cfg, err := parseConfig.ParseConfig(f)
 	if err != nil {
 		return nil, fmt.Errorf("parsing config failed with: %v", err)
 	}
 	return cfg, nil
 }
 
-func getNextAddr(addr parseConfig.AddrRange) (nextAddr []uint8) {
-	if addr.Incr == 0 {
-		return addr.Start
+func getNextAddr(addr *parseConfig.AddrRange) []uint8 {
+	if len(addr.Current) == 0 {
+		addr.Current = []uint8{0}
 	}
-	if len(addr.Start) == 0 {
-		return []uint8{0}
+	if addr.Incr == 0 {
+		return addr.Current
 	}
-	if bytes.Compare(addr.Start, addr.Min) > 0 || bytes.Compare(addr.Start, addr.Max) < 0 {
-		addr.Start = addr.Min
+	// if current < min or current > max, copy min to current
+	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
+		addr.Current = make([]byte, len(addr.Min))
+		copy(addr.Current, addr.Min)
 	}
-	nextAddr = addr.Start
+	// copy current to return
+	retAddr := make([]byte, len(addr.Current))
+	copy(retAddr, addr.Current)
+	// increment current
 	addressInt := big.NewInt(0)
-	addressInt.SetBytes(addr.Start)
+	addressInt.SetBytes(addr.Current)
 	addressInt.Add(big.NewInt(int64(addr.Incr)), addressInt)
 	newAddr := addressInt.Bytes()
-	if len(newAddr) >= len(addr.Start) {
-		copy(addr.Start[:], newAddr[len(newAddr)-len(addr.Start):])
-	} else {
-		copy(addr.Start[len(addr.Start)-len(newAddr):], newAddr[:])
-	}
-	if bytes.Compare(addr.Start, addr.Max) <= 0 {
-		addr.Start = addr.Min
-	}
-	return nextAddr
+	copyAddr(addr.Current, newAddr, len(addr.Current))
+	return retAddr
 }
 
 func copyAddr(destination []uint8, source []uint8, size int) {
@@ -125,29 +200,19 @@ func copyAddr(destination []uint8, source []uint8, size int) {
 	}
 }
 
-func cropAddr(addr []uint8, size int) []uint8 {
-	cropped := make([]uint8, size)
-	if size < len(addr) {
-		copy(cropped[:], addr[len(addr)-size:])
-	} else {
-		copy(cropped[size-len(addr):], addr[:])
-	}
-	return cropped
-}
-
-func getNextPort(port parseConfig.PortRange) (nextPort uint16) {
-	if len(port.Start) == 0 {
-		return 0
+func getNextPort(port *parseConfig.PortRange) (nextPort uint16) {
+	if len(port.Current) == 0 {
+		port.Current = []uint16{0}
 	}
-	if port.Start[0] < port.Min || port.Start[0] > port.Max {
-		port.Start[0] = port.Min
+	if port.Current[0] < port.Min || port.Current[0] > port.Max {
+		port.Current[0] = port.Min
 	}
-	nextPort = port.Start[0]
-	port.Start[0] += port.Incr
+	nextPort = port.Current[0]
+	port.Current[0] += port.Incr
 	return nextPort
 }
 
-func getNextSeqNumber(seq parseConfig.Sequence) (nextSeqNum uint32) {
+func getNextSeqNumber(seq *parseConfig.Sequence) (nextSeqNum uint32) {
 	if len(seq.Next) == 0 {
 		return 0
 	}
@@ -207,7 +272,7 @@ func generateData(configuration interface{}) ([]uint8, error) {
 	return nil, fmt.Errorf("unknown data type")
 }
 
-func getGenerator() (func(*packet.Packet, flow.UserContext), error) {
+func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet, *parseConfig.PacketConfig), error) {
 	switch l2 := (configuration.Data).(type) {
 	case parseConfig.EtherConfig:
 		switch l3 := l2.Data.(type) {
@@ -237,19 +302,88 @@ func getGenerator() (func(*packet.Packet, flow.UserContext), error) {
 }
 
 func checkFinish() {
-	if count >= totalPackets {
+	if isCycled {
+		return
+	}
+	if atomic.AddUint64(&count, 1) >= number {
 		time.Sleep(time.Second)
 		testDoneEvent.Signal()
 	}
 }
 
-func generateEther(pkt *packet.Packet, context flow.UserContext) {
+// one unit for each mix
+type generatorTableUnit struct {
+	have, need    uint32
+	generatorFunc func(*packet.Packet, *parseConfig.PacketConfig)
+	config        *parseConfig.PacketConfig
+}
+
+func (gtu *generatorTableUnit) String() string {
+	return fmt.Sprintf("need: %d, config: %v\n", gtu.need, gtu.config)
+}
+
+type genParameters struct {
+	table  []generatorTableUnit
+	next   []uint32
+	length uint32
+}
+
+func (gp *genParameters) String() string {
+	s := "genP:\n"
+	for _, tu := range gp.table {
+		s = s + fmt.Sprintf("\ttu: %v\n", tu.String())
+	}
+	s = s + fmt.Sprintf("next: %d, len: %d\n", gp.next, gp.length)
+	return s
+}
+
+func (gp genParameters) Copy() interface{} {
+	cpy := make([]generatorTableUnit, len(gp.table))
+	copy(cpy, gp.table)
+	return genParameters{table: cpy, next: []uint32{0}, length: gp.length}
+}
+
+func (gp genParameters) Delete() {
+}
+
+func getGeneratorContext(mixConfig []*parseConfig.MixConfig) (genParameters, error) {
+	var t []generatorTableUnit
+	for _, packetConfig := range mixConfig {
+		genFunc, err := getGenerator(packetConfig.Config)
+		if err != nil {
+			return genParameters{}, err
+		}
+		tu := generatorTableUnit{have: 0, need: packetConfig.Quantity, generatorFunc: genFunc, config: packetConfig.Config}
+		t = append(t, tu)
+	}
+	return genParameters{table: t, next: []uint32{0}, length: uint32(len(t))}, nil
+}
+
+func generate(pkt *packet.Packet, context flow.UserContext) {
+	genP := context.(genParameters)
+	table := genP.table
+	next := genP.next[0]
+	if genP.length > 1 {
+		if table[next].have == table[next].need {
+			context.(genParameters).table[next].have = 0
+			if next+1 < genP.length {
+				next++
+			} else {
+				next = 0
+			}
+			context.(genParameters).next[0] = next
+		}
+	}
+	table[next].generatorFunc(pkt, table[next].config)
+	context.(genParameters).table[next].have++
+}
+
+func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	data, err := generateData(l2.Data)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l2: %v", err))
@@ -262,18 +396,18 @@ func generateEther(pkt *packet.Packet, context flow.UserContext) {
 		panic(fmt.Sprintf("InitEmptyPacket failed"))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillEtherHdr(pkt, l2); err != nil {
+	if err := fillEtherHdr(pkt, &l2); err != nil {
 		panic(fmt.Sprintf("failed to fill ether header %v", err))
 	}
+	config.Data = l2
 }
 
-func generateIP(pkt *packet.Packet, context flow.UserContext) {
+func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	data, err := generateData(l3.Data)
 	if err != nil {
@@ -295,30 +429,29 @@ func generateIP(pkt *packet.Packet, context flow.UserContext) {
 		panic(fmt.Sprintf("fillPacketl3 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-
-	if err := fillIPHdr(pkt, l3); err != nil {
+	if err := fillIPHdr(pkt, &l3); err != nil {
 		panic(fmt.Sprintf("failed to fill ip header %v", err))
 	}
-	if err := fillEtherHdr(pkt, l2); err != nil {
+	if err := fillEtherHdr(pkt, &l2); err != nil {
 		panic(fmt.Sprintf("failed to fill ether header %v", err))
 	}
 	if l3.Version == 4 {
 		pkt.GetIPv4CheckVLAN().HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pkt.GetIPv4CheckVLAN()))
 	}
+	l2.Data = l3
+	config.Data = l2
 }
 
-func generateARP(pkt *packet.Packet, context flow.UserContext) {
+func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.ARPConfig)
 	var SHA, THA [common.EtherAddrLen]uint8
-	copyAddr(SHA[:], getNextAddr(l3.SHA), common.EtherAddrLen)
-	SPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(l3.SPA)).To4())
-
+	copyAddr(SHA[:], getNextAddr(&(l3.SHA)), common.EtherAddrLen)
+	SPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SPA))).To4())
 	switch l3.Operation {
 	case packet.ARPRequest:
 		if l3.Gratuitous {
@@ -326,7 +459,7 @@ func generateARP(pkt *packet.Packet, context flow.UserContext) {
 				panic(fmt.Sprintf("InitGARPAnnouncementRequestPacket returned false"))
 			}
 		} else {
-			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(l3.TPA)).To4())
+			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.TPA))).To4())
 			if !packet.InitARPRequestPacket(pkt, SHA, SPA, TPA) {
 				panic(fmt.Sprintf("InitARPRequestPacket returned false"))
 			}
@@ -337,8 +470,8 @@ func generateARP(pkt *packet.Packet, context flow.UserContext) {
 				panic(fmt.Sprintf("InitGARPAnnouncementReplyPacket returned false"))
 			}
 		} else {
-			copyAddr(THA[:], getNextAddr(l3.THA), common.EtherAddrLen)
-			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(l3.TPA)).To4())
+			copyAddr(THA[:], getNextAddr(&(l3.THA)), common.EtherAddrLen)
+			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.TPA))).To4())
 			if !packet.InitARPReplyPacket(pkt, SHA, THA, SPA, TPA) {
 				panic(fmt.Sprintf("InitARPReplyPacket returned false"))
 			}
@@ -346,23 +479,24 @@ func generateARP(pkt *packet.Packet, context flow.UserContext) {
 	default:
 		panic(fmt.Sprintf("unsupported operation code: %d", l3.Operation))
 	}
-	if len(l2.DAddr.Start) != 0 {
-		copyAddr(pkt.Ether.DAddr[:], getNextAddr(l2.DAddr), common.EtherAddrLen)
+	if len(l2.DAddr.Current) != 0 {
+		copyAddr(pkt.Ether.DAddr[:], getNextAddr(&(l2.DAddr)), common.EtherAddrLen)
 	}
 	if l2.VLAN != nil {
 		if !pkt.AddVLANTag(l2.VLAN.TCI) {
 			panic("failed to add vlan tag to arp packet")
 		}
 	}
+	l2.Data = l3
+	config.Data = l2
 }
 
-func generateTCPIP(pkt *packet.Packet, context flow.UserContext) {
+func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.TCPConfig)
 	data, err := generateData(l4.Data)
@@ -385,14 +519,13 @@ func generateTCPIP(pkt *packet.Packet, context flow.UserContext) {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-
-	if err := fillTCPHdr(pkt, l4); err != nil {
+	if err := fillTCPHdr(pkt, &l4); err != nil {
 		panic(fmt.Sprintf("failed to fill tcp header %v", err))
 	}
-	if err := fillIPHdr(pkt, l3); err != nil {
+	if err := fillIPHdr(pkt, &l3); err != nil {
 		panic(fmt.Sprintf("failed to fill ip header %v", err))
 	}
-	if err := fillEtherHdr(pkt, l2); err != nil {
+	if err := fillEtherHdr(pkt, &l2); err != nil {
 		panic(fmt.Sprintf("failed to fill ether header %v", err))
 	}
 	if l3.Version == 4 {
@@ -401,15 +534,17 @@ func generateTCPIP(pkt *packet.Packet, context flow.UserContext) {
 	} else if l3.Version == 6 {
 		pkt.GetTCPForIPv6().Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetTCPForIPv6(), pkt.Data))
 	}
+	l3.Data = l4
+	l2.Data = l3
+	config.Data = l2
 }
 
-func generateUDPIP(pkt *packet.Packet, context flow.UserContext) {
+func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.UDPConfig)
 	data, err := generateData(l4.Data)
@@ -432,14 +567,13 @@ func generateUDPIP(pkt *packet.Packet, context flow.UserContext) {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-
-	if err := fillUDPHdr(pkt, l4); err != nil {
+	if err := fillUDPHdr(pkt, &l4); err != nil {
 		panic(fmt.Sprintf("failed to fill udp header %v", err))
 	}
-	if err := fillIPHdr(pkt, l3); err != nil {
+	if err := fillIPHdr(pkt, &l3); err != nil {
 		panic(fmt.Sprintf("failed to fill ip header %v", err))
 	}
-	if err := fillEtherHdr(pkt, l2); err != nil {
+	if err := fillEtherHdr(pkt, &l2); err != nil {
 		panic(fmt.Sprintf("failed to fill ether header %v", err))
 	}
 	if l3.Version == 4 {
@@ -448,15 +582,17 @@ func generateUDPIP(pkt *packet.Packet, context flow.UserContext) {
 	} else if l3.Version == 6 {
 		pkt.GetUDPForIPv6().DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv6UDPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetUDPForIPv6(), pkt.Data))
 	}
+	l3.Data = l4
+	l2.Data = l3
+	config.Data = l2
 }
 
-func generateICMPIP(pkt *packet.Packet, context flow.UserContext) {
+func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
 	checkFinish()
-	atomic.AddUint64(&count, 1)
-	l2 := configuration.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.ICMPConfig)
 	data, err := generateData(l4.Data)
@@ -479,14 +615,13 @@ func generateICMPIP(pkt *packet.Packet, context flow.UserContext) {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-
-	if err := fillICMPHdr(pkt, l4); err != nil {
+	if err := fillICMPHdr(pkt, &l4); err != nil {
 		panic(fmt.Sprintf("failed to fill icmp header %v", err))
 	}
-	if err := fillIPHdr(pkt, l3); err != nil {
+	if err := fillIPHdr(pkt, &l3); err != nil {
 		panic(fmt.Sprintf("failed to fill ip header %v", err))
 	}
-	if err := fillEtherHdr(pkt, l2); err != nil {
+	if err := fillEtherHdr(pkt, &l2); err != nil {
 		panic(fmt.Sprintf("failed to fill ether header %v", err))
 	}
 	if l3.Version == 4 {
@@ -495,57 +630,60 @@ func generateICMPIP(pkt *packet.Packet, context flow.UserContext) {
 	} else if l3.Version == 6 {
 		pkt.GetICMPForIPv6().Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetICMPForIPv6(), pkt.Data))
 	}
+	l3.Data = l4
+	l2.Data = l3
+	config.Data = l2
 }
 
-func fillTCPHdr(pkt *packet.Packet, l4 parseConfig.TCPConfig) error {
+func fillTCPHdr(pkt *packet.Packet, l4 *parseConfig.TCPConfig) error {
 	emptyPacketTCP := (*packet.TCPHdr)(pkt.L4)
-	emptyPacketTCP.SrcPort = packet.SwapBytesUint16(getNextPort(l4.SPort))
-	emptyPacketTCP.DstPort = packet.SwapBytesUint16(getNextPort(l4.DPort))
-	emptyPacketTCP.SentSeq = packet.SwapBytesUint32(getNextSeqNumber(l4.Seq))
+	emptyPacketTCP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
+	emptyPacketTCP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
+	emptyPacketTCP.SentSeq = packet.SwapBytesUint32(getNextSeqNumber((&l4.Seq)))
 	emptyPacketTCP.TCPFlags = l4.Flags
 	return nil
 }
 
-func fillUDPHdr(pkt *packet.Packet, l4 parseConfig.UDPConfig) error {
+func fillUDPHdr(pkt *packet.Packet, l4 *parseConfig.UDPConfig) error {
 	emptyPacketUDP := (*packet.UDPHdr)(pkt.L4)
-	emptyPacketUDP.SrcPort = packet.SwapBytesUint16(getNextPort(l4.SPort))
-	emptyPacketUDP.DstPort = packet.SwapBytesUint16(getNextPort(l4.DPort))
+	emptyPacketUDP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
+	emptyPacketUDP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
 	return nil
 }
 
-func fillICMPHdr(pkt *packet.Packet, l4 parseConfig.ICMPConfig) error {
+func fillICMPHdr(pkt *packet.Packet, l4 *parseConfig.ICMPConfig) error {
 	emptyPacketICMP := (*packet.ICMPHdr)(pkt.L4)
 	emptyPacketICMP.Type = l4.Type
 	emptyPacketICMP.Code = l4.Code
 	emptyPacketICMP.Identifier = l4.Identifier
-	emptyPacketICMP.SeqNum = packet.SwapBytesUint16(uint16(getNextSeqNumber(l4.Seq)))
+	emptyPacketICMP.SeqNum = packet.SwapBytesUint16(uint16(getNextSeqNumber(&(l4.Seq))))
 	return nil
 }
 
-func fillIPHdr(pkt *packet.Packet, l3 parseConfig.IPConfig) error {
+func fillIPHdr(pkt *packet.Packet, l3 *parseConfig.IPConfig) error {
 	if l3.Version == 4 {
 		pktIP := pkt.GetIPv4()
-		pktIP.SrcAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(l3.SAddr)).To4())
-		pktIP.DstAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(l3.DAddr)).To4())
+		pktIP.SrcAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SAddr))).To4())
+		pktIP.DstAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.DAddr))).To4())
 		return nil
 	}
 	pktIP := pkt.GetIPv6()
-	nextAddr := getNextAddr(l3.SAddr)
+	nextAddr := getNextAddr(&(l3.SAddr))
 	copyAddr(pktIP.SrcAddr[:], nextAddr, common.IPv6AddrLen)
-	nextAddr = getNextAddr(l3.DAddr)
+	nextAddr = getNextAddr(&(l3.DAddr))
 	copyAddr(pktIP.DstAddr[:], nextAddr, common.IPv6AddrLen)
 	return nil
 }
 
-func fillEtherHdr(pkt *packet.Packet, l2 parseConfig.EtherConfig) error {
+func fillEtherHdr(pkt *packet.Packet, l2 *parseConfig.EtherConfig) error {
 	if l2.VLAN != nil {
 		if err := addVLAN(pkt, l2.VLAN.TCI); err != nil {
 			return err
 		}
 	}
-	nextAddr := getNextAddr(l2.DAddr)
+	nextAddr := getNextAddr(&(l2.DAddr))
 	copyAddr(pkt.Ether.DAddr[:], nextAddr, common.EtherAddrLen)
-	nextAddr = getNextAddr(l2.SAddr)
+	nextAddr = getNextAddr(&(l2.SAddr))
 	copyAddr(pkt.Ether.SAddr[:], nextAddr, common.EtherAddrLen)
 	return nil
 }
diff --git a/test/localTesting/pktgen/parseConfig/parseConfig.go b/test/localTesting/pktgen/parseConfig/parseConfig.go
index 1c0bdb49..ec3981b2 100644
--- a/test/localTesting/pktgen/parseConfig/parseConfig.go
+++ b/test/localTesting/pktgen/parseConfig/parseConfig.go
@@ -8,25 +8,40 @@ import (
 	"math/rand"
 	"net"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/intel-go/nff-go/common"
 )
 
+var mixPattern = regexp.MustCompile(`^mix[0-9]*$`)
+
 // AddrRange describes range of addresses.
 type AddrRange struct {
-	Min   []uint8
-	Max   []uint8
-	Start []uint8
-	Incr  uint64
+	Min     []uint8
+	Max     []uint8
+	Current []uint8
+	Incr    uint64
+}
+
+func (ar *AddrRange) String() string {
+	s := "addrRange:\n"
+	s += fmt.Sprintf("min: %v, max: %v, start: %v, incr: %d", ar.Min, ar.Max, ar.Current, ar.Incr)
+	return s
 }
 
 // PortRange describes range of ports.
 type PortRange struct {
-	Min   uint16
-	Max   uint16
-	Start []uint16
-	Incr  uint16
+	Min     uint16
+	Max     uint16
+	Current []uint16
+	Incr    uint16
+}
+
+func (pr *PortRange) String() string {
+	s := "portRange:\n"
+	s += fmt.Sprintf("min: %v, max: %v, start: %v, incr: %d", pr.Min, pr.Max, pr.Current, pr.Incr)
+	return s
 }
 
 // SequenceType used in enum below.
@@ -49,6 +64,16 @@ type PacketConfig struct {
 	Data interface{}
 }
 
+// MixConfig contains PacketConfigs with quantity.
+type MixConfig struct {
+	Config   *PacketConfig
+	Quantity uint32
+}
+
+func (mc *MixConfig) String() string {
+	return fmt.Sprintf("config: %v; quantity: %v\n", *(mc.Config), mc.Quantity)
+}
+
 // EtherConfig configures ether header.
 type EtherConfig struct {
 	DAddr AddrRange
@@ -123,31 +148,72 @@ type Raw struct {
 	Data string
 }
 
-// ParsePacketConfig parses json config and returns structure.
-func ParsePacketConfig(f *os.File) (*PacketConfig, error) {
+// ParseConfig parses json config and returns []*MixConfig.
+func ParseConfig(f *os.File) (config []*MixConfig, err error) {
 	r := bufio.NewReader(f)
 	var in map[string]interface{}
-	err := json.NewDecoder(r).Decode(&in)
+	err = json.NewDecoder(r).Decode(&in)
 	if err != nil {
 		return nil, fmt.Errorf("decoding input from file returned: %v", err)
 	}
 	for k, v := range in {
-		switch strings.ToLower(k) {
-		case "ether":
+		key := strings.ToLower(k)
+		switch {
+		case key == "ether":
 			etherConfig := v.(map[string]interface{})
 			ethHdr, err := parseEtherHdr(etherConfig)
 			if err != nil {
 				return nil, fmt.Errorf("parseEtherHdr returned: %v", err)
 			}
-			return &PacketConfig{Data: ethHdr}, nil
+			pktConfig := &PacketConfig{Data: ethHdr}
+			return append(config, &MixConfig{Config: pktConfig, Quantity: 1}), nil
+		case mixPattern.MatchString(key):
+			return ParseMixConfig(in)
 		default:
-			return nil, fmt.Errorf("unexpected key: %s", k)
+			return nil, fmt.Errorf("unexpected key: %s, expected mix[0-9]* or ether", k)
 		}
 	}
 
 	return nil, fmt.Errorf("expected 'ether' key , but did not get")
 }
 
+// ParseMixConfig parses json config and returns []*MixConfig.
+func ParseMixConfig(in map[string]interface{}) (config []*MixConfig, err error) {
+	for k, v := range in {
+		key := strings.ToLower(k)
+		switch {
+		case mixPattern.MatchString(key):
+			var (
+				pktConfig *PacketConfig
+				q         uint32
+			)
+			mixUnit := v.(map[string]interface{})
+			for kk, vv := range mixUnit {
+				switch strings.ToLower(kk) {
+				case "ether":
+					etherConfig := vv.(map[string]interface{})
+					ethHdr, err := parseEtherHdr(etherConfig)
+					if err != nil {
+						return nil, fmt.Errorf("parseEtherHdr returned: %v", err)
+					}
+					pktConfig = &PacketConfig{Data: ethHdr}
+				case "quantity", "q":
+					q = uint32(vv.(float64))
+				default:
+					return nil, fmt.Errorf("unexpected key: %s, expected ether and quantity or q", k)
+				}
+			}
+			if q == 0 || pktConfig == nil {
+				return nil, fmt.Errorf("some fields were not set")
+			}
+			config = append(config, &MixConfig{Config: pktConfig, Quantity: q})
+		default:
+			return nil, fmt.Errorf("unexpected key: %s, expected mix[0-9]*", k)
+		}
+	}
+	return config, nil
+}
+
 func parseEtherHdr(in map[string]interface{}) (EtherConfig, error) {
 	ethConfig := EtherConfig{DAddr: AddrRange{}, SAddr: AddrRange{}, VLAN: nil, Data: Raw{Data: ""}}
 	for k, v := range in {
@@ -550,7 +616,7 @@ func parseMacAddr(value interface{}) (AddrRange, error) {
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing mac saddr returned: %v", err)
 		}
-		return AddrRange{Min: saddr, Max: saddr, Start: saddr, Incr: 0}, nil
+		return AddrRange{Min: saddr, Max: saddr, Current: saddr, Incr: 0}, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
 }
@@ -591,7 +657,7 @@ func parseIPAddr(value interface{}) (AddrRange, error) {
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing ip addr returned: %v", err)
 		}
-		return AddrRange{Min: addr, Max: addr, Start: addr, Incr: 0}, nil
+		return AddrRange{Min: addr, Max: addr, Current: addr, Incr: 0}, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
 }
@@ -600,7 +666,7 @@ type fn func(string) ([]uint8, error)
 
 func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error) {
 	addr := AddrRange{Incr: 0}
-	wasMin, wasMax, wasStart := false, false, false
+	wasMin, wasMax, wasStart, wasIncr := false, false, false, false
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "min":
@@ -622,10 +688,11 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing start returned: %v", err)
 			}
-			addr.Start = start
+			addr.Current = start
 			wasStart = true
 		case "incr":
 			addr.Incr = (uint64)(v.(float64))
+			wasIncr = true
 		default:
 			return AddrRange{}, fmt.Errorf("unknown key %s", k)
 		}
@@ -637,10 +704,14 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 		return AddrRange{}, fmt.Errorf("Min value should be less than Max")
 	}
 	if !wasStart {
-		addr.Start = addr.Min
+		addr.Current = make([]byte, len(addr.Min))
+		copy(addr.Current, addr.Min)
 	}
-	if bytes.Compare(addr.Start, addr.Min) < 0 || bytes.Compare(addr.Start, addr.Max) > 0 {
-		return AddrRange{}, fmt.Errorf("Start value should be between min and max")
+	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
+		return AddrRange{}, fmt.Errorf(fmt.Sprintf("Start value should be between min and max: start=%v, min=%v, max=%v", addr.Current, addr.Min, addr.Max))
+	}
+	if !wasIncr {
+		addr.Incr = 1
 	}
 	return addr, nil
 }
@@ -665,14 +736,14 @@ func parsePort(in interface{}) (PortRange, error) {
 			return PortRange{}, fmt.Errorf("Port should be positive")
 		}
 		port := (uint16)(in.(float64))
-		return PortRange{Min: port, Max: port, Start: []uint16{port}, Incr: 0}, nil
+		return PortRange{Min: port, Max: port, Current: []uint16{port}, Incr: 0}, nil
 	}
 	return PortRange{}, fmt.Errorf("unknown type")
 }
 
 func parsePortRange(in map[string]interface{}) (PortRange, error) {
 	portRng := PortRange{Incr: 0}
-	wasMin, wasMax, wasStart := false, false, false
+	wasMin, wasMax, wasStart, wasIncr := false, false, false, false
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "min":
@@ -691,13 +762,14 @@ func parsePortRange(in map[string]interface{}) (PortRange, error) {
 			if v.(float64) < 0 {
 				return PortRange{}, fmt.Errorf("Start should be positive")
 			}
-			portRng.Start = []uint16{(uint16)(v.(float64))}
+			portRng.Current = []uint16{(uint16)(v.(float64))}
 			wasStart = true
 		case "incr":
 			if v.(float64) < 0 {
 				return PortRange{}, fmt.Errorf("Incr should be positive")
 			}
 			portRng.Incr = (uint16)(v.(float64))
+			wasIncr = true
 		default:
 			return PortRange{}, fmt.Errorf("unknown key %s", k)
 		}
@@ -709,10 +781,13 @@ func parsePortRange(in map[string]interface{}) (PortRange, error) {
 		return PortRange{}, fmt.Errorf("Min value should be <= Max value")
 	}
 	if !wasStart {
-		portRng.Start = []uint16{portRng.Min}
+		portRng.Current = []uint16{portRng.Min}
 	}
-	if portRng.Start[0] < portRng.Min || portRng.Start[0] > portRng.Max {
+	if portRng.Current[0] < portRng.Min || portRng.Current[0] > portRng.Max {
 		return PortRange{}, fmt.Errorf("Start should be in range of min and max")
 	}
+	if !wasIncr {
+		portRng.Incr = 1
+	}
 	return portRng, nil
 }
diff --git a/test/localTesting/pktgen/testing/mix.json b/test/localTesting/pktgen/testing/mix.json
new file mode 100644
index 00000000..23aa7a3b
--- /dev/null
+++ b/test/localTesting/pktgen/testing/mix.json
@@ -0,0 +1,110 @@
+{
+    "mix1": {
+            "ether": {
+                        "saddr": {
+                            "range": {
+                                "min": "00:25:96:FF:FE:12",
+                                "max": "00:FF:96:FF:FE:12",
+                                "incr": 10
+                            }
+                        },
+                        "daddr": "00:00:96:FF:00:00",
+                        "ip": {
+                            "version": 4,
+                            "saddr": "1.1.127.1",
+                            "daddr": {
+                                "range": {
+                                    "min": "1.1.1.1",
+                                    "max": "1.1.1.5"
+                                }
+                            },
+                            "icmp": {
+                                "type": 10,
+                                "code": 1,
+                                "seq": "increasing",
+                                "randbytes":    {
+                                    "size": 6,
+                                    "deviation": 0
+                                }
+                            }
+                        }
+                    },
+            "quantity": 1
+        },
+        "mix2": {
+            "ether": {
+                        "saddr": {
+                            "range": {
+                                "min": "00:25:96:FF:FE:12",
+                                "max": "00:FF:96:FF:FE:12",
+                                "incr": 10
+                            }
+                        },
+                        "daddr": "00:00:96:FF:00:00",
+                        "ip": {
+                            "version": 4,
+                            "saddr": "1.1.127.1",
+                            "daddr": {
+                                "range": {
+                                    "min": "1.1.1.1",
+                                    "max": "3.3.3.3"
+                                }
+                            },
+                            "udp": {
+                                "sport": {
+                                    "range": {
+                                        "min": 1,
+                                        "max": 8080,
+                                        "incr": 100
+                                    }
+                                },
+                                "dport": 2000,
+                                "randbytes":    {
+                                    "size": 542,
+                                    "deviation": 0
+                                }
+                            }
+                        }
+                    },
+            "quantity": 3
+        },
+        "mix3": {
+            "ether": {
+                        "saddr": {
+                            "range": {
+                                "min": "00:25:96:FF:FE:12",
+                                "max": "00:FF:96:FF:FE:12",
+                                "incr": 10
+                            }
+                        },
+                        "daddr": "00:00:96:FF:00:00",
+                        "ip": {
+                            "version": 4,
+                            "saddr": "1.1.127.1",
+                            "daddr": {
+                                "range": {
+                                    "min": "1.1.1.1",
+                                    "max": "1.1.1.3"
+                                }
+                            },
+                            "tcp": {
+                                "sport": {
+                                    "range": {
+                                        "min": 1,
+                                        "max": 8080,
+                                        "incr": 100
+                                    }
+                                },
+                                "dport": 2000,
+                                "seq": "increasing",
+                                "flags": ["ack", "fin", "syn"],
+                                "randbytes":    {
+                                    "size": 1466,
+                                    "deviation": 0
+                                }
+                            }
+                        }
+                    },
+            "quantity": 6
+        }
+}
diff --git a/test/localTesting/pktgen/testing/run.sh b/test/localTesting/pktgen/testing/run.sh
index bcfd857e..91b75d53 100755
--- a/test/localTesting/pktgen/testing/run.sh
+++ b/test/localTesting/pktgen/testing/run.sh
@@ -1,15 +1,15 @@
 #!/bin/bash
 
-./../generate -totalPackets 100000 -infile ether.json -outfile ether.pcap
-./../generate -totalPackets 10000 -infile ip4.json -outfile ip4.pcap
-./../generate -totalPackets 1000 -infile ip6.json -outfile ip6.pcap
-./../generate -totalPackets 100000 -infile ip4tcp.json -outfile ip4tcp.pcap
-./../generate -totalPackets 1000000 -infile ip6tcp.json -outfile ip6tcp.pcap
-./../generate -totalPackets 10000 -infile ip4udp.json -outfile ip4udp.pcap
-./../generate -totalPackets 1000 -infile ip6udp.json -outfile ip6udp.pcap
-./../generate -totalPackets 1000000 -infile ip4icmp.json -outfile ip4icmp.pcap
-./../generate -totalPackets 100000 -infile ip6icmp.json -outfile ip6icmp.pcap
-./../generate -totalPackets 1000 -infile arp.json -outfile arp.pcap
-./../generate -totalPackets 10 -infile vlanTag.json -outfile vlanTag.pcap
-./../generate -totalPackets 10 -infile arpVlan.json -outfile arpVlan.pcap
-./../generate -totalPackets 100
\ No newline at end of file
+./../generate -number 100000 -infile ether.json -outfile ether.pcap
+./../generate -number 10000 -infile ip4.json -outfile ip4.pcap
+./../generate -number 1000 -infile ip6.json -outfile ip6.pcap
+./../generate -number 100000 -infile ip4tcp.json -outfile ip4tcp.pcap
+./../generate -number 1000000 -infile ip6tcp.json -outfile ip6tcp.pcap
+./../generate -number 10000 -infile ip4udp.json -outfile ip4udp.pcap
+./../generate -number 1000 -infile ip6udp.json -outfile ip6udp.pcap
+./../generate -number 1000000 -infile ip4icmp.json -outfile ip4icmp.pcap
+./../generate -number 100000 -infile ip6icmp.json -outfile ip6icmp.pcap
+./../generate -number 1000 -infile arp.json -outfile arp.pcap
+./../generate -number 10 -infile vlanTag.json -outfile vlanTag.pcap
+./../generate -number 10 -infile arpVlan.json -outfile arpVlan.pcap
+./../generate -number 100
\ No newline at end of file

From 8fb9320b7a468c7e152e6184596e55d3e7db7cdc Mon Sep 17 00:00:00 2001
From: root <root@antsatel05.an.intel.com>
Date: Fri, 29 Jun 2018 08:02:24 -0500
Subject: [PATCH 05/50] Made debug build mode through makefile

---
 Makefile      |  3 +++
 README.md     | 16 ++++++++++++++++
 mk/include.mk |  5 ++++-
 mk/leaf.mk    | 10 +++++++---
 4 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 39f0cf93..4d6782d0 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,9 @@ TESTING_TARGETS = $(CI_TESTING_TARGETS) test/stability
 
 all: $(SUBDIRS)
 
+debug:
+	$(MAKE) all NFF_GO_DEBUG=y
+
 dpdk: nff-go-base
 
 test: dpdk
diff --git a/README.md b/README.md
index bf0d2a3b..3aa3b9d0 100644
--- a/README.md
+++ b/README.md
@@ -138,6 +138,22 @@ Use Go version 1.9 or higher. To check the version of Go, do:
         cd $GOPATH/src/github.com/intel-go/nff-go
         make -j8
 
+## Building NFF-GO in debug mode
+
+There are several ways.
+* To set environment variable:
+
+		export NFF_GO_DEBUG=y
+		make
+
+or
+
+        make -j8 NFF_GO_DEBUG=y
+
+* To build with debug target
+
+		 make debug -j8
+
 # Running NFF-GO
 
 ## Documentation 
diff --git a/mk/include.mk b/mk/include.mk
index 1d702142..dc855a9f 100644
--- a/mk/include.mk
+++ b/mk/include.mk
@@ -41,8 +41,11 @@ CFLAGS = -I$(RTE_SDK)/$(RTE_TARGET)/include	\
 	-DRTE_MACHINE_CPUFLAG_F16C		\
 	-include rte_config.h			\
 	-Wno-deprecated-declarations
+
+ifdef NFF_GO_DEBUG
 # DEBUG flags
-# export CFLAGS = -g -O0 -I$(RTE_SDK)/$(RTE_TARGET)/include -std=gnu11 -m64 -pthread -march=native -mno-fsgsbase -mno-f16c -include rte_config.h
+export CFLAGS = -g -O0 -I$(RTE_SDK)/$(RTE_TARGET)/include -std=gnu11 -m64 -pthread -march=native -mno-fsgsbase -mno-f16c -include rte_config.h
+endif
 
 HAVE_AVX2 := $(shell grep avx2 /proc/cpuinfo)
 ifdef HAVE_AVX2
diff --git a/mk/leaf.mk b/mk/leaf.mk
index e6f3dc72..c1228f9a 100644
--- a/mk/leaf.mk
+++ b/mk/leaf.mk
@@ -9,10 +9,14 @@ include $(PATH_TO_MK)/include.mk
 
 # Build all
 .PHONY: clean
+
+ifdef NFF_GO_DEBUG
+# Flags to build Go files without optimizations
+export GO_COMPILE_FLAGS=-gcflags '-N -l'
+endif
+
 $(EXECUTABLES) : % : %.go
-	go build $< $(COMMON_FILES)
-# Use the following line to build Go files without optimizations
-#	go build -gcflags '-N -l' $< $(COMMON_FILES)
+	go build $(GO_COMPILE_FLAGS) $< $(COMMON_FILES)
 
 ifndef NOCHECK_PKTGEN
 all: check-pktgen

From ff466bc7179917970e3bd95aefff17d705e99177 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Sun, 17 Jun 2018 20:06:04 +0300
Subject: [PATCH 06/50] Refactored packet initialization to remove duplicate
 code.

---
 packet/packet.go | 59 +++++++++++++++++++-----------------------------
 1 file changed, 23 insertions(+), 36 deletions(-)

diff --git a/packet/packet.go b/packet/packet.go
index 2873a1d4..c6d6b4ec 100644
--- a/packet/packet.go
+++ b/packet/packet.go
@@ -473,6 +473,18 @@ func InitEmptyPacket(packet *Packet, plSize uint) bool {
 	return true
 }
 
+func fillIPv4Default(packet *Packet, plLen uint16, nextProto uint8) {
+	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
+	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(plLen)
+	packet.GetIPv4NoCheck().NextProtoID = nextProto
+}
+
+func fillIPv6Default(packet *Packet, totalLen uint16, nextProto uint8) {
+	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(totalLen)
+	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
+	packet.GetIPv6NoCheck().Proto = nextProto
+}
+
 // InitEmptyIPv4Packet initializes input packet with preallocated plSize of bytes for payload
 // and init pointers to Ethernet and IPv4 headers.
 func InitEmptyIPv4Packet(packet *Packet, plSize uint) bool {
@@ -483,16 +495,12 @@ func InitEmptyIPv4Packet(packet *Packet, plSize uint) bool {
 		LogWarning(Debug, "InitEmptyIPv4Packet: Cannot append mbuf")
 		return false
 	}
-
-	// After packet is parsed, we can write to packet struct known protocol types
 	packet.Ether.EtherType = SwapBytesUint16(IPV4Number)
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv4MinLen)
 
-	// Next fields not required by pktgen to accept packet. But set anyway
 	packet.ParseL3()
-	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
-	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(uint16(IPv4MinLen + plSize))
-	packet.GetIPv4NoCheck().NextProtoID = NoNextHeader
+	fillIPv4Default(packet, uint16(IPv4MinLen+plSize), NoNextHeader)
+
 	if hwtxchecksum {
 		packet.GetIPv4NoCheck().HdrChecksum = 0
 		low.SetTXIPv4OLFlags(packet.CMbuf, EtherLen, IPv4MinLen)
@@ -512,10 +520,7 @@ func InitEmptyIPv6Packet(packet *Packet, plSize uint) bool {
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv6Len)
 
 	packet.ParseL3()
-	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(uint16(plSize))
-	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
-	packet.GetIPv6NoCheck().Proto = NoNextHeader
-
+	fillIPv6Default(packet, uint16(plSize), NoNextHeader)
 	return true
 }
 
@@ -546,15 +551,12 @@ func InitEmptyIPv4TCPPacket(packet *Packet, plSize uint) bool {
 	}
 	packet.Ether.EtherType = SwapBytesUint16(IPV4Number)
 
-	// Next fields not required by pktgen to accept packet. But set anyway
 	packet.ParseL3()
-	packet.GetIPv4NoCheck().NextProtoID = TCPNumber
-	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
-	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(uint16(IPv4MinLen + TCPMinLen + plSize))
-
+	fillIPv4Default(packet, uint16(IPv4MinLen+TCPMinLen+plSize), TCPNumber)
 	packet.ParseL4ForIPv4()
 	packet.GetTCPNoCheck().DataOff = TCPMinDataOffset
 	packet.Data = unsafe.Pointer(uintptr(packet.L4) + uintptr(packet.GetTCPNoCheck().DataOff&0xf0)>>2)
+
 	if hwtxchecksum {
 		packet.GetIPv4NoCheck().HdrChecksum = 0
 		low.SetTXIPv4TCPOLFlags(packet.CMbuf, EtherLen, IPv4MinLen)
@@ -575,12 +577,8 @@ func InitEmptyIPv4UDPPacket(packet *Packet, plSize uint) bool {
 	packet.Ether.EtherType = SwapBytesUint16(IPV4Number)
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv4MinLen + UDPLen)
 
-	// Next fields not required by pktgen to accept packet. But set anyway
 	packet.ParseL3()
-	packet.GetIPv4NoCheck().NextProtoID = UDPNumber
-	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
-	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(uint16(IPv4MinLen + UDPLen + plSize))
-
+	fillIPv4Default(packet, uint16(IPv4MinLen+UDPLen+plSize), UDPNumber)
 	packet.ParseL4ForIPv4()
 	packet.GetUDPNoCheck().DgramLen = SwapBytesUint16(uint16(UDPLen + plSize))
 
@@ -604,11 +602,8 @@ func InitEmptyIPv4ICMPPacket(packet *Packet, plSize uint) bool {
 	packet.Ether.EtherType = SwapBytesUint16(IPV4Number)
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv4MinLen + ICMPLen)
 
-	// Next fields not required by pktgen to accept packet. But set anyway
 	packet.ParseL3()
-	packet.GetIPv4NoCheck().NextProtoID = ICMPNumber
-	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
-	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(uint16(IPv4MinLen + ICMPLen + plSize))
+	fillIPv4Default(packet, uint16(IPv4MinLen+ICMPLen+plSize), ICMPNumber)
 	packet.ParseL4ForIPv4()
 	return true
 }
@@ -627,13 +622,11 @@ func InitEmptyIPv6TCPPacket(packet *Packet, plSize uint) bool {
 	packet.Ether.EtherType = SwapBytesUint16(IPV6Number)
 
 	packet.ParseL3()
-	packet.GetIPv6NoCheck().Proto = TCPNumber
-	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(uint16(TCPMinLen + plSize))
-	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
-
+	fillIPv6Default(packet, uint16(TCPMinLen+plSize), TCPNumber)
 	packet.ParseL4ForIPv6()
 	packet.GetTCPNoCheck().DataOff = TCPMinDataOffset
 	packet.Data = unsafe.Pointer(uintptr(packet.L4) + uintptr(packet.GetTCPNoCheck().DataOff&0xf0)>>2)
+
 	if hwtxchecksum {
 		low.SetTXIPv6TCPOLFlags(packet.CMbuf, EtherLen, IPv6Len)
 	}
@@ -654,10 +647,7 @@ func InitEmptyIPv6UDPPacket(packet *Packet, plSize uint) bool {
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv6Len + UDPLen)
 
 	packet.ParseL3()
-	packet.GetIPv6NoCheck().Proto = UDPNumber
-	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(uint16(UDPLen + plSize))
-	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
-
+	fillIPv6Default(packet, uint16(UDPLen+plSize), UDPNumber)
 	packet.ParseL4ForIPv6()
 	packet.GetUDPNoCheck().DgramLen = SwapBytesUint16(uint16(UDPLen + plSize))
 
@@ -678,11 +668,8 @@ func InitEmptyIPv6ICMPPacket(packet *Packet, plSize uint) bool {
 	packet.Ether.EtherType = SwapBytesUint16(IPV6Number)
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv6Len + ICMPLen)
 
-	// Next fields not required by pktgen to accept packet. But set anyway
 	packet.ParseL3()
-	packet.GetIPv6NoCheck().Proto = ICMPNumber
-	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(uint16(UDPLen + plSize))
-	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
+	fillIPv6Default(packet, uint16(ICMPLen+plSize), ICMPNumber)
 	packet.ParseL4ForIPv6()
 	return true
 }

From 6227ea50a49f467f156410745595c84b64de0013 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Tue, 10 Jul 2018 12:09:16 -0600
Subject: [PATCH 07/50] Fix for generate speed report

---
 flow/scheduler.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index 88d3837d..620e9658 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -552,9 +552,9 @@ func (ff *flowFunction) printDebug(schedTime uint) {
 		targetSpeed := (ff.Parameters.(*generateParameters)).targetSpeed
 		out := ff.instance[0].currentSpeed.normalize(schedTime)
 		if reportMbits {
-			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s, target speed is", int64(targetSpeed), "PKT/S")
+			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s, target speed is", uint64(targetSpeed), "PKT/S")
 		} else {
-			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S, target speed is", int64(targetSpeed), "PKT/S")
+			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S, target speed is", uint64(targetSpeed), "PKT/S")
 		}
 	case readWrite, generate, sendReceiveKNI, receiveRSS:
 	}

From c6de7a8c0e557bc2fcc9c0824306769f17df9028 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Tue, 10 Jul 2018 12:01:33 -0600
Subject: [PATCH 08/50] Fix number of report channels

---
 flow/scheduler.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index 88d3837d..9600615a 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -217,7 +217,7 @@ func (ff *flowFunction) startNewInstance(inIndex []int32, scheduler *scheduler)
 	ffi := new(instance)
 	common.LogDebug(common.Initialization, "Start new instance for", ff.name)
 	ffi.inIndex = inIndex
-	ffi.report = make(chan reportPair, 50)
+	ffi.report = make(chan reportPair, len(scheduler.cores) - 1)
 	ffi.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
 	ffi.ff = ff
 	err = ffi.startNewClone(scheduler, ff.instanceNumber)
@@ -583,6 +583,15 @@ func (ff *flowFunction) updateCurrentSpeed() {
 			ffi.currentSpeed.Packets += temp.Packets
 			ffi.currentSpeed.Bytes += temp.Bytes
 		}
+		// If any flow functions wait in a queue to put their
+		// reports immidiately after reading- we should remove them.
+		// They will put reports again after scheduling time.
+		t = len(ffi.report)
+		if t != 0 {
+			for k := 0; k < t; k++ {
+				<-ffi.report
+			}
+		}
 	}
 }
 

From dd70b48d23209556f90b6f4e6dec3745297e7c93 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Tue, 10 Jul 2018 13:02:33 -0600
Subject: [PATCH 09/50] Added assertion about minimal generate target speed

---
 flow/flow.go | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/flow/flow.go b/flow/flow.go
index 4eedbc85..1368247d 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -174,16 +174,25 @@ func addGenerator(out low.Rings, generateFunction GenerateFunction, context User
 }
 
 func addFastGenerator(out low.Rings, generateFunction GenerateFunction,
-	vectorGenerateFunction VectorGenerateFunction, targetSpeed uint64, context UserContext) {
+	vectorGenerateFunction VectorGenerateFunction, targetSpeed uint64, context UserContext) error {
+	fTargetSpeed := float64(targetSpeed)
+	if fTargetSpeed <= 0 {
+		return common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
+	} else if fTargetSpeed / (1000 /*milleseconds*/ / float64(schedTime)) < float64(burstSize) {
+		// TargetSpeed per schedTime should be more than burstSize because one burstSize packets in
+		// one schedTime seconds are out minimal scheduling part. We can't make generate speed less than this.
+		return common.WrapWithNFError(nil, "Target speed per schedTime should be more than burstSize", common.BadArgument)
+	}
 	par := new(generateParameters)
 	par.out = out
 	par.generateFunction = generateFunction
 	par.mempool = low.CreateMempool("fast generate")
 	par.vectorGenerateFunction = vectorGenerateFunction
-	par.targetSpeed = float64(targetSpeed)
+	par.targetSpeed = fTargetSpeed
 	ctx := make([]UserContext, 1, 1)
 	ctx[0] = context
 	schedState.addFF("fast generator", nil, nil, pFastGenerate, par, &ctx, fastGenerate, 0)
+	return nil
 }
 
 type sendParameters struct {
@@ -641,10 +650,8 @@ func SetReceiverKNI(kni *Kni) (OUT *Flow) {
 // Function tries to achieve target speed by cloning.
 func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
 	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
-	if targetSpeed > 0 {
-		addFastGenerator(rings, f, nil, targetSpeed, context)
-	} else {
-		return nil, common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
+	if err := addFastGenerator(rings, f, nil, targetSpeed, context); err != nil {
+		return nil, err
 	}
 	return newFlow(rings, 1), nil
 }
@@ -655,10 +662,8 @@ func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContex
 // Function tries to achieve target speed by cloning.
 func SetVectorFastGenerator(f VectorGenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
 	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
-	if targetSpeed > 0 {
-		addFastGenerator(rings, nil, f, targetSpeed, context)
-	} else {
-		return nil, common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
+	if err := addFastGenerator(rings, nil, f, targetSpeed, context); err != nil {
+		return nil, err
 	}
 	return newFlow(rings, 1), nil
 }

From e214be8c74d9365105f4540ea09bc434c33e7e94 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Wed, 11 Jul 2018 12:40:31 +0300
Subject: [PATCH 10/50] Fixed debug compilation

---
 README.md     | 14 +-------------
 dpdk/Makefile |  7 ++++---
 mk/include.mk |  4 +---
 3 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/README.md b/README.md
index 3aa3b9d0..16e720ad 100644
--- a/README.md
+++ b/README.md
@@ -140,19 +140,7 @@ Use Go version 1.9 or higher. To check the version of Go, do:
 
 ## Building NFF-GO in debug mode
 
-There are several ways.
-* To set environment variable:
-
-		export NFF_GO_DEBUG=y
-		make
-
-or
-
-        make -j8 NFF_GO_DEBUG=y
-
-* To build with debug target
-
-		 make debug -j8
+		make debug -j8
 
 # Running NFF-GO
 
diff --git a/dpdk/Makefile b/dpdk/Makefile
index 1ec1a889..c6634ddd 100644
--- a/dpdk/Makefile
+++ b/dpdk/Makefile
@@ -15,9 +15,10 @@ DPDK_INSTALL_DIR=$(RTE_TARGET)-install
 export WORKDIR=/workdir
 
 # Disable FSGSBASE and F16C to run in VMs and Docker containers.
-export EXTRA_CFLAGS=-mno-fsgsbase -mno-f16c
-# Uncomment to build DEBUG DPDK
-# export EXTRA_CFLAGS=-mno-fsgsbase -mno-f16c -g -O0
+export EXTRA_CFLAGS = -mno-fsgsbase -mno-f16c
+ifdef NFF_GO_DEBUG
+export EXTRA_CFLAGS += -g -O0
+endif
 
 all: pktgen
 	cp $(PKTGEN_DIR)/Pktgen.lua .
diff --git a/mk/include.mk b/mk/include.mk
index dc855a9f..0ff530ed 100644
--- a/mk/include.mk
+++ b/mk/include.mk
@@ -23,7 +23,6 @@ export RTE_TARGET = x86_64-native-linuxapp-gcc
 # VMs and Docker containers.
 CFLAGS = -I$(RTE_SDK)/$(RTE_TARGET)/include	\
 	-O3					\
-	-g					\
 	-std=gnu11				\
 	-m64					\
 	-pthread				\
@@ -43,8 +42,7 @@ CFLAGS = -I$(RTE_SDK)/$(RTE_TARGET)/include	\
 	-Wno-deprecated-declarations
 
 ifdef NFF_GO_DEBUG
-# DEBUG flags
-export CFLAGS = -g -O0 -I$(RTE_SDK)/$(RTE_TARGET)/include -std=gnu11 -m64 -pthread -march=native -mno-fsgsbase -mno-f16c -include rte_config.h
+export CFLAGS += -g -O0 -D DEBUG
 endif
 
 HAVE_AVX2 := $(shell grep avx2 /proc/cpuinfo)

From 0d0ce9c5b08c83afa8c5946b721903d20091a139 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Thu, 19 Jul 2018 11:50:48 -0600
Subject: [PATCH 11/50] Add timers

---
 examples/Dockerfile |  1 +
 examples/Makefile   |  2 +-
 examples/timer.go   | 52 +++++++++++++++++++++++++++++++++++++++++++++
 flow/flow.go        | 44 ++++++++++++++++++++++++++++++++++++++
 flow/scheduler.go   | 31 +++++++++++++++++++++++++++
 5 files changed, 129 insertions(+), 1 deletion(-)
 create mode 100644 examples/timer.go

diff --git a/examples/Dockerfile b/examples/Dockerfile
index 5c6d4b2a..809ed0bb 100644
--- a/examples/Dockerfile
+++ b/examples/Dockerfile
@@ -17,3 +17,4 @@ COPY sendFixedPktsNumber .
 COPY errorHandling .
 COPY gtpu .
 COPY pingReplay .
+COPY timer .
diff --git a/examples/Makefile b/examples/Makefile
index 93aa6bc1..99efc3f7 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -4,7 +4,7 @@
 
 PATH_TO_MK = ../mk
 IMAGENAME = nff-go-examples
-EXECUTABLES = dump clonablePcapDumper kni copy errorHandling \
+EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
 			  createPacket sendFixedPktsNumber gtpu pingReplay
 SUBDIRS = nat tutorial antiddos demo fileReadWrite firewall forwarding
 
diff --git a/examples/timer.go b/examples/timer.go
new file mode 100644
index 00000000..04cf9a75
--- /dev/null
+++ b/examples/timer.go
@@ -0,0 +1,52 @@
+// Copyright 2017 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+)
+
+var t *flow.Timer
+var check *bool
+var inport *uint
+var duration *time.Duration
+
+func main() {
+	inport = flag.Uint("inport", 0, "port for receiver")
+	duration = flag.Duration("duration", 2000 * time.Millisecond, "seconds to react")
+	flag.Parse()
+
+	flow.CheckFatal(flow.SystemInit(nil))
+
+	firstFlow, err := flow.SetReceiver(uint16(*inport))
+	flow.CheckFatal(err)
+	flow.CheckFatal(flow.SetHandler(firstFlow, handler, nil))
+	flow.CheckFatal(flow.SetStopper(firstFlow))
+
+	t = flow.AddTimer(*duration, react)
+	flow.CheckFatal(flow.SystemStart())
+}
+
+func handler(currentPacket *packet.Packet, context flow.UserContext) {
+	if check == nil {
+		check = t.AddVariant(nil)
+	}
+	*check = true
+}
+
+func react(context flow.UserContext) {
+	fmt.Println(*duration, "after last packet was arrived")
+
+	answerPacket, _ := packet.NewPacket()
+	packet.InitEmptyIPv4Packet(answerPacket, 64)
+	answerPacket.SendPacket(uint16(*inport))
+
+	check = nil
+}
diff --git a/flow/flow.go b/flow/flow.go
index 1368247d..0aa44126 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -47,6 +47,13 @@ var portPair map[uint32](*port)
 var schedState *scheduler
 var vEach [10][burstSize]uint8
 
+type Timer struct {
+	t *time.Ticker
+	handler func(UserContext)
+	contexts []UserContext
+	checks []*bool
+}
+
 type processSegment struct {
 	out      []low.Rings
 	contexts []UserContext
@@ -1532,6 +1539,43 @@ func FillSliceFromMask(input []uintptr, mask *[burstSize]bool, output []uintptr)
 	return uint8(count)
 }
 
+// AddTimer adds a timer which may call handler function every d milliseconds
+// It is required to add at least one variant of this timer for working
+// TODO d should be approximate as schedTime because handler will be call from scheduler
+// Return created timer
+func AddTimer(d time.Duration, handler func(UserContext)) *Timer {
+	t := new(Timer)
+	t.t = time.NewTicker(d)
+	t.handler = handler
+	t.contexts = make([]UserContext, 0, 0)
+	t.checks = make([]*bool, 0, 0)
+	schedState.Timers = append(schedState.Timers, t)
+	return t
+}
+
+// AddVariant adds a variant for an existing timer. Variant is a context parameter
+// which will be passed to handler callback from AddTimer function
+// Function return a pointer to variable which should be set to "true"
+// everytime to prevent timer from ping.
+// Timer variant automatically drops after timer invocation
+func (timer *Timer) AddVariant(context UserContext) *bool {
+	check := false
+	timer.contexts = append(timer.contexts, context)
+	timer.checks = append(timer.checks, &check)
+	return &check
+}
+
+// Stop removes timer with all its variants
+func (timer *Timer) Stop() {
+	timer.t.Stop()
+	for i, t := range schedState.Timers {
+		if t == timer {
+			schedState.Timers = append(schedState.Timers[:i], schedState.Timers[i+1:]...)
+			return
+		}
+	}
+}
+
 // CheckFatal is a default error handling function, which prints error message and
 // makes os.Exit in case of non nil error. Any other error handler can be used instead.
 func CheckFatal(err error) {
diff --git a/flow/scheduler.go b/flow/scheduler.go
index 7c31c01e..0928bf8a 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -153,6 +153,7 @@ type scheduler struct {
 	maxPacketsToClone uint32
 	stopFlag          int32
 	maxRecv           int
+	Timers		  []*Timer
 }
 
 type core struct {
@@ -321,6 +322,36 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 	checkRequired := false
 	for atomic.LoadInt32(&scheduler.stopFlag) == process {
 		time.Sleep(time.Millisecond * time.Duration(schedTime))
+		// We have an array of Timers which can be increated by AddTimer function
+		// Timer has duration and handler common for all Timer variants, so firstly
+		// we check that timer ticker channel is ready:
+		for _, t := range(scheduler.Timers) {
+			select {
+			case <-t.t.C:
+				// If this timer was ticked we check all checks of all variants
+				// Timer. We need variants because one TCP timer will handle
+				// different TCP sessions. Variant's context is a difference of
+				// current session from other sessions. Variant's check is a
+				// variable that user will set to "true" if he wants to reset timer
+				// for example if packet of this session was arrived.
+				for i, c := range t.checks {
+					// We set all checks to false. If we saw false check it means
+					// that user didn't see any event and timer handler should be called.
+					// We use boolean checks for performance reasons because reset timer
+					// after each packet is quite slow.
+					if *c == false {
+						t.handler(t.contexts[i])
+						// We remove current variant after handler calling, because this
+						// session was finished.
+						t.contexts = append(t.contexts[:i], t.contexts[i+1:]...)
+						t.checks = append(t.checks[:i], t.checks[i+1:]...)
+						continue
+					}
+					*c = false
+				}
+			default:
+			}
+		}
 		select {
 		case <-tick:
 			checkRequired = true

From 47bbd97a6dc34435775b739ba24b98e670735b87 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Wed, 11 Jul 2018 14:14:38 +0300
Subject: [PATCH 12/50] Extracted API for generator creation, created generator
 which gets packets back

---
 test/localTesting/pktgen/Makefile             |   2 +-
 test/localTesting/pktgen/README.md            |  79 +++++-
 .../{generate.go => generator/generator.go}   | 248 +++++-------------
 .../pktgen/parseConfig/parseConfig.go         |   4 +
 test/localTesting/pktgen/sendGetBack/Makefile |   9 +
 .../localTesting/pktgen/sendGetBack/README.md |  11 +
 .../pktgen/sendGetBack/sendGetBack.go         | 188 +++++++++++++
 7 files changed, 349 insertions(+), 192 deletions(-)
 rename test/localTesting/pktgen/{generate.go => generator/generator.go} (76%)
 create mode 100644 test/localTesting/pktgen/sendGetBack/Makefile
 create mode 100644 test/localTesting/pktgen/sendGetBack/README.md
 create mode 100644 test/localTesting/pktgen/sendGetBack/sendGetBack.go

diff --git a/test/localTesting/pktgen/Makefile b/test/localTesting/pktgen/Makefile
index 1ba2d0c3..55dd0674 100644
--- a/test/localTesting/pktgen/Makefile
+++ b/test/localTesting/pktgen/Makefile
@@ -4,6 +4,6 @@
 
 PATH_TO_MK = ../../../mk
 IMAGENAME = local-pktgen
-EXECUTABLES = generate
+SUBDIRS = sendGetBack
 
 include $(PATH_TO_MK)/leaf.mk
diff --git a/test/localTesting/pktgen/README.md b/test/localTesting/pktgen/README.md
index c5b0ca3c..93ee59f4 100644
--- a/test/localTesting/pktgen/README.md
+++ b/test/localTesting/pktgen/README.md
@@ -1,16 +1,77 @@
 # Packet generation to file
 
 ## What it is
-Pktgen parses config in json format and generates packets according to it in pcap file that can be read by NFF-GO reader, Wireshark, tcpdump and other tools reading pcap files.
+Pktgen parses config in json format and generates packets according to it and sends to port or pcap file that can be read by NFF-GO reader, Wireshark, tcpdump and other tools reading pcap files.
+Generator package has public API which can be used for own generator.
 
-### Command-line options:
-* --totalPackets sets the number of packets to generate, default value is 10000000
-* --infile sets the name of the file with packet configurations, default value is "config.json"
-* --outfile sets the name of the file to write output to, default value is "pkts_generated.pcap". Can be set with port (the output is copied).
-* --outport sets the port number to send output to. Can be used alone, with or instead of outfile.
-* --speed sets the speed of generator
-* --cycle sets cycle execution to generate infinite number of packets
-* --portConfig specifies config per port portNum: 'path', portNum2: 'path2'. For example: 1: 'ip4.json', 0: 'mix.json'
+### API:
+
+```go
+func GetGenerator() *generator 
+```
+returns generator object pointer which is singleton, so will be created only once.
+
+```go
+func (g *generator) GetGeneratedNumber() uint64
+```
+returns how much packets were generated.
+
+```go
+func (g *generator) ResetCounter() {
+```
+sets counter of generated packets to 0.
+
+```go
+func (g *generator) SetGenerateNumber(number uint64)
+```
+sets number to generate. When generated counter will reach it, generation will be stopped.
+
+```go
+func (g *generator) ResetGenerateNumber()
+```
+sets number to generate to infinity (by default).
+
+```go
+func ReadConfig(fileName string) ([]*parseConfig.MixConfig, error)
+```
+returns read and parsed config from file.
+
+```go
+func GetContext(mixConfig []*parseConfig.MixConfig) (genParameters, error)
+```
+returns context that chould be sent to generator according to configuration.
+
+```go
+func Generate(pkt *packet.Packet, context flow.UserContext)
+```
+is a main generator nunction. Context is obligatory.
+
+### Example of usage:
+```go
+    // parse config
+    configuration, err := generator.ReadConfig(pathToConfigJSON)
+    // check error
+	flow.CheckFatal(err)
+    // generate context by config
+    context, err := generator.GetContext(configuration)
+    // check error
+    flow.CheckFatal(err)
+    // set generator
+    outFlow, err := flow.SetFastGenerator(generator.Generate, speed, &context)
+    // check error
+    flow.CheckFatal(err)
+    // send
+    flow.CheckFatal(flow.SetSender(outFlow, uint16(port)))
+
+    // periodically print statistics
+	go func() {
+        g :=  generator.GetGenerator()
+		for {
+			println("Sent", g.GetGeneratedNumber(), "packets")
+			time.Sleep(time.Second * 5)
+		}
+	}()
+```
 
 ### Configuration syntax:
 File should be a structure containing structure with ethernet header or mix configuration:
diff --git a/test/localTesting/pktgen/generate.go b/test/localTesting/pktgen/generator/generator.go
similarity index 76%
rename from test/localTesting/pktgen/generate.go
rename to test/localTesting/pktgen/generator/generator.go
index 710f5b25..8115768b 100644
--- a/test/localTesting/pktgen/generate.go
+++ b/test/localTesting/pktgen/generator/generator.go
@@ -1,24 +1,19 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package generator
 
 import (
 	"bytes"
 	"encoding/binary"
-	"flag"
 	"fmt"
 	"math"
 	"math/big"
 	"math/rand"
 	"net"
 	"os"
-	"strconv"
-	"strings"
-	"sync"
 	"sync/atomic"
-	"time"
 
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/flow"
@@ -26,133 +21,40 @@ import (
 	"github.com/intel-go/nff-go/test/localTesting/pktgen/parseConfig"
 )
 
-var (
-	count         uint64
-	testDoneEvent *sync.Cond
-	number        uint64
-	isCycled      bool
-)
+var gen generator
 
-type mapFlags struct {
-	value map[int]string
-	set   bool
+type generator struct {
+	count    uint64
+	isFinite bool
+	number   uint64
 }
 
-func (m *mapFlags) String() string {
-	s := ""
-	for key, value := range (*m).value {
-		if s != "" {
-			s += ","
-		}
-		s += fmt.Sprintf("%d: '%s'", key, value)
-	}
-	return s
+// GetGenerator returns generator struct pointer
+// generator is single and created only once
+func GetGenerator() *generator {
+	return &gen
 }
 
-func (m *mapFlags) Set(s string) error {
-	ss := strings.Split(s, ",")
-	(*m).value = make(map[int]string)
-	for _, val := range ss {
-		val = strings.TrimSpace(val)
-		pair := strings.Split(val, ":")
-		key, err := strconv.Atoi(pair[0])
-		if err != nil {
-			return err
-		}
-		value := strings.Trim(pair[1], "' ")
-		(*m).value[key] = value
-	}
-	(*m).set = true
-	return nil
+// GetGeneratedNumber returns a number of packets generated
+func (g *generator) GetGeneratedNumber() uint64 {
+	return atomic.LoadUint64(&(g.count))
 }
 
-func main() {
-	var (
-		portflag           uint
-		outFile, inFile    string
-		outPort            uint16
-		usePort, useFile   bool
-		speed              uint64
-		m                  sync.Mutex
-		fileFlow, portFlow *flow.Flow
-		eachPortConfig     mapFlags
-	)
-	flag.StringVar(&outFile, "outfile", "", "file to write output to")
-	flag.UintVar(&portflag, "outport", 70000, "port to send output to")
-	flag.StringVar(&inFile, "infile", "config.json", "file with configurations for generator")
-	flag.Uint64Var(&number, "number", 10000000, "stop after generation number number")
-	flag.BoolVar(&isCycled, "cycle", false, "cycle execution and generate infinite number of packets")
-	flag.Uint64Var(&speed, "speed", 6000000, "speed of fast generator, Pkts/s")
-	flag.Var(&eachPortConfig, "portConfig", "specify config per port portNum: 'path', portNum2: 'path2'. For example: 1: 'ip4.json', 0: 'mix.json'")
-	flag.Parse()
-
-	// Init NFF-GO system at 16 available cores
-	config := flow.Config{}
-	flow.CheckFatal(flow.SystemInit(&config))
-
-	testDoneEvent = sync.NewCond(&m)
-
-	if eachPortConfig.set {
-		for key, value := range eachPortConfig.value {
-			configuration, err := ReadConfig(value)
-			if err != nil {
-				panic(fmt.Sprintf("%s config reading failed: %v", value, err))
-			}
-			context, err := getGeneratorContext(configuration)
-			flow.CheckFatal(err)
-			outFlow, err := flow.SetFastGenerator(generate, speed, &context)
-			flow.CheckFatal(err)
-			flow.CheckFatal(flow.SetSender(outFlow, uint16(key)))
-		}
-	} else {
-		if portflag != 70000 {
-			usePort = true
-			outPort = uint16(portflag)
-		}
-		if outFile != "" {
-			useFile = true
-		}
-		if !usePort && !useFile {
-			useFile = true
-			outFile = "pkts_generated.pcap"
-		}
-		configuration, err := ReadConfig(inFile)
-		if err != nil {
-			panic(fmt.Sprintf("config reading failed: %v", err))
-		}
-
-		context, err := getGeneratorContext(configuration)
-		flow.CheckFatal(err)
-		fileFlow, err = flow.SetFastGenerator(generate, speed, &context)
-		flow.CheckFatal(err)
-		if useFile && usePort {
-			portFlow, err = flow.SetCopier(fileFlow)
-			flow.CheckFatal(err)
-		} else {
-			if usePort {
-				portFlow = fileFlow
-			}
-		}
-		if useFile {
-			flow.CheckFatal(flow.SetSenderFile(fileFlow, outFile))
-		}
-		if usePort {
-			flow.CheckFatal(flow.SetSender(portFlow, outPort))
-		}
-	}
-
-	// Start pipeline
-	go func() {
-		flow.CheckFatal(flow.SystemStart())
-	}()
+// ResetCounter resets count of generated packets
+func (g *generator) ResetCounter() {
+	atomic.StoreUint64(&(g.count), 0)
+}
 
-	// Wait for enough packets to arrive
-	testDoneEvent.L.Lock()
-	testDoneEvent.Wait()
-	testDoneEvent.L.Unlock()
+// SetGenerateNumber sets a number to generate
+func (g *generator) SetGenerateNumber(number uint64) {
+	g.number = number
+	g.isFinite = true
+}
 
-	// Print report
-	println("Sent", atomic.LoadUint64(&count), "packets")
+// ResetGenerateNumber sets a number to generate to infinite
+func (g *generator) ResetGenerateNumber() {
+	g.number = 0
+	g.isFinite = false
 }
 
 // ReadConfig function reads and parses config file.
@@ -212,20 +114,20 @@ func getNextPort(port *parseConfig.PortRange) (nextPort uint16) {
 	return nextPort
 }
 
-func getNextSeqNumber(seq *parseConfig.Sequence) (nextSeqNum uint32) {
+func getNextSeqNumber(seq *parseConfig.Sequence, rnd *rand.Rand) (nextSeqNum uint32) {
 	if len(seq.Next) == 0 {
 		return 0
 	}
 	nextSeqNum = seq.Next[0]
 	if seq.Type == parseConfig.RANDOM {
-		seq.Next[0] = rand.Uint32()
+		seq.Next[0] = rnd.Uint32()
 	} else if seq.Type == parseConfig.INCREASING {
 		seq.Next[0]++
 	}
 	return nextSeqNum
 }
 
-func generateData(configuration interface{}) ([]uint8, error) {
+func generateData(configuration interface{}, rnd *rand.Rand) ([]uint8, error) {
 	switch data := configuration.(type) {
 	case parseConfig.Raw:
 		pktData := make([]uint8, len(data.Data))
@@ -234,20 +136,20 @@ func generateData(configuration interface{}) ([]uint8, error) {
 	case parseConfig.RandBytes:
 		maxZise := data.Size + data.Deviation
 		minSize := data.Size - data.Deviation
-		randSize := uint(rand.Float64()*float64(maxZise-minSize) + float64(minSize))
+		randSize := uint(rnd.Float64()*float64(maxZise-minSize) + float64(minSize))
 		pktData := make([]uint8, randSize)
 		for i := range pktData {
-			pktData[i] = byte(rand.Int())
+			pktData[i] = byte(rnd.Int())
 		}
 		return pktData, nil
 	case []parseConfig.PDistEntry:
 		prob := 0.0
-		rndN := math.Abs(rand.Float64())
+		rndN := math.Abs(rnd.Float64())
 		maxProb := parseConfig.PDistEntry{Probability: 0}
 		for _, item := range data {
 			prob += item.Probability
 			if rndN <= prob {
-				pktData, err := generateData(item.Data)
+				pktData, err := generateData(item.Data, rnd)
 				if err != nil {
 					return nil, fmt.Errorf("failed to fill data with pdist data type: %v", err)
 				}
@@ -263,7 +165,7 @@ func generateData(configuration interface{}) ([]uint8, error) {
 		// get the variant with max prob
 		// if something went wrong and rand did not match any prob
 		// may happen if sum of prob was not 1
-		pktData, err := generateData(maxProb.Data)
+		pktData, err := generateData(maxProb.Data, rnd)
 		if err != nil {
 			return nil, fmt.Errorf("failed to fill data with pdist data type: %v", err)
 		}
@@ -272,7 +174,7 @@ func generateData(configuration interface{}) ([]uint8, error) {
 	return nil, fmt.Errorf("unknown data type")
 }
 
-func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet, *parseConfig.PacketConfig), error) {
+func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet, *parseConfig.PacketConfig, *rand.Rand), error) {
 	switch l2 := (configuration.Data).(type) {
 	case parseConfig.EtherConfig:
 		switch l3 := l2.Data.(type) {
@@ -301,20 +203,10 @@ func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet,
 	}
 }
 
-func checkFinish() {
-	if isCycled {
-		return
-	}
-	if atomic.AddUint64(&count, 1) >= number {
-		time.Sleep(time.Second)
-		testDoneEvent.Signal()
-	}
-}
-
 // one unit for each mix
 type generatorTableUnit struct {
 	have, need    uint32
-	generatorFunc func(*packet.Packet, *parseConfig.PacketConfig)
+	generatorFunc func(*packet.Packet, *parseConfig.PacketConfig, *rand.Rand)
 	config        *parseConfig.PacketConfig
 }
 
@@ -326,27 +218,20 @@ type genParameters struct {
 	table  []generatorTableUnit
 	next   []uint32
 	length uint32
-}
-
-func (gp *genParameters) String() string {
-	s := "genP:\n"
-	for _, tu := range gp.table {
-		s = s + fmt.Sprintf("\ttu: %v\n", tu.String())
-	}
-	s = s + fmt.Sprintf("next: %d, len: %d\n", gp.next, gp.length)
-	return s
+	rnd    *rand.Rand
 }
 
 func (gp genParameters) Copy() interface{} {
 	cpy := make([]generatorTableUnit, len(gp.table))
 	copy(cpy, gp.table)
-	return genParameters{table: cpy, next: []uint32{0}, length: gp.length}
+	return genParameters{table: cpy, next: []uint32{0}, length: gp.length, rnd: rand.New(rand.NewSource(13))}
 }
 
 func (gp genParameters) Delete() {
 }
 
-func getGeneratorContext(mixConfig []*parseConfig.MixConfig) (genParameters, error) {
+// GetContext gets generator context according to config
+func GetContext(mixConfig []*parseConfig.MixConfig) (genParameters, error) {
 	var t []generatorTableUnit
 	for _, packetConfig := range mixConfig {
 		genFunc, err := getGenerator(packetConfig.Config)
@@ -356,10 +241,14 @@ func getGeneratorContext(mixConfig []*parseConfig.MixConfig) (genParameters, err
 		tu := generatorTableUnit{have: 0, need: packetConfig.Quantity, generatorFunc: genFunc, config: packetConfig.Config}
 		t = append(t, tu)
 	}
-	return genParameters{table: t, next: []uint32{0}, length: uint32(len(t))}, nil
+	return genParameters{table: t, next: []uint32{0}, length: uint32(len(t)), rnd: rand.New(rand.NewSource(13))}, nil
 }
 
-func generate(pkt *packet.Packet, context flow.UserContext) {
+// Generate is a main generatior func
+func Generate(pkt *packet.Packet, context flow.UserContext) {
+	if gen.isFinite && gen.count >= gen.number {
+		return
+	}
 	genP := context.(genParameters)
 	table := genP.table
 	next := genP.next[0]
@@ -374,17 +263,17 @@ func generate(pkt *packet.Packet, context flow.UserContext) {
 			context.(genParameters).next[0] = next
 		}
 	}
-	table[next].generatorFunc(pkt, table[next].config)
+	table[next].generatorFunc(pkt, table[next].config, genP.rnd)
+	atomic.AddUint64(&(gen.count), 1)
 	context.(genParameters).table[next].have++
 }
 
-func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
-	data, err := generateData(l2.Data)
+	data, err := generateData(l2.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l2: %v", err))
 	}
@@ -402,14 +291,13 @@ func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
-	data, err := generateData(l3.Data)
+	data, err := generateData(l3.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l3: %v", err))
 	}
@@ -442,11 +330,10 @@ func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.ARPConfig)
 	var SHA, THA [common.EtherAddrLen]uint8
@@ -491,15 +378,14 @@ func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.TCPConfig)
-	data, err := generateData(l4.Data)
+	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
 	}
@@ -519,7 +405,7 @@ func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillTCPHdr(pkt, &l4); err != nil {
+	if err := fillTCPHdr(pkt, &l4, rnd); err != nil {
 		panic(fmt.Sprintf("failed to fill tcp header %v", err))
 	}
 	if err := fillIPHdr(pkt, &l3); err != nil {
@@ -539,15 +425,14 @@ func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.UDPConfig)
-	data, err := generateData(l4.Data)
+	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
 	}
@@ -587,15 +472,14 @@ func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
+func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	checkFinish()
 	l2 := config.Data.(parseConfig.EtherConfig)
 	l3 := l2.Data.(parseConfig.IPConfig)
 	l4 := l3.Data.(parseConfig.ICMPConfig)
-	data, err := generateData(l4.Data)
+	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
 	}
@@ -615,7 +499,7 @@ func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
 	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillICMPHdr(pkt, &l4); err != nil {
+	if err := fillICMPHdr(pkt, &l4, rnd); err != nil {
 		panic(fmt.Sprintf("failed to fill icmp header %v", err))
 	}
 	if err := fillIPHdr(pkt, &l3); err != nil {
@@ -635,11 +519,11 @@ func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig) {
 	config.Data = l2
 }
 
-func fillTCPHdr(pkt *packet.Packet, l4 *parseConfig.TCPConfig) error {
+func fillTCPHdr(pkt *packet.Packet, l4 *parseConfig.TCPConfig, rnd *rand.Rand) error {
 	emptyPacketTCP := (*packet.TCPHdr)(pkt.L4)
 	emptyPacketTCP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
 	emptyPacketTCP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
-	emptyPacketTCP.SentSeq = packet.SwapBytesUint32(getNextSeqNumber((&l4.Seq)))
+	emptyPacketTCP.SentSeq = packet.SwapBytesUint32(getNextSeqNumber((&l4.Seq), rnd))
 	emptyPacketTCP.TCPFlags = l4.Flags
 	return nil
 }
@@ -651,12 +535,12 @@ func fillUDPHdr(pkt *packet.Packet, l4 *parseConfig.UDPConfig) error {
 	return nil
 }
 
-func fillICMPHdr(pkt *packet.Packet, l4 *parseConfig.ICMPConfig) error {
+func fillICMPHdr(pkt *packet.Packet, l4 *parseConfig.ICMPConfig, rnd *rand.Rand) error {
 	emptyPacketICMP := (*packet.ICMPHdr)(pkt.L4)
 	emptyPacketICMP.Type = l4.Type
 	emptyPacketICMP.Code = l4.Code
 	emptyPacketICMP.Identifier = l4.Identifier
-	emptyPacketICMP.SeqNum = packet.SwapBytesUint16(uint16(getNextSeqNumber(&(l4.Seq))))
+	emptyPacketICMP.SeqNum = packet.SwapBytesUint16(uint16(getNextSeqNumber(&(l4.Seq), rnd)))
 	return nil
 }
 
diff --git a/test/localTesting/pktgen/parseConfig/parseConfig.go b/test/localTesting/pktgen/parseConfig/parseConfig.go
index ec3981b2..800639ad 100644
--- a/test/localTesting/pktgen/parseConfig/parseConfig.go
+++ b/test/localTesting/pktgen/parseConfig/parseConfig.go
@@ -1,3 +1,7 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package parseConfig
 
 import (
diff --git a/test/localTesting/pktgen/sendGetBack/Makefile b/test/localTesting/pktgen/sendGetBack/Makefile
new file mode 100644
index 00000000..25fd101b
--- /dev/null
+++ b/test/localTesting/pktgen/sendGetBack/Makefile
@@ -0,0 +1,9 @@
+# Copyright 2017 Intel Corporation. 
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+PATH_TO_MK = ../../../../mk
+IMAGENAME = local-pktgen
+EXECUTABLES = sendGetBack
+
+include $(PATH_TO_MK)/leaf.mk
diff --git a/test/localTesting/pktgen/sendGetBack/README.md b/test/localTesting/pktgen/sendGetBack/README.md
new file mode 100644
index 00000000..c68cd74a
--- /dev/null
+++ b/test/localTesting/pktgen/sendGetBack/README.md
@@ -0,0 +1,11 @@
+# Packet generation, sending and receiving
+
+## What it is
+Parses config files, generates output and sends to file or port and counts them. Receives packets back and counts.
+
+### Command-line options:
+* --number sets the number of packets to get back and stop, default value is 10000000
+* --speed sets the speed of generator
+* --cycle sets cycle execution to generate infinite number of packets
+* --outConfig specifies config per port portNum or file: 'path', 'path3': 'path2'. For example: 1: 'ip4.json', 'mix.pcap': 'mix.json'
+* --inConfig specifies input ports and files: 'path', 'portNum2', 'path2'. For example: 1, 'ip4.pcap', 0, 'mix.pcap'
\ No newline at end of file
diff --git a/test/localTesting/pktgen/sendGetBack/sendGetBack.go b/test/localTesting/pktgen/sendGetBack/sendGetBack.go
new file mode 100644
index 00000000..ac5e9b1e
--- /dev/null
+++ b/test/localTesting/pktgen/sendGetBack/sendGetBack.go
@@ -0,0 +1,188 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+	"github.com/intel-go/nff-go/test/localTesting/pktgen/generator"
+)
+
+var (
+	number, recv  uint64
+	isCycled      bool
+	testDoneEvent *sync.Cond
+)
+
+type mapFlags struct {
+	value map[interface{}]string
+	set   bool
+}
+
+func parseIntOrStr(s string) interface{} {
+	var parsed interface{}
+	var err error
+	parsed, err = strconv.Atoi(s)
+	if err != nil {
+		parsed = strings.Trim(s, "' ")
+	}
+	return parsed
+}
+
+func (m *mapFlags) String() string {
+	s := ""
+	for key, value := range (*m).value {
+		if s != "" {
+			s += ","
+		}
+		if k, ok := key.(int); ok {
+			s += fmt.Sprintf("%d: '%s'", k, value)
+		} else {
+			s += fmt.Sprintf("'%v': '%s'", key, value)
+		}
+	}
+	return s
+}
+
+func (m *mapFlags) Set(s string) error {
+	ss := strings.Split(s, ",")
+	(*m).value = make(map[interface{}]string)
+	for _, val := range ss {
+		val = strings.TrimSpace(val)
+		pair := strings.Split(val, ":")
+		value := strings.Trim(pair[1], "' ")
+		(*m).value[parseIntOrStr(pair[0])] = value
+	}
+	(*m).set = true
+	return nil
+}
+
+type listFlags struct {
+	value []interface{}
+	set   bool
+}
+
+func (l *listFlags) String() string {
+	s := ""
+	for _, value := range (*l).value {
+		if s != "" {
+			s += ", "
+		}
+		if v, ok := value.(int); ok {
+			s += fmt.Sprintf("%d", v)
+		} else {
+			s += fmt.Sprintf("'%s'", value)
+		}
+	}
+	return s
+}
+
+func (l *listFlags) Set(s string) error {
+	ss := strings.Split(s, ",")
+	for _, val := range ss {
+		val = strings.TrimSpace(val)
+		(*l).value = append((*l).value, parseIntOrStr(val))
+	}
+	(*l).set = true
+	return nil
+}
+
+func main() {
+	var (
+		speed             uint64
+		m                 sync.Mutex
+		eachOutPortConfig mapFlags
+		eachInPortConfig  listFlags
+	)
+	flag.Uint64Var(&number, "number", 10000000, "stop after generated number")
+	flag.BoolVar(&isCycled, "cycle", false, "cycle execution")
+	flag.Uint64Var(&speed, "speed", 6000000, "speed of fast generator, Pkts/s")
+	flag.Var(&eachOutPortConfig, "outConfig", "specifies config per port portNum or file: 'path', 'pcapOut': 'path2'. For example: 1: 'ip4.json', 'mix.pcap': 'mix.json'")
+	flag.Var(&eachInPortConfig, "inConfig", "specifies input ports and files 'path', portNum2, 'path2'. For example: 1, 'ip4.pcap', 0, 'mix.pcap'")
+	flag.Parse()
+
+	// Init NFF-GO system at 16 available cores
+	config := flow.Config{}
+	flow.CheckFatal(flow.SystemInit(&config))
+
+	testDoneEvent = sync.NewCond(&m)
+	if !eachOutPortConfig.set {
+		eachOutPortConfig.Set("0: '../testing/config.json'")
+	}
+	for key, value := range eachOutPortConfig.value {
+		configuration, err := generator.ReadConfig(value)
+		if err != nil {
+			panic(fmt.Sprintf("%s config reading failed: %v", value, err))
+		}
+		context, err := generator.GetContext(configuration)
+		flow.CheckFatal(err)
+		outFlow, err := flow.SetFastGenerator(generator.Generate, speed, &context)
+		flow.CheckFatal(err)
+		switch t := key.(type) {
+		case int:
+			flow.CheckFatal(flow.SetSender(outFlow, uint16(t)))
+		case string:
+			flow.CheckFatal(flow.SetSenderFile(outFlow, t))
+		default:
+			panic(fmt.Sprintf("Unrecognized type in outConfig: %v", key))
+		}
+
+	}
+	for _, value := range eachInPortConfig.value {
+		var (
+			inFlow *flow.Flow
+			err    error
+		)
+		switch t := value.(type) {
+		case int:
+			inFlow, err = flow.SetReceiver(uint16(t))
+			flow.CheckFatal(err)
+		case string:
+			inFlow = flow.SetReceiverFile(t, -1)
+		default:
+			panic(fmt.Sprintf("Unrecognized type in inConfig: %v", value))
+		}
+		flow.CheckFatal(flow.SetHandler(inFlow, handleRecv, nil))
+		flow.CheckFatal(flow.SetStopper(inFlow))
+	}
+
+	// Start pipeline
+	go func() {
+		flow.CheckFatal(flow.SystemStart())
+	}()
+
+	g := generator.GetGenerator()
+	go func() {
+		for {
+			println("Sent/Got", g.GetGeneratedNumber(), "/", atomic.LoadUint64(&recv), "packets")
+			time.Sleep(time.Second * 5)
+		}
+	}()
+
+	// Wait for enough packets to arrive
+	testDoneEvent.L.Lock()
+	testDoneEvent.Wait()
+	testDoneEvent.L.Unlock()
+	println("Sent/Got", g.GetGeneratedNumber(), "/", atomic.LoadUint64(&recv), "packets")
+}
+
+func handleRecv(currentPacket *packet.Packet, context flow.UserContext) {
+	got := atomic.AddUint64(&recv, 1)
+	if isCycled {
+		return
+	}
+	if got >= number {
+		time.Sleep(time.Second)
+		testDoneEvent.Signal()
+	}
+}

From b207e1910cd57f155ada30ff62e196361196c980 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Fri, 29 Jun 2018 13:26:58 +0300
Subject: [PATCH 13/50] Extracted generator to examples, renamed it, udpated
 readme

---
 .travis.yml                                   |  2 +-
 examples/Makefile                             |  5 +-
 .../nffPktgen}/Makefile                       |  5 +-
 .../pktgen => examples/nffPktgen}/README.md   | 10 +-
 .../nffPktgen}/generator/generator.go         | 97 +++++++++----------
 .../nffPktgen/generator}/parseConfig.go       |  2 +-
 .../nffPktgen/sendGetBack}/Makefile           |  4 +-
 .../nffPktgen}/sendGetBack/README.md          |  9 +-
 .../nffPktgen}/sendGetBack/sendGetBack.go     | 13 ++-
 .../nffPktgen}/testing/arp.json               |  0
 .../nffPktgen}/testing/arpVlan.json           |  0
 .../nffPktgen}/testing/config.json            |  0
 .../nffPktgen}/testing/ether.json             |  0
 .../nffPktgen}/testing/ip4.json               |  0
 .../nffPktgen}/testing/ip4icmp.json           |  0
 .../nffPktgen}/testing/ip4tcp.json            |  0
 .../nffPktgen}/testing/ip4tcpVlan.json        |  0
 .../nffPktgen}/testing/ip4udp.json            |  0
 .../nffPktgen}/testing/ip4udpVlan.json        |  0
 .../nffPktgen}/testing/ip6.json               |  0
 .../nffPktgen}/testing/ip6icmp.json           |  0
 .../nffPktgen}/testing/ip6tcp.json            |  0
 .../nffPktgen}/testing/ip6udp.json            |  0
 .../nffPktgen}/testing/mix.json               |  0
 .../nffPktgen}/testing/run.sh                 |  0
 .../nffPktgen}/testing/vlanTag.json           |  0
 test/localTesting/README.md                   |  8 --
 test/localTesting/pktgen/sendGetBack/Makefile |  9 --
 28 files changed, 81 insertions(+), 83 deletions(-)
 rename {test/localTesting => examples/nffPktgen}/Makefile (68%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/README.md (96%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/generator/generator.go (85%)
 rename {test/localTesting/pktgen/parseConfig => examples/nffPktgen/generator}/parseConfig.go (99%)
 rename {test/localTesting/pktgen => examples/nffPktgen/sendGetBack}/Makefile (80%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/sendGetBack/README.md (61%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/sendGetBack/sendGetBack.go (94%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/arp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/arpVlan.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/config.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ether.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4icmp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4tcp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4tcpVlan.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4udp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip4udpVlan.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip6.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip6icmp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip6tcp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/ip6udp.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/mix.json (100%)
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/run.sh (100%)
 mode change 100755 => 100644
 rename {test/localTesting/pktgen => examples/nffPktgen}/testing/vlanTag.json (100%)
 delete mode 100644 test/localTesting/README.md
 delete mode 100644 test/localTesting/pktgen/sendGetBack/Makefile

diff --git a/.travis.yml b/.travis.yml
index 962448f6..0ee9041f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ script:
   - docker exec -i test-nff-go make
   # Build standalone examples
   - docker exec -i test-nff-go bash -c "cd examples && make gopacketParserExample && cd .."
-  - docker exec -i test-nff-go bash -c "cd test/localTesting && make && cd -"
+  - docker exec -i test-nff-go bash -c "cd examples && make nffPktgen && cd -"
   - docker exec -i test-nff-go make -C examples/dpi
   # Run unit tests
   - docker exec -i test-nff-go sysctl -w vm.nr_hugepages=1024
diff --git a/examples/Makefile b/examples/Makefile
index 99efc3f7..267d5bc0 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -14,9 +14,12 @@ gopacket:
 gopacketParserExample: gopacket
 	go build gopacketParserExample.go
 
-.PHONY: dpi
+.PHONY: dpi nffPktgen
 dpi:
 	$(MAKE) -C dpi
 
+nffPktgen:
+	$(MAKE) -C nffPktgen
+
 include $(PATH_TO_MK)/intermediate.mk
 include $(PATH_TO_MK)/leaf.mk
diff --git a/test/localTesting/Makefile b/examples/nffPktgen/Makefile
similarity index 68%
rename from test/localTesting/Makefile
rename to examples/nffPktgen/Makefile
index a66d4b90..94c7c788 100644
--- a/test/localTesting/Makefile
+++ b/examples/nffPktgen/Makefile
@@ -3,6 +3,7 @@
 # license that can be found in the LICENSE file.
 
 PATH_TO_MK = ../../mk
-SUBDIRS = pktgen
+IMAGENAME = nff-pktgen
+SUBDIRS = sendGetBack
 
-include $(PATH_TO_MK)/intermediate.mk
+include $(PATH_TO_MK)/leaf.mk
diff --git a/test/localTesting/pktgen/README.md b/examples/nffPktgen/README.md
similarity index 96%
rename from test/localTesting/pktgen/README.md
rename to examples/nffPktgen/README.md
index 93ee59f4..8745a322 100644
--- a/test/localTesting/pktgen/README.md
+++ b/examples/nffPktgen/README.md
@@ -1,11 +1,9 @@
-# Packet generation to file
+# Packet generator based on NFF-GO
 
 ## What it is
-Pktgen parses config in json format and generates packets according to it and sends to port or pcap file that can be read by NFF-GO reader, Wireshark, tcpdump and other tools reading pcap files.
+nffPktgen parses config in json format and generates packets according to it and either sends to port or writes to pcap file that can be read by NFF-GO reader, Wireshark, tcpdump and other tools reading pcap files.
 Generator package has public API which can be used for own generator.
-
 ### API:
-
 ```go
 func GetGenerator() *generator 
 ```
@@ -32,12 +30,12 @@ func (g *generator) ResetGenerateNumber()
 sets number to generate to infinity (by default).
 
 ```go
-func ReadConfig(fileName string) ([]*parseConfig.MixConfig, error)
+func ReadConfig(fileName string) ([]*MixConfig, error)
 ```
 returns read and parsed config from file.
 
 ```go
-func GetContext(mixConfig []*parseConfig.MixConfig) (genParameters, error)
+func GetContext(mixConfig []*MixConfig) (genParameters, error)
 ```
 returns context that chould be sent to generator according to configuration.
 
diff --git a/test/localTesting/pktgen/generator/generator.go b/examples/nffPktgen/generator/generator.go
similarity index 85%
rename from test/localTesting/pktgen/generator/generator.go
rename to examples/nffPktgen/generator/generator.go
index 8115768b..608dadfc 100644
--- a/test/localTesting/pktgen/generator/generator.go
+++ b/examples/nffPktgen/generator/generator.go
@@ -18,7 +18,6 @@ import (
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/flow"
 	"github.com/intel-go/nff-go/packet"
-	"github.com/intel-go/nff-go/test/localTesting/pktgen/parseConfig"
 )
 
 var gen generator
@@ -58,19 +57,19 @@ func (g *generator) ResetGenerateNumber() {
 }
 
 // ReadConfig function reads and parses config file.
-func ReadConfig(fileName string) ([]*parseConfig.MixConfig, error) {
+func ReadConfig(fileName string) ([]*MixConfig, error) {
 	f, err := os.Open(fileName)
 	if err != nil {
 		return nil, fmt.Errorf("opening file failed with: %v ", err)
 	}
-	cfg, err := parseConfig.ParseConfig(f)
+	cfg, err := ParseConfig(f)
 	if err != nil {
 		return nil, fmt.Errorf("parsing config failed with: %v", err)
 	}
 	return cfg, nil
 }
 
-func getNextAddr(addr *parseConfig.AddrRange) []uint8 {
+func getNextAddr(addr *AddrRange) []uint8 {
 	if len(addr.Current) == 0 {
 		addr.Current = []uint8{0}
 	}
@@ -102,7 +101,7 @@ func copyAddr(destination []uint8, source []uint8, size int) {
 	}
 }
 
-func getNextPort(port *parseConfig.PortRange) (nextPort uint16) {
+func getNextPort(port *PortRange) (nextPort uint16) {
 	if len(port.Current) == 0 {
 		port.Current = []uint16{0}
 	}
@@ -114,14 +113,14 @@ func getNextPort(port *parseConfig.PortRange) (nextPort uint16) {
 	return nextPort
 }
 
-func getNextSeqNumber(seq *parseConfig.Sequence, rnd *rand.Rand) (nextSeqNum uint32) {
+func getNextSeqNumber(seq *Sequence, rnd *rand.Rand) (nextSeqNum uint32) {
 	if len(seq.Next) == 0 {
 		return 0
 	}
 	nextSeqNum = seq.Next[0]
-	if seq.Type == parseConfig.RANDOM {
+	if seq.Type == RANDOM {
 		seq.Next[0] = rnd.Uint32()
-	} else if seq.Type == parseConfig.INCREASING {
+	} else if seq.Type == INCREASING {
 		seq.Next[0]++
 	}
 	return nextSeqNum
@@ -129,11 +128,11 @@ func getNextSeqNumber(seq *parseConfig.Sequence, rnd *rand.Rand) (nextSeqNum uin
 
 func generateData(configuration interface{}, rnd *rand.Rand) ([]uint8, error) {
 	switch data := configuration.(type) {
-	case parseConfig.Raw:
+	case Raw:
 		pktData := make([]uint8, len(data.Data))
 		copy(pktData[:], ([]uint8(data.Data)))
 		return pktData, nil
-	case parseConfig.RandBytes:
+	case RandBytes:
 		maxZise := data.Size + data.Deviation
 		minSize := data.Size - data.Deviation
 		randSize := uint(rnd.Float64()*float64(maxZise-minSize) + float64(minSize))
@@ -142,10 +141,10 @@ func generateData(configuration interface{}, rnd *rand.Rand) ([]uint8, error) {
 			pktData[i] = byte(rnd.Int())
 		}
 		return pktData, nil
-	case []parseConfig.PDistEntry:
+	case []PDistEntry:
 		prob := 0.0
 		rndN := math.Abs(rnd.Float64())
-		maxProb := parseConfig.PDistEntry{Probability: 0}
+		maxProb := PDistEntry{Probability: 0}
 		for _, item := range data {
 			prob += item.Probability
 			if rndN <= prob {
@@ -174,26 +173,26 @@ func generateData(configuration interface{}, rnd *rand.Rand) ([]uint8, error) {
 	return nil, fmt.Errorf("unknown data type")
 }
 
-func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet, *parseConfig.PacketConfig, *rand.Rand), error) {
+func getGenerator(configuration *PacketConfig) (func(*packet.Packet, *PacketConfig, *rand.Rand), error) {
 	switch l2 := (configuration.Data).(type) {
-	case parseConfig.EtherConfig:
+	case EtherConfig:
 		switch l3 := l2.Data.(type) {
-		case parseConfig.IPConfig:
+		case IPConfig:
 			switch l3.Data.(type) {
-			case parseConfig.TCPConfig:
+			case TCPConfig:
 				return generateTCPIP, nil
-			case parseConfig.UDPConfig:
+			case UDPConfig:
 				return generateUDPIP, nil
-			case parseConfig.ICMPConfig:
+			case ICMPConfig:
 				return generateICMPIP, nil
-			case parseConfig.Raw, parseConfig.RandBytes, []parseConfig.PDistEntry:
+			case Raw, RandBytes, []PDistEntry:
 				return generateIP, nil
 			default:
 				return nil, fmt.Errorf("unknown packet l4 configuration")
 			}
-		case parseConfig.ARPConfig:
+		case ARPConfig:
 			return generateARP, nil
-		case parseConfig.Raw, parseConfig.RandBytes, []parseConfig.PDistEntry:
+		case Raw, RandBytes, []PDistEntry:
 			return generateEther, nil
 		default:
 			return nil, fmt.Errorf("unknown packet l3 configuration")
@@ -206,8 +205,8 @@ func getGenerator(configuration *parseConfig.PacketConfig) (func(*packet.Packet,
 // one unit for each mix
 type generatorTableUnit struct {
 	have, need    uint32
-	generatorFunc func(*packet.Packet, *parseConfig.PacketConfig, *rand.Rand)
-	config        *parseConfig.PacketConfig
+	generatorFunc func(*packet.Packet, *PacketConfig, *rand.Rand)
+	config        *PacketConfig
 }
 
 func (gtu *generatorTableUnit) String() string {
@@ -231,7 +230,7 @@ func (gp genParameters) Delete() {
 }
 
 // GetContext gets generator context according to config
-func GetContext(mixConfig []*parseConfig.MixConfig) (genParameters, error) {
+func GetContext(mixConfig []*MixConfig) (genParameters, error) {
 	var t []generatorTableUnit
 	for _, packetConfig := range mixConfig {
 		genFunc, err := getGenerator(packetConfig.Config)
@@ -268,11 +267,11 @@ func Generate(pkt *packet.Packet, context flow.UserContext) {
 	context.(genParameters).table[next].have++
 }
 
-func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateEther(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
+	l2 := config.Data.(EtherConfig)
 	data, err := generateData(l2.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l2: %v", err))
@@ -291,12 +290,12 @@ func generateEther(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *ra
 	config.Data = l2
 }
 
-func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
-	l3 := l2.Data.(parseConfig.IPConfig)
+	l2 := config.Data.(EtherConfig)
+	l3 := l2.Data.(IPConfig)
 	data, err := generateData(l3.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l3: %v", err))
@@ -330,12 +329,12 @@ func generateIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.
 	config.Data = l2
 }
 
-func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateARP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
-	l3 := l2.Data.(parseConfig.ARPConfig)
+	l2 := config.Data.(EtherConfig)
+	l3 := l2.Data.(ARPConfig)
 	var SHA, THA [common.EtherAddrLen]uint8
 	copyAddr(SHA[:], getNextAddr(&(l3.SHA)), common.EtherAddrLen)
 	SPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SPA))).To4())
@@ -378,13 +377,13 @@ func generateARP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand
 	config.Data = l2
 }
 
-func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateTCPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
-	l3 := l2.Data.(parseConfig.IPConfig)
-	l4 := l3.Data.(parseConfig.TCPConfig)
+	l2 := config.Data.(EtherConfig)
+	l3 := l2.Data.(IPConfig)
+	l4 := l3.Data.(TCPConfig)
 	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
@@ -425,13 +424,13 @@ func generateTCPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *ra
 	config.Data = l2
 }
 
-func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateUDPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
-	l3 := l2.Data.(parseConfig.IPConfig)
-	l4 := l3.Data.(parseConfig.UDPConfig)
+	l2 := config.Data.(EtherConfig)
+	l3 := l2.Data.(IPConfig)
+	l4 := l3.Data.(UDPConfig)
 	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
@@ -472,13 +471,13 @@ func generateUDPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *ra
 	config.Data = l2
 }
 
-func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *rand.Rand) {
+func generateICMPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(parseConfig.EtherConfig)
-	l3 := l2.Data.(parseConfig.IPConfig)
-	l4 := l3.Data.(parseConfig.ICMPConfig)
+	l2 := config.Data.(EtherConfig)
+	l3 := l2.Data.(IPConfig)
+	l4 := l3.Data.(ICMPConfig)
 	data, err := generateData(l4.Data, rnd)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
@@ -519,7 +518,7 @@ func generateICMPIP(pkt *packet.Packet, config *parseConfig.PacketConfig, rnd *r
 	config.Data = l2
 }
 
-func fillTCPHdr(pkt *packet.Packet, l4 *parseConfig.TCPConfig, rnd *rand.Rand) error {
+func fillTCPHdr(pkt *packet.Packet, l4 *TCPConfig, rnd *rand.Rand) error {
 	emptyPacketTCP := (*packet.TCPHdr)(pkt.L4)
 	emptyPacketTCP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
 	emptyPacketTCP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
@@ -528,14 +527,14 @@ func fillTCPHdr(pkt *packet.Packet, l4 *parseConfig.TCPConfig, rnd *rand.Rand) e
 	return nil
 }
 
-func fillUDPHdr(pkt *packet.Packet, l4 *parseConfig.UDPConfig) error {
+func fillUDPHdr(pkt *packet.Packet, l4 *UDPConfig) error {
 	emptyPacketUDP := (*packet.UDPHdr)(pkt.L4)
 	emptyPacketUDP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
 	emptyPacketUDP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
 	return nil
 }
 
-func fillICMPHdr(pkt *packet.Packet, l4 *parseConfig.ICMPConfig, rnd *rand.Rand) error {
+func fillICMPHdr(pkt *packet.Packet, l4 *ICMPConfig, rnd *rand.Rand) error {
 	emptyPacketICMP := (*packet.ICMPHdr)(pkt.L4)
 	emptyPacketICMP.Type = l4.Type
 	emptyPacketICMP.Code = l4.Code
@@ -544,7 +543,7 @@ func fillICMPHdr(pkt *packet.Packet, l4 *parseConfig.ICMPConfig, rnd *rand.Rand)
 	return nil
 }
 
-func fillIPHdr(pkt *packet.Packet, l3 *parseConfig.IPConfig) error {
+func fillIPHdr(pkt *packet.Packet, l3 *IPConfig) error {
 	if l3.Version == 4 {
 		pktIP := pkt.GetIPv4()
 		pktIP.SrcAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SAddr))).To4())
@@ -559,7 +558,7 @@ func fillIPHdr(pkt *packet.Packet, l3 *parseConfig.IPConfig) error {
 	return nil
 }
 
-func fillEtherHdr(pkt *packet.Packet, l2 *parseConfig.EtherConfig) error {
+func fillEtherHdr(pkt *packet.Packet, l2 *EtherConfig) error {
 	if l2.VLAN != nil {
 		if err := addVLAN(pkt, l2.VLAN.TCI); err != nil {
 			return err
diff --git a/test/localTesting/pktgen/parseConfig/parseConfig.go b/examples/nffPktgen/generator/parseConfig.go
similarity index 99%
rename from test/localTesting/pktgen/parseConfig/parseConfig.go
rename to examples/nffPktgen/generator/parseConfig.go
index 800639ad..1551705c 100644
--- a/test/localTesting/pktgen/parseConfig/parseConfig.go
+++ b/examples/nffPktgen/generator/parseConfig.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package parseConfig
+package generator
 
 import (
 	"bufio"
diff --git a/test/localTesting/pktgen/Makefile b/examples/nffPktgen/sendGetBack/Makefile
similarity index 80%
rename from test/localTesting/pktgen/Makefile
rename to examples/nffPktgen/sendGetBack/Makefile
index 55dd0674..48b152d5 100644
--- a/test/localTesting/pktgen/Makefile
+++ b/examples/nffPktgen/sendGetBack/Makefile
@@ -3,7 +3,7 @@
 # license that can be found in the LICENSE file.
 
 PATH_TO_MK = ../../../mk
-IMAGENAME = local-pktgen
-SUBDIRS = sendGetBack
+IMAGENAME = nff-pktgen
+EXECUTABLES = sendGetBack
 
 include $(PATH_TO_MK)/leaf.mk
diff --git a/test/localTesting/pktgen/sendGetBack/README.md b/examples/nffPktgen/sendGetBack/README.md
similarity index 61%
rename from test/localTesting/pktgen/sendGetBack/README.md
rename to examples/nffPktgen/sendGetBack/README.md
index c68cd74a..e080d129 100644
--- a/test/localTesting/pktgen/sendGetBack/README.md
+++ b/examples/nffPktgen/sendGetBack/README.md
@@ -8,4 +8,11 @@ Parses config files, generates output and sends to file or port and counts them.
 * --speed sets the speed of generator
 * --cycle sets cycle execution to generate infinite number of packets
 * --outConfig specifies config per port portNum or file: 'path', 'path3': 'path2'. For example: 1: 'ip4.json', 'mix.pcap': 'mix.json'
-* --inConfig specifies input ports and files: 'path', 'portNum2', 'path2'. For example: 1, 'ip4.pcap', 0, 'mix.pcap'
\ No newline at end of file
+* --inConfig specifies input ports and files: 'path', 'portNum2', 'path2'. For example: 1, 'ip4.pcap', 0, 'mix.pcap'
+
+### Example of usage:
+To run generation from "mix.json" config to port number 1 and from "../../ip4tcp.json" to file "ip4tcp.pcap" and receive packets from 
+port 0 and file "../../ip4tcp.pcap" run:
+```
+    ./sendGetBack --outConfig "1: 'mix.json', ip4tcp.pcap: '../../ip4tcp.json'" --inConfig "0, '../../ip4tcp.json'"
+```
\ No newline at end of file
diff --git a/test/localTesting/pktgen/sendGetBack/sendGetBack.go b/examples/nffPktgen/sendGetBack/sendGetBack.go
similarity index 94%
rename from test/localTesting/pktgen/sendGetBack/sendGetBack.go
rename to examples/nffPktgen/sendGetBack/sendGetBack.go
index ac5e9b1e..7ba0d2a0 100644
--- a/test/localTesting/pktgen/sendGetBack/sendGetBack.go
+++ b/examples/nffPktgen/sendGetBack/sendGetBack.go
@@ -15,7 +15,7 @@ import (
 
 	"github.com/intel-go/nff-go/flow"
 	"github.com/intel-go/nff-go/packet"
-	"github.com/intel-go/nff-go/test/localTesting/pktgen/generator"
+	"github.com/intel-go/nff-go/examples/nffPktgen/generator"
 )
 
 var (
@@ -29,12 +29,19 @@ type mapFlags struct {
 	set   bool
 }
 
+func parseStr(s string) string {
+	indx := strings.Index(s, "'")
+	substr := s[indx+1:]
+	indx = strings.Index(substr, "'")
+	return substr[:indx]
+}
+
 func parseIntOrStr(s string) interface{} {
 	var parsed interface{}
 	var err error
 	parsed, err = strconv.Atoi(s)
 	if err != nil {
-		parsed = strings.Trim(s, "' ")
+		parsed  = parseStr(s)
 	}
 	return parsed
 }
@@ -60,7 +67,7 @@ func (m *mapFlags) Set(s string) error {
 	for _, val := range ss {
 		val = strings.TrimSpace(val)
 		pair := strings.Split(val, ":")
-		value := strings.Trim(pair[1], "' ")
+		value := parseStr(pair[1])
 		(*m).value[parseIntOrStr(pair[0])] = value
 	}
 	(*m).set = true
diff --git a/test/localTesting/pktgen/testing/arp.json b/examples/nffPktgen/testing/arp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/arp.json
rename to examples/nffPktgen/testing/arp.json
diff --git a/test/localTesting/pktgen/testing/arpVlan.json b/examples/nffPktgen/testing/arpVlan.json
similarity index 100%
rename from test/localTesting/pktgen/testing/arpVlan.json
rename to examples/nffPktgen/testing/arpVlan.json
diff --git a/test/localTesting/pktgen/testing/config.json b/examples/nffPktgen/testing/config.json
similarity index 100%
rename from test/localTesting/pktgen/testing/config.json
rename to examples/nffPktgen/testing/config.json
diff --git a/test/localTesting/pktgen/testing/ether.json b/examples/nffPktgen/testing/ether.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ether.json
rename to examples/nffPktgen/testing/ether.json
diff --git a/test/localTesting/pktgen/testing/ip4.json b/examples/nffPktgen/testing/ip4.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4.json
rename to examples/nffPktgen/testing/ip4.json
diff --git a/test/localTesting/pktgen/testing/ip4icmp.json b/examples/nffPktgen/testing/ip4icmp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4icmp.json
rename to examples/nffPktgen/testing/ip4icmp.json
diff --git a/test/localTesting/pktgen/testing/ip4tcp.json b/examples/nffPktgen/testing/ip4tcp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4tcp.json
rename to examples/nffPktgen/testing/ip4tcp.json
diff --git a/test/localTesting/pktgen/testing/ip4tcpVlan.json b/examples/nffPktgen/testing/ip4tcpVlan.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4tcpVlan.json
rename to examples/nffPktgen/testing/ip4tcpVlan.json
diff --git a/test/localTesting/pktgen/testing/ip4udp.json b/examples/nffPktgen/testing/ip4udp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4udp.json
rename to examples/nffPktgen/testing/ip4udp.json
diff --git a/test/localTesting/pktgen/testing/ip4udpVlan.json b/examples/nffPktgen/testing/ip4udpVlan.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip4udpVlan.json
rename to examples/nffPktgen/testing/ip4udpVlan.json
diff --git a/test/localTesting/pktgen/testing/ip6.json b/examples/nffPktgen/testing/ip6.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip6.json
rename to examples/nffPktgen/testing/ip6.json
diff --git a/test/localTesting/pktgen/testing/ip6icmp.json b/examples/nffPktgen/testing/ip6icmp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip6icmp.json
rename to examples/nffPktgen/testing/ip6icmp.json
diff --git a/test/localTesting/pktgen/testing/ip6tcp.json b/examples/nffPktgen/testing/ip6tcp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip6tcp.json
rename to examples/nffPktgen/testing/ip6tcp.json
diff --git a/test/localTesting/pktgen/testing/ip6udp.json b/examples/nffPktgen/testing/ip6udp.json
similarity index 100%
rename from test/localTesting/pktgen/testing/ip6udp.json
rename to examples/nffPktgen/testing/ip6udp.json
diff --git a/test/localTesting/pktgen/testing/mix.json b/examples/nffPktgen/testing/mix.json
similarity index 100%
rename from test/localTesting/pktgen/testing/mix.json
rename to examples/nffPktgen/testing/mix.json
diff --git a/test/localTesting/pktgen/testing/run.sh b/examples/nffPktgen/testing/run.sh
old mode 100755
new mode 100644
similarity index 100%
rename from test/localTesting/pktgen/testing/run.sh
rename to examples/nffPktgen/testing/run.sh
diff --git a/test/localTesting/pktgen/testing/vlanTag.json b/examples/nffPktgen/testing/vlanTag.json
similarity index 100%
rename from test/localTesting/pktgen/testing/vlanTag.json
rename to examples/nffPktgen/testing/vlanTag.json
diff --git a/test/localTesting/README.md b/test/localTesting/README.md
deleted file mode 100644
index e8d28b8c..00000000
--- a/test/localTesting/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Local testing tools
-
-## What it is
-Tools for testing and running nff-go applications on one local machine by writing to file and reading from file instead of sending and receiving packets.
-
-### Pktgen
-Pktgen parses config in json format and generates packets according to it in pcap file that can be read by NFF-GO reader, Wireshark, tcpdump and other tools reading pcap files.
-More detailed information can be found in [pktgen directory](https://github.com/intel-go/nff-go/tree/master/test/localTesting/pktgen)
diff --git a/test/localTesting/pktgen/sendGetBack/Makefile b/test/localTesting/pktgen/sendGetBack/Makefile
deleted file mode 100644
index 25fd101b..00000000
--- a/test/localTesting/pktgen/sendGetBack/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2017 Intel Corporation. 
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-PATH_TO_MK = ../../../../mk
-IMAGENAME = local-pktgen
-EXECUTABLES = sendGetBack
-
-include $(PATH_TO_MK)/leaf.mk

From f265de529bf4d43d3f2036c09c1dc421d42d411a Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Wed, 25 Jul 2018 12:44:29 +0300
Subject: [PATCH 14/50] Fixed generator performance problems

---
 examples/nffPktgen/Makefile                   |   2 +-
 examples/nffPktgen/generator/generator.go     | 431 +++++++-----------
 examples/nffPktgen/generator/parseConfig.go   | 171 ++++---
 examples/nffPktgen/perfTest/Makefile          |   9 +
 examples/nffPktgen/perfTest/perfTest.go       |  42 ++
 examples/nffPktgen/sendGetBack/sendGetBack.go |  25 +-
 examples/nffPktgen/testing/run.sh             |  29 +-
 7 files changed, 361 insertions(+), 348 deletions(-)
 create mode 100644 examples/nffPktgen/perfTest/Makefile
 create mode 100644 examples/nffPktgen/perfTest/perfTest.go

diff --git a/examples/nffPktgen/Makefile b/examples/nffPktgen/Makefile
index 94c7c788..65c5ecae 100644
--- a/examples/nffPktgen/Makefile
+++ b/examples/nffPktgen/Makefile
@@ -4,6 +4,6 @@
 
 PATH_TO_MK = ../../mk
 IMAGENAME = nff-pktgen
-SUBDIRS = sendGetBack
+SUBDIRS = sendGetBack perfTest
 
 include $(PATH_TO_MK)/leaf.mk
diff --git a/examples/nffPktgen/generator/generator.go b/examples/nffPktgen/generator/generator.go
index 608dadfc..5eb726cf 100644
--- a/examples/nffPktgen/generator/generator.go
+++ b/examples/nffPktgen/generator/generator.go
@@ -5,15 +5,14 @@
 package generator
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
 	"math/rand"
-	"net"
 	"os"
 	"sync/atomic"
+	"unsafe"
 
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/flow"
@@ -23,9 +22,7 @@ import (
 var gen generator
 
 type generator struct {
-	count    uint64
-	isFinite bool
-	number   uint64
+	count uint64
 }
 
 // GetGenerator returns generator struct pointer
@@ -39,23 +36,6 @@ func (g *generator) GetGeneratedNumber() uint64 {
 	return atomic.LoadUint64(&(g.count))
 }
 
-// ResetCounter resets count of generated packets
-func (g *generator) ResetCounter() {
-	atomic.StoreUint64(&(g.count), 0)
-}
-
-// SetGenerateNumber sets a number to generate
-func (g *generator) SetGenerateNumber(number uint64) {
-	g.number = number
-	g.isFinite = true
-}
-
-// ResetGenerateNumber sets a number to generate to infinite
-func (g *generator) ResetGenerateNumber() {
-	g.number = 0
-	g.isFinite = false
-}
-
 // ReadConfig function reads and parses config file.
 func ReadConfig(fileName string) ([]*MixConfig, error) {
 	f, err := os.Open(fileName)
@@ -70,27 +50,15 @@ func ReadConfig(fileName string) ([]*MixConfig, error) {
 }
 
 func getNextAddr(addr *AddrRange) []uint8 {
-	if len(addr.Current) == 0 {
-		addr.Current = []uint8{0}
-	}
 	if addr.Incr == 0 {
-		return addr.Current
+		return addr.Current.Bytes()
 	}
+	addr.Current.Add(big.NewInt(int64(addr.Incr)), addr.Current)
 	// if current < min or current > max, copy min to current
-	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
-		addr.Current = make([]byte, len(addr.Min))
-		copy(addr.Current, addr.Min)
-	}
-	// copy current to return
-	retAddr := make([]byte, len(addr.Current))
-	copy(retAddr, addr.Current)
-	// increment current
-	addressInt := big.NewInt(0)
-	addressInt.SetBytes(addr.Current)
-	addressInt.Add(big.NewInt(int64(addr.Incr)), addressInt)
-	newAddr := addressInt.Bytes()
-	copyAddr(addr.Current, newAddr, len(addr.Current))
-	return retAddr
+	if addr.Current.Cmp(addr.Min) < 0 || addr.Current.Cmp(addr.Max) > 0 {
+		addr.Current = addr.Min
+	}
+	return addr.Current.Bytes()
 }
 
 func copyAddr(destination []uint8, source []uint8, size int) {
@@ -102,97 +70,96 @@ func copyAddr(destination []uint8, source []uint8, size int) {
 }
 
 func getNextPort(port *PortRange) (nextPort uint16) {
-	if len(port.Current) == 0 {
-		port.Current = []uint16{0}
-	}
-	if port.Current[0] < port.Min || port.Current[0] > port.Max {
-		port.Current[0] = port.Min
+	if port.Current < port.Min || port.Current > port.Max {
+		port.Current = port.Min
 	}
-	nextPort = port.Current[0]
-	port.Current[0] += port.Incr
+	nextPort = port.Current
+	port.Current += port.Incr
 	return nextPort
 }
 
 func getNextSeqNumber(seq *Sequence, rnd *rand.Rand) (nextSeqNum uint32) {
-	if len(seq.Next) == 0 {
-		return 0
-	}
-	nextSeqNum = seq.Next[0]
+	nextSeqNum = seq.Next
 	if seq.Type == RANDOM {
-		seq.Next[0] = rnd.Uint32()
+		seq.Next = rnd.Uint32()
 	} else if seq.Type == INCREASING {
-		seq.Next[0]++
+		seq.Next++
 	}
 	return nextSeqNum
 }
 
-func generateData(configuration interface{}, rnd *rand.Rand) ([]uint8, error) {
-	switch data := configuration.(type) {
-	case Raw:
-		pktData := make([]uint8, len(data.Data))
-		copy(pktData[:], ([]uint8(data.Data)))
-		return pktData, nil
-	case RandBytes:
+type dataCopier func(unsafe.Pointer, uint, *rand.Rand, unsafe.Pointer)
+
+func copyRaw(configuration unsafe.Pointer, size uint, rnd *rand.Rand, copyTo unsafe.Pointer) {
+	data := (*Raw)(configuration)
+	copy((*[1 << 30]uint8)(copyTo)[0:size], ([]uint8(data.Data)))
+}
+
+func copyRand(configuration unsafe.Pointer, size uint, rnd *rand.Rand, copyTo unsafe.Pointer) {
+	packetData := (*[1 << 30]byte)(copyTo)[0:size]
+	for i := range packetData {
+		packetData[i] = byte(rnd.Int())
+	}
+}
+
+func getDataSizeType(configuration unsafe.Pointer, dtype DataType, rnd *rand.Rand) (uint, unsafe.Pointer, dataCopier) {
+	switch dtype {
+	case RAWDATA:
+		data := (*Raw)(configuration)
+		return uint(len(data.Data)), configuration, copyRaw
+	case RANDDATA:
+		data := (*RandBytes)(configuration)
 		maxZise := data.Size + data.Deviation
 		minSize := data.Size - data.Deviation
 		randSize := uint(rnd.Float64()*float64(maxZise-minSize) + float64(minSize))
-		pktData := make([]uint8, randSize)
-		for i := range pktData {
-			pktData[i] = byte(rnd.Int())
-		}
-		return pktData, nil
-	case []PDistEntry:
+		return randSize, configuration, copyRand
+	case PDISTDATA:
+		data := (*[]PDistEntry)(configuration)
 		prob := 0.0
 		rndN := math.Abs(rnd.Float64())
 		maxProb := PDistEntry{Probability: 0}
-		for _, item := range data {
+		for _, item := range *data {
 			prob += item.Probability
 			if rndN <= prob {
-				pktData, err := generateData(item.Data, rnd)
-				if err != nil {
-					return nil, fmt.Errorf("failed to fill data with pdist data type: %v", err)
-				}
-				return pktData, nil
+				return getDataSizeType(item.Data, item.DType, rnd)
 			}
 			if item.Probability > maxProb.Probability {
 				maxProb = item
 			}
 		}
 		if prob <= 0 || prob > 1 {
-			return nil, fmt.Errorf("sum of pdist probabilities is invalid, %f", prob)
+			panic(fmt.Sprintf("sum of pdist probabilities is invalid, %f", prob))
 		}
 		// get the variant with max prob
 		// if something went wrong and rand did not match any prob
 		// may happen if sum of prob was not 1
-		pktData, err := generateData(maxProb.Data, rnd)
-		if err != nil {
-			return nil, fmt.Errorf("failed to fill data with pdist data type: %v", err)
-		}
-		return pktData, nil
+		return getDataSizeType(maxProb.Data, maxProb.DType, rnd)
 	}
-	return nil, fmt.Errorf("unknown data type")
+	panic(fmt.Sprintf("unknown data type"))
 }
 
 func getGenerator(configuration *PacketConfig) (func(*packet.Packet, *PacketConfig, *rand.Rand), error) {
-	switch l2 := (configuration.Data).(type) {
-	case EtherConfig:
-		switch l3 := l2.Data.(type) {
-		case IPConfig:
-			switch l3.Data.(type) {
-			case TCPConfig:
+	switch configuration.DType {
+	case ETHERHDR:
+		l2 := (*EtherConfig)(configuration.Data)
+		switch l2.DType {
+		case IPHDR:
+			l3 := (*IPConfig)(l2.Data)
+			switch l3.DType {
+			case TCPHDR:
 				return generateTCPIP, nil
-			case UDPConfig:
+			case UDPHDR:
 				return generateUDPIP, nil
-			case ICMPConfig:
+			case ICMPHDR:
 				return generateICMPIP, nil
-			case Raw, RandBytes, []PDistEntry:
+			case RAWDATA, RANDDATA, PDISTDATA:
 				return generateIP, nil
 			default:
 				return nil, fmt.Errorf("unknown packet l4 configuration")
 			}
-		case ARPConfig:
+		case ARPHDR:
 			return generateARP, nil
-		case Raw, RandBytes, []PDistEntry:
+		case RAWDATA, RANDDATA, PDISTDATA:
 			return generateEther, nil
 		default:
 			return nil, fmt.Errorf("unknown packet l3 configuration")
@@ -215,95 +182,80 @@ func (gtu *generatorTableUnit) String() string {
 
 type genParameters struct {
 	table  []generatorTableUnit
-	next   []uint32
+	next   uint32
 	length uint32
 	rnd    *rand.Rand
 }
 
 func (gp genParameters) Copy() interface{} {
-	cpy := make([]generatorTableUnit, len(gp.table))
-	copy(cpy, gp.table)
-	return genParameters{table: cpy, next: []uint32{0}, length: gp.length, rnd: rand.New(rand.NewSource(13))}
+	ret := new(genParameters)
+	ret.table = make([]generatorTableUnit, len(gp.table))
+	copy(ret.table, gp.table)
+	ret.length = gp.length
+	ret.rnd = rand.New(rand.NewSource(13))
+	return ret
 }
 
 func (gp genParameters) Delete() {
 }
 
 // GetContext gets generator context according to config
-func GetContext(mixConfig []*MixConfig) (genParameters, error) {
+func GetContext(mixConfig []*MixConfig) (*genParameters, error) {
 	var t []generatorTableUnit
 	for _, packetConfig := range mixConfig {
 		genFunc, err := getGenerator(packetConfig.Config)
 		if err != nil {
-			return genParameters{}, err
+			return nil, err
 		}
 		tu := generatorTableUnit{have: 0, need: packetConfig.Quantity, generatorFunc: genFunc, config: packetConfig.Config}
 		t = append(t, tu)
 	}
-	return genParameters{table: t, next: []uint32{0}, length: uint32(len(t)), rnd: rand.New(rand.NewSource(13))}, nil
+	ret := new(genParameters)
+	ret.table = t
+	ret.length = uint32(len(t))
+	ret.rnd = rand.New(rand.NewSource(13))
+	return ret, nil
 }
 
 // Generate is a main generatior func
 func Generate(pkt *packet.Packet, context flow.UserContext) {
-	if gen.isFinite && gen.count >= gen.number {
-		return
-	}
-	genP := context.(genParameters)
-	table := genP.table
-	next := genP.next[0]
+	genP := context.(*genParameters)
 	if genP.length > 1 {
-		if table[next].have == table[next].need {
-			context.(genParameters).table[next].have = 0
-			if next+1 < genP.length {
-				next++
+		if genP.table[genP.next].have == genP.table[genP.next].need {
+			genP.table[genP.next].have = 0
+			if genP.next+1 < genP.length {
+				genP.next++
 			} else {
-				next = 0
+				genP.next = 0
 			}
-			context.(genParameters).next[0] = next
 		}
 	}
-	table[next].generatorFunc(pkt, table[next].config, genP.rnd)
+	genP.table[genP.next].generatorFunc(pkt, genP.table[genP.next].config, genP.rnd)
 	atomic.AddUint64(&(gen.count), 1)
-	context.(genParameters).table[next].have++
+	genP.table[genP.next].have++
 }
 
 func generateEther(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	data, err := generateData(l2.Data, rnd)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to parse data for l2: %v", err))
-	}
-	if data == nil {
-		panic(fmt.Sprintf("failed to generate data"))
-	}
-	size := uint(len(data))
+	l2 := (*EtherConfig)(config.Data)
+	size, dataConfig, copyDataFunc := getDataSizeType(l2.Data, l2.DType, rnd)
+
 	if !packet.InitEmptyPacket(pkt, size) {
 		panic(fmt.Sprintf("InitEmptyPacket failed"))
 	}
-	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillEtherHdr(pkt, &l2); err != nil {
-		panic(fmt.Sprintf("failed to fill ether header %v", err))
-	}
-	config.Data = l2
+	copyDataFunc(dataConfig, size, rnd, pkt.Data)
+	fillEtherHdr(pkt, l2)
 }
 
 func generateIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	l3 := l2.Data.(IPConfig)
-	data, err := generateData(l3.Data, rnd)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to parse data for l3: %v", err))
-	}
-	if data == nil {
-		panic(fmt.Sprintf("failed to generate data"))
-	}
-	size := uint(len(data))
+	l2 := (*EtherConfig)(config.Data)
+	l3 := (*IPConfig)(l2.Data)
+	size, dataConfig, copyDataFunc := getDataSizeType(l3.Data, l3.DType, rnd)
 	if l3.Version == 4 {
 		if !packet.InitEmptyIPv4Packet(pkt, size) {
 			panic(fmt.Sprintf("InitEmptyIPv4Packet returned false"))
@@ -315,29 +267,31 @@ func generateIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	} else {
 		panic(fmt.Sprintf("fillPacketl3 failed, unknovn version %d", l3.Version))
 	}
-	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillIPHdr(pkt, &l3); err != nil {
-		panic(fmt.Sprintf("failed to fill ip header %v", err))
-	}
-	if err := fillEtherHdr(pkt, &l2); err != nil {
-		panic(fmt.Sprintf("failed to fill ether header %v", err))
-	}
+	copyDataFunc(dataConfig, size, rnd, pkt.Data)
+	fillEtherHdr(pkt, l2)
+	fillIPHdr(pkt, l3)
 	if l3.Version == 4 {
-		pkt.GetIPv4CheckVLAN().HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pkt.GetIPv4CheckVLAN()))
+		pktIP := (*packet.IPv4Hdr)(pkt.L3)
+		pktIP.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pktIP))
 	}
-	l2.Data = l3
-	config.Data = l2
+}
+
+func To4(addr []byte) []byte {
+	if len(addr) > common.IPv4AddrLen {
+		return addr[len(addr)-common.IPv4AddrLen:]
+	}
+	return addr
 }
 
 func generateARP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	l3 := l2.Data.(ARPConfig)
+	l2 := (*EtherConfig)(config.Data)
+	l3 := (*ARPConfig)(l2.Data)
 	var SHA, THA [common.EtherAddrLen]uint8
 	copyAddr(SHA[:], getNextAddr(&(l3.SHA)), common.EtherAddrLen)
-	SPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SPA))).To4())
+	SPA := binary.LittleEndian.Uint32(To4(getNextAddr(&(l3.SPA))))
 	switch l3.Operation {
 	case packet.ARPRequest:
 		if l3.Gratuitous {
@@ -345,7 +299,7 @@ func generateARP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 				panic(fmt.Sprintf("InitGARPAnnouncementRequestPacket returned false"))
 			}
 		} else {
-			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.TPA))).To4())
+			TPA := binary.LittleEndian.Uint32(To4(getNextAddr(&(l3.TPA))))
 			if !packet.InitARPRequestPacket(pkt, SHA, SPA, TPA) {
 				panic(fmt.Sprintf("InitARPRequestPacket returned false"))
 			}
@@ -357,7 +311,7 @@ func generateARP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 			}
 		} else {
 			copyAddr(THA[:], getNextAddr(&(l3.THA)), common.EtherAddrLen)
-			TPA := binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.TPA))).To4())
+			TPA := binary.LittleEndian.Uint32(To4(getNextAddr(&(l3.TPA))))
 			if !packet.InitARPReplyPacket(pkt, SHA, THA, SPA, TPA) {
 				panic(fmt.Sprintf("InitARPReplyPacket returned false"))
 			}
@@ -365,33 +319,22 @@ func generateARP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	default:
 		panic(fmt.Sprintf("unsupported operation code: %d", l3.Operation))
 	}
-	if len(l2.DAddr.Current) != 0 {
-		copyAddr(pkt.Ether.DAddr[:], getNextAddr(&(l2.DAddr)), common.EtherAddrLen)
-	}
+	copyAddr(pkt.Ether.DAddr[:], getNextAddr(&(l2.DAddr)), common.EtherAddrLen)
 	if l2.VLAN != nil {
 		if !pkt.AddVLANTag(l2.VLAN.TCI) {
 			panic("failed to add vlan tag to arp packet")
 		}
 	}
-	l2.Data = l3
-	config.Data = l2
 }
 
 func generateTCPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	l3 := l2.Data.(IPConfig)
-	l4 := l3.Data.(TCPConfig)
-	data, err := generateData(l4.Data, rnd)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
-	}
-	if data == nil {
-		panic(fmt.Sprintf("failed to generate data"))
-	}
-	size := uint(len(data))
+	l2 := (*EtherConfig)(config.Data)
+	l3 := (*IPConfig)(l2.Data)
+	l4 := (*TCPConfig)(l3.Data)
+	size, dataConfig, copyDataFunc := getDataSizeType(l4.Data, l4.DType, rnd)
 	if l3.Version == 4 {
 		if !packet.InitEmptyIPv4TCPPacket(pkt, size) {
 			panic(fmt.Sprintf("InitEmptyIPv4TCPPacket returned false"))
@@ -403,42 +346,29 @@ func generateTCPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	} else {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
-	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillTCPHdr(pkt, &l4, rnd); err != nil {
-		panic(fmt.Sprintf("failed to fill tcp header %v", err))
-	}
-	if err := fillIPHdr(pkt, &l3); err != nil {
-		panic(fmt.Sprintf("failed to fill ip header %v", err))
-	}
-	if err := fillEtherHdr(pkt, &l2); err != nil {
-		panic(fmt.Sprintf("failed to fill ether header %v", err))
-	}
+	copyDataFunc(dataConfig, size, rnd, pkt.Data)
+	fillEtherHdr(pkt, l2)
+	fillIPHdr(pkt, l3)
+	fillTCPHdr(pkt, l4, rnd)
+	pktTCP := (*packet.TCPHdr)(pkt.L4)
 	if l3.Version == 4 {
-		pkt.GetIPv4CheckVLAN().HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pkt.GetIPv4CheckVLAN()))
-		pkt.GetTCPForIPv4().Cksum = packet.SwapBytesUint16(packet.CalculateIPv4TCPChecksum(pkt.GetIPv4CheckVLAN(), pkt.GetTCPForIPv4(), pkt.Data))
+		pktIP := (*packet.IPv4Hdr)(pkt.L3)
+		pktIP.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pktIP))
+		pktTCP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4TCPChecksum(pktIP, pktTCP, pkt.Data))
 	} else if l3.Version == 6 {
-		pkt.GetTCPForIPv6().Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetTCPForIPv6(), pkt.Data))
+		pktIP := (*packet.IPv6Hdr)(pkt.L3)
+		pktTCP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(pktIP, pktTCP, pkt.Data))
 	}
-	l3.Data = l4
-	l2.Data = l3
-	config.Data = l2
 }
 
 func generateUDPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	l3 := l2.Data.(IPConfig)
-	l4 := l3.Data.(UDPConfig)
-	data, err := generateData(l4.Data, rnd)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
-	}
-	if data == nil {
-		panic(fmt.Sprintf("failed to generate data"))
-	}
-	size := uint(len(data))
+	l2 := (*EtherConfig)(config.Data)
+	l3 := (*IPConfig)(l2.Data)
+	l4 := (*UDPConfig)(l3.Data)
+	size, dataConfig, copyDataFunc := getDataSizeType(l4.Data, l4.DType, rnd)
 	if l3.Version == 4 {
 		if !packet.InitEmptyIPv4UDPPacket(pkt, size) {
 			panic(fmt.Sprintf("InitEmptyIPv4UDPPacket returned false"))
@@ -450,42 +380,29 @@ func generateUDPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	} else {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
-	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillUDPHdr(pkt, &l4); err != nil {
-		panic(fmt.Sprintf("failed to fill udp header %v", err))
-	}
-	if err := fillIPHdr(pkt, &l3); err != nil {
-		panic(fmt.Sprintf("failed to fill ip header %v", err))
-	}
-	if err := fillEtherHdr(pkt, &l2); err != nil {
-		panic(fmt.Sprintf("failed to fill ether header %v", err))
-	}
+	copyDataFunc(dataConfig, size, rnd, pkt.Data)
+	fillEtherHdr(pkt, l2)
+	fillIPHdr(pkt, l3)
+	fillUDPHdr(pkt, l4)
+	pktUDP := (*packet.UDPHdr)(pkt.L4)
 	if l3.Version == 4 {
-		pkt.GetIPv4CheckVLAN().HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pkt.GetIPv4CheckVLAN()))
-		pkt.GetUDPForIPv4().DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv4UDPChecksum(pkt.GetIPv4CheckVLAN(), pkt.GetUDPForIPv4(), pkt.Data))
+		pktIP := (*packet.IPv4Hdr)(pkt.L3)
+		pktIP.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pktIP))
+		pktUDP.DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv4UDPChecksum(pktIP, pktUDP, pkt.Data))
 	} else if l3.Version == 6 {
-		pkt.GetUDPForIPv6().DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv6UDPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetUDPForIPv6(), pkt.Data))
+		pktIP := (*packet.IPv6Hdr)(pkt.L3)
+		pktUDP.DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv6UDPChecksum(pktIP, pktUDP, pkt.Data))
 	}
-	l3.Data = l4
-	l2.Data = l3
-	config.Data = l2
 }
 
 func generateICMPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	if pkt == nil {
 		panic("Failed to create new packet")
 	}
-	l2 := config.Data.(EtherConfig)
-	l3 := l2.Data.(IPConfig)
-	l4 := l3.Data.(ICMPConfig)
-	data, err := generateData(l4.Data, rnd)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to parse data for l4: %v", err))
-	}
-	if data == nil {
-		panic(fmt.Sprintf("failed to generate data"))
-	}
-	size := uint(len(data))
+	l2 := (*EtherConfig)(config.Data)
+	l3 := (*IPConfig)(l2.Data)
+	l4 := (*ICMPConfig)(l3.Data)
+	size, dataConfig, copyDataFunc := getDataSizeType(l4.Data, l4.DType, rnd)
 	if l3.Version == 4 {
 		if !packet.InitEmptyIPv4ICMPPacket(pkt, size) {
 			panic(fmt.Sprintf("InitEmptyIPv4ICMPPacket returned false"))
@@ -497,92 +414,74 @@ func generateICMPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 	} else {
 		panic(fmt.Sprintf("fill packet l4 failed, unknovn version %d", l3.Version))
 	}
-	copy((*[1 << 30]uint8)(pkt.Data)[0:size], data)
-	if err := fillICMPHdr(pkt, &l4, rnd); err != nil {
-		panic(fmt.Sprintf("failed to fill icmp header %v", err))
-	}
-	if err := fillIPHdr(pkt, &l3); err != nil {
-		panic(fmt.Sprintf("failed to fill ip header %v", err))
-	}
-	if err := fillEtherHdr(pkt, &l2); err != nil {
-		panic(fmt.Sprintf("failed to fill ether header %v", err))
-	}
+	copyDataFunc(dataConfig, size, rnd, pkt.Data)
+	fillEtherHdr(pkt, l2)
+	fillIPHdr(pkt, l3)
+	fillICMPHdr(pkt, l4, rnd)
+	pktICMP := (*packet.ICMPHdr)(pkt.L4)
 	if l3.Version == 4 {
-		pkt.GetIPv4CheckVLAN().HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pkt.GetIPv4CheckVLAN()))
-		pkt.GetICMPForIPv4().Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(pkt.GetIPv4CheckVLAN(), pkt.GetICMPForIPv4(), pkt.Data))
+		pktIP := (*packet.IPv4Hdr)(pkt.L3)
+		pktIP.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(pktIP))
+		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(pktIP, pktICMP, pkt.Data))
 	} else if l3.Version == 6 {
-		pkt.GetICMPForIPv6().Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pkt.GetIPv6CheckVLAN(), pkt.GetICMPForIPv6(), pkt.Data))
+		pktIP := (*packet.IPv6Hdr)(pkt.L3)
+		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pktIP, pktICMP, pkt.Data))
 	}
-	l3.Data = l4
-	l2.Data = l3
-	config.Data = l2
 }
 
-func fillTCPHdr(pkt *packet.Packet, l4 *TCPConfig, rnd *rand.Rand) error {
+func fillTCPHdr(pkt *packet.Packet, l4 *TCPConfig, rnd *rand.Rand) {
 	emptyPacketTCP := (*packet.TCPHdr)(pkt.L4)
 	emptyPacketTCP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
 	emptyPacketTCP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
 	emptyPacketTCP.SentSeq = packet.SwapBytesUint32(getNextSeqNumber((&l4.Seq), rnd))
 	emptyPacketTCP.TCPFlags = l4.Flags
-	return nil
 }
 
-func fillUDPHdr(pkt *packet.Packet, l4 *UDPConfig) error {
+func fillUDPHdr(pkt *packet.Packet, l4 *UDPConfig) {
 	emptyPacketUDP := (*packet.UDPHdr)(pkt.L4)
 	emptyPacketUDP.SrcPort = packet.SwapBytesUint16(getNextPort(&(l4.SPort)))
 	emptyPacketUDP.DstPort = packet.SwapBytesUint16(getNextPort(&(l4.DPort)))
-	return nil
 }
 
-func fillICMPHdr(pkt *packet.Packet, l4 *ICMPConfig, rnd *rand.Rand) error {
+func fillICMPHdr(pkt *packet.Packet, l4 *ICMPConfig, rnd *rand.Rand) {
 	emptyPacketICMP := (*packet.ICMPHdr)(pkt.L4)
 	emptyPacketICMP.Type = l4.Type
 	emptyPacketICMP.Code = l4.Code
 	emptyPacketICMP.Identifier = l4.Identifier
 	emptyPacketICMP.SeqNum = packet.SwapBytesUint16(uint16(getNextSeqNumber(&(l4.Seq), rnd)))
-	return nil
 }
 
-func fillIPHdr(pkt *packet.Packet, l3 *IPConfig) error {
+func fillIPHdr(pkt *packet.Packet, l3 *IPConfig) {
 	if l3.Version == 4 {
-		pktIP := pkt.GetIPv4()
-		pktIP.SrcAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.SAddr))).To4())
-		pktIP.DstAddr = binary.LittleEndian.Uint32(net.IP(getNextAddr(&(l3.DAddr))).To4())
-		return nil
+		pktIP := (*packet.IPv4Hdr)(pkt.L3)
+		pktIP.SrcAddr = binary.LittleEndian.Uint32(To4(getNextAddr(&(l3.SAddr))))
+		pktIP.DstAddr = binary.LittleEndian.Uint32(To4(getNextAddr(&(l3.DAddr))))
+		return
 	}
-	pktIP := pkt.GetIPv6()
-	nextAddr := getNextAddr(&(l3.SAddr))
-	copyAddr(pktIP.SrcAddr[:], nextAddr, common.IPv6AddrLen)
-	nextAddr = getNextAddr(&(l3.DAddr))
-	copyAddr(pktIP.DstAddr[:], nextAddr, common.IPv6AddrLen)
-	return nil
+	pktIP := (*packet.IPv6Hdr)(pkt.L3)
+	copyAddr(pktIP.SrcAddr[:], getNextAddr(&(l3.SAddr)), common.IPv6AddrLen)
+	copyAddr(pktIP.DstAddr[:], getNextAddr(&(l3.DAddr)), common.IPv6AddrLen)
 }
 
-func fillEtherHdr(pkt *packet.Packet, l2 *EtherConfig) error {
+func fillEtherHdr(pkt *packet.Packet, l2 *EtherConfig) {
 	if l2.VLAN != nil {
-		if err := addVLAN(pkt, l2.VLAN.TCI); err != nil {
-			return err
-		}
+		addVLAN(pkt, l2.VLAN.TCI)
 	}
-	nextAddr := getNextAddr(&(l2.DAddr))
-	copyAddr(pkt.Ether.DAddr[:], nextAddr, common.EtherAddrLen)
-	nextAddr = getNextAddr(&(l2.SAddr))
-	copyAddr(pkt.Ether.SAddr[:], nextAddr, common.EtherAddrLen)
-	return nil
+	copyAddr(pkt.Ether.DAddr[:], getNextAddr(&(l2.DAddr)), common.EtherAddrLen)
+	copyAddr(pkt.Ether.SAddr[:], getNextAddr(&(l2.SAddr)), common.EtherAddrLen)
 }
 
 // faster version of packet.AddVLANTag, can be used only
 // when ether src and dst are not set, but ether type is
 // (most InitEmptyPacket functions do so).
-func addVLAN(pkt *packet.Packet, tag uint16) error {
+func addVLAN(pkt *packet.Packet, tag uint16) {
 	if !pkt.EncapsulateHead(0, 4) {
-		return fmt.Errorf("failed to add vlan tag, EncapsulateHead returned false")
+		panic("failed to add vlan tag, EncapsulateHead returned false")
 	}
 	pkt.Ether.EtherType = packet.SwapBytesUint16(common.VLANNumber)
 	vhdr := pkt.GetVLAN()
 	if vhdr == nil {
-		return fmt.Errorf("failed to get vlan, GetVLAN returned nil")
+		panic("failed to get vlan, GetVLAN returned nil")
 	}
 	vhdr.TCI = packet.SwapBytesUint16(tag)
-	return nil
 }
diff --git a/examples/nffPktgen/generator/parseConfig.go b/examples/nffPktgen/generator/parseConfig.go
index 1551705c..d99c8557 100644
--- a/examples/nffPktgen/generator/parseConfig.go
+++ b/examples/nffPktgen/generator/parseConfig.go
@@ -6,14 +6,15 @@ package generator
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/big"
 	"math/rand"
 	"net"
 	"os"
 	"regexp"
 	"strings"
+	"unsafe"
 
 	"github.com/intel-go/nff-go/common"
 )
@@ -22,9 +23,9 @@ var mixPattern = regexp.MustCompile(`^mix[0-9]*$`)
 
 // AddrRange describes range of addresses.
 type AddrRange struct {
-	Min     []uint8
-	Max     []uint8
-	Current []uint8
+	Min     *big.Int
+	Max     *big.Int
+	Current *big.Int
 	Incr    uint64
 }
 
@@ -38,7 +39,7 @@ func (ar *AddrRange) String() string {
 type PortRange struct {
 	Min     uint16
 	Max     uint16
-	Current []uint16
+	Current uint16
 	Incr    uint16
 }
 
@@ -60,12 +61,28 @@ const (
 // Sequence contains type and next sequence value.
 type Sequence struct {
 	Type SequenceType
-	Next []uint32
+	Next uint32
 }
 
+type DataType int
+
+const (
+	ETHERHDR = iota
+	IPHDR
+	TCPHDR
+	UDPHDR
+	ICMPHDR
+	ARPHDR
+	PDISTDATA
+	RANDDATA
+	RAWDATA
+	NONE
+)
+
 // PacketConfig configures packet
 type PacketConfig struct {
-	Data interface{}
+	DType DataType
+	Data  unsafe.Pointer
 }
 
 // MixConfig contains PacketConfigs with quantity.
@@ -83,7 +100,8 @@ type EtherConfig struct {
 	DAddr AddrRange
 	SAddr AddrRange
 	VLAN  *VlanTagConfig
-	Data  interface{}
+	DType DataType
+	Data  unsafe.Pointer
 }
 
 // VlanTagConfig configures vlan tag
@@ -96,7 +114,8 @@ type IPConfig struct {
 	Version int
 	SAddr   AddrRange // source address
 	DAddr   AddrRange // destination address
-	Data    interface{}
+	DType   DataType
+	Data    unsafe.Pointer
 }
 
 // TCPConfig configures tcp header.
@@ -105,14 +124,16 @@ type TCPConfig struct {
 	DPort PortRange
 	Seq   Sequence
 	Flags common.TCPFlags
-	Data  interface{}
+	DType DataType
+	Data  unsafe.Pointer
 }
 
 // UDPConfig configures tcp header.
 type UDPConfig struct {
 	SPort PortRange
 	DPort PortRange
-	Data  interface{}
+	DType DataType
+	Data  unsafe.Pointer
 }
 
 // ICMPConfig configures tcp header.
@@ -121,7 +142,8 @@ type ICMPConfig struct {
 	Code       uint8
 	Identifier uint16
 	Seq        Sequence
-	Data       interface{}
+	DType      DataType
+	Data       unsafe.Pointer
 }
 
 // ARPConfig configures arp header.
@@ -137,7 +159,8 @@ type ARPConfig struct {
 // PDistEntry gives data with associated probability of being chosen.
 type PDistEntry struct {
 	Probability float64
-	Data        interface{}
+	DType       DataType
+	Data        unsafe.Pointer
 }
 
 // RandBytes gives  a payload of random bytes, of a given size.
@@ -169,7 +192,7 @@ func ParseConfig(f *os.File) (config []*MixConfig, err error) {
 			if err != nil {
 				return nil, fmt.Errorf("parseEtherHdr returned: %v", err)
 			}
-			pktConfig := &PacketConfig{Data: ethHdr}
+			pktConfig := &PacketConfig{Data: unsafe.Pointer(&ethHdr), DType: ETHERHDR}
 			return append(config, &MixConfig{Config: pktConfig, Quantity: 1}), nil
 		case mixPattern.MatchString(key):
 			return ParseMixConfig(in)
@@ -200,7 +223,7 @@ func ParseMixConfig(in map[string]interface{}) (config []*MixConfig, err error)
 					if err != nil {
 						return nil, fmt.Errorf("parseEtherHdr returned: %v", err)
 					}
-					pktConfig = &PacketConfig{Data: ethHdr}
+					pktConfig = &PacketConfig{Data: unsafe.Pointer(&ethHdr), DType: ETHERHDR}
 				case "quantity", "q":
 					q = uint32(vv.(float64))
 				default:
@@ -219,7 +242,7 @@ func ParseMixConfig(in map[string]interface{}) (config []*MixConfig, err error)
 }
 
 func parseEtherHdr(in map[string]interface{}) (EtherConfig, error) {
-	ethConfig := EtherConfig{DAddr: AddrRange{}, SAddr: AddrRange{}, VLAN: nil, Data: Raw{Data: ""}}
+	ethConfig := EtherConfig{DAddr: allocAddrRange(), SAddr: allocAddrRange(), VLAN: nil, DType: NONE}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "saddr":
@@ -242,7 +265,8 @@ func parseEtherHdr(in map[string]interface{}) (EtherConfig, error) {
 			if err != nil {
 				return EtherConfig{}, fmt.Errorf("parseIpHdr returned %v", err)
 			}
-			ethConfig.Data = ipConf
+			ethConfig.Data = unsafe.Pointer(&ipConf)
+			ethConfig.DType = IPHDR
 			break
 		case "arp":
 			arpConfig := v.(map[string]interface{})
@@ -250,51 +274,53 @@ func parseEtherHdr(in map[string]interface{}) (EtherConfig, error) {
 			if err != nil {
 				return EtherConfig{}, fmt.Errorf("parseARPHdr returned %v", err)
 			}
-			ethConfig.Data = arpConf
+			ethConfig.Data = unsafe.Pointer(&arpConf)
+			ethConfig.DType = ARPHDR
 			break
 		default:
-			data, err := parseData(map[string]interface{}{k: v})
+			data, dataType, err := parseData(map[string]interface{}{k: v})
 			if err != nil {
 				return EtherConfig{}, fmt.Errorf("parseData for ether returned: %v", err)
 			}
 			ethConfig.Data = data
+			ethConfig.DType = dataType
 			break
 		}
 	}
 	return ethConfig, nil
 }
 
-func parseData(in map[string]interface{}) (interface{}, error) {
+func parseData(in map[string]interface{}) (unsafe.Pointer, DataType, error) {
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "raw":
 			data, err := parseRaw(v.(map[string]interface{}))
 			if err != nil {
-				return nil, fmt.Errorf("parseRaw returned %v", err)
+				return nil, NONE, fmt.Errorf("parseRaw returned %v", err)
 			}
-			return data, nil
+			return unsafe.Pointer(&data), RAWDATA, nil
 		case "randbytes":
 			randBytes, err := parseRandBytes(v.(map[string]interface{}))
 			if err != nil {
-				return nil, fmt.Errorf("parseRandBytes returned %v", err)
+				return nil, NONE, fmt.Errorf("parseRandBytes returned %v", err)
 			}
-			return randBytes, nil
+			return unsafe.Pointer(&randBytes), RANDDATA, nil
 		case "pdist":
 			pdist, err := parsePdist(v.([]interface{}))
 			if err != nil {
-				return nil, fmt.Errorf("parsePdist returned %v", err)
+				return nil, NONE, fmt.Errorf("parsePdist returned %v", err)
 			}
-			return pdist, nil
+			return unsafe.Pointer(&pdist), PDISTDATA, nil
 		default:
 			fmt.Println("unknown key: ", k)
 			break
 		}
 	}
-	return nil, fmt.Errorf("failed to parse data")
+	return nil, NONE, fmt.Errorf("failed to parse data")
 }
 
 func parseIPHdr(in map[string]interface{}) (IPConfig, error) {
-	ipHdr := IPConfig{Version: 4, DAddr: AddrRange{}, SAddr: AddrRange{}, Data: Raw{Data: ""}}
+	ipHdr := IPConfig{Version: 4, DAddr: AddrRange{}, SAddr: AddrRange{}}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "version":
@@ -320,28 +346,32 @@ func parseIPHdr(in map[string]interface{}) (IPConfig, error) {
 			if err != nil {
 				return IPConfig{}, fmt.Errorf("parseTCPHdr returned %v", err)
 			}
-			ipHdr.Data = tcpConf
+			ipHdr.Data = unsafe.Pointer(&tcpConf)
+			ipHdr.DType = TCPHDR
 			break
 		case "udp":
 			udpConf, err := parseUDPHdr(v.(map[string]interface{}))
 			if err != nil {
 				return IPConfig{}, fmt.Errorf("parseUDPHdr returned %v", err)
 			}
-			ipHdr.Data = udpConf
+			ipHdr.Data = unsafe.Pointer(&udpConf)
+			ipHdr.DType = UDPHDR
 			break
 		case "icmp":
 			icmpConf, err := parseICMPHdr(v.(map[string]interface{}))
 			if err != nil {
 				return IPConfig{}, fmt.Errorf("parseICMPHdr returned %v", err)
 			}
-			ipHdr.Data = icmpConf
+			ipHdr.Data = unsafe.Pointer(&icmpConf)
+			ipHdr.DType = ICMPHDR
 			break
 		default:
-			data, err := parseData(map[string]interface{}{k: v})
+			data, dataType, err := parseData(map[string]interface{}{k: v})
 			if err != nil {
 				return IPConfig{}, fmt.Errorf("parseData for ip returned: %v", err)
 			}
 			ipHdr.Data = data
+			ipHdr.DType = dataType
 			break
 		}
 	}
@@ -392,7 +422,7 @@ func parseARPHdr(in map[string]interface{}) (ARPConfig, error) {
 }
 
 func parseTCPHdr(in map[string]interface{}) (TCPConfig, error) {
-	tcpHdr := TCPConfig{SPort: PortRange{}, DPort: PortRange{}, Data: Raw{Data: ""}}
+	tcpHdr := TCPConfig{SPort: PortRange{}, DPort: PortRange{}}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "sport":
@@ -420,11 +450,12 @@ func parseTCPHdr(in map[string]interface{}) (TCPConfig, error) {
 			}
 			tcpHdr.Flags = flags
 		default:
-			data, err := parseData(map[string]interface{}{k: v})
+			data, dataType, err := parseData(map[string]interface{}{k: v})
 			if err != nil {
 				return TCPConfig{}, fmt.Errorf("parseData for tcp returned: %v", err)
 			}
 			tcpHdr.Data = data
+			tcpHdr.DType = dataType
 			break
 		}
 	}
@@ -432,7 +463,7 @@ func parseTCPHdr(in map[string]interface{}) (TCPConfig, error) {
 }
 
 func parseUDPHdr(in map[string]interface{}) (UDPConfig, error) {
-	udpHdr := UDPConfig{SPort: PortRange{}, DPort: PortRange{}, Data: Raw{Data: ""}}
+	udpHdr := UDPConfig{SPort: PortRange{}, DPort: PortRange{}}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "sport":
@@ -448,11 +479,12 @@ func parseUDPHdr(in map[string]interface{}) (UDPConfig, error) {
 			}
 			udpHdr.DPort = dport
 		default:
-			data, err := parseData(map[string]interface{}{k: v})
+			data, dataType, err := parseData(map[string]interface{}{k: v})
 			if err != nil {
 				return UDPConfig{}, fmt.Errorf("parseData for udp returned: %v", err)
 			}
 			udpHdr.Data = data
+			udpHdr.DType = dataType
 			break
 		}
 	}
@@ -460,7 +492,7 @@ func parseUDPHdr(in map[string]interface{}) (UDPConfig, error) {
 }
 
 func parseICMPHdr(in map[string]interface{}) (ICMPConfig, error) {
-	icmpHdr := ICMPConfig{Data: Raw{Data: ""}}
+	icmpHdr := ICMPConfig{Data: unsafe.Pointer(&Raw{Data: ""})}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "type":
@@ -476,11 +508,12 @@ func parseICMPHdr(in map[string]interface{}) (ICMPConfig, error) {
 			}
 			icmpHdr.Seq = seq
 		default:
-			data, err := parseData(map[string]interface{}{k: v})
+			data, dataType, err := parseData(map[string]interface{}{k: v})
 			if err != nil {
 				return ICMPConfig{}, fmt.Errorf("parseData for icmp returned: %v", err)
 			}
 			icmpHdr.Data = data
+			icmpHdr.DType = dataType
 			break
 		}
 	}
@@ -517,9 +550,9 @@ func parseTCPFlags(in []interface{}) (ret common.TCPFlags, err error) {
 func parseSeq(in string) (Sequence, error) {
 	switch strings.ToLower(in) {
 	case "rand", "random":
-		return Sequence{Type: RANDOM, Next: []uint32{rand.Uint32()}}, nil
+		return Sequence{Type: RANDOM, Next: rand.Uint32()}, nil
 	case "incr", "increasing":
-		return Sequence{Type: INCREASING, Next: []uint32{0}}, nil
+		return Sequence{Type: INCREASING, Next: 0}, nil
 	}
 	return Sequence{}, fmt.Errorf("unknown key for tcp sequence: %s", in)
 }
@@ -585,13 +618,15 @@ func parsePdistEntry(in map[string]interface{}) (PDistEntry, error) {
 			if err != nil {
 				return PDistEntry{}, fmt.Errorf("parsing pdist entry: parseRaw returned %v", err)
 			}
-			pdistElem.Data = data
+			pdistElem.Data = unsafe.Pointer(&data)
+			pdistElem.DType = RAWDATA
 		case "randbytes":
 			randBytes, err := parseRandBytes(v.(map[string]interface{}))
 			if err != nil {
 				return PDistEntry{}, fmt.Errorf("parsing pdist entry: parseRandBytes returned %v", err)
 			}
-			pdistElem.Data = randBytes
+			pdistElem.Data = unsafe.Pointer(&randBytes)
+			pdistElem.DType = RANDDATA
 		default:
 			return PDistEntry{}, fmt.Errorf("unknown key %s", k)
 		}
@@ -620,7 +655,11 @@ func parseMacAddr(value interface{}) (AddrRange, error) {
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing mac saddr returned: %v", err)
 		}
-		return AddrRange{Min: saddr, Max: saddr, Current: saddr, Incr: 0}, nil
+		ret := allocAddrRange()
+		ret.Min.SetBytes(saddr)
+		ret.Max.SetBytes(saddr)
+		ret.Current.SetBytes(saddr)
+		return ret, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
 }
@@ -641,6 +680,14 @@ func parseRawMACAddr(rawAddr string) ([]uint8, error) {
 	return addr, nil
 }
 
+func allocAddrRange() AddrRange {
+	addr := AddrRange{}
+	addr.Min = big.NewInt(0)
+	addr.Max = big.NewInt(0)
+	addr.Current = big.NewInt(0)
+	return addr
+}
+
 func parseIPAddr(value interface{}) (AddrRange, error) {
 	switch v2 := value.(type) {
 	case map[string]interface{}:
@@ -657,11 +704,15 @@ func parseIPAddr(value interface{}) (AddrRange, error) {
 			}
 		}
 	case string:
-		addr, err := parseRawIPAddr(v2)
+		saddr, err := parseRawIPAddr(v2)
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing ip addr returned: %v", err)
 		}
-		return AddrRange{Min: addr, Max: addr, Current: addr, Incr: 0}, nil
+		ret := allocAddrRange()
+		ret.Min.SetBytes(saddr)
+		ret.Max.SetBytes(saddr)
+		ret.Current.SetBytes(saddr)
+		return ret, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
 }
@@ -669,7 +720,7 @@ func parseIPAddr(value interface{}) (AddrRange, error) {
 type fn func(string) ([]uint8, error)
 
 func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error) {
-	addr := AddrRange{Incr: 0}
+	addr := allocAddrRange()
 	wasMin, wasMax, wasStart, wasIncr := false, false, false, false
 	for k, v := range in {
 		switch strings.ToLower(k) {
@@ -678,21 +729,24 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing min returned: %v", err)
 			}
-			addr.Min = min
+			addr.Min = big.NewInt(0)
+			addr.Min.SetBytes(min)
 			wasMin = true
 		case "max":
 			max, err := parseFunc(v.(string))
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing max returned: %v", err)
 			}
-			addr.Max = max
+			addr.Max = big.NewInt(0)
+			addr.Max.SetBytes(max)
 			wasMax = true
 		case "start":
 			start, err := parseFunc(v.(string))
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing start returned: %v", err)
 			}
-			addr.Current = start
+			addr.Current = big.NewInt(0)
+			addr.Current.SetBytes(start)
 			wasStart = true
 		case "incr":
 			addr.Incr = (uint64)(v.(float64))
@@ -704,15 +758,16 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 	if !wasMax || !wasMin {
 		return AddrRange{}, fmt.Errorf("Min and max values should be given for range")
 	}
-	if bytes.Compare(addr.Max, addr.Min) < 0 {
+	if addr.Max.Cmp(addr.Min) < 0 {
 		return AddrRange{}, fmt.Errorf("Min value should be less than Max")
 	}
 	if !wasStart {
-		addr.Current = make([]byte, len(addr.Min))
-		copy(addr.Current, addr.Min)
+		addr.Current = big.NewInt(0)
+		*(addr.Current) = *(addr.Min)
 	}
-	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
-		return AddrRange{}, fmt.Errorf(fmt.Sprintf("Start value should be between min and max: start=%v, min=%v, max=%v", addr.Current, addr.Min, addr.Max))
+
+	if addr.Current.Cmp(addr.Min) < 0 || addr.Current.Cmp(addr.Max) > 0 {
+		return AddrRange{}, fmt.Errorf(fmt.Sprintf("Start value should be between min and max: start=%v, min=%v, max=%v", addr.Current.Bytes(), addr.Min.Bytes(), addr.Max.Bytes()))
 	}
 	if !wasIncr {
 		addr.Incr = 1
@@ -740,7 +795,7 @@ func parsePort(in interface{}) (PortRange, error) {
 			return PortRange{}, fmt.Errorf("Port should be positive")
 		}
 		port := (uint16)(in.(float64))
-		return PortRange{Min: port, Max: port, Current: []uint16{port}, Incr: 0}, nil
+		return PortRange{Min: port, Max: port, Current: port, Incr: 0}, nil
 	}
 	return PortRange{}, fmt.Errorf("unknown type")
 }
@@ -766,7 +821,7 @@ func parsePortRange(in map[string]interface{}) (PortRange, error) {
 			if v.(float64) < 0 {
 				return PortRange{}, fmt.Errorf("Start should be positive")
 			}
-			portRng.Current = []uint16{(uint16)(v.(float64))}
+			portRng.Current = (uint16)(v.(float64))
 			wasStart = true
 		case "incr":
 			if v.(float64) < 0 {
@@ -785,9 +840,9 @@ func parsePortRange(in map[string]interface{}) (PortRange, error) {
 		return PortRange{}, fmt.Errorf("Min value should be <= Max value")
 	}
 	if !wasStart {
-		portRng.Current = []uint16{portRng.Min}
+		portRng.Current = portRng.Min
 	}
-	if portRng.Current[0] < portRng.Min || portRng.Current[0] > portRng.Max {
+	if portRng.Current < portRng.Min || portRng.Current > portRng.Max {
 		return PortRange{}, fmt.Errorf("Start should be in range of min and max")
 	}
 	if !wasIncr {
diff --git a/examples/nffPktgen/perfTest/Makefile b/examples/nffPktgen/perfTest/Makefile
new file mode 100644
index 00000000..769c07ee
--- /dev/null
+++ b/examples/nffPktgen/perfTest/Makefile
@@ -0,0 +1,9 @@
+# Copyright 2017 Intel Corporation. 
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+PATH_TO_MK = ../../../mk
+IMAGENAME = nff-pktgen
+EXECUTABLES = perfTest
+
+include $(PATH_TO_MK)/leaf.mk
diff --git a/examples/nffPktgen/perfTest/perfTest.go b/examples/nffPktgen/perfTest/perfTest.go
new file mode 100644
index 00000000..01e4e733
--- /dev/null
+++ b/examples/nffPktgen/perfTest/perfTest.go
@@ -0,0 +1,42 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/intel-go/nff-go/examples/nffPktgen/generator"
+	"github.com/intel-go/nff-go/flow"
+)
+
+func main() {
+	var (
+		speed            uint64
+		genConfig, cores string
+		port             uint
+	)
+	flag.Uint64Var(&speed, "speed", 6000000, "speed of fast generator, Pkts/s")
+	flag.StringVar(&genConfig, "config", "../testing/ip4.json", "specifies config for generator")
+	flag.StringVar(&cores, "cores", "0-3", "specifies cores")
+	flag.UintVar(&port, "port", 1, "specifies output port")
+	flag.Parse()
+
+	// Init NFF-GO system at 16 available cores
+	config := flow.Config{
+		CPUList: cores,
+	}
+	flow.CheckFatal(flow.SystemInit(&config))
+
+	configuration, err := generator.ReadConfig(genConfig)
+	if err != nil {
+		panic(fmt.Sprintf("%s config reading failed: %v", genConfig, err))
+	}
+	context, err := generator.GetContext(configuration)
+	flow.CheckFatal(err)
+	outFlow, err := flow.SetFastGenerator(generator.Generate, speed, context)
+	flow.CheckFatal(err)
+	flow.CheckFatal(flow.SetSender(outFlow, uint16(port)))
+	flow.CheckFatal(flow.SystemStart())
+}
diff --git a/examples/nffPktgen/sendGetBack/sendGetBack.go b/examples/nffPktgen/sendGetBack/sendGetBack.go
index 7ba0d2a0..fa3ea22c 100644
--- a/examples/nffPktgen/sendGetBack/sendGetBack.go
+++ b/examples/nffPktgen/sendGetBack/sendGetBack.go
@@ -13,15 +13,15 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/intel-go/nff-go/examples/nffPktgen/generator"
 	"github.com/intel-go/nff-go/flow"
 	"github.com/intel-go/nff-go/packet"
-	"github.com/intel-go/nff-go/examples/nffPktgen/generator"
 )
 
 var (
-	number, recv  uint64
-	isCycled      bool
-	testDoneEvent *sync.Cond
+	number, numberGot, recv uint64
+	isCycled                bool
+	testDoneEvent           *sync.Cond
 )
 
 type mapFlags struct {
@@ -41,7 +41,7 @@ func parseIntOrStr(s string) interface{} {
 	var err error
 	parsed, err = strconv.Atoi(s)
 	if err != nil {
-		parsed  = parseStr(s)
+		parsed = parseStr(s)
 	}
 	return parsed
 }
@@ -111,7 +111,8 @@ func main() {
 		eachOutPortConfig mapFlags
 		eachInPortConfig  listFlags
 	)
-	flag.Uint64Var(&number, "number", 10000000, "stop after generated number")
+	flag.Uint64Var(&number, "number", 0, "stop after generated number")
+	flag.Uint64Var(&numberGot, "numberGot", 10000000, "stop after got number")
 	flag.BoolVar(&isCycled, "cycle", false, "cycle execution")
 	flag.Uint64Var(&speed, "speed", 6000000, "speed of fast generator, Pkts/s")
 	flag.Var(&eachOutPortConfig, "outConfig", "specifies config per port portNum or file: 'path', 'pcapOut': 'path2'. For example: 1: 'ip4.json', 'mix.pcap': 'mix.json'")
@@ -119,7 +120,9 @@ func main() {
 	flag.Parse()
 
 	// Init NFF-GO system at 16 available cores
-	config := flow.Config{}
+	config := flow.Config{
+		CPUList: "0-43",
+	}
 	flow.CheckFatal(flow.SystemInit(&config))
 
 	testDoneEvent = sync.NewCond(&m)
@@ -143,7 +146,6 @@ func main() {
 		default:
 			panic(fmt.Sprintf("Unrecognized type in outConfig: %v", key))
 		}
-
 	}
 	for _, value := range eachInPortConfig.value {
 		var (
@@ -171,8 +173,11 @@ func main() {
 	g := generator.GetGenerator()
 	go func() {
 		for {
+			if g.GetGeneratedNumber() >= number {
+				testDoneEvent.Signal()
+			}
 			println("Sent/Got", g.GetGeneratedNumber(), "/", atomic.LoadUint64(&recv), "packets")
-			time.Sleep(time.Second * 5)
+			time.Sleep(time.Second)
 		}
 	}()
 
@@ -188,7 +193,7 @@ func handleRecv(currentPacket *packet.Packet, context flow.UserContext) {
 	if isCycled {
 		return
 	}
-	if got >= number {
+	if got >= numberGot {
 		time.Sleep(time.Second)
 		testDoneEvent.Signal()
 	}
diff --git a/examples/nffPktgen/testing/run.sh b/examples/nffPktgen/testing/run.sh
index 91b75d53..38bc1bc3 100644
--- a/examples/nffPktgen/testing/run.sh
+++ b/examples/nffPktgen/testing/run.sh
@@ -1,15 +1,18 @@
 #!/bin/bash
 
-./../generate -number 100000 -infile ether.json -outfile ether.pcap
-./../generate -number 10000 -infile ip4.json -outfile ip4.pcap
-./../generate -number 1000 -infile ip6.json -outfile ip6.pcap
-./../generate -number 100000 -infile ip4tcp.json -outfile ip4tcp.pcap
-./../generate -number 1000000 -infile ip6tcp.json -outfile ip6tcp.pcap
-./../generate -number 10000 -infile ip4udp.json -outfile ip4udp.pcap
-./../generate -number 1000 -infile ip6udp.json -outfile ip6udp.pcap
-./../generate -number 1000000 -infile ip4icmp.json -outfile ip4icmp.pcap
-./../generate -number 100000 -infile ip6icmp.json -outfile ip6icmp.pcap
-./../generate -number 1000 -infile arp.json -outfile arp.pcap
-./../generate -number 10 -infile vlanTag.json -outfile vlanTag.pcap
-./../generate -number 10 -infile arpVlan.json -outfile arpVlan.pcap
-./../generate -number 100
\ No newline at end of file
+../sendGetBack/sendGetBack -number 100000 -outConfig "'ether.pcap': 'ether.json'"
+../sendGetBack/sendGetBack -number 10000 -outConfig "'ip4.pcap':'ip4.json'"
+../sendGetBack/sendGetBack -number 1000 -outConfig "'ip6.pcap':'ip6.json'"
+../sendGetBack/sendGetBack -number 100000 -outConfig "'ip4tcp.pcap':'ip4tcp.json'"
+../sendGetBack/sendGetBack -number 100000 -outConfig "'ip4tcpVlan.pcap':'ip4tcpVlan.json'"
+../sendGetBack/sendGetBack -number 1000000 -outConfig "'ip6tcp.pcap':'ip6tcp.json'"
+../sendGetBack/sendGetBack -number 10000 -outConfig "'ip4udp.pcap':'ip4udp.json'"
+../sendGetBack/sendGetBack -number 10000 -outConfig "'ip4udpVlan.pcap':'ip4udpVlan.json'"
+../sendGetBack/sendGetBack -number 1000 -outConfig "'ip6udp.pcap':'ip6udp.json'"
+../sendGetBack/sendGetBack -number 1000000 -outConfig "'ip4icmp.pcap':'ip4icmp.json'"
+../sendGetBack/sendGetBack -number 100000 -outConfig "'ip6icmp.pcap':'ip6icmp.json'"
+../sendGetBack/sendGetBack -number 1000 -outConfig "'arp.pcap':'arp.json'"
+../sendGetBack/sendGetBack -number 100 -outConfig "'vlanTag.pcap':'vlanTag.json'"
+../sendGetBack/sendGetBack -number 100 -outConfig "'arpVlan.pcap':'arpVlan.json'"
+../sendGetBack/sendGetBack -number 100 -outConfig "'config.pcap': 'config.json'"
+../sendGetBack/sendGetBack -number 100 -outConfig "'mix.pcap': 'mix.json'"
\ No newline at end of file

From 53c16896109e33871d4bdf30c462ee68032c1eee Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Fri, 27 Jul 2018 12:23:15 -0600
Subject: [PATCH 15/50] Scheduler corrections

---
 examples/.gitignore |  1 +
 flow/flow.go        |  9 +++++++--
 flow/scheduler.go   | 13 ++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/examples/.gitignore b/examples/.gitignore
index 7a4de026..ea817fee 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -8,3 +8,4 @@ createPacket
 sendFixedPktsNumber
 gtpu
 pingReplay
+timer
diff --git a/flow/flow.go b/flow/flow.go
index 0aa44126..e8bb2db8 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -1045,9 +1045,14 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 			for q := int32(1); q < inIndex[0]+1; q++ {
 				n := IN[inIndex[q]].DequeueBurst(InputMbufs, burstSize)
 				if n == 0 {
+					// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
+					// we just want to slow goroutine down without parking, so loop is OK for this.
+					// time.Now lasts approximately 70ns and this satisfies us
 					if pause != 0 {
 						// pause should be non 0 only if function works with ONE inIndex
-						time.Sleep(time.Duration(pause) * time.Nanosecond)
+						a := time.Now()
+						for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
+						}
 					}
 					continue
 				}
@@ -1263,7 +1268,7 @@ func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report
 				// we just want to slow goroutine down without parking, so loop is OK for this.
 				// time.Now lasts approximately 70ns and this satisfies us
 				if pause != 0 {
-					// pause should be 0 only if function works with ONE inIndex
+					// pause should be non 0 only if function works with ONE inIndex
 					a := time.Now()
 					for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
 					}
diff --git a/flow/scheduler.go b/flow/scheduler.go
index 0928bf8a..6243ae4f 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -216,7 +216,7 @@ func (scheduler *scheduler) systemStart() (err error) {
 
 func (ff *flowFunction) startNewInstance(inIndex []int32, scheduler *scheduler) (err error) {
 	ffi := new(instance)
-	common.LogDebug(common.Initialization, "Start new instance for", ff.name)
+	common.LogDebug(common.Debug, "Start new instance for", ff.name)
 	ffi.inIndex = inIndex
 	ffi.report = make(chan reportPair, len(scheduler.cores) - 1)
 	ffi.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
@@ -236,7 +236,7 @@ func (ffi *instance) startNewClone(scheduler *scheduler, n int) (err error) {
 		common.LogWarning(common.Debug, "Can't start new clone for", ff.name, "instance", n)
 		return err
 	}
-	common.LogDebug(common.Initialization, "Start new clone for", ff.name, "instance", n, "at", core, "core")
+	common.LogDebug(common.Debug, "Start new clone for", ff.name, "instance", n, "at", core, "core")
 	ffi.clone = append(ffi.clone, &clonePair{index, [2]chan int{nil, nil}, process})
 	ffi.cloneNumber++
 	if ff.fType != receiveRSS && ff.fType != sendReceiveKNI {
@@ -246,7 +246,7 @@ func (ffi *instance) startNewClone(scheduler *scheduler, n int) (err error) {
 	go func() {
 		if ff.fType != receiveRSS && ff.fType != sendReceiveKNI {
 			if err := low.SetAffinity(core); err != nil {
-				common.LogFatal(common.Initialization, "Failed to set affinity to", core, "core: ", err)
+				common.LogFatal(common.Debug, "Failed to set affinity to", core, "core: ", err)
 			}
 			if ff.fType == segmentCopy || ff.fType == fastGenerate || ff.fType == generate {
 				ff.cloneFunction(ff.Parameters, ffi.inIndex, ffi.clone[ffi.cloneNumber-1].channel, ffi.report, cloneContext(ff.context))
@@ -261,6 +261,7 @@ func (ffi *instance) startNewClone(scheduler *scheduler, n int) (err error) {
 }
 
 func (ff *flowFunction) stopClone(ffi *instance, scheduler *scheduler) {
+	common.LogDebug(common.Debug, "Stop clone")
 	if ffi.clone[ffi.cloneNumber-1].channel[0] != nil {
 		ffi.clone[ffi.cloneNumber-1].channel[0] <- -1
 		<-ffi.clone[ffi.cloneNumber-1].channel[1]
@@ -278,7 +279,7 @@ func (ff *flowFunction) stopClone(ffi *instance, scheduler *scheduler) {
 }
 
 func (ff *flowFunction) stopInstance(from int, to int, scheduler *scheduler) {
-	common.LogDebug(common.Initialization, "Stop instance for", ff.name)
+	common.LogDebug(common.Debug, "Stop instance for", ff.name)
 	fromInstance := ff.instance[from]
 	for fromInstance.cloneNumber != 0 {
 		ff.stopClone(fromInstance, scheduler)
@@ -396,6 +397,8 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 							}
 						}
 					}
+					// TODO this is wrong due to different speeds of different flows. Also we should take into account only
+					// instances without clones here.
 					if ff.instanceNumber > 1 {
 						min0 := 0
 						min1 := 1
@@ -576,7 +579,7 @@ func (ff *flowFunction) printDebug(schedTime uint) {
 			if reportMbits {
 				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s")
 			} else {
-				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S, cloneNumber:", ff.instance[q].cloneNumber)
+				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S, cloneNumber:", ff.instance[q].cloneNumber, "queue number:", ff.instance[q].inIndex[0])
 			}
 		}
 	case fastGenerate:

From 81322fc3fd02bfaef0e256c89f29752a059b0d6b Mon Sep 17 00:00:00 2001
From: Maria Bulatova <maria.bulatova@intel.com>
Date: Fri, 6 Jul 2018 15:49:57 +0300
Subject: [PATCH 16/50] Netlink example: can cache kernel routes into lpm,
 update it and lookup routes

---
 examples/Makefile      |   3 +-
 examples/netlink.go    | 179 +++++++++++++++++++++++++++++++++++++++++
 scripts/get-depends.sh |   1 +
 3 files changed, 182 insertions(+), 1 deletion(-)
 create mode 100755 examples/netlink.go

diff --git a/examples/Makefile b/examples/Makefile
index 267d5bc0..cce9d5f6 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -5,7 +5,8 @@
 PATH_TO_MK = ../mk
 IMAGENAME = nff-go-examples
 EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
-			  createPacket sendFixedPktsNumber gtpu pingReplay
+			  createPacket sendFixedPktsNumber gtpu pingReplay \
+			  netlink
 SUBDIRS = nat tutorial antiddos demo fileReadWrite firewall forwarding
 
 gopacket:
diff --git a/examples/netlink.go b/examples/netlink.go
new file mode 100755
index 00000000..ff2a0269
--- /dev/null
+++ b/examples/netlink.go
@@ -0,0 +1,179 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+	"golang.org/x/sys/unix"
+
+	"github.com/vishvananda/netlink"
+)
+
+// internal structures:
+// LPM rules storage
+var lpm *packet.LPM
+
+// LPM binds ip with number and below
+// is a struct to get ip by binded value
+var portToIP []uint32
+
+// length of portToIp
+var lenP2IP uint32
+
+func main() {
+	inport := flag.Uint("inport", 0, "port for receiver")
+	lpmSocket := flag.Uint("lpmSocket", 0, "create lpm structure at given socket")
+	lpmMaxRules := flag.Uint64("lpmMaxRules", 100, "maximum number of LPM rules inside table")
+	lpmMaxNumberTbl8 := flag.Uint64("lpmMaxNumberTbl8", 256*256, "maximum number of LPM rules with mask length more than 24 bits")
+	cores := flag.String("cores", "0-7", "Specify CPU cores to use")
+	flag.Parse()
+
+	config := flow.Config{
+		CPUList: *cores,
+	}
+	flow.CheckFatal(flow.SystemInit(&config))
+
+	initRouteCache(uint8(*lpmSocket), uint32(*lpmMaxRules), uint32(*lpmMaxNumberTbl8))
+
+	go updateRouteCache()
+
+	inputFlow, err := flow.SetReceiver(uint16(*inport))
+	flow.CheckFatal(err)
+
+	flow.CheckFatal(flow.SetHandler(inputFlow, handler, nil))
+
+	flow.CheckFatal(flow.SetStopper(inputFlow))
+
+	flow.CheckFatal(flow.SystemStart())
+}
+
+func handler(current *packet.Packet, ctx flow.UserContext) {
+	ipv4, _, _ := current.ParseAllKnownL3()
+	if ipv4 != nil {
+		var next uint32
+		if lpm.Lookup(ipv4.DstAddr, &next) {
+			common.LogDebug(common.Debug, "gateway for packet: ", packet.IPv4ToBytes(portToIP[next]))
+		}
+	}
+}
+
+func netToNffIPv4(netIP net.IP) uint32 {
+	if netIP == nil || len(netIP) != 4 {
+		return 0
+	}
+	return packet.BytesToIPv4(netIP[0], netIP[1], netIP[2], netIP[3])
+}
+
+func getIPAndDepth(netIP *net.IPNet) (uint32, uint8, error) {
+	var (
+		ip    uint32
+		depth int
+	)
+	if netIP != nil {
+		if len(netIP.IP) == 4 {
+			ip = netToNffIPv4(netIP.IP)
+			depth, _ = netIP.Mask.Size()
+		} else {
+			return 0, 0, fmt.Errorf("IPv6 is not supported, fail to add route, skip")
+		}
+	}
+	if depth == 0 {
+		depth = 1
+	}
+	return ip, uint8(depth), nil
+}
+
+func addLPM(v netlink.Route) {
+	gw := netToNffIPv4(v.Gw)
+	ip, depth, err := getIPAndDepth(v.Dst)
+	if err != nil {
+		common.LogWarning(common.Debug, err)
+		return
+	}
+	portToIP = append(portToIP, gw)
+	lenP2IP++
+	common.LogDebug(common.Debug, "adding: ", ip, depth, lenP2IP-1)
+	if err := lpm.Add(ip, depth, lenP2IP-1); err < 0 {
+		common.LogFatal(common.Debug, "failed to add to lpm with ", err)
+	}
+}
+
+func deleteLPM(v netlink.Route) {
+	ip, depth, err := getIPAndDepth(v.Dst)
+	if err != nil {
+		common.LogWarning(common.Debug, err)
+		return
+	}
+	if lpm.Delete(ip, depth) < 0 {
+		common.LogFatal(common.Debug, "failed to delete from lpm")
+	}
+}
+
+func initRouteCache(socket uint8, maxRules uint32, numberTbl8 uint32) {
+	common.LogDebug(common.Debug, "------- Init Route Cache -------", socket, maxRules, numberTbl8)
+
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		common.LogFatal(common.Debug, "Cannot list routs")
+	}
+
+	lpm = packet.CreateLPM("lpm", socket, maxRules, numberTbl8)
+	if lpm == nil {
+		common.LogFatal(common.Debug, "failed to create LPM struct")
+	}
+	for k, v := range routes {
+		common.LogDebug(common.Debug, "Add route: ", k, v)
+		addLPM(v)
+	}
+}
+
+func printKernelRouteTable() {
+	common.LogDebug(common.Debug, "------- Print Kernel Route Table -------")
+
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		common.LogFatal(common.Debug, "Cannot list routs")
+	}
+
+	for k, v := range routes {
+		common.LogDebug(common.Debug, "route: ", k, v)
+	}
+}
+
+func updateRouteCache() {
+	ch := make(chan netlink.RouteUpdate)
+	done := make(chan struct{})
+
+	defer close(done)
+
+	if err := netlink.RouteSubscribe(ch, done); err != nil {
+		common.LogFatal(common.Debug, "Cannot subscribe:", err)
+	}
+
+	for {
+		timeout := time.After(2 * time.Second)
+		select {
+		case update := <-ch:
+			if update.Type == unix.RTM_NEWROUTE {
+				common.LogDebug(common.Debug, "========== Route added: ", update)
+				addLPM(update.Route)
+			}
+			if update.Type == unix.RTM_DELROUTE && update.Route.Dst != nil && update.Route.Gw != nil {
+				common.LogDebug(common.Debug, "========== Route deleted: ", update)
+				deleteLPM(update.Route)
+			}
+		case <-timeout:
+			common.LogDebug(common.Debug, "===== No changes after 2s =====")
+			printKernelRouteTable()
+		}
+	}
+}
diff --git a/scripts/get-depends.sh b/scripts/get-depends.sh
index dcc3b111..37ec4f73 100755
--- a/scripts/get-depends.sh
+++ b/scripts/get-depends.sh
@@ -7,4 +7,5 @@ go get -v github.com/pkg/errors
 go get -v golang.org/x/net/proxy
 go get -v github.com/docker/go-connections
 go get -v golang.org/x/tools/cmd/stringer
+go get -v github.com/vishvananda/netlink
 

From b21716bf71c4063f64dd013fdf4fe72698fb73a8 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Mon, 6 Aug 2018 10:36:03 -0500
Subject: [PATCH 17/50] Fixed renaming system connection when initially
 creating VM

---
 vagrant/Vagrantfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 453ad063..b4e021f7 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -75,8 +75,9 @@ sudo systemctl start network-manager
 SHELL
 
 $provision_common = <<SHELL
-echo Reassigning \'Wired connection 1\' interface to system name
-sudo nmcli c mod 'Wired connection 1' connection.id 'System connection'
+syscon=$(nmcli --fields device,name,type,state -t c s | grep activated | grep -v bridge | cut -d: -f2)
+echo Reassigning "${syscon}" interface to system name
+sudo nmcli c mod "${syscon}" connection.id 'System connection'
 
 echo Unpacking Go language into /opt
 (cd /opt; sudo sh -c 'curl -L -s https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz | tar zx')

From b40c71f8155622388168d1e2c9ec13ab00ce39e9 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Mon, 6 Aug 2018 20:42:35 +0300
Subject: [PATCH 18/50] Added gitignore and docker files

---
 examples/.gitignore                       |  1 +
 examples/Dockerfile                       |  1 +
 examples/nffPktgen/.gitignore             |  1 +
 examples/nffPktgen/perfTest/.gitignore    |  1 +
 examples/nffPktgen/perfTest/Dockerfile    | 11 +++++++++++
 examples/nffPktgen/sendGetBack/.gitignore |  1 +
 examples/nffPktgen/sendGetBack/Dockerfile | 11 +++++++++++
 7 files changed, 27 insertions(+)
 create mode 100644 examples/nffPktgen/.gitignore
 create mode 100644 examples/nffPktgen/perfTest/.gitignore
 create mode 100644 examples/nffPktgen/perfTest/Dockerfile
 create mode 100644 examples/nffPktgen/sendGetBack/.gitignore
 create mode 100644 examples/nffPktgen/sendGetBack/Dockerfile

diff --git a/examples/.gitignore b/examples/.gitignore
index ea817fee..5946c6e8 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -9,3 +9,4 @@ sendFixedPktsNumber
 gtpu
 pingReplay
 timer
+netlink
diff --git a/examples/Dockerfile b/examples/Dockerfile
index 809ed0bb..dd089d3d 100644
--- a/examples/Dockerfile
+++ b/examples/Dockerfile
@@ -18,3 +18,4 @@ COPY errorHandling .
 COPY gtpu .
 COPY pingReplay .
 COPY timer .
+COPY netlink .
diff --git a/examples/nffPktgen/.gitignore b/examples/nffPktgen/.gitignore
new file mode 100644
index 00000000..84650ba7
--- /dev/null
+++ b/examples/nffPktgen/.gitignore
@@ -0,0 +1 @@
+*.pcap
diff --git a/examples/nffPktgen/perfTest/.gitignore b/examples/nffPktgen/perfTest/.gitignore
new file mode 100644
index 00000000..fe034ff6
--- /dev/null
+++ b/examples/nffPktgen/perfTest/.gitignore
@@ -0,0 +1 @@
+perfTest
diff --git a/examples/nffPktgen/perfTest/Dockerfile b/examples/nffPktgen/perfTest/Dockerfile
new file mode 100644
index 00000000..170a9213
--- /dev/null
+++ b/examples/nffPktgen/perfTest/Dockerfile
@@ -0,0 +1,11 @@
+# Copyright 2017 Intel Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+ARG USER_NAME
+FROM ${USER_NAME}/nff-go-base
+
+LABEL RUN docker run -it --privileged -v /sys/bus/pci/drivers:/sys/bus/pci/drivers -v /sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages -v /sys/devices/system/node:/sys/devices/system/node -v /dev:/dev --name NAME -e NAME=NAME -e IMAGE=IMAGE IMAGE
+
+WORKDIR /workdir
+COPY perfTest .
diff --git a/examples/nffPktgen/sendGetBack/.gitignore b/examples/nffPktgen/sendGetBack/.gitignore
new file mode 100644
index 00000000..820c9c8e
--- /dev/null
+++ b/examples/nffPktgen/sendGetBack/.gitignore
@@ -0,0 +1 @@
+sendGetBack
diff --git a/examples/nffPktgen/sendGetBack/Dockerfile b/examples/nffPktgen/sendGetBack/Dockerfile
new file mode 100644
index 00000000..83a9e9b9
--- /dev/null
+++ b/examples/nffPktgen/sendGetBack/Dockerfile
@@ -0,0 +1,11 @@
+# Copyright 2017 Intel Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+ARG USER_NAME
+FROM ${USER_NAME}/nff-go-base
+
+LABEL RUN docker run -it --privileged -v /sys/bus/pci/drivers:/sys/bus/pci/drivers -v /sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages -v /sys/devices/system/node:/sys/devices/system/node -v /dev:/dev --name NAME -e NAME=NAME -e IMAGE=IMAGE IMAGE
+
+WORKDIR /workdir
+COPY sendGetBack .

From 1a6d3833f1e1438b02a3736d25b9d73082004fb2 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Tue, 7 Aug 2018 15:56:44 +0300
Subject: [PATCH 19/50] Added linking with dpdk ena lib for aws

---
 low/low.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/low/low.go b/low/low.go
index fdaf1c01..029969dc 100644
--- a/low/low.go
+++ b/low/low.go
@@ -13,7 +13,7 @@ package low
 // it increases executable size and build time.
 
 /*
-#cgo LDFLAGS: -lrte_distributor -lrte_reorder -lrte_kni -lrte_pipeline -lrte_table -lrte_port -lrte_timer -lrte_jobstats -lrte_lpm -lrte_power -lrte_acl -lrte_meter -lrte_sched -lrte_vhost -lrte_ip_frag -lrte_cfgfile -Wl,--whole-archive -Wl,--start-group -lrte_kvargs -lrte_mbuf -lrte_hash -lrte_ethdev -lrte_mempool -lrte_ring -lrte_mempool_ring -lrte_eal -lrte_cmdline -lrte_net -lrte_bus_pci -lrte_pci -lrte_bus_vdev -lrte_pmd_bond -lrte_pmd_vmxnet3_uio -lrte_pmd_virtio -lrte_pmd_cxgbe -lrte_pmd_enic -lrte_pmd_i40e -lrte_pmd_fm10k -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -lrte_pmd_af_packet -lrte_pmd_null -Wl,--end-group -Wl,--no-whole-archive -lrt -lm -ldl -lnuma
+#cgo LDFLAGS: -lrte_distributor -lrte_reorder -lrte_kni -lrte_pipeline -lrte_table -lrte_port -lrte_timer -lrte_jobstats -lrte_lpm -lrte_power -lrte_acl -lrte_meter -lrte_sched -lrte_vhost -lrte_ip_frag -lrte_cfgfile -Wl,--whole-archive -Wl,--start-group -lrte_kvargs -lrte_mbuf -lrte_hash -lrte_ethdev -lrte_mempool -lrte_ring -lrte_mempool_ring -lrte_eal -lrte_cmdline -lrte_net -lrte_bus_pci -lrte_pci -lrte_bus_vdev -lrte_pmd_bond -lrte_pmd_vmxnet3_uio -lrte_pmd_virtio -lrte_pmd_cxgbe -lrte_pmd_enic -lrte_pmd_i40e -lrte_pmd_fm10k -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ena -lrte_pmd_ring -lrte_pmd_af_packet -lrte_pmd_null -Wl,--end-group -Wl,--no-whole-archive -lrt -lm -ldl -lnuma
 #include "low.h"
 */
 import "C"

From 90eb9739358773d489f638de0f1f965dfda39909 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Wed, 8 Aug 2018 09:23:59 -0600
Subject: [PATCH 20/50] gofmt cleanup

---
 examples/timer.go         |  2 +-
 flow/flow.go              | 42 +++++++++++++++++++--------------------
 flow/scheduler.go         |  6 +++---
 low/low.go                |  1 -
 test/performance/ipsec.go |  2 +-
 5 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/examples/timer.go b/examples/timer.go
index 04cf9a75..b656b4aa 100644
--- a/examples/timer.go
+++ b/examples/timer.go
@@ -20,7 +20,7 @@ var duration *time.Duration
 
 func main() {
 	inport = flag.Uint("inport", 0, "port for receiver")
-	duration = flag.Duration("duration", 2000 * time.Millisecond, "seconds to react")
+	duration = flag.Duration("duration", 2000*time.Millisecond, "seconds to react")
 	flag.Parse()
 
 	flow.CheckFatal(flow.SystemInit(nil))
diff --git a/flow/flow.go b/flow/flow.go
index e8bb2db8..a63998b8 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -48,10 +48,10 @@ var schedState *scheduler
 var vEach [10][burstSize]uint8
 
 type Timer struct {
-	t *time.Ticker
-	handler func(UserContext)
+	t        *time.Ticker
+	handler  func(UserContext)
 	contexts []UserContext
-	checks []*bool
+	checks   []*bool
 }
 
 type processSegment struct {
@@ -63,9 +63,9 @@ type processSegment struct {
 // Flow is an abstraction for connecting flow functions with each other.
 // Flow shouldn't be understood in any way beyond this.
 type Flow struct {
-	current  low.Rings
-	segment  *processSegment
-	previous **Func
+	current       low.Rings
+	segment       *processSegment
+	previous      **Func
 	inIndexNumber int32
 }
 
@@ -185,7 +185,7 @@ func addFastGenerator(out low.Rings, generateFunction GenerateFunction,
 	fTargetSpeed := float64(targetSpeed)
 	if fTargetSpeed <= 0 {
 		return common.WrapWithNFError(nil, "Target speed value should be > 0", common.BadArgument)
-	} else if fTargetSpeed / (1000 /*milleseconds*/ / float64(schedTime)) < float64(burstSize) {
+	} else if fTargetSpeed/(1000 /*milleseconds*/ /float64(schedTime)) < float64(burstSize) {
 		// TargetSpeed per schedTime should be more than burstSize because one burstSize packets in
 		// one schedTime seconds are out minimal scheduling part. We can't make generate speed less than this.
 		return common.WrapWithNFError(nil, "Target speed per schedTime should be more than burstSize", common.BadArgument)
@@ -377,7 +377,7 @@ type port struct {
 	willKNI        bool // will this port has assigned KNI device
 	port           uint16
 	MAC            [common.EtherAddrLen]uint8
-	InIndex     int32
+	InIndex        int32
 }
 
 // Config is a struct with all parameters, which user can pass to NFF-GO library
@@ -535,7 +535,7 @@ func SystemInit(args *Config) error {
 	portPair = make(map[uint32](*port))
 	// Init scheduler
 	common.LogTitle(common.Initialization, "------------***------ Initializing scheduler -----***------------")
-	StopRing := low.CreateRings(burstSize * sizeMultiplier, 50 /*Maximum possible rings*/)
+	StopRing := low.CreateRings(burstSize*sizeMultiplier, 50 /*Maximum possible rings*/)
 	common.LogDebug(common.Initialization, "Scheduler can use cores:", cpus)
 	schedState = newScheduler(cpus, schedulerOff, schedulerOffRemove, stopDedicatedCore, StopRing, checkTime, debugTime, maxPacketsToClone, maxRecv, anyway)
 	// Init packet processing
@@ -618,7 +618,7 @@ func SetSenderFile(IN *Flow, filename string) error {
 // file is read infinitely in circle.
 // Returns new opened flow with read packets.
 func SetReceiverFile(filename string, repcount int32) (OUT *Flow) {
-	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	rings := low.CreateRings(burstSize*sizeMultiplier, 1)
 	addReader(filename, rings, repcount)
 	return newFlow(rings, 1)
 }
@@ -636,7 +636,7 @@ func SetReceiver(portId uint16) (OUT *Flow, err error) {
 	}
 	createdPorts[portId].wasRequested = true
 	createdPorts[portId].willReceive = true
-	rings := low.CreateRings(burstSize * sizeMultiplier, createdPorts[portId].InIndex)
+	rings := low.CreateRings(burstSize*sizeMultiplier, createdPorts[portId].InIndex)
 	addReceiver(portId, false, rings, createdPorts[portId].InIndex)
 	return newFlow(rings, createdPorts[portId].InIndex), nil
 }
@@ -646,7 +646,7 @@ func SetReceiver(portId uint16) (OUT *Flow, err error) {
 // Receive queue will be added to port automatically.
 // Returns new opened flow with received packets
 func SetReceiverKNI(kni *Kni) (OUT *Flow) {
-	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	rings := low.CreateRings(burstSize*sizeMultiplier, 1)
 	addReceiver(kni.portId, true, rings, 1)
 	return newFlow(rings, 1)
 }
@@ -656,7 +656,7 @@ func SetReceiverKNI(kni *Kni) (OUT *Flow) {
 // Returns new open flow with generated packets.
 // Function tries to achieve target speed by cloning.
 func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
-	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	rings := low.CreateRings(burstSize*sizeMultiplier, 1)
 	if err := addFastGenerator(rings, f, nil, targetSpeed, context); err != nil {
 		return nil, err
 	}
@@ -668,7 +668,7 @@ func SetFastGenerator(f GenerateFunction, targetSpeed uint64, context UserContex
 // Returns new open flow with generated packets.
 // Function tries to achieve target speed by cloning.
 func SetVectorFastGenerator(f VectorGenerateFunction, targetSpeed uint64, context UserContext) (OUT *Flow, err error) {
-	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	rings := low.CreateRings(burstSize*sizeMultiplier, 1)
 	if err := addFastGenerator(rings, nil, f, targetSpeed, context); err != nil {
 		return nil, err
 	}
@@ -681,7 +681,7 @@ func SetVectorFastGenerator(f VectorGenerateFunction, targetSpeed uint64, contex
 // Single packet non-clonable flow function will be added. It can be used for waiting of
 // input user packets.
 func SetGenerator(f GenerateFunction, context UserContext) (OUT *Flow) {
-	rings := low.CreateRings(burstSize * sizeMultiplier, 1)
+	rings := low.CreateRings(burstSize*sizeMultiplier, 1)
 	addGenerator(rings, f, context)
 	return newFlow(rings, 1)
 }
@@ -719,12 +719,12 @@ func SetCopier(IN *Flow) (OUT *Flow, err error) {
 	if err := checkFlow(IN); err != nil {
 		return nil, err
 	}
-	ringFirst := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
-	ringSecond := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
+	ringFirst := low.CreateRings(burstSize*sizeMultiplier, IN.inIndexNumber)
+	ringSecond := low.CreateRings(burstSize*sizeMultiplier, IN.inIndexNumber)
 	if IN.segment == nil {
 		addCopier(IN.current, ringFirst, ringSecond, IN.inIndexNumber)
 	} else {
-		tRing := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
+		tRing := low.CreateRings(burstSize*sizeMultiplier, IN.inIndexNumber)
 		ms := makeSlice(tRing, IN.segment)
 		segmentInsert(IN, ms, false, nil, 0, 0)
 		addCopier(tRing, ringFirst, ringSecond, IN.inIndexNumber)
@@ -892,7 +892,7 @@ func SetMerger(InArray ...*Flow) (OUT *Flow, err error) {
 			max = InArray[i].inIndexNumber
 		}
 	}
-	rings := low.CreateRings(burstSize * sizeMultiplier, max)
+	rings := low.CreateRings(burstSize*sizeMultiplier, max)
 	for i := range InArray {
 		if err := checkFlow(InArray[i]); err != nil {
 			return nil, err
@@ -948,7 +948,7 @@ func finishFlow(IN *Flow) low.Rings {
 		ring = IN.current
 		closeFlow(IN)
 	} else {
-		ring = low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
+		ring = low.CreateRings(burstSize*sizeMultiplier, IN.inIndexNumber)
 		ms := makeSlice(ring, IN.segment)
 		segmentInsert(IN, ms, true, nil, 0, 0)
 	}
@@ -971,7 +971,7 @@ func segmentInsert(IN *Flow, f *Func, willClose bool, context UserContext, setTy
 	} else {
 		if setType > 0 && IN.segment.stype > 0 && setType != IN.segment.stype {
 			// Try to combine scalar and vector code. Start new segment
-			ring := low.CreateRings(burstSize * sizeMultiplier, IN.inIndexNumber)
+			ring := low.CreateRings(burstSize*sizeMultiplier, IN.inIndexNumber)
 			ms := makeSlice(ring, IN.segment)
 			segmentInsert(IN, ms, false, nil, 0, 0)
 			IN.segment = nil
diff --git a/flow/scheduler.go b/flow/scheduler.go
index 6243ae4f..a69d4bea 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -153,7 +153,7 @@ type scheduler struct {
 	maxPacketsToClone uint32
 	stopFlag          int32
 	maxRecv           int
-	Timers		  []*Timer
+	Timers            []*Timer
 }
 
 type core struct {
@@ -218,7 +218,7 @@ func (ff *flowFunction) startNewInstance(inIndex []int32, scheduler *scheduler)
 	ffi := new(instance)
 	common.LogDebug(common.Debug, "Start new instance for", ff.name)
 	ffi.inIndex = inIndex
-	ffi.report = make(chan reportPair, len(scheduler.cores) - 1)
+	ffi.report = make(chan reportPair, len(scheduler.cores)-1)
 	ffi.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
 	ffi.ff = ff
 	err = ffi.startNewClone(scheduler, ff.instanceNumber)
@@ -326,7 +326,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 		// We have an array of Timers which can be increated by AddTimer function
 		// Timer has duration and handler common for all Timer variants, so firstly
 		// we check that timer ticker channel is ready:
-		for _, t := range(scheduler.Timers) {
+		for _, t := range scheduler.Timers {
 			select {
 			case <-t.t.C:
 				// If this timer was ticked we check all checks of all variants
diff --git a/low/low.go b/low/low.go
index fdaf1c01..a127bedc 100644
--- a/low/low.go
+++ b/low/low.go
@@ -558,7 +558,6 @@ func CreateMempools(name string, inIndex int32) []*Mempool {
 	return m
 }
 
-
 // SetAffinity sets cpu affinity mask.
 func SetAffinity(coreID int) error {
 	// go tool trace shows that each proc executes different goroutine. However it is expected behavior
diff --git a/test/performance/ipsec.go b/test/performance/ipsec.go
index 9f9f5910..cf7cb9ec 100644
--- a/test/performance/ipsec.go
+++ b/test/performance/ipsec.go
@@ -27,7 +27,7 @@ func main() {
 	config := flow.Config{
 		DisableScheduler: *noscheduler,
 		DPDKArgs:         []string{*dpdkLogLevel},
-		CPUList:	  *cores,
+		CPUList:          *cores,
 	}
 	flow.CheckFatal(flow.SystemInit(&config))
 

From 5b6375eccb5232f1841ec2c7534b1e88ef1c4537 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Mon, 13 Aug 2018 08:38:57 -0600
Subject: [PATCH 21/50] Should change existing inIndex slice instead of making
 new one

---
 flow/scheduler.go | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index a69d4bea..d14990ae 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -481,7 +481,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 							if ffi.inIndex[0] > 1 {
 								ff.oneCoreSpeed = ffi.currentSpeed
 								if ff.startNewInstance(constructZeroIndex(ffi.inIndex), scheduler) == nil {
-									ff.instance[ff.instanceNumber-1].inIndex = constructDuplicatedIndex(ffi.inIndex)
+									constructDuplicatedIndex(ffi.inIndex, ff.instance[ff.instanceNumber-1].inIndex)
 									a = false
 								}
 							}
@@ -535,7 +535,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 						for q := 0; q < ff.instanceNumber; q++ {
 							if ff.instance[q].checkInputRingClonable(RSSCloneMax) && ff.instance[q].checkOutputRingClonable(scheduler.maxPacketsToClone) {
 								if ff.startNewInstance(constructZeroIndex(ff.instance[q].inIndex), scheduler) == nil {
-									ff.instance[ff.instanceNumber-1].inIndex = constructDuplicatedIndex(ff.instance[q].inIndex)
+									constructDuplicatedIndex(ff.instance[q].inIndex, ff.instance[ff.instanceNumber-1].inIndex)
 								}
 								break
 							}
@@ -705,8 +705,7 @@ func constructZeroIndex(old []int32) []int32 {
 	return make([]int32, len(old), len(old))
 }
 
-func constructDuplicatedIndex(old []int32) []int32 {
-	newIndex := make([]int32, len(old), len(old))
+func constructDuplicatedIndex(old []int32, newIndex []int32) {
 	oldLen := old[0]/2 + old[0]%2
 	newLen := old[0] / 2
 	for q := int32(0); q < newLen; q++ {
@@ -714,7 +713,6 @@ func constructDuplicatedIndex(old []int32) []int32 {
 	}
 	newIndex[0] = newLen
 	old[0] = oldLen
-	return newIndex
 }
 
 func constructNewIndex(inIndexNumber int32) []int32 {

From 443224f2ecc931b508aed01b54c3cdcd1f827aea Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Mon, 13 Aug 2018 15:09:57 -0600
Subject: [PATCH 22/50] More accurate synchronization between scheduler and
 handlers

---
 flow/flow.go      | 34 ++++++++++++++++++++++++++++------
 flow/scheduler.go | 10 ++++++++--
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/flow/flow.go b/flow/flow.go
index a63998b8..863559c7 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -1010,7 +1010,6 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 		countOfPackets[index] = 0
 	}
 	var currentSpeed reportPair
-	tick := time.Tick(time.Duration(schedTime) * time.Millisecond)
 	var pause int
 	firstFunc := lp.firstFunc
 	// For scalar part
@@ -1024,10 +1023,13 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 	def := make([]pair, 30, 30)
 	var currentMask [burstSize]bool
 	var answers [burstSize]uint8
+	tick := time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+	stopper[1] <- 2 // Answer that function is ready
 
 	for {
 		select {
 		case pause = <-stopper[0]:
+			tick.Stop()
 			if pause == -1 {
 				// It is time to close this clone
 				for i := range context {
@@ -1037,8 +1039,13 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 				}
 				stopper[1] <- 1
 				return
+			} else {
+				// For any events with this function we should restart timer
+				// We don't do it regularly without any events due to performance
+				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+				currentSpeed = reportPair{}
 			}
-		case <-tick:
+		case <-tick.C:
 			report <- currentSpeed
 			currentSpeed = reportPair{}
 		default:
@@ -1136,6 +1143,7 @@ func pGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int, rep
 	gp := parameters.(*generateParameters)
 	OUT := gp.out
 	generateFunction := gp.generateFunction
+	stopper[1] <- 2 // Answer that function is ready
 	for {
 		select {
 		case <-stopper[0]:
@@ -1168,11 +1176,13 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 	var tempPacket *packet.Packet
 	tempPackets := make([]*packet.Packet, burstSize)
 	var currentSpeed reportPair
-	tick := time.Tick(time.Duration(schedTime) * time.Millisecond)
 	var pause int
+	tick := time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+	stopper[1] <- 2 // Answer that function is ready
 	for {
 		select {
 		case pause = <-stopper[0]:
+			tick.Stop()
 			if pause == -1 {
 				// It is time to close this clone
 				if context[0] != nil {
@@ -1180,8 +1190,13 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 				}
 				stopper[1] <- 1
 				return
+			} else {
+				// For any events with this function we should restart timer
+				// We don't do it regularly without any events due to performance
+				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+				currentSpeed = reportPair{}
 			}
-		case <-tick:
+		case <-tick.C:
 			report <- currentSpeed
 			currentSpeed = reportPair{}
 		default:
@@ -1230,18 +1245,25 @@ func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report
 	var tempPacket1 *packet.Packet
 	var tempPacket2 *packet.Packet
 	var currentSpeed reportPair
-	tick := time.Tick(time.Duration(schedTime) * time.Millisecond)
 	var pause int
+	tick := time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+	stopper[1] <- 2 // Answer that function is ready
 
 	for {
 		select {
 		case pause = <-stopper[0]:
+			tick.Stop()
 			if pause == -1 {
 				// It is time to remove this clone
 				stopper[1] <- 1
 				return
+			} else {
+				// For any events with this function we should restart timer
+				// We don't do it regularly without any events due to performance
+				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
+				currentSpeed = reportPair{}
 			}
-		case <-tick:
+		case <-tick.C:
 			report <- currentSpeed
 			currentSpeed = reportPair{}
 		default:
diff --git a/flow/scheduler.go b/flow/scheduler.go
index d14990ae..bc768e63 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -257,6 +257,9 @@ func (ffi *instance) startNewClone(scheduler *scheduler, n int) (err error) {
 			ff.cFunction(ff.Parameters, ffi.inIndex, &ffi.clone[ffi.cloneNumber-1].flag, core)
 		}
 	}()
+	if ff.fType == segmentCopy || ff.fType == fastGenerate || ff.fType == generate {
+		<-ffi.clone[ffi.cloneNumber-1].channel[1]
+	}
 	return nil
 }
 
@@ -393,7 +396,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 								// Save current speed as speed of flow function with this number of clones before removing
 								ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
 								ff.stopClone(ffi, scheduler)
-								ffi.updatePause(ffi.cloneNumber)
+								ffi.updatePause(ffi.cloneNumber - 1)
 							}
 						}
 					}
@@ -417,7 +420,9 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 							}
 						}
 						if ff.instance[min0].currentSpeed.Packets+ff.instance[min1].currentSpeed.Packets < ff.oneCoreSpeed.Packets {
+							toInstance := ff.instance[min1]
 							ff.stopInstance(min0, min1, scheduler)
+							toInstance.updatePause(0)
 						}
 					}
 				case fastGenerate: // Only clones, no instances
@@ -482,6 +487,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 								ff.oneCoreSpeed = ffi.currentSpeed
 								if ff.startNewInstance(constructZeroIndex(ffi.inIndex), scheduler) == nil {
 									constructDuplicatedIndex(ffi.inIndex, ff.instance[ff.instanceNumber-1].inIndex)
+									ffi.updatePause(0)
 									a = false
 								}
 							}
@@ -498,7 +504,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 									ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
 									if ffi.startNewClone(scheduler, q) == nil {
 										// Add a pause to all clones. This pause depends on the number of clones.
-										ffi.updatePause(ffi.cloneNumber)
+										ffi.updatePause(ffi.cloneNumber - 1)
 									}
 									continue
 								}

From 5052129d498674ef7fa6f6f4afdb9a27580ab789 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Tue, 14 Aug 2018 13:46:42 -0600
Subject: [PATCH 23/50] Move mempools reporting to verbose plus remove
 redundant performance scenarios

---
 low/low.go                    |   2 +-
 test/framework/main/perf.json | 768 ----------------------------------
 2 files changed, 1 insertion(+), 769 deletions(-)

diff --git a/low/low.go b/low/low.go
index 633767b0..02f1242b 100644
--- a/low/low.go
+++ b/low/low.go
@@ -626,7 +626,7 @@ func Statistics(N float32) {
 func ReportMempoolsState() {
 	for _, m := range usedMempools {
 		use := C.getMempoolSpace(m.mempool)
-		common.LogDebug(common.Debug, "Mempool usage", m.name, use, "from", mbufNumberT)
+		common.LogDebug(common.Verbose, "Mempool usage", m.name, use, "from", mbufNumberT)
 		if float32(mbufNumberT-uint(use))/float32(mbufNumberT)*100 < 10 {
 			common.LogDrop(common.Debug, m.name, "mempool has less than 10% free space. This can lead to dropping packets while receive.")
 		}
diff --git a/test/framework/main/perf.json b/test/framework/main/perf.json
index ecfc09a4..8fd33213 100644
--- a/test/framework/main/perf.json
+++ b/test/framework/main/perf.json
@@ -1876,262 +1876,6 @@
                 "measure-for": 25000000000
             }
         },
-        {
-            "name": "perf_seq_50_2_posl_off_64",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_on_64",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_off_128",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_on_128",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_off_256",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_on_256",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_off_1518",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_50_2_posl_on_1518",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=50", "-mode=2", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
         {
             "name": "perf_seq_50_3_posl_off_64",
             "test-time": 32000000000,
@@ -2388,518 +2132,6 @@
                 "measure-for": 25000000000
             }
         },
-        {
-            "name": "perf_seq_25_3_posl_off_64",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_on_64",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_off_128",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_on_128",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_off_256",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_on_256",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_off_1518",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_posl_on_1518",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=3", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_off_64",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_on_64",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 64);",
-                    "pktgen.pkt_size(\"0\", \"max\", 64);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_off_128",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_on_128",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 128);",
-                    "pktgen.pkt_size(\"0\", \"max\", 128);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_off_256",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_on_256",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 256);",
-                    "pktgen.pkt_size(\"0\", \"max\", 256);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_off_1518",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "--no-scheduler", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "perf_seq_25_3_par_on_1518",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-performance",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./perf_seq", "-load=25", "-mode=13", "-inport=INPORT1", "-outport1=OUTPORT1_1", "-outport2=OUTPORT1_2", "-cores=CORES"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "PKTGENPORT", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.pkt_size(\"0\", \"min\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"max\", 1518);",
-                    "pktgen.pkt_size(\"0\", \"inc\", 64);",
-                    "pktgen.set_range(\"0\", \"on\");",
-                    "pktgen.start(0);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
         {
             "name": "perf_seq_25_4_posl_off_64",
             "test-time": 32000000000,

From 660ccf2433d652e7736567bb7f90b751507fdb5a Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Tue, 14 Aug 2018 15:40:04 -0600
Subject: [PATCH 24/50] Remove redundant constants from stop function

---
 flow/flow.go | 2 +-
 low/low.go   | 2 +-
 low/low.h    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flow/flow.go b/flow/flow.go
index 863559c7..4699d740 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -535,7 +535,7 @@ func SystemInit(args *Config) error {
 	portPair = make(map[uint32](*port))
 	// Init scheduler
 	common.LogTitle(common.Initialization, "------------***------ Initializing scheduler -----***------------")
-	StopRing := low.CreateRings(burstSize*sizeMultiplier, 50 /*Maximum possible rings*/)
+	StopRing := low.CreateRings(burstSize*sizeMultiplier, maxInIndex)
 	common.LogDebug(common.Initialization, "Scheduler can use cores:", cpus)
 	schedState = newScheduler(cpus, schedulerOff, schedulerOffRemove, stopDedicatedCore, StopRing, checkTime, debugTime, maxPacketsToClone, maxRecv, anyway)
 	// Init packet processing
diff --git a/low/low.go b/low/low.go
index 02f1242b..b7a3feda 100644
--- a/low/low.go
+++ b/low/low.go
@@ -470,7 +470,7 @@ func Send(port uint16, queue int16, IN Rings, inIndexNumber int32, flag *int32,
 
 // Stop - dequeue and free packets.
 func Stop(IN Rings, flag *int32, coreID int) {
-	C.nff_go_stop(C.extractDPDKRings((**C.struct_nff_go_ring)(unsafe.Pointer(&(IN[0]))), C.int32_t(len(IN))), (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
+	C.nff_go_stop(C.extractDPDKRings((**C.struct_nff_go_ring)(unsafe.Pointer(&(IN[0]))), C.int32_t(len(IN))), C.int(len(IN)), (*C.int)(unsafe.Pointer(flag)), C.int(coreID))
 }
 
 // InitDPDKArguments allocates and initializes arguments for dpdk.
diff --git a/low/low.h b/low/low.h
index 7fa49f01..65dc876e 100644
--- a/low/low.h
+++ b/low/low.h
@@ -402,14 +402,14 @@ void nff_go_send(uint16_t port, int16_t queue, struct rte_ring **in_rings, int32
 	*flag = wasStopped;
 }
 
-void nff_go_stop(struct rte_ring **in_rings, volatile int *flag, int coreId) {
+void nff_go_stop(struct rte_ring **in_rings, int len, volatile int *flag, int coreId) {
 	setAffinity(coreId);
 	struct rte_mbuf *bufs[BURST_SIZE];
 	uint16_t buf;
 	// Flag is used for both scheduler and stop.
 	// stopRequest will stop scheduler and this loop will stop with stopRequest+1
 	while (*flag == process || *flag == stopRequest) {
-		for (int q = 0; q < 50 /*Maximum possible rings*/; q++) {
+		for (int q = 0; q < len; q++) {
 			// Get packets for freeing from ring
 			uint16_t pkts_for_free_number = rte_ring_mc_dequeue_burst(in_rings[q], (void*)bufs, BURST_SIZE, NULL);
 

From 090cb6f971907d45b483a7b12a61115c43dac47e Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Tue, 7 Aug 2018 19:01:20 +0300
Subject: [PATCH 25/50] Make init return error instead of fail

---
 common/common.go                              |  2 ++
 flow/flow.go                                  |  8 ++++++--
 low/Makefile                                  |  2 +-
 low/low.go                                    | 19 ++++++++++++++-----
 low/low.h                                     | 12 +++++++-----
 low/low_test.go                               | 15 ++++++++++-----
 packet/utils_for_test.go                      |  4 +++-
 test/stability/testCksum/checksum_test.go     |  5 +++--
 test/stability/testMerge/merge_test.go        |  5 +++--
 .../singleWorkingFF_test.go                   |  6 +++---
 10 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/common/common.go b/common/common.go
index a9af639b..eb5a1d06 100644
--- a/common/common.go
+++ b/common/common.go
@@ -147,6 +147,8 @@ const (
 	MultipleReceivePort
 	MultipleKNIPort
 	WrongPort
+	FailToInitDPDK
+	FailToCreateKNI
 )
 
 // NFError is error type returned by nff-go functions
diff --git a/flow/flow.go b/flow/flow.go
index 4699d740..d0b9cc3d 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -521,7 +521,9 @@ func SystemInit(args *Config) error {
 	// TODO all low level initialization here! Now everything is default.
 	// Init eal
 	common.LogTitle(common.Initialization, "------------***-------- Initializing DPDK --------***------------")
-	low.InitDPDK(argc, argv, burstSize, mbufNumber, mbufCacheSize, needKNI)
+	if err := low.InitDPDK(argc, argv, burstSize, mbufNumber, mbufCacheSize, needKNI); err != nil {
+		return err
+	}
 	// Init Ports
 	createdPorts = make([]port, low.GetPortsNumber(), low.GetPortsNumber())
 	for i := range createdPorts {
@@ -1545,7 +1547,9 @@ func CreateKniDevice(portId uint16, name string) (*Kni, error) {
 	if core, _, err := schedState.getCore(); err != nil {
 		return nil, err
 	} else {
-		low.CreateKni(portId, uint(core), name)
+		if err := low.CreateKni(portId, uint(core), name); err != nil {
+			return nil, err
+		}
 	}
 	kni := new(Kni)
 	// Port will be identifier of this KNI
diff --git a/low/Makefile b/low/Makefile
index 050b11bb..f2d34ca8 100644
--- a/low/Makefile
+++ b/low/Makefile
@@ -7,4 +7,4 @@ include $(PATH_TO_MK)/include.mk
 
 .PHONY: testing
 testing: check-pktgen
-	go test
+	go test -v
diff --git a/low/low.go b/low/low.go
index b7a3feda..d5527865 100644
--- a/low/low.go
+++ b/low/low.go
@@ -487,11 +487,17 @@ func InitDPDKArguments(args []string) (C.int, **C.char) {
 }
 
 // InitDPDK initializes the Environment Abstraction Layer (EAL) in DPDK.
-func InitDPDK(argc C.int, argv **C.char, burstSize uint, mbufNumber uint, mbufCacheSize uint, needKNI int) {
-	C.eal_init(argc, argv, C.uint32_t(burstSize), C.int32_t(needKNI))
-
+func InitDPDK(argc C.int, argv **C.char, burstSize uint, mbufNumber uint, mbufCacheSize uint, needKNI int) error {
+	ret := C.eal_init(argc, argv, C.uint32_t(burstSize), C.int32_t(needKNI))
+	if ret < 0 {
+		return common.WrapWithNFError(nil, "Error with EAL initialization\n", common.FailToInitDPDK)
+	}
+	if ret > 0 {
+		return common.WrapWithNFError(nil, "rte_eal_init can't parse all parameters\n", common.FailToInitDPDK)
+	}
 	mbufNumberT = mbufNumber
 	mbufCacheSizeT = mbufCacheSize
+	return nil
 }
 
 func StopDPDK() {
@@ -634,9 +640,12 @@ func ReportMempoolsState() {
 }
 
 // CreateKni creates a KNI device
-func CreateKni(portId uint16, core uint, name string) {
+func CreateKni(portId uint16, core uint, name string) error {
 	mempool := (*C.struct_rte_mempool)(CreateMempool("KNI"))
-	C.create_kni(C.uint16_t(portId), C.uint32_t(core), C.CString(name), mempool)
+	if C.create_kni(C.uint16_t(portId), C.uint32_t(core), C.CString(name), mempool) != 0 {
+		common.WrapWithNFError(nil, "Error with KNI allocation\n", common.FailToCreateKNI)
+	}
+	return nil
 }
 
 // CreateLPM creates LPM table
diff --git a/low/low.h b/low/low.h
index 65dc876e..9df0fd5f 100644
--- a/low/low.h
+++ b/low/low.h
@@ -106,7 +106,7 @@ void setAffinity(int coreId) {
 	sched_setaffinity(0, sizeof(cpuset), &cpuset);
 }
 
-void create_kni(uint16_t port, uint32_t core, char *name, struct rte_mempool *mbuf_pool) {
+int create_kni(uint16_t port, uint32_t core, char *name, struct rte_mempool *mbuf_pool) {
 	struct rte_eth_dev_info dev_info;
 	memset(&dev_info, 0, sizeof(dev_info));
 	rte_eth_dev_info_get(port, &dev_info);
@@ -138,8 +138,9 @@ void create_kni(uint16_t port, uint32_t core, char *name, struct rte_mempool *mb
 	// version of these structures
 	kni[port] = rte_kni_alloc(mbuf_pool, &conf_default, &ops);
 	if (kni[port] == NULL) {
-		rte_exit(EXIT_FAILURE, "Error with KNI allocation\n");
+		return -1;
 	}
+	return 0;
 }
 
 int checkRSSPacketCount(struct cPort *port, int16_t queue) {
@@ -480,13 +481,13 @@ void statistics(float N) {
 }
 
 // Initialize the Environment Abstraction Layer (EAL) in DPDK.
-void eal_init(int argc, char *argv[], uint32_t burstSize, int32_t needKNI)
+int eal_init(int argc, char *argv[], uint32_t burstSize, int32_t needKNI)
 {
 	int ret = rte_eal_init(argc, argv);
 	if (ret < 0)
-		rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
+		return -1;
 	if (ret < argc-1)
-		rte_exit(EXIT_FAILURE, "rte_eal_init can't parse all parameters\n");
+		return 1;
 	free(argv[argc-1]);
 	free(argv);
 	BURST_SIZE = burstSize;
@@ -496,6 +497,7 @@ void eal_init(int argc, char *argv[], uint32_t burstSize, int32_t needKNI)
 	if (needKNI != 0) {
 		rte_kni_init(MAX_KNI);
 	}
+	return 0;
 }
 
 int allocateMbufs(struct rte_mempool *mempool, struct rte_mbuf **bufs, unsigned count);
diff --git a/low/low_test.go b/low/low_test.go
index 38674537..7be01d0f 100644
--- a/low/low_test.go
+++ b/low/low_test.go
@@ -4,15 +4,20 @@
 
 package low
 
-import "unsafe"
-import "testing"
-import "math/rand"
-import "time"
+import (
+	"log"
+	"math/rand"
+	"testing"
+	"time"
+	"unsafe"
+)
 
 func init() {
 	argc, argv := InitDPDKArguments([]string{})
 	// Default: burstSize=32, mbufNumber=8191, mbufCacheSize=250
-	InitDPDK(argc, argv, 32, 8191, 250, 0)
+	if err := InitDPDK(argc, argv, 32, 8191, 250, 0); err != nil {
+		log.Fatalf("fail to initialize with error: %+v\n", err)
+	}
 	rand.Seed(time.Now().UTC().UnixNano())
 }
 
diff --git a/packet/utils_for_test.go b/packet/utils_for_test.go
index 1f02dbdc..8839747b 100644
--- a/packet/utils_for_test.go
+++ b/packet/utils_for_test.go
@@ -20,7 +20,9 @@ func tInitDPDK() {
 	if isInit != true {
 		argc, argv := low.InitDPDKArguments([]string{})
 		// burstSize=32, mbufNumber=8191, mbufCacheSize=250
-		low.InitDPDK(argc, argv, 32, 8191, 250, 0)
+		if err := low.InitDPDK(argc, argv, 32, 8191, 250, 0); err != nil {
+			log.Fatal(err)
+		}
 		nonPerfMempool = low.CreateMempool("Test")
 		isInit = true
 	}
diff --git a/test/stability/testCksum/checksum_test.go b/test/stability/testCksum/checksum_test.go
index e940baba..a649b346 100644
--- a/test/stability/testCksum/checksum_test.go
+++ b/test/stability/testCksum/checksum_test.go
@@ -5,12 +5,13 @@
 package main
 
 import (
+	"log"
 	"testing"
 )
 
-func TestInit(t *testing.T) {
+func init() {
 	if err := initTest(); err != nil {
-		t.Fatalf("fail to initialize with error: %+v\n", err)
+		log.Fatalf("fail to initialize with error: %+v\n", err)
 	}
 }
 
diff --git a/test/stability/testMerge/merge_test.go b/test/stability/testMerge/merge_test.go
index 9e5107f5..9d7bd599 100644
--- a/test/stability/testMerge/merge_test.go
+++ b/test/stability/testMerge/merge_test.go
@@ -5,12 +5,13 @@
 package main
 
 import (
+	"log"
 	"testing"
 )
 
-func TestInit(t *testing.T) {
+func init() {
 	if err := initDPDK(); err != nil {
-		t.Fatalf("fail to initialize with error: %+v\n", err)
+		log.Fatalf("fail to initialize with error: %+v\n", err)
 	}
 }
 
diff --git a/test/stability/testSingleWorkingFF/singleWorkingFF_test.go b/test/stability/testSingleWorkingFF/singleWorkingFF_test.go
index c2170850..2f3c3bb7 100644
--- a/test/stability/testSingleWorkingFF/singleWorkingFF_test.go
+++ b/test/stability/testSingleWorkingFF/singleWorkingFF_test.go
@@ -5,16 +5,16 @@
 package main
 
 import (
+	"log"
 	"testing"
 )
 
 // TODO unfortunately this test can't be executed as go test right now due to https://github.com/intel-go/nff-go/issues/301
 // If you want to test it via go test you should firstly comment FreeMempools function at flow/flow.go
 // You will have multiple mempools however test will pass.
-func TestInit(t *testing.T) {
+func init() {
 	if err := initDPDK(); err != nil {
-		t.Logf("fail: %+v\n", err)
-		t.Fail()
+		log.Fatalf("fail: %+v\n", err)
 	}
 }
 

From 64a9e9f0b3d0ef948745a0a3675f7df38a0576e3 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Thu, 16 Aug 2018 16:20:06 +0300
Subject: [PATCH 26/50] Changed address arithmetic, removed make call

---
 examples/nffPktgen/Makefile                   |  2 +-
 examples/nffPktgen/generator/generator.go     | 36 +++++++---
 examples/nffPktgen/generator/parseConfig.go   | 69 +++++++++----------
 examples/nffPktgen/sendGetBack/sendGetBack.go |  2 +-
 4 files changed, 64 insertions(+), 45 deletions(-)

diff --git a/examples/nffPktgen/Makefile b/examples/nffPktgen/Makefile
index 65c5ecae..69366737 100644
--- a/examples/nffPktgen/Makefile
+++ b/examples/nffPktgen/Makefile
@@ -6,4 +6,4 @@ PATH_TO_MK = ../../mk
 IMAGENAME = nff-pktgen
 SUBDIRS = sendGetBack perfTest
 
-include $(PATH_TO_MK)/leaf.mk
+include $(PATH_TO_MK)/intermediate.mk
diff --git a/examples/nffPktgen/generator/generator.go b/examples/nffPktgen/generator/generator.go
index 5eb726cf..d416e219 100644
--- a/examples/nffPktgen/generator/generator.go
+++ b/examples/nffPktgen/generator/generator.go
@@ -5,10 +5,10 @@
 package generator
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math"
-	"math/big"
 	"math/rand"
 	"os"
 	"sync/atomic"
@@ -49,16 +49,36 @@ func ReadConfig(fileName string) ([]*MixConfig, error) {
 	return cfg, nil
 }
 
-func getNextAddr(addr *AddrRange) []uint8 {
-	if addr.Incr == 0 {
-		return addr.Current.Bytes()
+func addAddr(a *[]byte, b []byte) {
+	var offset int
+	aLen := len(*a)
+	bLen := len(b)
+	if aLen < bLen {
+		add := make([]byte, bLen-aLen)
+		*a = append(add, *a...)
+	} else {
+		offset = aLen - bLen
+	}
+	var next byte
+	for i := bLen - 1; i >= 0; i-- {
+		add := (*a)[i+offset] + b[i] + next
+		if add > 255 {
+			next = 1
+			(*a)[i+offset] = byte(255) - add
+		} else {
+			(*a)[i+offset] = add
+			next = 0
+		}
 	}
-	addr.Current.Add(big.NewInt(int64(addr.Incr)), addr.Current)
+}
+
+func getNextAddr(addr *AddrRange) []uint8 {
+	addAddr(&(addr.Current), addr.Incr)
 	// if current < min or current > max, copy min to current
-	if addr.Current.Cmp(addr.Min) < 0 || addr.Current.Cmp(addr.Max) > 0 {
-		addr.Current = addr.Min
+	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
+		copy(addr.Current, addr.Min)
 	}
-	return addr.Current.Bytes()
+	return addr.Current
 }
 
 func copyAddr(destination []uint8, source []uint8, size int) {
diff --git a/examples/nffPktgen/generator/parseConfig.go b/examples/nffPktgen/generator/parseConfig.go
index d99c8557..84e7090f 100644
--- a/examples/nffPktgen/generator/parseConfig.go
+++ b/examples/nffPktgen/generator/parseConfig.go
@@ -6,6 +6,7 @@ package generator
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"math/big"
@@ -23,10 +24,10 @@ var mixPattern = regexp.MustCompile(`^mix[0-9]*$`)
 
 // AddrRange describes range of addresses.
 type AddrRange struct {
-	Min     *big.Int
-	Max     *big.Int
-	Current *big.Int
-	Incr    uint64
+	Min     []byte
+	Max     []byte
+	Current []byte
+	Incr    []byte
 }
 
 func (ar *AddrRange) String() string {
@@ -242,7 +243,7 @@ func ParseMixConfig(in map[string]interface{}) (config []*MixConfig, err error)
 }
 
 func parseEtherHdr(in map[string]interface{}) (EtherConfig, error) {
-	ethConfig := EtherConfig{DAddr: allocAddrRange(), SAddr: allocAddrRange(), VLAN: nil, DType: NONE}
+	ethConfig := EtherConfig{DAddr: AddrRange{}, SAddr: AddrRange{}, VLAN: nil, DType: NONE}
 	for k, v := range in {
 		switch strings.ToLower(k) {
 		case "saddr":
@@ -655,10 +656,13 @@ func parseMacAddr(value interface{}) (AddrRange, error) {
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing mac saddr returned: %v", err)
 		}
-		ret := allocAddrRange()
-		ret.Min.SetBytes(saddr)
-		ret.Max.SetBytes(saddr)
-		ret.Current.SetBytes(saddr)
+		ret := AddrRange{}
+		ret.Min = make([]byte, len(saddr))
+		copy(ret.Min, saddr)
+		ret.Max = make([]byte, len(saddr))
+		copy(ret.Max, saddr)
+		ret.Current = make([]byte, len(saddr))
+		copy(ret.Current, saddr)
 		return ret, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
@@ -680,14 +684,6 @@ func parseRawMACAddr(rawAddr string) ([]uint8, error) {
 	return addr, nil
 }
 
-func allocAddrRange() AddrRange {
-	addr := AddrRange{}
-	addr.Min = big.NewInt(0)
-	addr.Max = big.NewInt(0)
-	addr.Current = big.NewInt(0)
-	return addr
-}
-
 func parseIPAddr(value interface{}) (AddrRange, error) {
 	switch v2 := value.(type) {
 	case map[string]interface{}:
@@ -708,10 +704,13 @@ func parseIPAddr(value interface{}) (AddrRange, error) {
 		if err != nil {
 			return AddrRange{}, fmt.Errorf("parsing ip addr returned: %v", err)
 		}
-		ret := allocAddrRange()
-		ret.Min.SetBytes(saddr)
-		ret.Max.SetBytes(saddr)
-		ret.Current.SetBytes(saddr)
+		ret := AddrRange{}
+		ret.Min = make([]byte, len(saddr))
+		copy(ret.Min, saddr)
+		ret.Max = make([]byte, len(saddr))
+		copy(ret.Max, saddr)
+		ret.Current = make([]byte, len(saddr))
+		copy(ret.Current, saddr)
 		return ret, nil
 	}
 	return AddrRange{}, fmt.Errorf("unknown type")
@@ -720,7 +719,7 @@ func parseIPAddr(value interface{}) (AddrRange, error) {
 type fn func(string) ([]uint8, error)
 
 func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error) {
-	addr := allocAddrRange()
+	addr := AddrRange{}
 	wasMin, wasMax, wasStart, wasIncr := false, false, false, false
 	for k, v := range in {
 		switch strings.ToLower(k) {
@@ -729,27 +728,27 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing min returned: %v", err)
 			}
-			addr.Min = big.NewInt(0)
-			addr.Min.SetBytes(min)
+			addr.Min = make([]byte, len(min))
+			copy(addr.Min, min)
 			wasMin = true
 		case "max":
 			max, err := parseFunc(v.(string))
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing max returned: %v", err)
 			}
-			addr.Max = big.NewInt(0)
-			addr.Max.SetBytes(max)
+			addr.Max = make([]byte, len(max))
+			copy(addr.Max, max)
 			wasMax = true
 		case "start":
 			start, err := parseFunc(v.(string))
 			if err != nil {
 				return AddrRange{}, fmt.Errorf("parsing start returned: %v", err)
 			}
-			addr.Current = big.NewInt(0)
-			addr.Current.SetBytes(start)
+			addr.Current = make([]byte, len(start))
+			copy(addr.Current, start)
 			wasStart = true
 		case "incr":
-			addr.Incr = (uint64)(v.(float64))
+			addr.Incr = big.NewInt((int64)(v.(float64))).Bytes()
 			wasIncr = true
 		default:
 			return AddrRange{}, fmt.Errorf("unknown key %s", k)
@@ -758,19 +757,19 @@ func parseAddrRange(in map[string]interface{}, parseFunc fn) (AddrRange, error)
 	if !wasMax || !wasMin {
 		return AddrRange{}, fmt.Errorf("Min and max values should be given for range")
 	}
-	if addr.Max.Cmp(addr.Min) < 0 {
+	if bytes.Compare(addr.Max, addr.Min) < 0 {
 		return AddrRange{}, fmt.Errorf("Min value should be less than Max")
 	}
 	if !wasStart {
-		addr.Current = big.NewInt(0)
-		*(addr.Current) = *(addr.Min)
+		addr.Current = make([]byte, len(addr.Min))
+		copy(addr.Current, addr.Min)
 	}
 
-	if addr.Current.Cmp(addr.Min) < 0 || addr.Current.Cmp(addr.Max) > 0 {
-		return AddrRange{}, fmt.Errorf(fmt.Sprintf("Start value should be between min and max: start=%v, min=%v, max=%v", addr.Current.Bytes(), addr.Min.Bytes(), addr.Max.Bytes()))
+	if bytes.Compare(addr.Current, addr.Min) < 0 || bytes.Compare(addr.Current, addr.Max) > 0 {
+		return AddrRange{}, fmt.Errorf(fmt.Sprintf("Start value should be between min and max: start=%v, min=%v, max=%v", addr.Current, addr.Min, addr.Max))
 	}
 	if !wasIncr {
-		addr.Incr = 1
+		addr.Incr = []byte{1}
 	}
 	return addr, nil
 }
diff --git a/examples/nffPktgen/sendGetBack/sendGetBack.go b/examples/nffPktgen/sendGetBack/sendGetBack.go
index fa3ea22c..003831bd 100644
--- a/examples/nffPktgen/sendGetBack/sendGetBack.go
+++ b/examples/nffPktgen/sendGetBack/sendGetBack.go
@@ -136,7 +136,7 @@ func main() {
 		}
 		context, err := generator.GetContext(configuration)
 		flow.CheckFatal(err)
-		outFlow, err := flow.SetFastGenerator(generator.Generate, speed, &context)
+		outFlow, err := flow.SetFastGenerator(generator.Generate, speed, context)
 		flow.CheckFatal(err)
 		switch t := key.(type) {
 		case int:

From 6d2a0a84a7188a61c3dd7e2d026f9e3ad3e19039 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Mon, 13 Aug 2018 11:02:53 -0600
Subject: [PATCH 27/50] Core scheduler changes

Before
Reported from handler:
- current speed
Was used in heuristics:
- clone removing:    as vector of speeds for all number of clones, removed clone is current is less than saved
                     really unstable heuristic
- clone adding:      as speed for +1 clone. Do not add clone if saved increased speed is less than current
- instance removing: usage was wrong
- instance adding:   no usage

After
Reported from handler:
- current speed
Is used in heuristics:
- clone removing:    as "decreased speed". If current speed right after addition is less than saved - remove
- clone adding:      as "increased speed" in above - no changes
+
Reported from handler:
- current number of zero attempts to get packets from input ring
Is used in heuristics:
- clone removing:    if "summ of all zero attempts" * "time per attempt (previously calculated)" > "schedtime" - remove
- instance removing: 1) get number of zero attempts from the ring with less number of them at each instance
                     2) convert this number to time by multiplying each instance number to corresponding time per attempt
                     3) get two instances with the biggest times
                     4) if summ of thier times is bigger than schedTime - combine them at one core
Logic for removing is the following. Each core has useful time and vacant time.
Vacant time is determined as time for zero attempts. If summ of vacant times of two cores is more than summ of their
usefull times then one core will be able to handle tasks of both cores. They can be combined.
We need to measure "time per attempT" before using it - it is quite flexible.
---
 flow/flow.go      |  40 ++++-----
 flow/scheduler.go | 212 ++++++++++++++++++++++++++++++----------------
 2 files changed, 160 insertions(+), 92 deletions(-)

diff --git a/flow/flow.go b/flow/flow.go
index 4699d740..6b4c6d59 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -1009,7 +1009,7 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 		OutputMbufs[index] = make([]uintptr, burstSize)
 		countOfPackets[index] = 0
 	}
-	var currentSpeed reportPair
+	var currentState reportPair
 	var pause int
 	firstFunc := lp.firstFunc
 	// For scalar part
@@ -1043,11 +1043,11 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 				// For any events with this function we should restart timer
 				// We don't do it regularly without any events due to performance
 				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
-				currentSpeed = reportPair{}
+				currentState = reportPair{}
 			}
 		case <-tick.C:
-			report <- currentSpeed
-			currentSpeed = reportPair{}
+			report <- currentState
+			currentState = reportPair{}
 		default:
 			for q := int32(1); q < inIndex[0]+1; q++ {
 				n := IN[inIndex[q]].DequeueBurst(InputMbufs, burstSize)
@@ -1061,6 +1061,7 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 						for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
 						}
 					}
+					currentState.ZeroAttempts[q-1]++
 					continue
 				}
 				if scalar { // Scalar code
@@ -1074,7 +1075,7 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 								OutputMbufs[nextIndex][countOfPackets[nextIndex]] = InputMbufs[i]
 								countOfPackets[nextIndex]++
 								if reportMbits {
-									currentSpeed.Bytes += uint64(tempPacket.GetPacketLen())
+									currentState.V.Bytes += uint64(tempPacket.GetPacketLen())
 								}
 								break
 							}
@@ -1086,7 +1087,7 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 							continue
 						}
 						safeEnqueue(OUT[index][inIndex[q]], OutputMbufs[index], uint(countOfPackets[index]))
-						currentSpeed.Packets += uint64(countOfPackets[index])
+						currentState.V.Packets += uint64(countOfPackets[index])
 						countOfPackets[index] = 0
 					}
 				} else { // Vector code
@@ -1103,7 +1104,7 @@ func segmentProcess(parameters interface{}, inIndex []int32, stopper [2]chan int
 							// We have constructSlice -> put packets inside ring, it is an end of segment
 							count := FillSliceFromMask(InputMbufs, &def[st].mask, OutputMbufs[0])
 							safeEnqueue(OUT[answers[0]][inIndex[q]], OutputMbufs[0], uint(count))
-							currentSpeed.Packets += uint64(count)
+							currentState.V.Packets += uint64(count)
 						} else if cur.followingNumber == 1 {
 							// We have simple handle. Mask will remain the same, current function will be changed
 							def[st].f = cur.next[0]
@@ -1175,7 +1176,7 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 	bufs := make([]uintptr, burstSize)
 	var tempPacket *packet.Packet
 	tempPackets := make([]*packet.Packet, burstSize)
-	var currentSpeed reportPair
+	var currentState reportPair
 	var pause int
 	tick := time.NewTicker(time.Duration(schedTime) * time.Millisecond)
 	stopper[1] <- 2 // Answer that function is ready
@@ -1194,11 +1195,11 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 				// For any events with this function we should restart timer
 				// We don't do it regularly without any events due to performance
 				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
-				currentSpeed = reportPair{}
+				currentState = reportPair{}
 			}
 		case <-tick.C:
-			report <- currentSpeed
-			currentSpeed = reportPair{}
+			report <- currentState
+			currentState = reportPair{}
 		default:
 			err := low.AllocateMbufs(bufs, mempool, burstSize)
 			if err != nil {
@@ -1211,7 +1212,7 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 					tempPacket = packet.ExtractPacket(bufs[i])
 					generateFunction(tempPacket, context[0])
 					if reportMbits {
-						currentSpeed.Bytes += uint64(tempPacket.GetPacketLen())
+						currentState.V.Bytes += uint64(tempPacket.GetPacketLen())
 					}
 				}
 			} else {
@@ -1219,7 +1220,7 @@ func pFastGenerate(parameters interface{}, inIndex []int32, stopper [2]chan int,
 				vectorGenerateFunction(tempPackets, context[0])
 			}
 			safeEnqueue(OUT[0], bufs, burstSize)
-			currentSpeed.Packets += uint64(burstSize)
+			currentState.V.Packets += uint64(burstSize)
 			// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
 			// we just want to slow goroutine down without parking, so loop is OK for this.
 			// time.Now lasts approximately 70ns and this satisfies us
@@ -1244,7 +1245,7 @@ func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report
 	bufs2 := make([]uintptr, burstSize)
 	var tempPacket1 *packet.Packet
 	var tempPacket2 *packet.Packet
-	var currentSpeed reportPair
+	var currentState reportPair
 	var pause int
 	tick := time.NewTicker(time.Duration(schedTime) * time.Millisecond)
 	stopper[1] <- 2 // Answer that function is ready
@@ -1261,11 +1262,11 @@ func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report
 				// For any events with this function we should restart timer
 				// We don't do it regularly without any events due to performance
 				tick = time.NewTicker(time.Duration(schedTime) * time.Millisecond)
-				currentSpeed = reportPair{}
+				currentState = reportPair{}
 			}
 		case <-tick.C:
-			report <- currentSpeed
-			currentSpeed = reportPair{}
+			report <- currentState
+			currentState = reportPair{}
 		default:
 			for q := int32(1); q < inIndex[0]+1; q++ {
 				n := IN[inIndex[q]].DequeueBurst(bufs1, burstSize)
@@ -1279,17 +1280,18 @@ func pcopy(parameters interface{}, inIndex []int32, stopper [2]chan int, report
 						tempPacket2 = packet.ExtractPacket(bufs2[i])
 						packet.GeneratePacketFromByte(tempPacket2, tempPacket1.GetRawPacketBytes())
 						if reportMbits {
-							currentSpeed.Bytes += uint64(tempPacket1.GetPacketLen())
+							currentState.V.Bytes += uint64(tempPacket1.GetPacketLen())
 						}
 					}
 					safeEnqueue(OUT[inIndex[q]], bufs1, uint(n))
 					safeEnqueue(OUTCopy[inIndex[q]], bufs2, uint(n))
-					currentSpeed.Packets += uint64(n)
+					currentState.V.Packets += uint64(n)
 				}
 				// GO parks goroutines while Sleep. So Sleep lasts more time than our precision
 				// we just want to slow goroutine down without parking, so loop is OK for this.
 				// time.Now lasts approximately 70ns and this satisfies us
 				if pause != 0 {
+					currentState.ZeroAttempts[q-1]++
 					// pause should be non 0 only if function works with ONE inIndex
 					a := time.Now()
 					for time.Since(a) < time.Duration(pause*int(burstSize))*time.Nanosecond {
diff --git a/flow/scheduler.go b/flow/scheduler.go
index bc768e63..cf5675b6 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -42,11 +42,16 @@ const RSSCloneMin = 5
 const RSSCloneMax = 39
 
 // Tuple of current speed in packets and bytes
-type reportPair struct {
+type speedPair struct {
 	Packets uint64
 	Bytes   uint64
 }
 
+type reportPair struct {
+	V            speedPair
+	ZeroAttempts [32]uint64
+}
+
 // Clones of flow functions. They are determined
 // by their core and their stopper channel
 type clonePair struct {
@@ -56,19 +61,24 @@ type clonePair struct {
 }
 
 type instance struct {
+	// Indexes of input rings that are handle by this instance
+	// inIndex[0] - number of this input rings
 	inIndex []int32
 	// Clones of this instance
 	clone []*clonePair
-	// Channel for reporting current speed.
-	report       chan reportPair
-	currentSpeed reportPair
-	// Current saved speed when making a clone.
-	// It will be compared with immediate current speed to stop a clone.
-	previousSpeed []reportPair
+	// Channel for returning current state of this instance
+	report chan reportPair
+	// Current state of this instance, returned from working thread
+	reportedState reportPair
+	// Speed with "+1" number of clones. Set while removing clone
+	increasedSpeed uint64
+	// Speed with "-1" number of clones. Set while adding clone
+	decreasedSpeed uint64
 	// Number of clones of this instance
 	cloneNumber int
 	pause       int
 	ff          *flowFunction
+	removed     bool
 }
 
 // UserContext is used inside flow packet and is going for user via it
@@ -135,6 +145,9 @@ func (scheduler *scheduler) addFF(name string, ucfn uncloneFlowFunction, Cfn cFl
 	ff.context = context
 	ff.fType = fType
 	ff.inIndexNumber = inIndexNumber
+	if inIndexNumber > scheduler.maxInIndex {
+		scheduler.maxInIndex = inIndexNumber
+	}
 	scheduler.ff = append(scheduler.ff, ff)
 }
 
@@ -154,6 +167,10 @@ type scheduler struct {
 	stopFlag          int32
 	maxRecv           int
 	Timers            []*Timer
+	nAttempts         []uint64
+	pAttempts         []uint64
+	maxInIndex        int32
+	measureRings      low.Rings
 }
 
 type core struct {
@@ -172,13 +189,14 @@ func newScheduler(cpus []int, schedulerOff bool, schedulerOffRemove bool, stopDe
 	}
 	scheduler.off = schedulerOff
 	scheduler.offRemove = schedulerOff || schedulerOffRemove
-	scheduler.StopRing = stopRing
 	scheduler.stopDedicatedCore = stopDedicatedCore
+	scheduler.StopRing = stopRing
 	scheduler.checkTime = checkTime
 	scheduler.debugTime = debugTime
 	scheduler.maxPacketsToClone = maxPacketsToClone
 	scheduler.maxRecv = maxRecv
 	scheduler.anyway = anyway
+	scheduler.pAttempts = make([]uint64, len(scheduler.cores), len(scheduler.cores))
 
 	return scheduler
 }
@@ -209,8 +227,11 @@ func (scheduler *scheduler) systemStart() (err error) {
 			return err
 		}
 	}
-	// We need this to get a chance to all started goroutines to log their warnings.
-	time.Sleep(time.Millisecond)
+	scheduler.measureRings = low.CreateRings(burstSize*sizeMultiplier, scheduler.maxInIndex+1)
+	scheduler.nAttempts = make([]uint64, scheduler.maxInIndex+1, scheduler.maxInIndex+1)
+	for i := int32(1); i < scheduler.maxInIndex+1; i++ {
+		scheduler.nAttempts[i] = scheduler.measure(int32(i), 1)
+	}
 	return nil
 }
 
@@ -219,7 +240,6 @@ func (ff *flowFunction) startNewInstance(inIndex []int32, scheduler *scheduler)
 	common.LogDebug(common.Debug, "Start new instance for", ff.name)
 	ffi.inIndex = inIndex
 	ffi.report = make(chan reportPair, len(scheduler.cores)-1)
-	ffi.previousSpeed = make([]reportPair, len(scheduler.cores), len(scheduler.cores))
 	ffi.ff = ff
 	err = ffi.startNewClone(scheduler, ff.instanceNumber)
 	if err == nil {
@@ -279,6 +299,7 @@ func (ff *flowFunction) stopClone(ffi *instance, scheduler *scheduler) {
 	scheduler.setCoreByIndex(ffi.clone[ffi.cloneNumber-1].index)
 	ffi.clone = ffi.clone[:len(ffi.clone)-1]
 	ffi.cloneNumber--
+	ffi.removed = true
 }
 
 func (ff *flowFunction) stopInstance(from int, to int, scheduler *scheduler) {
@@ -361,6 +382,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 			checkRequired = true
 		case <-debugTick:
 			// Report current state of system
+			common.LogDebug(common.Debug, "---------------")
 			common.LogDebug(common.Debug, "System is using", scheduler.usedCores, "cores now.", uint8(len(scheduler.cores))-scheduler.usedCores, "cores are left available.")
 			low.Statistics(float32(scheduler.debugTime) / 1000)
 			for i := range scheduler.ff {
@@ -373,14 +395,13 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 				scheduler.Dropped = 0
 			}
 			low.ReportMempoolsState()
-			common.LogDebug(common.Debug, "---------------")
 		default:
 		}
 		// Procced with each flow function
 		for i := range scheduler.ff {
 			ff := scheduler.ff[i]
 			if ff.fType == segmentCopy || ff.fType == fastGenerate {
-				ff.updateCurrentSpeed()
+				ff.updateReportedState() // TODO also for debug
 			}
 			// Firstly we check removing clones. We can remove one clone if:
 			// 1. flow function has clones or it is fastGenerate
@@ -388,46 +409,44 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 			if scheduler.offRemove == false {
 				switch ff.fType {
 				case segmentCopy: // Both instances and clones
+					maxZeroAttempts0 := -1
+					maxZeroAttempts1 := -1
 					for q := 0; q < ff.instanceNumber; q++ {
 						ffi := ff.instance[q]
 						if ffi.cloneNumber > 1 {
-							// 3. current speed of flow function is lower, than saved speed with less number of clones
-							if ffi.currentSpeed.Packets < ffi.previousSpeed[ffi.cloneNumber-1].Packets {
+							ffi.reportedState.ZeroAttempts[0] = ffi.reportedState.ZeroAttempts[0] * scheduler.pAttempts[ffi.cloneNumber]
+							if ffi.reportedState.ZeroAttempts[0] > uint64(schedTime)*uint64(1000000*1.05) || ffi.decreasedSpeed > ffi.reportedState.V.Packets {
 								// Save current speed as speed of flow function with this number of clones before removing
-								ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
+								ffi.increasedSpeed = ffi.reportedState.V.Packets
+								ffi.decreasedSpeed = 0
 								ff.stopClone(ffi, scheduler)
 								ffi.updatePause(ffi.cloneNumber - 1)
 							}
-						}
-					}
-					// TODO this is wrong due to different speeds of different flows. Also we should take into account only
-					// instances without clones here.
-					if ff.instanceNumber > 1 {
-						min0 := 0
-						min1 := 1
-						if ff.instance[0].currentSpeed.Packets > ff.instance[1].currentSpeed.Packets {
-							min0 = 1
-							min1 = 0
-						}
-						for q := 2; q < ff.instanceNumber; q++ {
-							if ff.instance[q].currentSpeed.Packets < ff.instance[min0].currentSpeed.Packets {
-								min1 = min0
-								min0 = q
-								continue
+						} else {
+							for z := int32(0); z < ffi.inIndex[0]; z++ {
+								if ffi.reportedState.ZeroAttempts[z] < ffi.reportedState.ZeroAttempts[0] {
+									ffi.reportedState.ZeroAttempts[0] = ffi.reportedState.ZeroAttempts[z]
+								}
 							}
-							if ff.instance[q].currentSpeed.Packets < ff.instance[min1].currentSpeed.Packets {
-								min1 = q
+							ffi.reportedState.ZeroAttempts[0] = ffi.reportedState.ZeroAttempts[0] * scheduler.nAttempts[ffi.inIndex[0]]
+							if maxZeroAttempts0 == -1 || ffi.reportedState.ZeroAttempts[0] > ff.instance[maxZeroAttempts0].reportedState.ZeroAttempts[0] {
+								maxZeroAttempts1 = maxZeroAttempts0
+								maxZeroAttempts0 = q
+							} else if maxZeroAttempts1 == -1 || ffi.reportedState.ZeroAttempts[0] > ff.instance[maxZeroAttempts1].reportedState.ZeroAttempts[0] {
+								maxZeroAttempts1 = q
 							}
 						}
-						if ff.instance[min0].currentSpeed.Packets+ff.instance[min1].currentSpeed.Packets < ff.oneCoreSpeed.Packets {
-							toInstance := ff.instance[min1]
-							ff.stopInstance(min0, min1, scheduler)
+					}
+					if maxZeroAttempts0 != -1 && maxZeroAttempts1 != -1 && ff.instance[maxZeroAttempts1].reportedState.ZeroAttempts[0] != 0 {
+						if ff.instance[maxZeroAttempts0].reportedState.ZeroAttempts[0]+ff.instance[maxZeroAttempts1].reportedState.ZeroAttempts[0] > uint64(schedTime)*uint64(1.05*1000000) {
+							toInstance := ff.instance[maxZeroAttempts1]
+							ff.stopInstance(maxZeroAttempts0, maxZeroAttempts1, scheduler)
 							toInstance.updatePause(0)
 						}
 					}
 				case fastGenerate: // Only clones, no instances
 					targetSpeed := (ff.Parameters.(*generateParameters)).targetSpeed
-					speedPKTS := float64(ff.instance[0].currentSpeed.normalize(schedTime).Packets)
+					speedPKTS := float64(ff.instance[0].reportedState.V.normalize(schedTime).Packets)
 					// 3. Current speed is much bigger than target speed
 					if speedPKTS > 1.1*targetSpeed {
 						ffi := ff.instance[0]
@@ -482,49 +501,46 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 					a := true
 					for q := 0; q < l; q++ {
 						ffi := ff.instance[q]
-						if ffi.checkInputRingClonable(scheduler.maxPacketsToClone) {
-							if ffi.inIndex[0] > 1 {
-								ff.oneCoreSpeed = ffi.currentSpeed
-								if ff.startNewInstance(constructZeroIndex(ffi.inIndex), scheduler) == nil {
-									constructDuplicatedIndex(ffi.inIndex, ff.instance[ff.instanceNumber-1].inIndex)
-									ffi.updatePause(0)
-									a = false
-								}
+						if ffi.inIndex[0] > 1 && ffi.checkInputRingClonable(scheduler.maxPacketsToClone) {
+							if ff.startNewInstance(constructZeroIndex(ffi.inIndex), scheduler) == nil {
+								constructDuplicatedIndex(ffi.inIndex, ff.instance[ff.instanceNumber-1].inIndex)
+								a = false
+								ffi.updatePause(0)
 							}
 						}
 					}
 					if a == true {
 						for q := 0; q < l; q++ {
 							ffi := ff.instance[q]
-							if ffi.checkInputRingClonable(scheduler.maxPacketsToClone) {
-								if ffi.inIndex[0] == 1 && scheduler.anyway && ffi.checkOutputRingClonable(scheduler.maxPacketsToClone) &&
-									(ffi.previousSpeed[ffi.cloneNumber+1].Packets == 0 ||
-										ffi.previousSpeed[ffi.cloneNumber+1].Packets > ffi.currentSpeed.Packets) {
-									// Save current speed as speed of flow function with this number of clones before adding
-									ffi.previousSpeed[ffi.cloneNumber] = ffi.currentSpeed
-									if ffi.startNewClone(scheduler, q) == nil {
-										// Add a pause to all clones. This pause depends on the number of clones.
-										ffi.updatePause(ffi.cloneNumber - 1)
-									}
+							if ffi.removed {
+								ffi.removed = false
+								continue
+							}
+							if ffi.inIndex[0] == 1 && scheduler.anyway && ffi.checkInputRingClonable(scheduler.maxPacketsToClone) &&
+								ffi.checkOutputRingClonable(scheduler.maxPacketsToClone) &&
+								(ffi.increasedSpeed == 0 || ffi.increasedSpeed > ffi.reportedState.V.Packets) {
+								if scheduler.pAttempts[ffi.cloneNumber+1] == 0 {
+									scheduler.pAttempts[ffi.cloneNumber+1] = scheduler.measure(1, ffi.cloneNumber+1)
+								}
+								if ffi.startNewClone(scheduler, q) == nil {
+									ffi.decreasedSpeed = ffi.reportedState.V.Packets
+									ffi.increasedSpeed = 0
+									ffi.updatePause(ffi.cloneNumber - 1)
 									continue
 								}
 							}
-							// Scheduler can't add new clones if saved flow function speed with more
+							// Scheduler can't add new clones if saved instance speed with more
 							// clones is slower than current. However this speed can be changed due to
-							// external reasons. So flow function should check and update this speed
-							// regular time.
+							// external reasons. So flow function should "forget" this speed regularly
+							// to try to clone function and get new increasedSpeed.
 							if checkRequired == true {
-								// Regular flow function speed check after each checkTime milliseconds,
-								// it is done by erasing previously measured speed data. After this scheduler can
-								// add new clone. If performance decreases with added clone, it will be stopped
-								// with setting speed data.
-								ffi.previousSpeed[ffi.cloneNumber+1].Packets = 0
+								ffi.increasedSpeed = 0
 							}
 						}
 					}
 				case fastGenerate: // Only clones, no instances
 					// 3. speed is not enough
-					if float64(ff.instance[0].currentSpeed.normalize(schedTime).Packets) < (ff.Parameters.(*generateParameters)).targetSpeed {
+					if float64(ff.instance[0].reportedState.V.normalize(schedTime).Packets) < (ff.Parameters.(*generateParameters)).targetSpeed {
 						if ff.instance[0].pause != 0 {
 							ff.instance[0].updatePause(int((1 - generatePauseStep) * float64(ff.instance[0].pause)))
 						} else if ff.instance[0].checkOutputRingClonable(scheduler.maxPacketsToClone) {
@@ -581,7 +597,7 @@ func (ff *flowFunction) printDebug(schedTime uint) {
 	switch ff.fType {
 	case segmentCopy:
 		for q := 0; q < ff.instanceNumber; q++ {
-			out := ff.instance[q].currentSpeed.normalize(schedTime)
+			out := ff.instance[q].reportedState.V.normalize(schedTime)
 			if reportMbits {
 				common.LogDebug(common.Debug, "Current speed of", q, "instance of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s")
 			} else {
@@ -590,7 +606,7 @@ func (ff *flowFunction) printDebug(schedTime uint) {
 		}
 	case fastGenerate:
 		targetSpeed := (ff.Parameters.(*generateParameters)).targetSpeed
-		out := ff.instance[0].currentSpeed.normalize(schedTime)
+		out := ff.instance[0].reportedState.V.normalize(schedTime)
 		if reportMbits {
 			common.LogDebug(common.Debug, "Current speed of", ff.name, "is", out.Packets, "PKT/S,", out.Bytes, "Mbits/s, target speed is", uint64(targetSpeed), "PKT/S")
 		} else {
@@ -600,28 +616,33 @@ func (ff *flowFunction) printDebug(schedTime uint) {
 	}
 }
 
-func (current reportPair) normalize(schedTime uint) reportPair {
+func (V speedPair) normalize(schedTime uint) speedPair {
 	// We should multiply by 1000 because schedTime is in milliseconds
 	// We multiply by 8 to translate bytes to bits
 	// We add 24 because 4 is checksum + 20 is before/after packet gaps in 40GB ethernet
 	// We divide by 1000 and 1000 because networking suppose that megabit has 1000 kilobits instead of 1024
-	return reportPair{current.Packets * 1000 / uint64(schedTime), (current.Bytes + current.Packets*24) * 1000 / uint64(schedTime) * 8 / 1000 / 1000}
+	return speedPair{V.Packets * 1000 / uint64(schedTime), (V.Bytes + V.Packets*24) * 1000 / uint64(schedTime) * 8 / 1000 / 1000}
 }
 
-func (ff *flowFunction) updateCurrentSpeed() {
+func (ff *flowFunction) updateReportedState() {
 	// Gather and sum current speeds from all clones of current flow function
 	// Flow function itself and all clones put their speeds in one channel
 	for q := 0; q < ff.instanceNumber; q++ {
 		ffi := ff.instance[q]
-		ffi.currentSpeed = reportPair{}
+		ffi.reportedState = reportPair{}
 		t := len(ffi.report) - ffi.cloneNumber
 		for k := 0; k < t; k++ {
 			<-ffi.report
 		}
 		for k := 0; k < ffi.cloneNumber; k++ {
 			temp := <-ffi.report
-			ffi.currentSpeed.Packets += temp.Packets
-			ffi.currentSpeed.Bytes += temp.Bytes
+			ffi.reportedState.V.Packets += temp.V.Packets
+			ffi.reportedState.V.Bytes += temp.V.Bytes
+			if ff.fType == segmentCopy {
+				for z := int32(0); z < ffi.inIndex[0]; z++ {
+					ffi.reportedState.ZeroAttempts[z] += temp.ZeroAttempts[z]
+				}
+			}
 		}
 		// If any flow functions wait in a queue to put their
 		// reports immidiately after reading- we should remove them.
@@ -732,3 +753,48 @@ func constructNewIndex(inIndexNumber int32) []int32 {
 	newIndex[0] = inIndexNumber
 	return newIndex
 }
+
+func (scheduler *scheduler) measure(N int32, clones int) uint64 {
+	core, index, err := scheduler.getCore()
+	if err != nil {
+		return 0
+	}
+	par := new(segmentParameters)
+	par.in = scheduler.measureRings
+	out := make([]low.Rings, 0, 0)
+	par.out = &out
+	stype := uint8(0)
+	par.stype = &stype
+	inIndex := constructNewIndex(N)
+	var stopper [2]chan int
+	stopper[0] = make(chan int)
+	stopper[1] = make(chan int)
+	report := make(chan reportPair, 20)
+	var avg uint64
+	t := schedTime
+	pause := clones - 1
+	if pause == 0 {
+		schedTime = 5
+	} else {
+		schedTime = 15
+	}
+	for o := 0; o < 5; o++ {
+		go func() {
+			low.SetAffinity(core)
+			segmentProcess(par, inIndex, stopper, report, nil)
+		}()
+		<-stopper[1]
+		stopper[0] <- pause
+		reportedState := <-report
+		reportedState = <-report
+		stopper[0] <- -1
+		<-stopper[1]
+		avg += uint64(schedTime) * 1000000 / reportedState.ZeroAttempts[0]
+	}
+	scheduler.setCoreByIndex(index)
+	schedTime = t
+	close(stopper[0])
+	close(stopper[1])
+	close(report)
+	return avg / 5
+}

From 014ae7457a40e3731f6971d4ec36332c1c7d5299 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Fri, 17 Aug 2018 16:35:07 -0500
Subject: [PATCH 28/50] Improve output of Ether.String() function with ethernet
 protocol name

---
 packet/packet.go | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/packet/packet.go b/packet/packet.go
index c6d6b4ec..662e2a93 100644
--- a/packet/packet.go
+++ b/packet/packet.go
@@ -75,10 +75,31 @@ type EtherHdr struct {
 }
 
 func (hdr *EtherHdr) String() string {
-	r0 := fmt.Sprintf("L2 protocol: Ethernet\nEtherType: 0x%02x\n", hdr.EtherType)
-	r1 := "Ethernet Source: " + MACToString(hdr.SAddr) + "\n"
-	r2 := "Ethernet Source: " + MACToString(hdr.DAddr) + "\n"
-	return r0 + r1 + r2
+	return fmt.Sprintf(`L2 protocol: Ethernet, EtherType: 0x%04x (%s)
+Ethernet Source: %s
+Ethernet Destination: %s
+`,
+		hdr.EtherType, getEtherTypeName(hdr.EtherType),
+		MACToString(hdr.SAddr),
+		MACToString(hdr.DAddr))
+}
+
+var (
+	etherTypeNameLookupTable = map[uint16]string{
+		SwapIPV4Number: "IPv4",
+		SwapARPNumber:  "ARP",
+		SwapVLANNumber: "VLAN",
+		SwapMPLSNumber: "MPLS",
+		SwapIPV6Number: "IPv6",
+	}
+)
+
+func getEtherTypeName(et uint16) string {
+	ret, ok := etherTypeNameLookupTable[et]
+	if !ok {
+		return "unknown"
+	}
+	return ret
 }
 
 // MACToString return MAC address like string

From f7ac24320d17ecf8910c9f04218975f5bd825863 Mon Sep 17 00:00:00 2001
From: Maria Bulatova <maria.bulatova@intel.com>
Date: Wed, 22 Aug 2018 17:42:38 +0300
Subject: [PATCH 29/50] Switch off dpi from CI build

due to https://github.com/flier/gohs/issues/12
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 0ee9041f..78ee1034 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ script:
   # Build standalone examples
   - docker exec -i test-nff-go bash -c "cd examples && make gopacketParserExample && cd .."
   - docker exec -i test-nff-go bash -c "cd examples && make nffPktgen && cd -"
-  - docker exec -i test-nff-go make -C examples/dpi
+  #- docker exec -i test-nff-go make -C examples/dpi
   # Run unit tests
   - docker exec -i test-nff-go sysctl -w vm.nr_hugepages=1024
   - docker exec -i test-nff-go sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'

From ac9bec283658bafd3b068a4b302168b4abdc1124 Mon Sep 17 00:00:00 2001
From: Kolistratova <darkol1312@gmail.com>
Date: Thu, 23 Aug 2018 20:42:54 +0300
Subject: [PATCH 30/50] Fixed dpi with hyperscan tag

---
 .travis.yml                   | 2 +-
 examples/dpi/Makefile         | 2 ++
 examples/dpi/main/Makefile    | 2 +-
 examples/dpi/pattern/Makefile | 2 +-
 mk/leaf.mk                    | 2 +-
 5 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 78ee1034..0ee9041f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ script:
   # Build standalone examples
   - docker exec -i test-nff-go bash -c "cd examples && make gopacketParserExample && cd .."
   - docker exec -i test-nff-go bash -c "cd examples && make nffPktgen && cd -"
-  #- docker exec -i test-nff-go make -C examples/dpi
+  - docker exec -i test-nff-go make -C examples/dpi
   # Run unit tests
   - docker exec -i test-nff-go sysctl -w vm.nr_hugepages=1024
   - docker exec -i test-nff-go sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
diff --git a/examples/dpi/Makefile b/examples/dpi/Makefile
index 3d33897b..dafe4ca5 100644
--- a/examples/dpi/Makefile
+++ b/examples/dpi/Makefile
@@ -5,4 +5,6 @@
 PATH_TO_MK = ../../mk
 SUBDIRS = main pattern
 
+export GO_COMPILE_FLAGS += -tags hyperscan_v4
+
 include $(PATH_TO_MK)/intermediate.mk
diff --git a/examples/dpi/main/Makefile b/examples/dpi/main/Makefile
index 3c3cc54b..65919230 100644
--- a/examples/dpi/main/Makefile
+++ b/examples/dpi/main/Makefile
@@ -9,7 +9,7 @@ EXECUTABLES = dpi
 dpi: ../pattern gohs
 
 gohs:
-	go get -v github.com/flier/gohs/hyperscan
+	go get $(GO_COMPILE_FLAGS) -v github.com/flier/gohs/hyperscan
 
 COMMON_FILES = handlers.go
 
diff --git a/examples/dpi/pattern/Makefile b/examples/dpi/pattern/Makefile
index f8f977bf..56a177a8 100644
--- a/examples/dpi/pattern/Makefile
+++ b/examples/dpi/pattern/Makefile
@@ -6,4 +6,4 @@ PATH_TO_MK = ../../../mk
 IMAGENAME = dpi
 
 pattern:
-	go install
+	go install $(GO_COMPILE_FLAGS)
diff --git a/mk/leaf.mk b/mk/leaf.mk
index c1228f9a..9443300d 100644
--- a/mk/leaf.mk
+++ b/mk/leaf.mk
@@ -12,7 +12,7 @@ include $(PATH_TO_MK)/include.mk
 
 ifdef NFF_GO_DEBUG
 # Flags to build Go files without optimizations
-export GO_COMPILE_FLAGS=-gcflags '-N -l'
+export GO_COMPILE_FLAGS += -gcflags '-N -l'
 endif
 
 $(EXECUTABLES) : % : %.go

From 7ab61f47db587ddca1ff943b717d378e15528db9 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Wed, 15 Aug 2018 20:49:50 +0000
Subject: [PATCH 31/50] Added KNI, port forwarding and full ARP support to NAT

- Cosmetic code refactoring into smaller separate functions
- Implemented KNI and port forwarding config
- Added creation of KNI devices and flows (no traffic forwarded yet)
- Implemented port forwarding (including KNI). It works to ssh from public interface.
- Direct all ARP and ICMP traffic to KNI if it is present on interface
- Fixed handling of ICMP and directing all ICMP traffic to KNI if it is present
- Fixed directing traffic to KNI in private subnet
- Implemented ARP requests when target destination MAC is unknown
- Fixed getting ARP response when KNI is enabled on interface
- Fixed directing traffic to KNI when it originates from NAT host
- Fixed printing IPv4 packet address and header
- Switched configuration to multiple hosts in local subnet
- Fixed bug with forwarded port reuse
---
 common/common.go                    |   6 +-
 examples/nat/config.go              | 305 +++++++++++++++++-----
 examples/nat/main/config-kni.json   |  50 ++++
 examples/nat/main/config-vlan.json  |   3 +-
 examples/nat/main/config.json       |  25 +-
 examples/nat/main/config2ports.json |   6 +-
 examples/nat/main/nat.go            |   1 +
 examples/nat/portalloc.go           |  12 +-
 examples/nat/translation.go         | 384 +++++++++++++++++-----------
 examples/nat/util.go                |  72 ++----
 flow/flow.go                        |   4 +-
 packet/packet.go                    |  10 +-
 vagrant/Vagrantfile                 |  29 +--
 13 files changed, 607 insertions(+), 300 deletions(-)
 create mode 100644 examples/nat/main/config-kni.json

diff --git a/common/common.go b/common/common.go
index eb5a1d06..f3f5923a 100644
--- a/common/common.go
+++ b/common/common.go
@@ -92,11 +92,11 @@ const (
 	// No - no output even after fatal errors
 	No LogType = 1 << iota
 	// Initialization - output during system initialization
-	Initialization
+	Initialization = 2
 	// Debug - output during execution one time per time period (scheduler ticks)
-	Debug
+	Debug = 4
 	// Verbose - output during execution as soon as something happens. Can influence performance
-	Verbose
+	Verbose = 8
 )
 
 // TCPFlags contains set TCP flags.
diff --git a/examples/nat/config.go b/examples/nat/config.go
index bb8d354e..0cff06e4 100644
--- a/examples/nat/config.go
+++ b/examples/nat/config.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -7,7 +7,6 @@ package nat
 import (
 	"encoding/json"
 	"errors"
-	"log"
 	"net"
 	"os"
 	"strconv"
@@ -16,6 +15,7 @@ import (
 
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
 )
 
 type terminationDirection uint8
@@ -28,15 +28,70 @@ const (
 	iPUBLIC  interfaceType = 0
 	iPRIVATE interfaceType = 1
 
+	dirDROP = 0
+	dirSEND = 1
+	dirKNI  = 2
+
 	connectionTimeout time.Duration = 1 * time.Minute
 	portReuseTimeout  time.Duration = 1 * time.Second
 )
 
+type hostPort struct {
+	Addr uint32
+	Port uint16
+}
+
+type protocolId uint8
+
+type forwardedPort struct {
+	Port         uint16     `json:"port"`
+	Destination  hostPort   `json:"destination"`
+	Protocol     protocolId `json:"protocol"`
+	forwardToKNI bool
+}
+
+var protocolIdLookup map[string]protocolId = map[string]protocolId{
+	"TCP": common.TCPNumber,
+	"UDP": common.UDPNumber,
+}
+
+func (out *protocolId) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	result, ok := protocolIdLookup[s]
+	if !ok {
+		return errors.New("Bad protocol name: " + s)
+	}
+
+	*out = result
+	return nil
+}
+
 type ipv4Subnet struct {
 	Addr uint32
 	Mask uint32
 }
 
+func (subnet *ipv4Subnet) String() string {
+	// Count most significant set bits
+	mask := uint32(1) << 31
+	i := 0
+	for ; i <= 32; i++ {
+		if subnet.Mask&mask == 0 {
+			break
+		}
+		mask >>= 1
+	}
+	return packet.IPv4ToString(subnet.Addr) + "/" + strconv.Itoa(i)
+}
+
+func (subnet *ipv4Subnet) checkAddrWithingSubnet(addr uint32) bool {
+	return addr&subnet.Mask == subnet.Addr&subnet.Mask
+}
+
 type macAddress [common.EtherAddrLen]uint8
 
 type portMapEntry struct {
@@ -44,35 +99,39 @@ type portMapEntry struct {
 	addr                 uint32
 	finCount             uint8
 	terminationDirection terminationDirection
+	static               bool
 }
 
 // Type describing a network port
 type ipv4Port struct {
-	Index         uint16     `json:"index"`
-	DstMACAddress macAddress `json:"dst-mac"`
-	Subnet        ipv4Subnet `json:"subnet"`
-	Vlan          uint16     `json:"vlan-tag"`
+	Index         uint16          `json:"index"`
+	Subnet        ipv4Subnet      `json:"subnet"`
+	Vlan          uint16          `json:"vlan-tag"`
+	KNIName       string          `json:"kni-name"`
+	ForwardPorts  []forwardedPort `json:"forward-ports"`
 	SrcMACAddress macAddress
+	Type          interfaceType
+	// Pointer to an opposite port in a pair
+	opposite *ipv4Port
+	// Map of allocated IP ports on public interface
+	portmap [][]portMapEntry
+	// Main lookup table which contains entries for packets coming at this port
+	translationTable []*sync.Map
 	// ARP lookup table
-	ArpTable sync.Map
+	arpTable sync.Map
+	// Debug dump stuff
+	fdump    [dirKNI + 1]*os.File
+	dumpsync [dirKNI + 1]sync.Mutex
 }
 
 // Config for one port pair.
 type portPair struct {
 	PrivatePort ipv4Port `json:"private-port"`
 	PublicPort  ipv4Port `json:"public-port"`
-	// Main lookup table which contains entries for private to public translation
-	pri2pubTable []sync.Map
-	// Main lookup table which contains entries for public to private translation
-	pub2priTable []sync.Map
 	// Synchronization point for lookup table modifications
 	mutex sync.Mutex
-	// Map of allocated IP ports on public interface
-	portmap [][]portMapEntry
 	// Port that was allocated last
 	lastport int
-	// Maximum allowed port number
-	maxport int
 }
 
 // Config for NAT.
@@ -94,17 +153,11 @@ var (
 	// HWTXChecksum is a flag whether checksums calculation should be
 	// offloaded to HW.
 	NoHWTXChecksum bool
+	NeedKNI        bool
 
 	// Debug variables
 	debugDump = false
 	debugDrop = false
-	// Controls whether debug dump files are separate for private and
-	// public interface or both traces are dumped in the same file.
-	dumptogether = false
-	fdump        []*os.File
-	dumpsync     []sync.Mutex
-	fdrop        []*os.File
-	dropsync     []sync.Mutex
 )
 
 func (pi pairIndex) Copy() interface{} {
@@ -155,6 +208,41 @@ func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
 	return errors.New("Failed to parse address " + s)
 }
 
+// UnmarshalJSON parses ipv4 host:port string. Port may be omitted and
+// is set to zero in this case.
+func (out *hostPort) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	hostStr, portStr, err := net.SplitHostPort(s)
+	if err != nil {
+		return err
+	}
+
+	ipArray := net.ParseIP(hostStr)
+	if ipArray == nil {
+		return errors.New("Bad IPv4 address specified: " + hostStr)
+	}
+	out.Addr, err = convertIPv4(ipArray.To4())
+	if err != nil {
+		return err
+	}
+
+	if portStr != "" {
+		port, err := strconv.ParseInt(portStr, 10, 32)
+		if err != nil {
+			return err
+		}
+		out.Port = uint16(port)
+	} else {
+		out.Port = 0
+	}
+
+	return nil
+}
+
 // UnmarshalJSON parses MAC address.
 func (out *macAddress) UnmarshalJSON(b []byte) error {
 	var s string
@@ -186,6 +274,10 @@ func ReadConfig(fileName string) error {
 
 	for i := range Natconfig.PortPairs {
 		pp := &Natconfig.PortPairs[i]
+
+		pp.PrivatePort.Type = iPRIVATE
+		pp.PublicPort.Type = iPUBLIC
+
 		if pp.PrivatePort.Vlan == 0 && pp.PublicPort.Vlan != 0 {
 			return errors.New("Private port with index " +
 				strconv.Itoa(int(pp.PrivatePort.Index)) +
@@ -199,6 +291,43 @@ func ReadConfig(fileName string) error {
 				strconv.Itoa(int(pp.PublicPort.Index)) +
 				" has zero vlan tag. Transition between VLAN-enabled and VLAN-disabled networks is not supported yet.")
 		}
+
+		port := &pp.PrivatePort
+		opposite := &pp.PublicPort
+		for pi := 0; pi < 2; pi++ {
+			for fpi := range port.ForwardPorts {
+				fp := &port.ForwardPorts[fpi]
+				if fp.Destination.Addr == 0 {
+					if port.KNIName == "" {
+						return errors.New("Port with index " +
+							strconv.Itoa(int(port.Index)) +
+							" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0")
+					}
+					fp.forwardToKNI = true
+					if fp.Destination.Port != fp.Port {
+						return errors.New("When address 0.0.0.0 is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
+							strconv.Itoa(int(fp.Port)) + " and " +
+							strconv.Itoa(int(fp.Destination.Port)))
+					}
+					NeedKNI = true
+				} else {
+					if pi == 0 {
+						return errors.New("Only KNI port forwarding is allowed on private port. All translated connections from private to public network can be initiated without any forwarding rules.")
+					}
+					if !opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr) {
+						return errors.New("Destination address " +
+							packet.IPv4ToString(fp.Destination.Addr) +
+							" should be within subnet " +
+							opposite.Subnet.String())
+					}
+					if fp.Destination.Port == 0 {
+						fp.Destination.Port = fp.Port
+					}
+				}
+			}
+			port = &pp.PublicPort
+			opposite = &pp.PrivatePort
+		}
 	}
 
 	return nil
@@ -210,18 +339,43 @@ func (pp *portPair) initLocalMACs() {
 	pp.PrivatePort.SrcMACAddress = flow.GetPortMACAddress(pp.PrivatePort.Index)
 }
 
-func (pp *portPair) allocatePortMap() {
-	pp.maxport = portEnd
-	pp.portmap = make([][]portMapEntry, common.UDPNumber+1)
-	pp.portmap[common.ICMPNumber] = make([]portMapEntry, pp.maxport)
-	pp.portmap[common.TCPNumber] = make([]portMapEntry, pp.maxport)
-	pp.portmap[common.UDPNumber] = make([]portMapEntry, pp.maxport)
-	pp.lastport = portStart
+func (port *ipv4Port) allocatePublicPortPortMap() {
+	port.portmap = make([][]portMapEntry, common.UDPNumber+1)
+	port.portmap[common.ICMPNumber] = make([]portMapEntry, portEnd)
+	port.portmap[common.TCPNumber] = make([]portMapEntry, portEnd)
+	port.portmap[common.UDPNumber] = make([]portMapEntry, portEnd)
 }
 
-func (pp *portPair) allocateLookupMap() {
-	pp.pri2pubTable = make([]sync.Map, common.UDPNumber+1)
-	pp.pub2priTable = make([]sync.Map, common.UDPNumber+1)
+func (port *ipv4Port) allocateLookupMap() {
+	port.translationTable = make([]*sync.Map, common.UDPNumber+1)
+	for i := range port.translationTable {
+		port.translationTable[i] = new(sync.Map)
+	}
+}
+
+func (port *ipv4Port) initPublicPortPortForwardingEntries() {
+	// Initialize port forwarding rules on public interface
+	for _, fp := range port.ForwardPorts {
+		keyEntry := Tuple{
+			addr: port.Subnet.Addr,
+			port: fp.Port,
+		}
+		valEntry := Tuple{
+			addr: fp.Destination.Addr,
+			port: fp.Destination.Port,
+		}
+		port.translationTable[fp.Protocol].Store(keyEntry, valEntry)
+		if fp.Destination.Addr != 0 {
+			port.opposite.translationTable[fp.Protocol].Store(valEntry, keyEntry)
+		}
+		port.portmap[fp.Protocol][fp.Port] = portMapEntry{
+			lastused:             time.Now(),
+			addr:                 fp.Destination.Addr,
+			finCount:             0,
+			terminationDirection: 0,
+			static:               true,
+		}
+	}
 }
 
 // InitFlows initializes flow graph for all interface pairs.
@@ -229,54 +383,85 @@ func InitFlows() {
 	for i := range Natconfig.PortPairs {
 		pp := &Natconfig.PortPairs[i]
 
+		pp.PublicPort.opposite = &pp.PrivatePort
+		pp.PrivatePort.opposite = &pp.PublicPort
+
 		// Init port pairs state
 		pp.initLocalMACs()
-		pp.allocatePortMap()
-		pp.allocateLookupMap()
+		pp.PrivatePort.allocateLookupMap()
+		pp.PublicPort.allocateLookupMap()
+		pp.PublicPort.allocatePublicPortPortMap()
+		pp.lastport = portStart
+		pp.PublicPort.initPublicPortPortForwardingEntries()
 
 		// Handler context with handler index
 		context := new(pairIndex)
 		context.index = i
 
+		var fromPubKNI, fromPrivKNI, toPub, toPriv *flow.Flow
+		var pubKNI, privKNI *flow.Kni
+		var outsPub = uint(2)
+		var outsPriv = uint(2)
+
 		// Initialize public to private flow
 		publicToPrivate, err := flow.SetReceiver(pp.PublicPort.Index)
-		if err != nil {
-			log.Fatal(err)
+		flow.CheckFatal(err)
+		if pp.PublicPort.KNIName != "" {
+			outsPub = 3
 		}
-		if flow.SetHandlerDrop(publicToPrivate, PublicToPrivateTranslation, context) != nil {
-			log.Fatal(err)
+		pubTranslationOut, err := flow.SetSplitter(publicToPrivate, PublicToPrivateTranslation, outsPub, context)
+		flow.CheckFatal(err)
+		flow.CheckFatal(flow.SetStopper(pubTranslationOut[dirDROP]))
+
+		// Initialize public KNI interface if requested
+		if pp.PublicPort.KNIName != "" {
+			pubKNI, err = flow.CreateKniDevice(pp.PublicPort.Index, pp.PublicPort.KNIName)
+			flow.CheckFatal(err)
+			flow.CheckFatal(flow.SetSenderKNI(pubTranslationOut[dirKNI], pubKNI))
+			fromPubKNI = flow.SetReceiverKNI(pubKNI)
 		}
 
 		// Initialize private to public flow
 		privateToPublic, err := flow.SetReceiver(pp.PrivatePort.Index)
-		if err != nil {
-			log.Fatal(err)
+		flow.CheckFatal(err)
+		if pp.PrivatePort.KNIName != "" {
+			outsPriv = 3
 		}
-		if flow.SetHandlerDrop(privateToPublic, PrivateToPublicTranslation, context) != nil {
-			log.Fatal(err)
+		privTranslationOut, err := flow.SetSplitter(privateToPublic, PrivateToPublicTranslation, outsPriv, context)
+		flow.CheckFatal(err)
+		flow.CheckFatal(flow.SetStopper(privTranslationOut[dirDROP]))
+
+		// Initialize private KNI interface if requested
+		if pp.PrivatePort.KNIName != "" {
+			privKNI, err = flow.CreateKniDevice(pp.PrivatePort.Index, pp.PrivatePort.KNIName)
+			flow.CheckFatal(err)
+			flow.CheckFatal(flow.SetSenderKNI(privTranslationOut[dirKNI], privKNI))
+			fromPrivKNI = flow.SetReceiverKNI(privKNI)
 		}
 
-		err = flow.SetSender(publicToPrivate, pp.PrivatePort.Index)
-		if err != nil {
-			log.Fatal(err)
+		// Merge traffic coming from public KNI with translated
+		// traffic from private side
+		if fromPubKNI != nil {
+			toPub, err = flow.SetMerger(fromPubKNI, privTranslationOut[dirSEND])
+			flow.CheckFatal(err)
+		} else {
+			toPub = privTranslationOut[dirSEND]
 		}
-		err = flow.SetSender(privateToPublic, pp.PublicPort.Index)
-		if err != nil {
-			log.Fatal(err)
+
+		// Merge traffic coming from private KNI with translated
+		// traffic from public side
+		if fromPrivKNI != nil {
+			toPriv, err = flow.SetMerger(fromPrivKNI, pubTranslationOut[dirSEND])
+			flow.CheckFatal(err)
+		} else {
+			toPriv = pubTranslationOut[dirSEND]
 		}
-	}
 
-	asize := len(Natconfig.PortPairs)
-	if !dumptogether {
-		asize *= 2
-	}
-	if debugDump {
-		fdump = make([]*os.File, asize)
-		dumpsync = make([]sync.Mutex, asize)
-	}
-	if debugDrop {
-		fdrop = make([]*os.File, asize)
-		dropsync = make([]sync.Mutex, asize)
+		// Set senders to output packets
+		err = flow.SetSender(toPriv, pp.PrivatePort.Index)
+		flow.CheckFatal(err)
+		err = flow.SetSender(toPub, pp.PublicPort.Index)
+		flow.CheckFatal(err)
 	}
 }
 
diff --git a/examples/nat/main/config-kni.json b/examples/nat/main/config-kni.json
new file mode 100644
index 00000000..e5ad5272
--- /dev/null
+++ b/examples/nat/main/config-kni.json
@@ -0,0 +1,50 @@
+{
+    "port-pairs": [
+        {
+            "private-port": {
+                "index": 0,
+                "subnet": "192.168.14.1/24",
+                "kni-name": "priv0",
+                "forward-ports": [
+                    {
+                        "port": 22,
+                        "destination": "0.0.0.0:22",
+                        "protocol": "TCP"
+                    }
+                ]
+            },
+            "public-port": {
+                "index": 1,
+                "subnet": "192.168.16.1/24",
+                "kni-name": "pub1",
+                "forward-ports": [
+                    {
+                        "port": 22,
+                        "destination": "0.0.0.0:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8080,
+                        "destination": "192.168.14.3:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2222,
+                        "destination": "192.168.14.3:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8081,
+                        "destination": "192.168.14.4:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "192.168.14.4:22",
+                        "protocol": "TCP"
+                    }
+                ]
+            }
+        }
+    ]
+}
diff --git a/examples/nat/main/config-vlan.json b/examples/nat/main/config-vlan.json
index 4ebc6729..f4f55e26 100644
--- a/examples/nat/main/config-vlan.json
+++ b/examples/nat/main/config-vlan.json
@@ -8,8 +8,7 @@
             },
             "public-port": {
                 "index": 1,
-                "dst-mac": "3C:FD:FE:A4:DD:F0",
-                "subnet": "192.168.116.1",
+                "subnet": "192.168.116.1/24",
                 "vlan-tag": 116
             }
         }
diff --git a/examples/nat/main/config.json b/examples/nat/main/config.json
index 3fed6f95..427586c1 100644
--- a/examples/nat/main/config.json
+++ b/examples/nat/main/config.json
@@ -7,8 +7,29 @@
             },
             "public-port": {
                 "index": 1,
-                "dst-mac": "3C:FD:FE:A4:DD:F0",
-                "subnet": "192.168.16.1"
+                "subnet": "192.168.16.1/24",
+                "forward-ports": [
+                    {
+                        "port": 8080,
+                        "destination": "192.168.14.3:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2222,
+                        "destination": "192.168.14.3:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8081,
+                        "destination": "192.168.14.4:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "192.168.14.4:22",
+                        "protocol": "TCP"
+                    }
+                ]
             }
         }
     ]
diff --git a/examples/nat/main/config2ports.json b/examples/nat/main/config2ports.json
index 45f80c45..68dbb398 100644
--- a/examples/nat/main/config2ports.json
+++ b/examples/nat/main/config2ports.json
@@ -7,8 +7,7 @@
             },
             "public-port": {
                 "index": 3,
-                "dst-mac": "3c:fd:fe:a5:4d:68",
-                "subnet": "192.168.16.1"
+                "subnet": "192.168.16.1/24"
             }
         },
         {
@@ -19,8 +18,7 @@
             },
             "public-port": {
                 "index": 5,
-                "dst-mac": "3c:fd:fe:a5:4d:6a",
-                "subnet": "192.168.46.1",
+                "subnet": "192.168.46.1/24",
                 "vlan-tag": 146
             }
         }
diff --git a/examples/nat/main/nat.go b/examples/nat/main/nat.go
index c0b4eb93..708acef2 100644
--- a/examples/nat/main/nat.go
+++ b/examples/nat/main/nat.go
@@ -37,6 +37,7 @@ func main() {
 		HWTXChecksum:     !nat.NoHWTXChecksum,
 		DPDKArgs:         []string{*dpdkLogLevel},
 		DisableScheduler: *noscheduler,
+		NeedKNI:          nat.NeedKNI,
 	}
 
 	flow.CheckFatal(flow.SystemInit(&nffgoconfig))
diff --git a/examples/nat/portalloc.go b/examples/nat/portalloc.go
index 86f1564c..e21a2860 100644
--- a/examples/nat/portalloc.go
+++ b/examples/nat/portalloc.go
@@ -29,8 +29,8 @@ func (dir terminationDirection) String() string {
 }
 
 func (pp *portPair) deleteOldConnection(protocol uint8, port int) {
-	pubTable := &pp.pub2priTable[protocol]
-	pm := pp.portmap[protocol]
+	pubTable := pp.PublicPort.translationTable[protocol]
+	pm := pp.PublicPort.portmap[protocol]
 
 	pub2priKey := Tuple{
 		addr: pm[port].addr,
@@ -39,7 +39,7 @@ func (pp *portPair) deleteOldConnection(protocol uint8, port int) {
 	pri2pubKey, found := pubTable.Load(pub2priKey)
 
 	if found {
-		pp.pri2pubTable[protocol].Delete(pri2pubKey)
+		pp.PrivatePort.translationTable[protocol].Delete(pri2pubKey)
 		pubTable.Delete(pub2priKey)
 	}
 	pm[port] = portMapEntry{}
@@ -48,10 +48,10 @@ func (pp *portPair) deleteOldConnection(protocol uint8, port int) {
 // This function currently is not thread safe and should be executed
 // under a global lock
 func (pp *portPair) allocNewPort(protocol uint8) (int, error) {
-	pm := pp.portmap[protocol]
+	pm := pp.PublicPort.portmap[protocol]
 	for {
 		for p := pp.lastport; p < portEnd; p++ {
-			if time.Since(pm[p].lastused) > connectionTimeout {
+			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
 				pp.lastport = p
 				pp.deleteOldConnection(protocol, p)
 				return p, nil
@@ -59,7 +59,7 @@ func (pp *portPair) allocNewPort(protocol uint8) (int, error) {
 		}
 
 		for p := portStart; p < pp.lastport; p++ {
-			if time.Since(pm[p].lastused) > connectionTimeout {
+			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
 				pp.lastport = p
 				pp.deleteOldConnection(protocol, p)
 				return p, nil
diff --git a/examples/nat/translation.go b/examples/nat/translation.go
index 64726d64..cc98a981 100644
--- a/examples/nat/translation.go
+++ b/examples/nat/translation.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -18,11 +18,7 @@ type Tuple struct {
 	port uint16
 }
 
-var (
-	emptyEntry = Tuple{addr: 0, port: 0}
-)
-
-func (pp *portPair) allocateNewEgressConnection(protocol uint8, privEntry Tuple) (Tuple, error) {
+func (pp *portPair) allocateNewEgressConnection(protocol uint8, privEntry *Tuple) (Tuple, error) {
 	pp.mutex.Lock()
 
 	port, err := pp.allocNewPort(protocol)
@@ -37,214 +33,257 @@ func (pp *portPair) allocateNewEgressConnection(protocol uint8, privEntry Tuple)
 		port: uint16(port),
 	}
 
-	pp.portmap[protocol][port] = portMapEntry{
+	pp.PublicPort.portmap[protocol][port] = portMapEntry{
 		lastused:             time.Now(),
 		addr:                 publicAddr,
 		finCount:             0,
 		terminationDirection: 0,
+		static:               false,
 	}
 
 	// Add lookup entries for packet translation
-	pp.pri2pubTable[protocol].Store(privEntry, pubEntry)
-	pp.pub2priTable[protocol].Store(pubEntry, privEntry)
+	pp.PublicPort.translationTable[protocol].Store(pubEntry, *privEntry)
+	pp.PrivatePort.translationTable[protocol].Store(*privEntry, pubEntry)
 
 	pp.mutex.Unlock()
 	return pubEntry, nil
 }
 
 // PublicToPrivateTranslation does ingress translation.
-func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) bool {
+func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 	pi := ctx.(pairIndex)
 	pp := &Natconfig.PortPairs[pi.index]
+	port := &pp.PublicPort
 
-	dumpPacket(pkt, pi.index, iPUBLIC)
+	port.dumpPacket(pkt, dirSEND)
 
 	// Parse packet type and address
-	pktVLAN := pkt.ParseL3CheckVLAN()
-	pktIPv4 := pkt.GetIPv4CheckVLAN()
+	dir, pktVLAN, pktIPv4 := port.parsePacketAndCheckARP(pkt)
 	if pktIPv4 == nil {
-		arp := pkt.GetARPCheckVLAN()
-		if arp != nil {
-			if pp.PublicPort.handleARP(pkt, pi.index, iPUBLIC) == false {
-				dumpDrop(pkt, pi.index, iPUBLIC)
-			}
-			return false
-		}
-		// We don't currently support anything except for IPv4 and ARP
-		dumpDrop(pkt, pi.index, iPUBLIC)
-		return false
+		return dir
 	}
 
+	// Create a lookup key from packet destination address and port
 	pktTCP, pktUDP, pktICMP := pkt.ParseAllKnownL4ForIPv4()
-	// Create a lookup key
 	protocol := pktIPv4.NextProtoID
-	pub2priKey := Tuple{
-		addr: packet.SwapBytesUint32(pktIPv4.DstAddr),
-	}
-	// Parse packet destination port
-	if pktTCP != nil {
-		pub2priKey.port = packet.SwapBytesUint16(pktTCP.DstPort)
-	} else if pktUDP != nil {
-		pub2priKey.port = packet.SwapBytesUint16(pktUDP.DstPort)
-	} else if pktICMP != nil {
-		// Check if this ICMP packet destination is NAT itself. If
-		// yes, reply back with ICMP and stop packet processing.
-		if pp.PublicPort.handleICMP(pkt, pi.index, iPUBLIC) == false {
-			return false
-		}
-		pub2priKey.port = packet.SwapBytesUint16(pktICMP.Identifier)
-	} else {
-		dumpDrop(pkt, pi.index, iPUBLIC)
-		return false
+	pub2priKey, dir := port.generateLookupKeyFromDstAndHandleICMP(pkt, pktIPv4, pktTCP, pktUDP, pktICMP)
+	if pub2priKey == nil {
+		return dir
 	}
 
 	// Do lookup
-	v, found := pp.pub2priTable[protocol].Load(pub2priKey)
+	v, found := port.translationTable[protocol].Load(*pub2priKey)
 	// For ingress connections packets are allowed only if a
 	// connection has been previosly established with a egress
 	// (private to public) packet. So if lookup fails, this incoming
-	// packet is ignored.
+	// packet is ignored unless there is a KNI interface. If KNI is
+	// present, traffic is directed there.
 	if !found {
-		dumpDrop(pkt, pi.index, iPUBLIC)
-		return false
+		if port.KNIName != "" {
+			dir = dirKNI
+		} else {
+			dir = dirDROP
+		}
+		port.dumpPacket(pkt, dir)
+		return dir
 	}
 	value := v.(Tuple)
 
 	// Check whether connection is too old
-	if time.Since(pp.portmap[protocol][pub2priKey.port].lastused) <= connectionTimeout {
-		pp.portmap[protocol][pub2priKey.port].lastused = time.Now()
+	if port.portmap[protocol][pub2priKey.port].static || time.Since(port.portmap[protocol][pub2priKey.port].lastused) <= connectionTimeout {
+		port.portmap[protocol][pub2priKey.port].lastused = time.Now()
 	} else {
 		// There was no transfer on this port for too long
 		// time. We don't allow it any more
 		pp.mutex.Lock()
 		pp.deleteOldConnection(protocol, int(pub2priKey.port))
 		pp.mutex.Unlock()
-		dumpDrop(pkt, pi.index, iPUBLIC)
-		return false
+		port.dumpPacket(pkt, dirDROP)
+		return dirDROP
 	}
 
-	// Check whether TCP connection could be reused
-	if protocol == common.TCPNumber {
-		pp.checkTCPTermination(pktTCP, int(pub2priKey.port), pub2pri)
-	}
+	if value.addr != 0 {
+		// Check whether TCP connection could be reused
+		if pktTCP != nil && !pp.PublicPort.portmap[protocol][value.port].static {
+			pp.checkTCPTermination(pktTCP, int(pub2priKey.port), pub2pri)
+		}
 
-	// Do packet translation
-	pkt.Ether.DAddr = pp.PrivatePort.getMACForIP(value.addr)
-	pkt.Ether.SAddr = pp.PrivatePort.SrcMACAddress
-	if pktVLAN != nil {
-		pktVLAN.SetVLANTagIdentifier(pp.PrivatePort.Vlan)
-	}
-	pktIPv4.DstAddr = packet.SwapBytesUint32(value.addr)
+		// Do packet translation
+		mac, found := port.opposite.getMACForIP(value.addr)
+		if !found {
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
+		}
+		pkt.Ether.DAddr = mac
+		pkt.Ether.SAddr = port.SrcMACAddress
+		if pktVLAN != nil {
+			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
+		}
+		pktIPv4.DstAddr = packet.SwapBytesUint32(value.addr)
+		setPacketDstPort(pkt, value.port, pktTCP, pktUDP, pktICMP)
 
-	if pktTCP != nil {
-		pktTCP.DstPort = packet.SwapBytesUint16(value.port)
-		setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	} else if pktUDP != nil {
-		pktUDP.DstPort = packet.SwapBytesUint16(value.port)
-		setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		port.dumpPacket(pkt, dirSEND)
+		return dirSEND
 	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(value.port)
-		setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		port.dumpPacket(pkt, dirKNI)
+		return dirKNI
 	}
-
-	dumpPacket(pkt, pi.index, iPRIVATE)
-	return true
 }
 
 // PrivateToPublicTranslation does egress translation.
-func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) bool {
+func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 	pi := ctx.(pairIndex)
 	pp := &Natconfig.PortPairs[pi.index]
+	port := &pp.PrivatePort
 
-	dumpPacket(pkt, pi.index, iPRIVATE)
+	port.dumpPacket(pkt, dirSEND)
 
 	// Parse packet type and address
-	pktVLAN := pkt.ParseL3CheckVLAN()
-	pktIPv4 := pkt.GetIPv4CheckVLAN()
+	dir, pktVLAN, pktIPv4 := port.parsePacketAndCheckARP(pkt)
 	if pktIPv4 == nil {
-		arp := pkt.GetARPCheckVLAN()
-		if arp != nil {
-			if pp.PrivatePort.handleARP(pkt, pi.index, iPRIVATE) == false {
-				dumpDrop(pkt, pi.index, iPRIVATE)
-			}
-			return false
-		}
-		// We don't currently support anything except for IPv4 and ARP
-		dumpDrop(pkt, pi.index, iPRIVATE)
-		return false
+		return dir
 	}
 
+	// Create a lookup key from packet source address and port
 	pktTCP, pktUDP, pktICMP := pkt.ParseAllKnownL4ForIPv4()
-
-	// Create a lookup key
 	protocol := pktIPv4.NextProtoID
-	pri2pubKey := Tuple{
-		addr: packet.SwapBytesUint32(pktIPv4.SrcAddr),
+	pri2pubKey, dir := port.generateLookupKeyFromSrcAndHandleICMP(pkt, pktIPv4, pktTCP, pktUDP, pktICMP)
+	if pri2pubKey == nil {
+		return dir
 	}
 
-	// Parse packet source port
-	if pktTCP != nil {
-		pri2pubKey.port = packet.SwapBytesUint16(pktTCP.SrcPort)
-	} else if pktUDP != nil {
-		pri2pubKey.port = packet.SwapBytesUint16(pktUDP.SrcPort)
-	} else if pktICMP != nil {
-		// Check if this ICMP packet destination is NAT itself. If
-		// yes, reply back with ICMP and stop packet processing.
-		if pp.PrivatePort.handleICMP(pkt, pi.index, iPRIVATE) == false {
-			return false
-		}
-		pri2pubKey.port = packet.SwapBytesUint16(pktICMP.Identifier)
-	} else {
-		dumpDrop(pkt, pi.index, iPRIVATE)
-		return false
+	// If traffic is directed at private interface IP and KNI is
+	// present, this traffic is directed to KNI
+	if port.KNIName != "" && port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr) {
+		port.dumpPacket(pkt, dirKNI)
+		return dirKNI
 	}
 
 	// Do lookup
 	var value Tuple
-	v, found := pp.pri2pubTable[protocol].Load(pri2pubKey)
+	v, found := port.translationTable[protocol].Load(*pri2pubKey)
 	if !found {
 		var err error
 		// Store new local network entry in ARP cache
-		pp.PrivatePort.ArpTable.Store(pri2pubKey.addr, pkt.Ether.SAddr)
+		port.arpTable.Store(pri2pubKey.addr, pkt.Ether.SAddr)
 		// Allocate new connection from private to public network
 		value, err = pp.allocateNewEgressConnection(protocol, pri2pubKey)
 
 		if err != nil {
 			println("Warning! Failed to allocate new connection", err)
-			dumpDrop(pkt, pi.index, iPRIVATE)
-			return false
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
 		}
 	} else {
 		value = v.(Tuple)
-		pp.portmap[protocol][value.port].lastused = time.Now()
+		pp.PublicPort.portmap[protocol][value.port].lastused = time.Now()
+	}
+
+	if value.addr != 0 {
+		// Check whether TCP connection could be reused
+		if pktTCP != nil && !pp.PublicPort.portmap[protocol][value.port].static {
+			pp.checkTCPTermination(pktTCP, int(value.port), pri2pub)
+		}
+
+		// Do packet translation
+		mac, found := port.opposite.getMACForIP(packet.SwapBytesUint32(pktIPv4.DstAddr))
+		if !found {
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
+		}
+		pkt.Ether.DAddr = mac
+		pkt.Ether.SAddr = port.SrcMACAddress
+		if pktVLAN != nil {
+			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
+		}
+		pktIPv4.SrcAddr = packet.SwapBytesUint32(value.addr)
+		setPacketSrcPort(pkt, value.port, pktTCP, pktUDP, pktICMP)
+
+		port.dumpPacket(pkt, dirSEND)
+		return dirSEND
+	} else {
+		port.dumpPacket(pkt, dirKNI)
+		return dirKNI
+	}
+}
+
+// Used to generate key in public to private translation
+func (port *ipv4Port) generateLookupKeyFromDstAndHandleICMP(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) (*Tuple, uint) {
+	key := Tuple{
+		addr: packet.SwapBytesUint32(pktIPv4.DstAddr),
 	}
 
-	// Check whether TCP connection could be reused
+	// Parse packet destination port
 	if pktTCP != nil {
-		pp.checkTCPTermination(pktTCP, int(value.port), pri2pub)
+		key.port = packet.SwapBytesUint16(pktTCP.DstPort)
+	} else if pktUDP != nil {
+		key.port = packet.SwapBytesUint16(pktUDP.DstPort)
+	} else if pktICMP != nil {
+		// Check if this ICMP packet destination is NAT itself. If
+		// yes, reply back with ICMP and stop packet processing.
+		key.port = packet.SwapBytesUint16(pktICMP.Identifier)
+		dir := port.handleICMP(pkt, &key)
+		if dir != dirSEND {
+			return nil, dir
+		}
+	} else {
+		port.dumpPacket(pkt, dirDROP)
+		return nil, dirDROP
 	}
+	return &key, dirSEND
+}
 
-	// Do packet translation
-	pkt.Ether.DAddr = pp.PublicPort.DstMACAddress
-	pkt.Ether.SAddr = pp.PublicPort.SrcMACAddress
-	if pktVLAN != nil {
-		pktVLAN.SetVLANTagIdentifier(pp.PublicPort.Vlan)
+// Used to generate key in private to public translation
+func (port *ipv4Port) generateLookupKeyFromSrcAndHandleICMP(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) (*Tuple, uint) {
+	key := Tuple{
+		addr: packet.SwapBytesUint32(pktIPv4.SrcAddr),
 	}
-	pktIPv4.SrcAddr = packet.SwapBytesUint32(value.addr)
 
+	// Parse packet source port
 	if pktTCP != nil {
-		pktTCP.SrcPort = packet.SwapBytesUint16(value.port)
+		key.port = packet.SwapBytesUint16(pktTCP.SrcPort)
+	} else if pktUDP != nil {
+		key.port = packet.SwapBytesUint16(pktUDP.SrcPort)
+	} else if pktICMP != nil {
+		// Check if this ICMP packet destination is NAT itself. If
+		// yes, reply back with ICMP and stop packet processing or
+		// direct to KNI if KNI is present.
+		dir := port.handleICMP(pkt, nil)
+		if dir != dirSEND {
+			return nil, dir
+		}
+		key.port = packet.SwapBytesUint16(pktICMP.Identifier)
+	} else {
+		port.dumpPacket(pkt, dirDROP)
+		return nil, dirDROP
+	}
+	return &key, dirSEND
+}
+
+func setPacketDstPort(pkt *packet.Packet, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
+	if pktTCP != nil {
+		pktTCP.DstPort = packet.SwapBytesUint16(port)
 		setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
 	} else if pktUDP != nil {
-		pktUDP.SrcPort = packet.SwapBytesUint16(value.port)
+		pktUDP.DstPort = packet.SwapBytesUint16(port)
 		setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
 	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(value.port)
+		pktICMP.Identifier = packet.SwapBytesUint16(port)
 		setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
 	}
+}
 
-	dumpPacket(pkt, pi.index, iPUBLIC)
-	return true
+func setPacketSrcPort(pkt *packet.Packet, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
+	if pktTCP != nil {
+		pktTCP.SrcPort = packet.SwapBytesUint16(port)
+		setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+	} else if pktUDP != nil {
+		pktUDP.SrcPort = packet.SwapBytesUint16(port)
+		setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+	} else {
+		pktICMP.Identifier = packet.SwapBytesUint16(port)
+		setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+	}
 }
 
 // Simple check for FIN or RST in TCP
@@ -253,7 +292,7 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 		// First check for FIN
 		pp.mutex.Lock()
 
-		pme := &pp.portmap[common.TCPNumber][port]
+		pme := &pp.PublicPort.portmap[common.TCPNumber][port]
 		if pme.finCount == 0 {
 			pme.finCount = 1
 			pme.terminationDirection = dir
@@ -273,7 +312,7 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 		// FIN
 		pp.mutex.Lock()
 
-		pme := &pp.portmap[common.TCPNumber][port]
+		pme := &pp.PublicPort.portmap[common.TCPNumber][port]
 		if pme.finCount == 2 {
 			pp.deleteOldConnection(common.TCPNumber, port)
 			// Set some time while port cannot be used before
@@ -285,12 +324,40 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 	}
 }
 
-func (port *ipv4Port) handleARP(pkt *packet.Packet, index int, isPrivate interfaceType) bool {
+func (port *ipv4Port) parsePacketAndCheckARP(pkt *packet.Packet) (dir uint, vhdr *packet.VLANHdr, iphdr *packet.IPv4Hdr) {
+	pktVLAN := pkt.ParseL3CheckVLAN()
+	pktIPv4 := pkt.GetIPv4CheckVLAN()
+	if pktIPv4 == nil {
+		arp := pkt.GetARPCheckVLAN()
+		if arp != nil {
+			dir := port.handleARP(pkt)
+			port.dumpPacket(pkt, dir)
+			return dir, pktVLAN, nil
+		}
+		// We don't currently support anything except for IPv4 and ARP
+		port.dumpPacket(pkt, dirDROP)
+		return dirDROP, pktVLAN, nil
+	}
+	return dirSEND, pktVLAN, pktIPv4
+}
+
+func (port *ipv4Port) handleARP(pkt *packet.Packet) uint {
 	arp := pkt.GetARPNoCheck()
 
 	if packet.SwapBytesUint16(arp.Operation) != packet.ARPRequest {
-		// We don't care about replies so far
-		return false
+		if packet.SwapBytesUint16(arp.Operation) == packet.ARPReply {
+			ipv4 := packet.SwapBytesUint32(packet.ArrayToIPv4(arp.SPA))
+			port.arpTable.Store(ipv4, arp.SHA)
+		}
+		if port.KNIName != "" {
+			return dirKNI
+		}
+		return dirDROP
+	}
+
+	// If there is a KNI interface, direct all ARP traffic to it
+	if port.KNIName != "" {
+		return dirKNI
 	}
 
 	// Check that someone is asking about MAC of my IP address and HW
@@ -299,12 +366,12 @@ func (port *ipv4Port) handleARP(pkt *packet.Packet, index int, isPrivate interfa
 		println("Warning! Got an ARP packet with target IPv4 address", StringIPv4Array(arp.TPA),
 			"different from IPv4 address on interface. Should be", StringIPv4Int(port.Subnet.Addr),
 			". ARP request ignored.")
-		return false
+		return dirDROP
 	}
 	if arp.THA != [common.EtherAddrLen]byte{} {
 		println("Warning! Got an ARP packet with non-zero MAC address", StringMAC(arp.THA),
 			". ARP request ignored.")
-		return false
+		return dirDROP
 	}
 
 	// Prepare an answer to this request
@@ -319,39 +386,66 @@ func (port *ipv4Port) handleARP(pkt *packet.Packet, index int, isPrivate interfa
 		answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
 	}
 
-	dumpPacket(answerPacket, index, isPrivate)
+	port.dumpPacket(answerPacket, dirSEND)
 	answerPacket.SendPacket(port.Index)
 
-	return true
+	return dirDROP
 }
 
-func (port *ipv4Port) getMACForIP(ip uint32) macAddress {
-	v, found := port.ArpTable.Load(ip)
+func (port *ipv4Port) getMACForIP(ip uint32) (macAddress, bool) {
+	v, found := port.arpTable.Load(ip)
 	if found {
-		return macAddress(v.([common.EtherAddrLen]byte))
+		return macAddress(v.([common.EtherAddrLen]byte)), true
+	}
+	port.sendARPRequest(ip)
+	return macAddress{}, false
+}
+
+func (port *ipv4Port) sendARPRequest(ip uint32) {
+	// Prepare an answer to this request
+	requestPacket, err := packet.NewPacket()
+	if err != nil {
+		common.LogFatal(common.Debug, err)
 	}
-	println("Warning! IP address",
-		byte(ip), ".", byte(ip>>8), ".", byte(ip>>16), ".", byte(ip>>24),
-		"not found in ARP cache on port", port.Index)
-	return macAddress{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+
+	packet.InitARPRequestPacket(requestPacket, port.SrcMACAddress,
+		packet.SwapBytesUint32(port.Subnet.Addr), packet.SwapBytesUint32(ip))
+	if port.Vlan != 0 {
+		requestPacket.AddVLANTag(port.Vlan)
+	}
+
+	port.dumpPacket(requestPacket, dirSEND)
+	requestPacket.SendPacket(port.Index)
 }
 
-func (port *ipv4Port) handleICMP(pkt *packet.Packet, index int, isPrivate interfaceType) bool {
+func (port *ipv4Port) handleICMP(pkt *packet.Packet, key *Tuple) uint {
 	ipv4 := pkt.GetIPv4NoCheck()
 
-	// Check that received ICMP packet is addressed at this host
+	// Check that received ICMP packet is addressed at this host. If
+	// not, packet should be translated
 	if packet.SwapBytesUint32(ipv4.DstAddr) != port.Subnet.Addr {
-		return true
+		return dirSEND
 	}
 
 	icmp := pkt.GetICMPNoCheck()
 
+	// If there is KNI interface, direct all ICMP traffic which
+	// doesn't have an active translation entry
+	if port.KNIName != "" {
+		if key != nil {
+			_, ok := port.translationTable[common.ICMPNumber].Load(*key)
+			if !ok || time.Since(port.portmap[common.ICMPNumber][key.port].lastused) > connectionTimeout {
+				return dirKNI
+			}
+		}
+	}
+
 	// Check that received ICMP packet is echo request packet. We
 	// don't support any other messages yet, so process them in normal
 	// NAT way. Maybe these are packets which should be passed through
 	// translation.
 	if icmp.Type != common.ICMPTypeEchoRequest || icmp.Code != 0 {
-		return true
+		return dirSEND
 	}
 
 	// Return a packet back to sender
@@ -367,7 +461,7 @@ func (port *ipv4Port) handleICMP(pkt *packet.Packet, index int, isPrivate interf
 	(answerPacket.GetICMPNoCheck()).Type = common.ICMPTypeEchoResponse
 	setIPv4ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
 
-	dumpPacket(answerPacket, index, isPrivate)
+	port.dumpPacket(answerPacket, dirSEND)
 	answerPacket.SendPacket(port.Index)
-	return false
+	return dirDROP
 }
diff --git a/examples/nat/util.go b/examples/nat/util.go
index 69c4d998..b942f289 100644
--- a/examples/nat/util.go
+++ b/examples/nat/util.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -45,18 +45,15 @@ func swapAddrIPv4(pkt *packet.Packet) {
 	ipv4.SrcAddr, ipv4.DstAddr = ipv4.DstAddr, ipv4.SrcAddr
 }
 
-func startTrace(name string, aindex, index int, isPrivate interfaceType) *os.File {
-	var fname string
-	if !dumptogether {
-		if isPrivate != 0 {
-			fname = fmt.Sprintf("%dpriv%s.pcap", index, name)
-		} else {
-			fname = fmt.Sprintf("%dpub%s.pcap", index, name)
-		}
-	} else {
-		fname = fmt.Sprintf("%d%s.pcap", index, name)
+func (port *ipv4Port) startTrace(dir uint) *os.File {
+	dumpNameLookup := [dirKNI + 1]string{
+		"drop",
+		"dump",
+		"kni",
 	}
 
+	fname := fmt.Sprintf("%s-%d-%s.pcap", dumpNameLookup[dir], port.Index, packet.MACToString(port.SrcMACAddress))
+
 	file, err := os.Create(fname)
 	if err != nil {
 		log.Fatal(err)
@@ -65,60 +62,33 @@ func startTrace(name string, aindex, index int, isPrivate interfaceType) *os.Fil
 	return file
 }
 
-func dumpPacket(pkt *packet.Packet, index int, isPrivate interfaceType) {
+func (port *ipv4Port) dumpPacket(pkt *packet.Packet, dir uint) {
 	if debugDump {
-		aindex := index
-		if !dumptogether {
-			aindex = index*2 + int(isPrivate)
-		}
-
-		dumpsync[aindex].Lock()
-		if fdump[aindex] == nil {
-			fdump[aindex] = startTrace("dump", aindex, index, isPrivate)
+		port.dumpsync[dir].Lock()
+		if port.fdump[dir] == nil {
+			port.fdump[dir] = port.startTrace(dir)
 		}
 
-		err := pkt.WritePcapOnePacket(fdump[aindex])
+		err := pkt.WritePcapOnePacket(port.fdump[dir])
 		if err != nil {
 			log.Fatal(err)
 		}
-		dumpsync[aindex].Unlock()
+		port.dumpsync[dir].Unlock()
 	}
 }
 
-func dumpDrop(pkt *packet.Packet, index int, isPrivate interfaceType) {
-	if debugDrop {
-		aindex := index
-		if !dumptogether {
-			aindex = index*2 + int(isPrivate)
-		}
-		dropsync[aindex].Lock()
-		if fdrop[aindex] == nil {
-			fdrop[aindex] = startTrace("drop", aindex, index, isPrivate)
-		}
-		err := pkt.WritePcapOnePacket(fdrop[aindex])
-		if err != nil {
-			log.Fatal(err)
+func (port *ipv4Port) closePortTraces() {
+	for _, f := range port.fdump {
+		if f != nil {
+			f.Close()
 		}
-		dropsync[aindex].Unlock()
 	}
 }
 
 // CloseAllDumpFiles closes all debug dump files.
 func CloseAllDumpFiles() {
-	if debugDump {
-		debugDump = false
-		for i := range fdump {
-			if fdump[i] != nil {
-				fdump[i].Close()
-			}
-		}
-	}
-	if debugDrop {
-		debugDrop = false
-		for i := range fdrop {
-			if fdump[i] != nil {
-				fdump[i].Close()
-			}
-		}
+	for i := range Natconfig.PortPairs {
+		Natconfig.PortPairs[i].PrivatePort.closePortTraces()
+		Natconfig.PortPairs[i].PublicPort.closePortTraces()
 	}
 }
diff --git a/flow/flow.go b/flow/flow.go
index 5bbef1b9..fed8d989 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -1614,8 +1614,8 @@ func (timer *Timer) Stop() {
 func CheckFatal(err error) {
 	if err != nil {
 		if nfErr := common.GetNFError(err); nfErr != nil {
-			common.LogFatalf(common.Debug, "failed with message and code: %+v\n", nfErr)
+			common.LogFatalf(common.No, "failed with message and code: %+v\n", nfErr)
 		}
-		common.LogFatalf(common.Debug, "failed with message: %s\n", err.Error())
+		common.LogFatalf(common.No, "failed with message: %s\n", err.Error())
 	}
 }
diff --git a/packet/packet.go b/packet/packet.go
index c6d6b4ec..0367c447 100644
--- a/packet/packet.go
+++ b/packet/packet.go
@@ -100,12 +100,14 @@ type IPv4Hdr struct {
 	DstAddr        uint32 // destination address
 }
 
+func IPv4ToString(addr uint32) string {
+	return fmt.Sprintf("%d.%d.%d.%d", byte(addr), byte(addr>>8), byte(addr>>16), byte(addr>>24))
+}
+
 func (hdr *IPv4Hdr) String() string {
 	r0 := "    L3 protocol: IPv4\n"
-	s := hdr.SrcAddr
-	r1 := fmt.Sprintln("    IPv4 Source:", byte(s), ":", byte(s>>8), ":", byte(s>>16), ":", byte(s>>24))
-	d := hdr.DstAddr
-	r2 := fmt.Sprintln("    IPv4 Destination:", byte(d), ":", byte(d>>8), ":", byte(d>>16), ":", byte(d>>24))
+	r1 := "    IPv4 Source: " + IPv4ToString(hdr.SrcAddr) + "\n"
+	r2 := "    IPv4 Destination: " + IPv4ToString(hdr.DstAddr) + "\n"
 	return r0 + r1 + r2
 }
 
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index b4e021f7..72a6411e 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -120,27 +120,14 @@ SHELL
       if i % vm_group_size != 0
         # Define backward inter-VM virtual network links
         (1..vm_links_number).each do |j|
-          if (i % vm_group_size) == vm_group_size - 1 && j == 1
-            # Workaround hardcoded MAC address for server VM until we implement ARP completely
-            node.vm.network "private_network",
-                            auto_config: false,
-                            virtualbox__intnet: "#{vm_name}-link-#{user}-#{i}-#{j}",
-                            :mac => '3cfdfea4ddf0',
-                            :model_type => 'virtio',
-                            :libvirt__forward_mode => 'none',
-                            :libvirt__tunnel_type => 'udp',
-                            :libvirt__tunnel_local_port => vm_second_port_base + i * vm_links_number + j,
-                            :libvirt__tunnel_port => vm_port_base + i * vm_links_number + j
-          else
-            node.vm.network "private_network",
-                            auto_config: false,
-                            virtualbox__intnet: "#{vm_name}-link-#{user}-#{i}-#{j}",
-                            :model_type => 'virtio',
-                            :libvirt__forward_mode => 'none',
-                            :libvirt__tunnel_type => 'udp',
-                            :libvirt__tunnel_local_port => vm_second_port_base + i * vm_links_number + j,
-                            :libvirt__tunnel_port => vm_port_base + i * vm_links_number + j
-          end
+          node.vm.network "private_network",
+                          auto_config: false,
+                          virtualbox__intnet: "#{vm_name}-link-#{user}-#{i}-#{j}",
+                          :model_type => 'virtio',
+                          :libvirt__forward_mode => 'none',
+                          :libvirt__tunnel_type => 'udp',
+                          :libvirt__tunnel_local_port => vm_second_port_base + i * vm_links_number + j,
+                          :libvirt__tunnel_port => vm_port_base + i * vm_links_number + j
         end
       end
 

From cee7e7bb43e02a37cae1db225272419c0ddbf8cb Mon Sep 17 00:00:00 2001
From: mkfsn <p408865@gmail.com>
Date: Wed, 29 Aug 2018 13:53:41 +0800
Subject: [PATCH 32/50] fix: overflow problem as C.checkRSSPacketCount() is
 returning C.Int and the value could be -1

---
 flow/scheduler.go | 2 +-
 low/low.go        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index cf5675b6..a79d5611 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -474,7 +474,7 @@ func (scheduler *scheduler) schedule(schedTime uint) {
 						for q := 0; q < ff.instanceNumber; q++ {
 							var q1 int32
 							for q1 = 1; q1 < ff.instance[q].inIndex[0]+1; q1++ {
-								if low.CheckRSSPacketCount(ff.Parameters.(*receiveParameters).port, int16(ff.instance[q].inIndex[q1])) > RSSCloneMin {
+								if low.CheckRSSPacketCount(ff.Parameters.(*receiveParameters).port, int16(ff.instance[q].inIndex[q1])) > int64(RSSCloneMin) {
 									break
 								}
 							}
diff --git a/low/low.go b/low/low.go
index d5527865..97da29d5 100644
--- a/low/low.go
+++ b/low/low.go
@@ -71,8 +71,8 @@ func GetPort(n uint16) *Port {
 	return p
 }
 
-func CheckRSSPacketCount(p *Port, queue int16) uint32 {
-	return uint32(C.checkRSSPacketCount((*C.struct_cPort)(p), (C.int16_t(queue))))
+func CheckRSSPacketCount(p *Port, queue int16) int64 {
+	return int64(C.checkRSSPacketCount((*C.struct_cPort)(p), (C.int16_t(queue))))
 }
 
 // GetPortMACAddress gets MAC address of given port.

From b21db124f0c28eb9db25f7d0913e8d3f218c2b7d Mon Sep 17 00:00:00 2001
From: mkfsn <p408865@gmail.com>
Date: Wed, 29 Aug 2018 14:18:29 +0800
Subject: [PATCH 33/50] fix: mismatched types int64 and uint32

---
 flow/scheduler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index a79d5611..36e7375e 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -688,7 +688,7 @@ func (ffi *instance) checkInputRingClonable(min uint32) bool {
 		}
 	case *receiveParameters:
 		for q := int32(0); q < ffi.inIndex[0]; q++ {
-			if low.CheckRSSPacketCount(ffi.ff.Parameters.(*receiveParameters).port, int16(ffi.inIndex[q+1])) > min {
+			if low.CheckRSSPacketCount(ffi.ff.Parameters.(*receiveParameters).port, int16(ffi.inIndex[q+1])) > int64(min) {
 				return true
 			}
 		}

From 633dce75d0646587a04a867739465de95fb39e6b Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Wed, 29 Aug 2018 20:12:21 +0000
Subject: [PATCH 34/50] Implemented GRPC control over dump, subnet address and
 port forwarding

---
 .travis.yml                            |   1 +
 examples/nat/Makefile                  |   2 +-
 examples/nat/client/.gitignore         |   1 +
 examples/nat/client/Makefile           |  13 ++
 examples/nat/client/client.go          | 240 +++++++++++++++++++++++++
 examples/nat/config.go                 | 144 +++++++++------
 examples/nat/grpc.go                   | 100 +++++++++++
 examples/nat/main/Makefile             |   7 +-
 examples/nat/main/nat.go               |   3 +
 examples/nat/updatecfg/.gitignore      |   1 +
 examples/nat/updatecfg/updatecfg.proto |  64 +++++++
 examples/nat/util.go                   |  35 +++-
 scripts/get-depends.sh                 |   6 +
 vagrant/Vagrantfile                    |   4 +-
 14 files changed, 557 insertions(+), 64 deletions(-)
 create mode 100644 examples/nat/client/.gitignore
 create mode 100644 examples/nat/client/Makefile
 create mode 100644 examples/nat/client/client.go
 create mode 100644 examples/nat/grpc.go
 create mode 100644 examples/nat/updatecfg/.gitignore
 create mode 100644 examples/nat/updatecfg/updatecfg.proto

diff --git a/.travis.yml b/.travis.yml
index 0ee9041f..5c9d4cee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,6 +18,7 @@ before_script:
 
 script:
   - docker exec -i test-nff-go ./scripts/get-depends.sh
+  - docker exec -i test-nff-go apt-get install -y protobuf-compiler
   - docker exec -i test-nff-go make
   # Build standalone examples
   - docker exec -i test-nff-go bash -c "cd examples && make gopacketParserExample && cd .."
diff --git a/examples/nat/Makefile b/examples/nat/Makefile
index 8131e6f6..4a03d043 100644
--- a/examples/nat/Makefile
+++ b/examples/nat/Makefile
@@ -3,6 +3,6 @@
 # license that can be found in the LICENSE file.
 
 PATH_TO_MK = ../../mk
-SUBDIRS = main
+SUBDIRS = main client
 
 include $(PATH_TO_MK)/intermediate.mk
diff --git a/examples/nat/client/.gitignore b/examples/nat/client/.gitignore
new file mode 100644
index 00000000..b051c6c5
--- /dev/null
+++ b/examples/nat/client/.gitignore
@@ -0,0 +1 @@
+client
diff --git a/examples/nat/client/Makefile b/examples/nat/client/Makefile
new file mode 100644
index 00000000..ad800782
--- /dev/null
+++ b/examples/nat/client/Makefile
@@ -0,0 +1,13 @@
+# Copyright 2018 Intel Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+PATH_TO_MK = ../../../mk
+EXECUTABLES = client
+
+nat: client.go ../updatecfg/updatecfg.pb.go
+
+../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
+	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
+
+include $(PATH_TO_MK)/leaf.mk
diff --git a/examples/nat/client/client.go b/examples/nat/client/client.go
new file mode 100644
index 00000000..8597df8c
--- /dev/null
+++ b/examples/nat/client/client.go
@@ -0,0 +1,240 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
+)
+
+type dumpRequestArray []*upd.DumpControlRequest
+type addresChangeRequestArray []*upd.InterfaceAddressChangeRequest
+type portForwardRequestArray []*upd.PortForwardingChangeRequest
+
+var (
+	dumpRequests         dumpRequestArray
+	addresChangeRequests addresChangeRequestArray
+	portForwardRequests  portForwardRequestArray
+)
+
+func (dra *dumpRequestArray) String() string {
+	res := ""
+	for _, r := range *dra {
+		res += r.String() + "\n"
+	}
+	return res
+}
+
+func (dra *dumpRequestArray) Set(value string) error {
+	req, ok := map[string]upd.DumpControlRequest{
+		"+d": upd.DumpControlRequest{
+			EnableTrace: true,
+			TraceType:   upd.TraceType_DUMP_DROP,
+		},
+		"-d": upd.DumpControlRequest{
+			EnableTrace: false,
+			TraceType:   upd.TraceType_DUMP_DROP,
+		},
+		"+t": upd.DumpControlRequest{
+			EnableTrace: true,
+			TraceType:   upd.TraceType_DUMP_TRANSLATE,
+		},
+		"-t": upd.DumpControlRequest{
+			EnableTrace: false,
+			TraceType:   upd.TraceType_DUMP_TRANSLATE,
+		},
+		"+k": upd.DumpControlRequest{
+			EnableTrace: true,
+			TraceType:   upd.TraceType_DUMP_KNI,
+		},
+		"-k": upd.DumpControlRequest{
+			EnableTrace: false,
+			TraceType:   upd.TraceType_DUMP_KNI,
+		},
+	}[value]
+
+	if !ok {
+		return fmt.Errorf("Bad dump control specification \"%s\"", value)
+	}
+	*dra = append(*dra, &req)
+	return nil
+}
+
+func (acra *addresChangeRequestArray) String() string {
+	res := ""
+	for _, r := range *acra {
+		res += r.String() + "\n"
+	}
+	return res
+}
+
+func (acra *addresChangeRequestArray) Set(value string) error {
+	parts := strings.Split(value, ":")
+	if len(parts) != 2 {
+		return fmt.Errorf("Bad port index and subnet address specified \"%s\"", value)
+	}
+	index, err := strconv.ParseUint(parts[0], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	ip, ipnet, err := net.ParseCIDR(parts[1])
+	if err != nil {
+		return err
+	}
+	if ip.To4() == nil {
+		return fmt.Errorf("Only IPv4 addresses are supported yet: %s", parts[1])
+	}
+	ones, _ := ipnet.Mask.Size()
+
+	*acra = append(*acra, &upd.InterfaceAddressChangeRequest{
+		InterfaceId: uint32(index),
+		PortSubnet: &upd.Subnet{
+			Address: &upd.IPAddress{
+				Address: ip.To4(),
+			},
+			MaskBitsNumber: uint32(ones),
+		},
+	})
+	return nil
+}
+
+func (pfra *portForwardRequestArray) String() string {
+	res := ""
+	for _, r := range *pfra {
+		res += r.String() + "\n"
+	}
+	return res
+}
+
+func (pfra *portForwardRequestArray) Set(value string) error {
+	parts := strings.Split(value, ":")
+	if len(parts) != 6 {
+		return fmt.Errorf("Bad port forwarding specification \"%s\"", value)
+	}
+
+	enable, ok := map[string]bool{
+		"+": true,
+		"-": false,
+	}[parts[0]]
+	if !ok {
+		return fmt.Errorf("Bad port forwarding enable sign string \"%s\"", parts[0])
+	}
+
+	index, err := strconv.ParseUint(parts[1], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	proto, ok := upd.Protocol_value[parts[2]]
+	if !ok || proto == int32(upd.Protocol_UNKNOWN) {
+		return fmt.Errorf("Bad protocol specified \"%s\"", parts[0])
+	}
+
+	sport, err := strconv.ParseUint(parts[3], 10, 16)
+	if err != nil {
+		return err
+	}
+
+	ip := net.ParseIP(parts[4])
+	if ip == nil {
+		return fmt.Errorf("Bad IP address specified \"%s\"", parts[4])
+	}
+	if ip.To4() == nil {
+		return fmt.Errorf("Only IPv4 addresses are supported yet: %s", parts[1])
+	}
+
+	tport, err := strconv.ParseUint(parts[5], 10, 16)
+	if err != nil {
+		return err
+	}
+
+	*pfra = append(*pfra, &upd.PortForwardingChangeRequest{
+		EnableForwarding: enable,
+		InterfaceId:      uint32(index),
+		Port: &upd.ForwardedPort{
+			SourcePortNumber: uint32(sport),
+			TargetAddress: &upd.IPAddress{
+				Address: ip.To4(),
+			},
+			TargetPortNumber: uint32(tport),
+			Protocol:         upd.Protocol(proto),
+		},
+	})
+	return nil
+}
+
+func main() {
+	flag.Usage = func() {
+		fmt.Printf(`Usage: client [-a server:port] [-d {+|-}{d|t|k}] [-s index:subnet] [-p {TCP|UDP}:port number:target IP address:target port]
+
+Client sends GRPS requests to NAT server controlling packets trace dump,
+ports subnet adresses and forwarded ports. Multiple requests of the same
+type are allowed and are processed in the following order: all dump, all
+subnet, all port forwarding requests.
+
+`)
+		flag.PrintDefaults()
+	}
+	address := flag.String("a", "localhost:60602", "Specifies server address")
+	flag.Var(&dumpRequests, "d", `Control dump trace output in a form of +/- and letter,
+e.g. +d or -t or +k:
+    + and - mean to enable or disable corresponding trace,
+    d means to trace dropped packets,
+    t means to trace translated (normally sent) packets,
+    k means to trace packets that were sent to KNI interface.`)
+	flag.Var(&addresChangeRequests, "s", `Control network interface subnet in a form of index:subnet,
+e.g. 1:192.168.5.1/24. Port index is DPDK port number. Subnet
+is given in form of port IP address and prefix bits.`)
+	flag.Var(&portForwardRequests, "p", `Control TCP and UDP port forwarding in a form of
++/-:index:protocol:source port:target IP address:target port, e.g.
++:1:TCP:2222:192.168.5.7:22 or -:0:TCP:22:0.0.0.0:0. If target
+address is zero, it means that port is forwarded to corresponding
+network port KNI interface. Port forwarding to a non-zero
+target address (not to a KNI interface) is possible only for
+public network port.`)
+	flag.Parse()
+
+	// Set up a connection to the server.
+	conn, err := grpc.Dial(*address, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer conn.Close()
+	c := upd.NewUpdaterClient(conn)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	for _, r := range dumpRequests {
+		reply, err := c.ControlDump(ctx, r)
+		if err != nil {
+			log.Fatalf("could not update: %v", err)
+		}
+		log.Printf("update successful: \"%s\"", reply.String())
+	}
+
+	for _, r := range addresChangeRequests {
+		reply, err := c.ChangeInterfaceAddress(ctx, r)
+		if err != nil {
+			log.Fatalf("could not update: %v", err)
+		}
+		log.Printf("update successful: \"%s\"", reply.String())
+	}
+
+	for _, r := range portForwardRequests {
+		reply, err := c.ChangePortForwarding(ctx, r)
+		if err != nil {
+			log.Fatalf("could not update: %v", err)
+		}
+		log.Printf("update successful: \"%s\"", reply.String())
+	}
+}
diff --git a/examples/nat/config.go b/examples/nat/config.go
index 0cff06e4..ea2ba7f1 100644
--- a/examples/nat/config.go
+++ b/examples/nat/config.go
@@ -7,6 +7,7 @@ package nat
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net"
 	"os"
 	"strconv"
@@ -16,6 +17,8 @@ import (
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/flow"
 	"github.com/intel-go/nff-go/packet"
+
+	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
 )
 
 type terminationDirection uint8
@@ -28,9 +31,9 @@ const (
 	iPUBLIC  interfaceType = 0
 	iPRIVATE interfaceType = 1
 
-	dirDROP = 0
-	dirSEND = 1
-	dirKNI  = 2
+	dirDROP = uint(upd.TraceType_DUMP_DROP)
+	dirSEND = uint(upd.TraceType_DUMP_TRANSLATE)
+	dirKNI  = uint(upd.TraceType_DUMP_KNI)
 
 	connectionTimeout time.Duration = 1 * time.Minute
 	portReuseTimeout  time.Duration = 1 * time.Second
@@ -75,6 +78,10 @@ type ipv4Subnet struct {
 	Mask uint32
 }
 
+func (fp *forwardedPort) String() string {
+	return fmt.Sprintf("Port:%d, Destination:%+v, Protocol: %d", fp.Port, packet.IPv4ToString(fp.Destination.Addr), fp.Protocol)
+}
+
 func (subnet *ipv4Subnet) String() string {
 	// Count most significant set bits
 	mask := uint32(1) << 31
@@ -85,7 +92,7 @@ func (subnet *ipv4Subnet) String() string {
 		}
 		mask >>= 1
 	}
-	return packet.IPv4ToString(subnet.Addr) + "/" + strconv.Itoa(i)
+	return packet.IPv4ToString(packet.SwapBytesUint32(subnet.Addr)) + "/" + strconv.Itoa(i)
 }
 
 func (subnet *ipv4Subnet) checkAddrWithingSubnet(addr uint32) bool {
@@ -156,8 +163,7 @@ var (
 	NeedKNI        bool
 
 	// Debug variables
-	debugDump = false
-	debugDrop = false
+	DumpEnabled [dirKNI + 1]bool
 )
 
 func (pi pairIndex) Copy() interface{} {
@@ -171,7 +177,7 @@ func (pi pairIndex) Delete() {
 
 func convertIPv4(in []byte) (uint32, error) {
 	if in == nil || len(in) > 4 {
-		return 0, errors.New("Only IPv4 addresses are supported now")
+		return 0, fmt.Errorf("Only IPv4 addresses are supported now while your address has %d bytes", len(in))
 	}
 
 	addr := (uint32(in[0]) << 24) | (uint32(in[1]) << 16) |
@@ -277,6 +283,8 @@ func ReadConfig(fileName string) error {
 
 		pp.PrivatePort.Type = iPRIVATE
 		pp.PublicPort.Type = iPUBLIC
+		pp.PublicPort.opposite = &pp.PrivatePort
+		pp.PrivatePort.opposite = &pp.PublicPort
 
 		if pp.PrivatePort.Vlan == 0 && pp.PublicPort.Vlan != 0 {
 			return errors.New("Private port with index " +
@@ -293,46 +301,52 @@ func ReadConfig(fileName string) error {
 		}
 
 		port := &pp.PrivatePort
-		opposite := &pp.PublicPort
 		for pi := 0; pi < 2; pi++ {
 			for fpi := range port.ForwardPorts {
 				fp := &port.ForwardPorts[fpi]
-				if fp.Destination.Addr == 0 {
-					if port.KNIName == "" {
-						return errors.New("Port with index " +
-							strconv.Itoa(int(port.Index)) +
-							" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0")
-					}
-					fp.forwardToKNI = true
-					if fp.Destination.Port != fp.Port {
-						return errors.New("When address 0.0.0.0 is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
-							strconv.Itoa(int(fp.Port)) + " and " +
-							strconv.Itoa(int(fp.Destination.Port)))
-					}
-					NeedKNI = true
-				} else {
-					if pi == 0 {
-						return errors.New("Only KNI port forwarding is allowed on private port. All translated connections from private to public network can be initiated without any forwarding rules.")
-					}
-					if !opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr) {
-						return errors.New("Destination address " +
-							packet.IPv4ToString(fp.Destination.Addr) +
-							" should be within subnet " +
-							opposite.Subnet.String())
-					}
-					if fp.Destination.Port == 0 {
-						fp.Destination.Port = fp.Port
-					}
+				err := port.checkPortForwarding(fp)
+				if err != nil {
+					return err
 				}
 			}
 			port = &pp.PublicPort
-			opposite = &pp.PrivatePort
 		}
 	}
 
 	return nil
 }
 
+func (port *ipv4Port) checkPortForwarding(fp *forwardedPort) error {
+	if fp.Destination.Addr == 0 {
+		if port.KNIName == "" {
+			return errors.New("Port with index " +
+				strconv.Itoa(int(port.Index)) +
+				" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0")
+		}
+		fp.forwardToKNI = true
+		if fp.Destination.Port != fp.Port {
+			return errors.New("When address 0.0.0.0 is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
+				strconv.Itoa(int(fp.Port)) + " and " +
+				strconv.Itoa(int(fp.Destination.Port)))
+		}
+		NeedKNI = true
+	} else {
+		if port.Type == iPRIVATE {
+			return errors.New("Only KNI port forwarding is allowed on private port. All translated connections from private to public network can be initiated without any forwarding rules.")
+		}
+		if !port.opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr) {
+			return errors.New("Destination address " +
+				packet.IPv4ToString(fp.Destination.Addr) +
+				" should be within subnet " +
+				port.opposite.Subnet.String())
+		}
+		if fp.Destination.Port == 0 {
+			fp.Destination.Port = fp.Port
+		}
+	}
+	return nil
+}
+
 // Reads MAC addresses for local interfaces into pair ports.
 func (pp *portPair) initLocalMACs() {
 	pp.PublicPort.SrcMACAddress = flow.GetPortMACAddress(pp.PublicPort.Index)
@@ -355,26 +369,30 @@ func (port *ipv4Port) allocateLookupMap() {
 
 func (port *ipv4Port) initPublicPortPortForwardingEntries() {
 	// Initialize port forwarding rules on public interface
-	for _, fp := range port.ForwardPorts {
-		keyEntry := Tuple{
-			addr: port.Subnet.Addr,
-			port: fp.Port,
-		}
-		valEntry := Tuple{
-			addr: fp.Destination.Addr,
-			port: fp.Destination.Port,
-		}
-		port.translationTable[fp.Protocol].Store(keyEntry, valEntry)
-		if fp.Destination.Addr != 0 {
-			port.opposite.translationTable[fp.Protocol].Store(valEntry, keyEntry)
-		}
-		port.portmap[fp.Protocol][fp.Port] = portMapEntry{
-			lastused:             time.Now(),
-			addr:                 fp.Destination.Addr,
-			finCount:             0,
-			terminationDirection: 0,
-			static:               true,
-		}
+	for i := range port.ForwardPorts {
+		port.enableStaticPortForward(&port.ForwardPorts[i])
+	}
+}
+
+func (port *ipv4Port) enableStaticPortForward(fp *forwardedPort) {
+	keyEntry := Tuple{
+		addr: port.Subnet.Addr,
+		port: fp.Port,
+	}
+	valEntry := Tuple{
+		addr: fp.Destination.Addr,
+		port: fp.Destination.Port,
+	}
+	port.translationTable[fp.Protocol].Store(keyEntry, valEntry)
+	if fp.Destination.Addr != 0 {
+		port.opposite.translationTable[fp.Protocol].Store(valEntry, keyEntry)
+	}
+	port.portmap[fp.Protocol][fp.Port] = portMapEntry{
+		lastused:             time.Now(),
+		addr:                 fp.Destination.Addr,
+		finCount:             0,
+		terminationDirection: 0,
+		static:               true,
 	}
 }
 
@@ -383,9 +401,6 @@ func InitFlows() {
 	for i := range Natconfig.PortPairs {
 		pp := &Natconfig.PortPairs[i]
 
-		pp.PublicPort.opposite = &pp.PrivatePort
-		pp.PrivatePort.opposite = &pp.PublicPort
-
 		// Init port pairs state
 		pp.initLocalMACs()
 		pp.PrivatePort.allocateLookupMap()
@@ -475,3 +490,16 @@ func CheckHWOffloading() bool {
 
 	return flow.CheckHWCapability(flow.HWTXChecksumCapability, ports)
 }
+
+func (c *Config) getPortAndPairByID(portId uint32) (*ipv4Port, *portPair) {
+	for i := range c.PortPairs {
+		pp := &c.PortPairs[i]
+		if uint32(pp.PublicPort.Index) == portId {
+			return &pp.PublicPort, pp
+		}
+		if uint32(pp.PrivatePort.Index) == portId {
+			return &pp.PrivatePort, pp
+		}
+	}
+	return nil, nil
+}
diff --git a/examples/nat/grpc.go b/examples/nat/grpc.go
new file mode 100644
index 00000000..3eb58302
--- /dev/null
+++ b/examples/nat/grpc.go
@@ -0,0 +1,100 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"fmt"
+	"net"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
+
+	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
+
+	"github.com/intel-go/nff-go/common"
+)
+
+const (
+	GRPCServerPort = ":60602"
+)
+
+type server struct{}
+
+func StartGRPCServer() error {
+	lis, err := net.Listen("tcp", GRPCServerPort)
+	if err != nil {
+		return err
+	}
+	s := grpc.NewServer()
+	upd.RegisterUpdaterServer(s, &server{})
+	// Register reflection service on gRPC server.
+	reflection.Register(s)
+
+	go func() {
+		if err := s.Serve(lis); err != nil {
+			common.LogWarning(common.Initialization, "Error while serving GRPC requests:", err)
+		}
+	}()
+	return nil
+}
+
+func (s *server) ControlDump(ctx context.Context, in *upd.DumpControlRequest) (*upd.Reply, error) {
+	enable := in.GetEnableTrace()
+	dumpType := in.GetTraceType()
+	if dumpType < upd.TraceType_DUMP_DROP || dumpType > upd.TraceType_DUMP_KNI {
+		return nil, fmt.Errorf("Bad value of dump type: %d", dumpType)
+	}
+	DumpEnabled[dumpType] = enable
+
+	return &upd.Reply{
+		Msg: "Success",
+	}, nil
+}
+
+func (s *server) ChangeInterfaceAddress(ctx context.Context, in *upd.InterfaceAddressChangeRequest) (*upd.Reply, error) {
+	portId := in.GetInterfaceId()
+	port, _ := Natconfig.getPortAndPairByID(portId)
+	if port == nil {
+		return nil, fmt.Errorf("Interface with ID %d not found", portId)
+	}
+	subnet, err := convertSubnet(in.GetPortSubnet())
+	if err != nil {
+		return nil, err
+	}
+	port.Subnet = *subnet
+
+	return &upd.Reply{
+		Msg: fmt.Sprintf("Successfully set port %d subnet to %s", portId, subnet.String()),
+	}, nil
+}
+
+func (s *server) ChangePortForwarding(ctx context.Context, in *upd.PortForwardingChangeRequest) (*upd.Reply, error) {
+	portId := in.GetInterfaceId()
+	port, pp := Natconfig.getPortAndPairByID(portId)
+	if port == nil {
+		return nil, fmt.Errorf("Interface with ID %d not found", portId)
+	}
+
+	fp, err := convertForwardedPort(in.GetPort())
+	if err != nil {
+		return nil, err
+	}
+	err = port.checkPortForwarding(fp)
+	if err != nil {
+		return nil, err
+	}
+
+	pp.mutex.Lock()
+	pp.deleteOldConnection(uint8(fp.Protocol), int(fp.Port))
+	if in.GetEnableForwarding() {
+		port.enableStaticPortForward(fp)
+	}
+	pp.mutex.Unlock()
+
+	return &upd.Reply{
+		Msg: "Success",
+	}, nil
+}
diff --git a/examples/nat/main/Makefile b/examples/nat/main/Makefile
index 92531516..655895ff 100644
--- a/examples/nat/main/Makefile
+++ b/examples/nat/main/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2017 Intel Corporation.
+# Copyright 2017-2018 Intel Corporation.
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
@@ -6,6 +6,9 @@ PATH_TO_MK = ../../../mk
 IMAGENAME = nff-go-nat
 EXECUTABLES = nat
 
-nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go
+nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../grpc.go ../updatecfg/updatecfg.pb.go
+
+../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
+	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
 
 include $(PATH_TO_MK)/leaf.mk
diff --git a/examples/nat/main/nat.go b/examples/nat/main/nat.go
index 708acef2..b66f85d8 100644
--- a/examples/nat/main/nat.go
+++ b/examples/nat/main/nat.go
@@ -52,6 +52,9 @@ func main() {
 	// Initialize flows and necessary state
 	nat.InitFlows()
 
+	// Start GRPC server
+	flow.CheckFatal(nat.StartGRPCServer())
+
 	// Start flow scheduler
 	go func() {
 		flow.CheckFatal(flow.SystemStart())
diff --git a/examples/nat/updatecfg/.gitignore b/examples/nat/updatecfg/.gitignore
new file mode 100644
index 00000000..fa6b8979
--- /dev/null
+++ b/examples/nat/updatecfg/.gitignore
@@ -0,0 +1 @@
+updatecfg.pb.go
diff --git a/examples/nat/updatecfg/updatecfg.proto b/examples/nat/updatecfg/updatecfg.proto
new file mode 100644
index 00000000..6a0f9d8b
--- /dev/null
+++ b/examples/nat/updatecfg/updatecfg.proto
@@ -0,0 +1,64 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_outer_classname = "UpdateNatCfg";
+
+package updatecfg;
+
+service Updater {
+  rpc ControlDump (DumpControlRequest) returns (Reply) {}
+  rpc ChangeInterfaceAddress (InterfaceAddressChangeRequest) returns (Reply) {}
+  rpc ChangePortForwarding (PortForwardingChangeRequest) returns (Reply) {}
+}
+
+enum TraceType {
+  DUMP_DROP = 0;
+  DUMP_TRANSLATE = 1;
+  DUMP_KNI = 2;
+}
+
+message DumpControlRequest {
+  bool enable_trace = 1;
+  TraceType trace_type = 2;
+}
+
+enum Protocol {
+  UNKNOWN = 0;
+  TCP = 0x06;
+  UDP = 0x11;
+}
+
+message IPAddress {
+  bytes address = 1;
+}
+
+message Subnet {
+  IPAddress address = 1;
+  uint32 mask_bits_number = 2;
+}
+
+message InterfaceAddressChangeRequest {
+  uint32 interface_id = 1;
+  Subnet port_subnet = 2;
+}
+
+message ForwardedPort {
+  uint32 source_port_number = 1;
+  IPAddress target_address = 2;
+  uint32 target_port_number = 3;
+  Protocol protocol = 4;
+}
+
+message PortForwardingChangeRequest {
+  bool enable_forwarding = 1;
+  uint32 interface_id = 2;
+  ForwardedPort port = 3;
+}
+
+message Reply {
+  string msg = 2;
+}
diff --git a/examples/nat/util.go b/examples/nat/util.go
index b942f289..e36cf3e0 100644
--- a/examples/nat/util.go
+++ b/examples/nat/util.go
@@ -9,6 +9,8 @@ import (
 	"log"
 	"os"
 
+	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
+
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/packet"
 )
@@ -63,7 +65,7 @@ func (port *ipv4Port) startTrace(dir uint) *os.File {
 }
 
 func (port *ipv4Port) dumpPacket(pkt *packet.Packet, dir uint) {
-	if debugDump {
+	if DumpEnabled[dir] {
 		port.dumpsync[dir].Lock()
 		if port.fdump[dir] == nil {
 			port.fdump[dir] = port.startTrace(dir)
@@ -92,3 +94,34 @@ func CloseAllDumpFiles() {
 		Natconfig.PortPairs[i].PublicPort.closePortTraces()
 	}
 }
+
+func convertSubnet(s *upd.Subnet) (*ipv4Subnet, error) {
+	addr, err := convertIPv4(s.GetAddress().GetAddress())
+	if err != nil {
+		return nil, err
+	}
+
+	return &ipv4Subnet{
+		Addr: addr,
+		Mask: uint32(0xffffffff) << (32 - s.GetMaskBitsNumber()),
+	}, nil
+}
+
+func convertForwardedPort(p *upd.ForwardedPort) (*forwardedPort, error) {
+	addr, err := convertIPv4(p.GetTargetAddress().GetAddress())
+	if err != nil {
+		return nil, err
+	}
+	if p.GetProtocol() != common.TCPNumber && p.GetProtocol() != common.UDPNumber {
+		return nil, fmt.Errorf("Bad protocol identifier %d", p.GetProtocol())
+	}
+
+	return &forwardedPort{
+		Port: uint16(p.GetSourcePortNumber()),
+		Destination: hostPort{
+			Addr: addr,
+			Port: uint16(p.GetTargetPortNumber()),
+		},
+		Protocol: protocolId(p.GetProtocol()),
+	}, nil
+}
diff --git a/scripts/get-depends.sh b/scripts/get-depends.sh
index 37ec4f73..6aaa435c 100755
--- a/scripts/get-depends.sh
+++ b/scripts/get-depends.sh
@@ -9,3 +9,9 @@ go get -v github.com/docker/go-connections
 go get -v golang.org/x/tools/cmd/stringer
 go get -v github.com/vishvananda/netlink
 
+# GRPC support for NAT example
+go get -v google.golang.org/grpc
+go get -v github.com/golang/protobuf/protoc-gen-go
+if ! command -v protoc &> /dev/null; then
+    echo You should install protobuf compiler package, e.g. \"sudo dnf install protobuf-compiler\" or \"sudo apt-get install protobuf-compiler\"
+fi
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 72a6411e..ed389cdb 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -61,7 +61,7 @@ fi
 
 echo Installing system packages
 sudo dnf update -y
-sudo dnf install -y python make gcc git numactl-devel libpcap-devel elfutils-libelf-devel NetworkManager net-tools redhat-lsb-core pciutils kernel-modules kernel-devel wget vim
+sudo dnf install -y python make gcc git numactl-devel libpcap-devel elfutils-libelf-devel NetworkManager net-tools redhat-lsb-core pciutils kernel-modules kernel-devel wget vim protobuf-compiler
 sudo systemctl enable NetworkManager
 sudo systemctl start NetworkManager
 SHELL
@@ -69,7 +69,7 @@ SHELL
 $provision_ubuntu = <<SHELL
 echo Installing system packages
 sudo apt-get update
-sudo apt-get install -y python make gcc git libnuma-dev libpcap0.8-dev libelf-dev network-manager
+sudo apt-get install -y python make gcc git libnuma-dev libpcap0.8-dev libelf-dev network-manager protobuf-compiler
 sudo systemctl enable network-manager
 sudo systemctl start network-manager
 SHELL

From ad84a6f70784eefc23731a7c370bd2a74e08a9f3 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Mon, 10 Sep 2018 14:07:36 -0600
Subject: [PATCH 35/50] Scheduler fix

Removed situation with zero division from measue function.
THis function itself can be removed for 210nn per 32 packets.
---
 flow/scheduler.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/flow/scheduler.go b/flow/scheduler.go
index 36e7375e..4ccb7a4f 100644
--- a/flow/scheduler.go
+++ b/flow/scheduler.go
@@ -754,6 +754,10 @@ func constructNewIndex(inIndexNumber int32) []int32 {
 	return newIndex
 }
 
+// This function is used for measuring "wasted" time that framework
+// spends on one "zero get from ring attempt". This function can be
+// replaced by "inline" time measurements which will cost
+// 3 time.Now = ~210nn per each burstSize (32) packets.
 func (scheduler *scheduler) measure(N int32, clones int) uint64 {
 	core, index, err := scheduler.getCore()
 	if err != nil {
@@ -770,6 +774,7 @@ func (scheduler *scheduler) measure(N int32, clones int) uint64 {
 	stopper[0] = make(chan int)
 	stopper[1] = make(chan int)
 	report := make(chan reportPair, 20)
+	var reportedState reportPair
 	var avg uint64
 	t := schedTime
 	pause := clones - 1
@@ -785,11 +790,17 @@ func (scheduler *scheduler) measure(N int32, clones int) uint64 {
 		}()
 		<-stopper[1]
 		stopper[0] <- pause
-		reportedState := <-report
-		reportedState = <-report
+		<-report
+		for reportedState.ZeroAttempts[0] == 0 {
+			reportedState = <-report
+		}
 		stopper[0] <- -1
 		<-stopper[1]
+		for len(report) > 0 {
+			<-report
+		}
 		avg += uint64(schedTime) * 1000000 / reportedState.ZeroAttempts[0]
+		reportedState.ZeroAttempts[0] = 0
 	}
 	scheduler.setCoreByIndex(index)
 	schedTime = t

From 902cc4ad3363c8a817016a98c7e9443088284692 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Mon, 10 Sep 2018 15:19:29 -0500
Subject: [PATCH 36/50] Trivial version bump for fedora image

---
 vagrant/Vagrantfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index ed389cdb..012531db 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -20,7 +20,7 @@ Vagrant.configure(2) do |config|
   vm_second_port_base = vm_port_base + (vm_links_number + 1) * vm_total_number
 
 #  config.vm.box = "ubuntu/xenial64"
-  config.vm.box = "fedora/27-cloud-base"
+  config.vm.box = "fedora/28-cloud-base"
 
   # Docker server port
   config.vm.network "forwarded_port", guest: 2375, host: 2375, auto_correct: true

From d03240ab61629548e0e767a53bf2b2a96093e5b3 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Tue, 11 Sep 2018 09:38:21 -0500
Subject: [PATCH 37/50] Fixed docker images build

---
 examples/nat/client/Makefile | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/examples/nat/client/Makefile b/examples/nat/client/Makefile
index ad800782..dd68f254 100644
--- a/examples/nat/client/Makefile
+++ b/examples/nat/client/Makefile
@@ -2,12 +2,18 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-PATH_TO_MK = ../../../mk
-EXECUTABLES = client
+% : %.go
+	go build $<
 
-nat: client.go ../updatecfg/updatecfg.pb.go
+all: client ../updatecfg/updatecfg.pb.go
 
 ../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
 	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
 
-include $(PATH_TO_MK)/leaf.mk
+clean:
+	-rm client
+
+images: all
+deploy: all
+clean-images: clean
+cleanall: clean

From 84a2aa481e64a1e82135bf99fc5140c1d1268e3e Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Wed, 5 Sep 2018 21:15:28 +0000
Subject: [PATCH 38/50] Implemented DHCP client functionality to acquire IPv4
 address

---
 examples/Makefile                      |   8 +-
 examples/nat/config.go                 |  43 +++--
 examples/nat/dhcp.go                   | 236 +++++++++++++++++++++++++
 examples/nat/main/Makefile             |   2 +-
 examples/nat/main/config-dhcp.json     |  37 ++++
 examples/nat/main/config-kni-dhcp.json |  51 ++++++
 examples/nat/main/nat.go               |  11 +-
 examples/nat/translation.go            |  43 ++++-
 flow/flow.go                           |  28 ++-
 scripts/get-depends.sh                 |   1 +
 10 files changed, 430 insertions(+), 30 deletions(-)
 create mode 100644 examples/nat/dhcp.go
 create mode 100644 examples/nat/main/config-dhcp.json
 create mode 100644 examples/nat/main/config-kni-dhcp.json

diff --git a/examples/Makefile b/examples/Makefile
index cce9d5f6..83f55d77 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -6,15 +6,9 @@ PATH_TO_MK = ../mk
 IMAGENAME = nff-go-examples
 EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
 			  createPacket sendFixedPktsNumber gtpu pingReplay \
-			  netlink
+			  netlink gopacketParserExample
 SUBDIRS = nat tutorial antiddos demo fileReadWrite firewall forwarding
 
-gopacket:
-	go get -v github.com/google/gopacket
-
-gopacketParserExample: gopacket
-	go build gopacketParserExample.go
-
 .PHONY: dpi nffPktgen
 dpi:
 	$(MAKE) -C dpi
diff --git a/examples/nat/config.go b/examples/nat/config.go
index ea2ba7f1..1aabc420 100644
--- a/examples/nat/config.go
+++ b/examples/nat/config.go
@@ -74,8 +74,10 @@ func (out *protocolId) UnmarshalJSON(b []byte) error {
 }
 
 type ipv4Subnet struct {
-	Addr uint32
-	Mask uint32
+	Addr            uint32
+	Mask            uint32
+	addressAcquired bool
+	ds              dhcpState
 }
 
 func (fp *forwardedPort) String() string {
@@ -83,16 +85,19 @@ func (fp *forwardedPort) String() string {
 }
 
 func (subnet *ipv4Subnet) String() string {
-	// Count most significant set bits
-	mask := uint32(1) << 31
-	i := 0
-	for ; i <= 32; i++ {
-		if subnet.Mask&mask == 0 {
-			break
+	if subnet.addressAcquired {
+		// Count most significant set bits
+		mask := uint32(1) << 31
+		i := 0
+		for ; i <= 32; i++ {
+			if subnet.Mask&mask == 0 {
+				break
+			}
+			mask >>= 1
 		}
-		mask >>= 1
+		return packet.IPv4ToString(packet.SwapBytesUint32(subnet.Addr)) + "/" + strconv.Itoa(i)
 	}
-	return packet.IPv4ToString(packet.SwapBytesUint32(subnet.Addr)) + "/" + strconv.Itoa(i)
+	return "DHCP address not acquired"
 }
 
 func (subnet *ipv4Subnet) checkAddrWithingSubnet(addr uint32) bool {
@@ -143,6 +148,7 @@ type portPair struct {
 
 // Config for NAT.
 type Config struct {
+	HostName  string     `json:"host-name"`
 	PortPairs []portPair `json:"port-pairs"`
 }
 
@@ -161,6 +167,7 @@ var (
 	// offloaded to HW.
 	NoHWTXChecksum bool
 	NeedKNI        bool
+	NeedDHCP       bool
 
 	// Debug variables
 	DumpEnabled [dirKNI + 1]bool
@@ -193,6 +200,13 @@ func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
 		return err
 	}
 
+	if s == "dhcp" {
+		out.Addr = uint32(0)
+		out.Mask = uint32(0)
+		out.addressAcquired = false
+		return nil
+	}
+
 	if ip, ipnet, err := net.ParseCIDR(s); err == nil {
 		if out.Addr, err = convertIPv4(ip.To4()); err != nil {
 			return err
@@ -200,6 +214,7 @@ func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
 		if out.Mask, err = convertIPv4(ipnet.Mask); err != nil {
 			return err
 		}
+		out.addressAcquired = true
 		return nil
 	}
 
@@ -209,6 +224,7 @@ func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
 			return err
 		}
 		out.Mask = 0xffffffff
+		out.addressAcquired = true
 		return nil
 	}
 	return errors.New("Failed to parse address " + s)
@@ -302,6 +318,13 @@ func ReadConfig(fileName string) error {
 
 		port := &pp.PrivatePort
 		for pi := 0; pi < 2; pi++ {
+			if !port.Subnet.addressAcquired {
+				if Natconfig.HostName == "" {
+					return fmt.Errorf("DHCP option for port %d requires that you set host-name configuration option", port.Index)
+				}
+				NeedDHCP = true
+			}
+
 			for fpi := range port.ForwardPorts {
 				fp := &port.ForwardPorts[fpi]
 				err := port.checkPortForwarding(fp)
diff --git a/examples/nat/dhcp.go b/examples/nat/dhcp.go
new file mode 100644
index 00000000..98667e00
--- /dev/null
+++ b/examples/nat/dhcp.go
@@ -0,0 +1,236 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"math/rand"
+	"net"
+	"time"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+
+	"github.com/vishvananda/netlink"
+
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
+)
+
+type dhcpState struct {
+	lastDHCPPacketTypeSent layers.DHCPMsgType
+	dhcpTransactionId      uint32
+}
+
+const (
+	requestInterval = 10 * time.Second
+	DHCPSrcPort     = 68
+	DHCPDstPort     = 67
+	BroadcastIPv4   = uint32(0xffffffff)
+)
+
+var (
+	rnd          = rand.New(rand.NewSource(time.Now().UnixNano()))
+	BroadcastMAC = [common.EtherAddrLen]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+	dhcpOptions  = []layers.DHCPOption{
+		layers.NewDHCPOption(layers.DHCPOptParamsRequest,
+			[]byte{
+				byte(layers.DHCPOptSubnetMask),
+				byte(layers.DHCPOptBroadcastAddr),
+				byte(layers.DHCPOptTimeOffset),
+				byte(layers.DHCPOptRouter),
+				byte(layers.DHCPOptDomainName),
+				byte(layers.DHCPOptDNS),
+				byte(layers.DHCPOptDomainSearch),
+				byte(layers.DHCPOptHostname),
+				byte(layers.DHCPOptInterfaceMTU),
+			},
+		),
+	}
+	dhcpRequestPacket = layers.DHCPv4{
+		Operation:    layers.DHCPOpRequest,
+		HardwareType: layers.LinkTypeEthernet,
+		ClientIP: net.IP{
+			0, 0, 0, 0,
+		},
+		YourClientIP: net.IP{
+			0, 0, 0, 0,
+		},
+		NextServerIP: net.IP{
+			0, 0, 0, 0,
+		},
+		RelayAgentIP: net.IP{
+			0, 0, 0, 0,
+		},
+	}
+)
+
+func StartDHCPClient() {
+	go func() {
+		sendDHCPRequests()
+	}()
+}
+
+func sendDHCPRequests() {
+	// Endless loop of sending DHCP requests
+	for {
+		for i := range Natconfig.PortPairs {
+			pp := &Natconfig.PortPairs[i]
+			if !pp.PublicPort.Subnet.addressAcquired {
+				pp.PublicPort.sendDHCPDiscoverRequest()
+			}
+			if !pp.PrivatePort.Subnet.addressAcquired {
+				pp.PrivatePort.sendDHCPDiscoverRequest()
+			}
+		}
+		time.Sleep(requestInterval)
+	}
+}
+
+func getDHCPOption(dhcp *layers.DHCPv4, optionType layers.DHCPOpt) *layers.DHCPOption {
+	for i := range dhcp.Options {
+		if dhcp.Options[i].Type == optionType {
+			return &dhcp.Options[i]
+		}
+	}
+	return nil
+}
+
+func (port *ipv4Port) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, options []layers.DHCPOption) {
+	hwa := make([]byte, common.EtherAddrLen)
+	copy(hwa, port.SrcMACAddress[:])
+
+	buf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		FixLengths:       true,
+		ComputeChecksums: true,
+	}
+
+	// Make local copy for modifications
+	dhcp := dhcpRequestPacket
+	dhcp.Xid = port.Subnet.ds.dhcpTransactionId
+	dhcp.ClientHWAddr = hwa
+	options = append(options,
+		layers.NewDHCPOption(layers.DHCPOptMessageType, []byte{byte(packetType)}),
+		layers.NewDHCPOption(layers.DHCPOptHostname, []byte(Natconfig.HostName)))
+	dhcp.Options = options
+	err := gopacket.SerializeLayers(buf, opts, &dhcp)
+	if err != nil {
+		common.LogFatal(common.No, err)
+	}
+
+	// Convert gopacket data structure into NFF-Go packet and send it
+	pkt, err := packet.NewPacket()
+	if err != nil {
+		println(err)
+	}
+	payloadBuffer := buf.Bytes()
+	packet.InitEmptyIPv4UDPPacket(pkt, uint(len(payloadBuffer)))
+	if port.Vlan != 0 {
+		pkt.AddVLANTag(port.Vlan)
+	}
+	payload, _ := pkt.GetPacketPayload()
+	copy(payload, payloadBuffer)
+
+	pkt.Ether.SAddr = port.SrcMACAddress
+	pkt.Ether.DAddr = BroadcastMAC
+	pkt.GetIPv4NoCheck().SrcAddr = uint32(0)
+	pkt.GetIPv4NoCheck().DstAddr = BroadcastIPv4
+	pkt.GetUDPNoCheck().SrcPort = packet.SwapBytesUint16(DHCPSrcPort)
+	pkt.GetUDPNoCheck().DstPort = packet.SwapBytesUint16(DHCPDstPort)
+
+	setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+	port.dumpPacket(pkt, dirSEND)
+	pkt.SendPacket(port.Index)
+
+	port.Subnet.ds.lastDHCPPacketTypeSent = packetType
+}
+
+func (port *ipv4Port) sendDHCPDiscoverRequest() {
+	port.Subnet.ds.dhcpTransactionId = rnd.Uint32()
+	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeDiscover, dhcpOptions)
+}
+
+func (port *ipv4Port) sendDHCPRequestRequest(serverIP, clientIP []byte) {
+	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeRequest, append(dhcpOptions,
+		layers.NewDHCPOption(layers.DHCPOptServerID, serverIP),
+		layers.NewDHCPOption(layers.DHCPOptRequestIP, clientIP)))
+}
+
+func (port *ipv4Port) handleDHCP(pkt *packet.Packet) bool {
+	if port.Subnet.addressAcquired {
+		// Port already has address, ignore this traffic
+		return false
+	}
+
+	// Check that this is DHCP offer or acknowledgement traffic
+	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPSrcPort) ||
+		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPDstPort) {
+		return false
+	}
+
+	var dhcp layers.DHCPv4
+	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeDHCPv4, &dhcp)
+	payload, _ := pkt.GetPacketPayload()
+	decoded := []gopacket.LayerType{}
+	err := parser.DecodeLayers(payload, &decoded)
+
+	if err != nil || len(decoded) != 1 || decoded[0] != layers.LayerTypeDHCPv4 {
+		println("Warning! Failed to parse DHCP packet", err)
+		return false
+	}
+
+	dhcpMessageType := getDHCPOption(&dhcp, layers.DHCPOptMessageType)
+	if dhcpMessageType == nil {
+		println("Warning! DHCP packet without message type received")
+		return false
+	}
+
+	if port.Subnet.ds.lastDHCPPacketTypeSent == layers.DHCPMsgTypeDiscover &&
+		dhcpMessageType.Data[0] == byte(layers.DHCPMsgTypeOffer) {
+		port.handleDHCPOffer(pkt, &dhcp)
+	} else if port.Subnet.ds.lastDHCPPacketTypeSent == layers.DHCPMsgTypeRequest &&
+		dhcpMessageType.Data[0] == byte(layers.DHCPMsgTypeAck) {
+		port.handleDHCPAck(pkt, &dhcp)
+	} else {
+		println("Warning! Received some bad response from DHCP server. Trying again with discover request.")
+		port.Subnet.addressAcquired = false
+		port.Subnet.ds = dhcpState{}
+	}
+	return true
+}
+
+func (port *ipv4Port) handleDHCPOffer(pkt *packet.Packet, dhcp *layers.DHCPv4) {
+	port.sendDHCPRequestRequest(dhcp.NextServerIP, dhcp.YourClientIP)
+}
+
+func (port *ipv4Port) handleDHCPAck(pkt *packet.Packet, dhcp *layers.DHCPv4) {
+	maskOption := getDHCPOption(dhcp, layers.DHCPOptSubnetMask)
+	if maskOption == nil {
+		println("Warning! Received a DHCP response without subnet mask! Trying again with discover request.")
+		port.Subnet.addressAcquired = false
+		port.Subnet.ds = dhcpState{}
+		return
+	}
+	port.Subnet.Addr, _ = convertIPv4(dhcp.YourClientIP.To4())
+	port.Subnet.Mask, _ = convertIPv4(maskOption.Data)
+	port.Subnet.addressAcquired = true
+	println("Successfully acquired IP address:", port.Subnet.String())
+
+	// Set address on KNI interface if present
+	if port.KNIName != "" {
+		myKNI, err := netlink.LinkByName(port.KNIName)
+		if err != nil {
+			println("Failed to get KNI interface", port.KNIName, ":", err)
+			return
+		}
+		addr := &netlink.Addr{
+			IPNet: &net.IPNet{
+				IP:   dhcp.YourClientIP,
+				Mask: maskOption.Data,
+			},
+		}
+		netlink.AddrAdd(myKNI, addr)
+	}
+}
diff --git a/examples/nat/main/Makefile b/examples/nat/main/Makefile
index 655895ff..7ac82cbc 100644
--- a/examples/nat/main/Makefile
+++ b/examples/nat/main/Makefile
@@ -6,7 +6,7 @@ PATH_TO_MK = ../../../mk
 IMAGENAME = nff-go-nat
 EXECUTABLES = nat
 
-nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../grpc.go ../updatecfg/updatecfg.pb.go
+nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../grpc.go ../dhcp.go ../updatecfg/updatecfg.pb.go
 
 ../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
 	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
diff --git a/examples/nat/main/config-dhcp.json b/examples/nat/main/config-dhcp.json
new file mode 100644
index 00000000..65c44392
--- /dev/null
+++ b/examples/nat/main/config-dhcp.json
@@ -0,0 +1,37 @@
+{
+    "host-name": "nat",
+    "port-pairs": [
+        {
+            "private-port": {
+                "index": 0,
+                "subnet": "dhcp"
+            },
+            "public-port": {
+                "index": 1,
+                "subnet": "dhcp",
+                "forward-ports": [
+                    {
+                        "port": 8080,
+                        "destination": "192.168.14.3:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2222,
+                        "destination": "192.168.14.3:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8081,
+                        "destination": "192.168.14.4:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "192.168.14.4:22",
+                        "protocol": "TCP"
+                    }
+                ]
+            }
+        }
+    ]
+}
diff --git a/examples/nat/main/config-kni-dhcp.json b/examples/nat/main/config-kni-dhcp.json
new file mode 100644
index 00000000..aad31117
--- /dev/null
+++ b/examples/nat/main/config-kni-dhcp.json
@@ -0,0 +1,51 @@
+{
+    "host-name": "nat",
+    "port-pairs": [
+        {
+            "private-port": {
+                "index": 0,
+                "subnet": "dhcp",
+                "kni-name": "priv0",
+                "forward-ports": [
+                    {
+                        "port": 22,
+                        "destination": "0.0.0.0:22",
+                        "protocol": "TCP"
+                    }
+                ]
+            },
+            "public-port": {
+                "index": 1,
+                "subnet": "dhcp",
+                "kni-name": "pub1",
+                "forward-ports": [
+                    {
+                        "port": 22,
+                        "destination": "0.0.0.0:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8080,
+                        "destination": "192.168.14.3:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2222,
+                        "destination": "192.168.14.3:22",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 8081,
+                        "destination": "192.168.14.4:80",
+                        "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "192.168.14.4:22",
+                        "protocol": "TCP"
+                    }
+                ]
+            }
+        }
+    ]
+}
diff --git a/examples/nat/main/nat.go b/examples/nat/main/nat.go
index b66f85d8..790c3be4 100644
--- a/examples/nat/main/nat.go
+++ b/examples/nat/main/nat.go
@@ -55,9 +55,18 @@ func main() {
 	// Start GRPC server
 	flow.CheckFatal(nat.StartGRPCServer())
 
+	// Perform all network initialization so that DHCP client could
+	// start sending packets
+	flow.CheckFatal(flow.SystemInitPortsAndMemory())
+
+	// Start DHCP client
+	if nat.NeedDHCP {
+		nat.StartDHCPClient()
+	}
+
 	// Start flow scheduler
 	go func() {
-		flow.CheckFatal(flow.SystemStart())
+		flow.CheckFatal(flow.SystemStartScheduler())
 	}()
 
 	// Wait for interrupt
diff --git a/examples/nat/translation.go b/examples/nat/translation.go
index cc98a981..53e5ab4f 100644
--- a/examples/nat/translation.go
+++ b/examples/nat/translation.go
@@ -71,15 +71,28 @@ func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 		return dir
 	}
 
+	// Check for DHCP traffic. We need to get an address if it not set yet
+	if pktUDP != nil {
+		if port.handleDHCP(pkt) {
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
+		}
+	}
+
 	// Do lookup
 	v, found := port.translationTable[protocol].Load(*pub2priKey)
-	// For ingress connections packets are allowed only if a
-	// connection has been previosly established with a egress
-	// (private to public) packet. So if lookup fails, this incoming
-	// packet is ignored unless there is a KNI interface. If KNI is
-	// present, traffic is directed there.
+
 	if !found {
-		if port.KNIName != "" {
+		// Store new local network entry in ARP cache
+		port.arpTable.Store(pktIPv4.SrcAddr, pkt.Ether.SAddr)
+
+		// For ingress connections packets are allowed only if a
+		// connection has been previosly established with a egress
+		// (private to public) packet. So if lookup fails, this
+		// incoming packet is ignored unless there is a KNI
+		// interface. If KNI is present and its IP address is known,
+		// traffic is directed there.
+		if port.KNIName != "" && port.Subnet.addressAcquired {
 			dir = dirKNI
 		} else {
 			dir = dirDROP
@@ -152,9 +165,17 @@ func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 		return dir
 	}
 
+	// Check for DHCP traffic. We need to get an address if it not set yet
+	if pktUDP != nil {
+		if port.handleDHCP(pkt) {
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
+		}
+	}
+
 	// If traffic is directed at private interface IP and KNI is
 	// present, this traffic is directed to KNI
-	if port.KNIName != "" && port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr) {
+	if port.KNIName != "" && port.Subnet.addressAcquired && port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr) {
 		port.dumpPacket(pkt, dirKNI)
 		return dirKNI
 	}
@@ -162,10 +183,18 @@ func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 	// Do lookup
 	var value Tuple
 	v, found := port.translationTable[protocol].Load(*pri2pubKey)
+
 	if !found {
 		var err error
 		// Store new local network entry in ARP cache
 		port.arpTable.Store(pri2pubKey.addr, pkt.Ether.SAddr)
+
+		if !port.Subnet.addressAcquired || !port.opposite.Subnet.addressAcquired {
+			// No packets are allowed yet because ports address is not
+			// known yet
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP
+		}
 		// Allocate new connection from private to public network
 		value, err = pp.allocateNewEgressConnection(protocol, pri2pubKey)
 
diff --git a/flow/flow.go b/flow/flow.go
index fed8d989..f676fe59 100644
--- a/flow/flow.go
+++ b/flow/flow.go
@@ -550,10 +550,9 @@ func SystemInit(args *Config) error {
 	return nil
 }
 
-// SystemStart starts system - begin packet receiving and packet sending.
-// This functions should be always called after flow graph construction.
-// Function can panic during execution.
-func SystemStart() error {
+// SystemInitPortsAndMemory performs all initialization necessary to
+// create and send new packets before scheduler may be started.
+func SystemInitPortsAndMemory() error {
 	if openFlowsNumber != 0 {
 		return common.WrapWithNFError(nil, "Some flows are left open at the end of configuration!", common.OpenedFlowAtTheEnd)
 	}
@@ -571,6 +570,12 @@ func SystemStart() error {
 	common.LogTitle(common.Initialization, "------------***------ Starting FlowFunctions -----***------------")
 	// Init low performance mempool
 	packet.SetNonPerfMempool(low.CreateMempool("slow operations"))
+	return nil
+}
+
+// SystemStartScheduler starts scheduler packet processing. Function
+// does not return.
+func SystemStartScheduler() error {
 	if err := schedState.systemStart(); err != nil {
 		return common.WrapWithNFError(err, "scheduler start failed", common.Fail)
 	}
@@ -579,6 +584,21 @@ func SystemStart() error {
 	return nil
 }
 
+// SystemStart starts system - begin packet receiving and packet sending.
+// This functions should be always called after flow graph construction.
+// Function can panic during execution.
+func SystemStart() error {
+	err := SystemInitPortsAndMemory()
+	if err != nil {
+		return err
+	}
+	err = SystemStartScheduler()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // SystemStop stops the system. All Flow functions plus resource releasing
 // Doesn't cleanup DPDK
 func SystemStop() {
diff --git a/scripts/get-depends.sh b/scripts/get-depends.sh
index 6aaa435c..1d43abc3 100755
--- a/scripts/get-depends.sh
+++ b/scripts/get-depends.sh
@@ -8,6 +8,7 @@ go get -v golang.org/x/net/proxy
 go get -v github.com/docker/go-connections
 go get -v golang.org/x/tools/cmd/stringer
 go get -v github.com/vishvananda/netlink
+go get -v github.com/google/gopacket
 
 # GRPC support for NAT example
 go get -v google.golang.org/grpc

From a8ee3ac26fbb9bc9c12231b179ab4b018b6df0c0 Mon Sep 17 00:00:00 2001
From: Ilia Filippov <ilia.filippov@intel.com>
Date: Mon, 17 Sep 2018 12:03:05 -0600
Subject: [PATCH 39/50] Only supported RSS types

---
 low/low.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/low/low.h b/low/low.h
index 9df0fd5f..24b63674 100644
--- a/low/low.h
+++ b/low/low.h
@@ -183,7 +183,7 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 			.mq_mode = ETH_MQ_RX_RSS    },
 	    .txmode = { .mq_mode = ETH_MQ_TX_NONE, },
 	    .rx_adv_conf.rss_conf.rss_key = NULL,
-	    .rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK
+	    .rx_adv_conf.rss_conf.rss_hf = dev_info.flow_type_rss_offloads
 	};
 
 	/* Configure the Ethernet device. */

From 8bb091ff946fb6dca0fe56eea8a912b6a2518091 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <marcus@glasnostic.com>
Date: Sat, 22 Sep 2018 04:02:44 +0800
Subject: [PATCH 40/50] F 1 Binding a device by NIC (#476)

* [devices] move devices package from crimea to nff-go
* [devices] add a devbind example
---
 devices/bind.go     |  75 +++++++++++++++
 devices/consts.go   | 112 +++++++++++++++++++++++
 devices/errno.go    |  21 +++++
 devices/misc.go     |  98 ++++++++++++++++++++
 devices/pci.go      | 218 ++++++++++++++++++++++++++++++++++++++++++++
 devices/vmbus.go    | 169 ++++++++++++++++++++++++++++++++++
 examples/.gitignore |   1 +
 examples/Makefile   |   2 +-
 examples/devbind.go |  71 +++++++++++++++
 9 files changed, 766 insertions(+), 1 deletion(-)
 create mode 100644 devices/bind.go
 create mode 100644 devices/consts.go
 create mode 100644 devices/errno.go
 create mode 100644 devices/misc.go
 create mode 100644 devices/pci.go
 create mode 100644 devices/vmbus.go
 create mode 100644 examples/devbind.go

diff --git a/devices/bind.go b/devices/bind.go
new file mode 100644
index 00000000..31758b7a
--- /dev/null
+++ b/devices/bind.go
@@ -0,0 +1,75 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+// Device is a DPDK compatible device and should be able to bind, unbind and
+// probe.
+type Device interface {
+	// Binds a driver to the device
+	Bind(driver string) error
+	// Unbinds the current driver from the device
+	Unbind() error
+	// Returns the name of the driver that is currently bound
+	CurrentDriver() (string, error)
+	// Probes the currently bound driver and checks if there is an error
+	Probe() error
+	// Returns the ID of the device
+	ID() string
+}
+
+// New returns a corresponding device by given input
+func New(input string) (Device, error) {
+	switch {
+	case IsPciID.Match([]byte(input)):
+		return NewDeviceByPciID(input)
+	case IsUUID.Match([]byte(input)):
+		return NewDeviceByVmbusID(input)
+	default:
+		return NewDeviceByNicName(input)
+	}
+}
+
+// NewDeviceByPciID returns a PCI device by given PCI ID
+func NewDeviceByPciID(pciID string) (Device, error) {
+	device, err := GetPciDeviceByPciID(pciID)
+	if err != nil {
+		return nil, err
+	}
+
+	return device, nil
+}
+
+// NewDeviceByVmbusID returns a VMBus device by given UUID
+func NewDeviceByVmbusID(uuid string) (Device, error) {
+	device, err := GetVmbusDeviceByUUID(uuid)
+	if err != nil {
+		return nil, err
+	}
+
+	return device, nil
+}
+
+// NewDeviceByNicName returns a device by given NIC name, e.g. eth0.
+func NewDeviceByNicName(nicName string) (Device, error) {
+	devID, err := GetDeviceID(nicName)
+	if err != nil {
+		return nil, err
+	}
+
+	device, err := newDevice(devID)
+	if err != nil {
+		return nil, err
+	}
+
+	return device, nil
+}
+
+func newDevice(id string) (Device, error) {
+	if IsPciID.Match([]byte(id)) {
+		return GetPciDeviceByPciID(id)
+	}
+	return GetVmbusDeviceByUUID(id)
+}
diff --git a/devices/consts.go b/devices/consts.go
new file mode 100644
index 00000000..62be880e
--- /dev/null
+++ b/devices/consts.go
@@ -0,0 +1,112 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+import (
+	"fmt"
+	"regexp"
+)
+
+// Driver names
+const (
+	DriverHvNetvcs      = "hv_netvcs"
+	DriverUioPciGeneric = "uio_pci_generic"
+	DriverIgUio         = "ig_uio"
+	DriverVfioPci       = "vfio-pci"
+	DriverUioHvGeneric  = "uio_hv_generic"
+)
+
+// Path to PCI
+const (
+	PathSysPciDevices     = "/sys/bus/pci/devices"
+	PathSysPciDrivers     = "/sys/bus/pci/drivers"
+	PathSysPciDriverProbe = "/sys/bus/pci/drivers_probe"
+)
+
+// Path to VMBus
+const (
+	PathSysVmbusDevices = "/sys/bus/vmbus/devices"
+	PathSysVmbusDrivers = "/sys/bus/vmbus/drivers"
+)
+
+// Path to net
+const (
+	PathSysClassNet = "/sys/class/net"
+)
+
+// Regular expressions for PCI-ID and UUID
+var (
+	IsPciID *regexp.Regexp
+	IsUUID  *regexp.Regexp
+)
+
+// DPDK related drivers
+var (
+	DefaultDpdkDriver = DriverUioPciGeneric
+	DpdkDrivers       = [...]string{DriverUioPciGeneric, DriverIgUio, DriverVfioPci, DriverUioHvGeneric}
+	DpdkPciDrivers    = [...]string{DriverUioPciGeneric, DriverIgUio, DriverVfioPci}
+	DpdkVmbusDrivers  = [...]string{DriverUioHvGeneric}
+)
+
+type stringBuilder string
+
+func (s stringBuilder) With(args ...interface{}) string {
+	return fmt.Sprintf(string(s), args...)
+}
+
+var (
+	pathSysPciDevicesBind           stringBuilder = PathSysPciDevices + "/%s/driver/bind"
+	pathSysPciDevicesUnbind         stringBuilder = PathSysPciDevices + "/%s/driver/unbind"
+	pathSysPciDevicesOverrideDriver stringBuilder = PathSysPciDevices + "/%s/driver_override"
+
+	pathSysPciDriversBind   stringBuilder = PathSysPciDrivers + "/%s/bind"
+	pathSysPciDriversUnbind stringBuilder = PathSysPciDrivers + "/%s/unbind"
+	pathSysPciDriversNewID  stringBuilder = PathSysPciDrivers + "/%s/new_id"
+)
+
+var (
+	pathSysVmbusDriversBind   stringBuilder = PathSysVmbusDrivers + "/%s/bind"
+	pathSysVmbusDriversUnbind stringBuilder = PathSysVmbusDrivers + "/%s/unbind"
+	pathSysVmbusDriversNewID  stringBuilder = PathSysVmbusDrivers + "/%s/new_id"
+)
+
+var (
+	pathSysClassNetDeviceDriver stringBuilder = PathSysClassNet + "/%s/device/driver"
+	pathSysClassNetDevice       stringBuilder = PathSysClassNet + "/%s/device"
+)
+
+var (
+	vmbusDeviceStringer stringBuilder = "ID: %s\nClass:\t%s\nDriver:\t%s"
+	pciDeviceStringer   stringBuilder = "ID: %s\nClass:\t%s\nVendor:\t%s\nDevice:\t%s\nDriver:\t%s"
+)
+
+var (
+	// for ethtool output
+	rPciIDForEthtool *regexp.Regexp
+
+	// for lspci output
+	rPciClass  *regexp.Regexp
+	rPciVendor *regexp.Regexp
+	rPciDevice *regexp.Regexp
+	rPciDriver *regexp.Regexp
+
+	// for find output
+	rVmbusDriver *regexp.Regexp
+)
+
+func init() {
+	rPciIDForEthtool = regexp.MustCompile("bus-info:\\s([0-9:.]+)")
+
+	rPciClass = regexp.MustCompile("[Cc]lass:\\s([0-9a-zA-Z]{4})")
+	rPciVendor = regexp.MustCompile("[Vv]endor:\\s([0-9a-zA-Z]{4})")
+	rPciDevice = regexp.MustCompile("[Dd]evice:\\s([0-9a-zA-Z]{4})")
+	rPciDriver = regexp.MustCompile("[Dd]river:\\s(\\S+)")
+
+	rVmbusDriver = regexp.MustCompile("/sys/bus/vmbus/drivers/(\\S+)/")
+
+	IsPciID = regexp.MustCompile("^\\d{4}:\\d{2}:\\d{2}.\\d$")
+	IsUUID = regexp.MustCompile("^[[:xdigit:]]{8}-[[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$")
+}
diff --git a/devices/errno.go b/devices/errno.go
new file mode 100644
index 00000000..88bf8f79
--- /dev/null
+++ b/devices/errno.go
@@ -0,0 +1,21 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+import (
+	"errors"
+)
+
+// Errors of devices package
+var (
+	ErrNoBoundDriver         = errors.New("no driver is bound to the device")
+	ErrAlreadyBoundDriver    = errors.New("device has already bound the selected driver")
+	ErrBind                  = errors.New("fail to bind the driver")
+	ErrUnbind                = errors.New("fail to unbind the driver")
+	ErrUnsupportedDriver     = errors.New("unsupported DPDK driver")
+	ErrNotProbe              = errors.New("device doesn't support 'drive_probe'")
+	ErrKernelModuleNotLoaded = errors.New("kernel module is not loaded")
+)
diff --git a/devices/misc.go b/devices/misc.go
new file mode 100644
index 00000000..e8e80bb3
--- /dev/null
+++ b/devices/misc.go
@@ -0,0 +1,98 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var defaultTimeoutLimitation = 5 * time.Second
+
+// FindDefaultDpdkDriver returns a default DPDK driver that the given NIC can
+// use.
+func FindDefaultDpdkDriver(nicName string) string {
+	driver, err := readlinkBaseCmd(pathSysClassNetDeviceDriver.With(nicName))
+	if err != nil {
+		return DefaultDpdkDriver
+	}
+	switch driver {
+	case DriverHvNetvcs:
+		return DriverUioHvGeneric
+	default:
+		return DefaultDpdkDriver
+	}
+}
+
+// GetDeviceID returns the device ID of given NIC name.
+func GetDeviceID(nicName string) (string, error) {
+	// DEV_ID=$(basename $(readlink /sys/class/net/<nicName>/device))
+	return readlinkBaseCmd(pathSysClassNetDevice.With(nicName))
+}
+
+// IsModuleLoaded checks if the kernel has already loaded the driver or not.
+func IsModuleLoaded(driver string) bool {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "lsmod")
+	if err != nil {
+		// Can't run lsmod, return false
+		return false
+	}
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		if checkModuleLine(line, driver) {
+			return true
+		}
+	}
+	return false
+}
+
+func writeToTargetWithData(sysfs string, flag int, mode os.FileMode, data string) error {
+	writer, err := os.OpenFile(sysfs, flag, mode)
+	if err != nil {
+		return fmt.Errorf("OpenFile failed: %s", err.Error())
+	}
+	defer writer.Close()
+
+	_, err = writer.Write([]byte(data))
+	if err != nil {
+		return fmt.Errorf("WriteFile failed: %s", err.Error())
+	}
+
+	return nil
+}
+
+func readlinkBaseCmd(path string) (string, error) {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "readlink", path)
+	if err != nil {
+		return "", fmt.Errorf("Cmd Execute readlink failed: %s", err.Error())
+	}
+	outputStr := strings.Trim(string(output), "\n")
+	result := filepath.Base(outputStr)
+	return result, nil
+}
+
+func checkModuleLine(line, driver string) bool {
+	if !strings.Contains(line, driver) {
+		return false
+	}
+	elements := strings.Split(line, " ")
+	return elements[0] == driver
+}
+
+func cmdOutputWithTimeout(duration time.Duration, cmd string, parameters ...string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), duration)
+	defer cancel()
+	output, err := exec.CommandContext(ctx, cmd, parameters...).Output()
+	if err != nil {
+		return nil, err
+	}
+	return output, nil
+}
diff --git a/devices/pci.go b/devices/pci.go
new file mode 100644
index 00000000..a73783ff
--- /dev/null
+++ b/devices/pci.go
@@ -0,0 +1,218 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+import (
+	"fmt"
+	"os"
+)
+
+type pciDevice struct {
+	id     string
+	class  string
+	vendor string
+	device string
+	driver string
+}
+
+// GetPciDeviceByPciID gets device info by PCI bus id.
+func GetPciDeviceByPciID(pciID string) (Device, error) {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "lspci", "-Dvmmnks", pciID)
+	if err != nil {
+		return nil, err
+	}
+
+	info := &pciDevice{id: pciID}
+
+	match := rPciClass.FindSubmatch(output)
+	if len(match) < 2 {
+		return nil, fmt.Errorf("Bad lspci output: %s", output)
+	}
+	info.class = string(match[1][:])
+
+	match = rPciVendor.FindSubmatch(output)
+	if len(match) < 2 {
+		return nil, fmt.Errorf("Bad lspci output: %s", output)
+	}
+	info.vendor = string(match[1][:])
+
+	match = rPciDevice.FindSubmatch(output)
+	if len(match) < 2 {
+		return nil, fmt.Errorf("Bad lspci output: %s", output)
+	}
+	info.device = string(match[1][:])
+
+	match = rPciDriver.FindSubmatch(output)
+	// driver may be empty
+	if len(match) >= 2 {
+		info.driver = string(match[1][:])
+	}
+
+	return info, nil
+}
+
+func (p *pciDevice) Bind(driver string) error {
+	var err error
+	p.driver, err = BindPci(p.id, driver, p.vendor, p.device)
+	return err
+}
+
+func (p *pciDevice) Unbind() error {
+	return UnbindPci(p.id, p.driver)
+}
+
+func (p *pciDevice) Probe() error {
+	var err error
+	p.driver, err = ProbePci(p.id)
+	return err
+}
+
+func (p *pciDevice) CurrentDriver() (string, error) {
+	return GetCurrentPciDriver(p.id)
+}
+
+func (p *pciDevice) ID() string {
+	return p.id
+}
+
+func (p *pciDevice) String() string {
+	return pciDeviceStringer.With(p.ID, p.class, p.vendor, p.device, p.driver)
+}
+
+// BindPci binds the driver to the given device ID
+func BindPci(devID, driver, vendor, device string) (string, error) {
+	current, err := GetCurrentPciDriver(devID)
+	if err != nil {
+		return "", err
+	}
+
+	switch current {
+	case driver:
+		// already binding the same driver, skip binding it
+		return driver, nil
+	case "":
+		// if not binding to any driver, continue to bind pci device driver
+	default:
+		// if there already binding to other driver, unbind it first
+		if err := unbindPciDeviceDriver(devID, current); err != nil {
+			return "", err
+		}
+	}
+
+	if err := bindPciDeviceDriver(devID, driver, vendor, device); err != nil {
+		return "", err
+	}
+
+	return driver, nil
+}
+
+// UnbindPci unbinds the driver that is bound to the given device ID
+func UnbindPci(devID, driver string) error {
+	current, err := GetCurrentPciDriver(devID)
+	if err != nil {
+		return err
+	} else if current == "" {
+		// this device is already not bound to any driver
+		return nil
+	}
+
+	return unbindPciDeviceDriver(devID, current)
+}
+
+func ProbePci(devID string) (string, error) {
+	if err := probePciDriver(devID); err != nil {
+		return "", err
+	}
+
+	return GetCurrentPciDriver(devID)
+}
+
+// GetCurrentPciDriver returns the current driver that device bound to.
+func GetCurrentPciDriver(devID string) (string, error) {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "lspci", "-Dvmmnks", devID)
+	if err != nil {
+		return "", fmt.Errorf("Cmd Execute lspci failed: %s", err.Error())
+	}
+
+	match := rPciDriver.FindSubmatch(output)
+	if len(match) >= 2 {
+		return string(match[1][:]), nil
+	}
+
+	return "", nil
+}
+
+// bindPciDeviceDriver binds driver to device in the follow flow:
+// 0. make sure device already unbound to any driver
+// 1. set sysfs device driver_override to target driver
+// 2. try binding driver
+// 3. clean up device driver_override
+// 4. if driver_override failed, try to use sysfs driver/.../new_id to bind device
+func bindPciDeviceDriver(devID, driver, vendor, device string) error {
+	if err := overrideDriver(devID, driver); err == nil {
+		defer cleanOverrideDriver(devID)
+
+		// normal way to bind pci driver
+		if err := writeToTargetWithData(pathSysPciDriversBind.With(driver), os.O_WRONLY, 0200, devID); err != nil {
+			return err
+		}
+
+		if current, _ := GetCurrentPciDriver(devID); current != driver {
+			return ErrBind
+		}
+		return nil
+	}
+
+	// NOTE if driver_override failed, it means kernel version is less than
+	// 3.15, so we need to use sysfs drivers/.../new_id to bind device
+	return addToDriver(devID, driver, vendor, device)
+}
+
+func unbindPciDeviceDriver(devID, driver string) error {
+	// first trying write to pathSysPciDevicesUnbindWithDevID, if fails, try next, write to pathSysPciDriversUnbindWithDriver
+	if err := writeToTargetWithData(pathSysPciDevicesUnbind.With(devID), os.O_WRONLY, 0200, devID); err != nil {
+		if err := writeToTargetWithData(pathSysPciDriversUnbind.With(driver), os.O_WRONLY, 0200, devID); err != nil {
+			return err
+		}
+	}
+
+	// check if unbind success
+	current, err := GetCurrentPciDriver(devID)
+	if err != nil {
+		return err
+	}
+
+	if current != "" {
+		return ErrUnbind
+	}
+
+	return nil
+}
+
+func probePciDriver(devID string) error {
+	return writeToTargetWithData(PathSysPciDriverProbe, os.O_WRONLY, 0200, devID)
+}
+
+func overrideDriver(devID, driver string) error {
+	return writeToTargetWithData(pathSysPciDevicesOverrideDriver.With(devID), os.O_WRONLY|os.O_TRUNC, 0755, driver)
+}
+
+func cleanOverrideDriver(devID string) error {
+	return overrideDriver(devID, "\x00")
+}
+
+func addToDriver(devID, driver, vendor, device string) error {
+	// see https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-bus-pci
+	// The format of the device ID is: VVVV DDDD SVVV SDDD CCCC MMMM PPPP.
+	// VVVV Vendor ID
+	// DDDD Device ID
+	// SVVV Subsystem Vendor ID
+	// SDDD ubsystem Device ID
+	// CCCC Class
+	// MMMM Class Mask
+	// PPPP Private Driver Data
+	return writeToTargetWithData(pathSysPciDriversNewID.With(driver), os.O_WRONLY, 0200, vendor+" "+device)
+}
diff --git a/devices/vmbus.go b/devices/vmbus.go
new file mode 100644
index 00000000..06c9c047
--- /dev/null
+++ b/devices/vmbus.go
@@ -0,0 +1,169 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package devices helps to query DPDK compatibles devices and to bind/unbind drivers
+package devices
+
+import (
+	"fmt"
+	"os"
+)
+
+/*
+   DEV_UUID=$(basename $(readlink /sys/class/net/eth1/device))
+
+   # only on kernel 4.18 or later
+   driverctl -b vmbus set-override $DEV_UUID uio_hv_generic
+
+   # others
+   modprobe uio_hv_generic
+   NET_UUID="f8615163-df3e-46c5-913f-f2d2f965ed0e"
+   echo $NET_UUID > /sys/bus/vmbus/drivers/uio_hv_generic/new_id // only run once
+   echo $DEV_UUID > /sys/bus/vmbus/drivers/hv_netvsc/unbind
+   echo $DEV_UUID > /sys/bus/vmbus/drivers/uio_hv_generic/bind
+*/
+
+type prepareFunc func() error
+
+type vmbusDevice struct {
+	UUID   string
+	Driver string
+}
+
+// GetVmbusDeviceByUUID returns a VMBus device by given UUID.
+func GetVmbusDeviceByUUID(uuid string) (Device, error) {
+	result := &vmbusDevice{UUID: uuid}
+	return result, nil
+}
+
+func (v *vmbusDevice) Bind(driver string) error {
+	current, err := GetCurrentVmbusDriver(v.UUID)
+	if err != nil {
+		return fmt.Errorf("GetCurrentVmbusDriver: %s", err.Error())
+	}
+	var prepare prepareFunc
+
+	switch current {
+	case driver:
+		// already binding the same driver, skip binding it
+		return nil
+	case "":
+		// if not binding to any driver, continue to bind pci device driver
+	default:
+		// if there already binding to other driver, unbind it first
+		prepare = func() error {
+			if err := unbindVmbusDeviceDriver(v.UUID, current); err != nil {
+				return fmt.Errorf("unbindVmbusDeviceDriver: %s", err.Error())
+			}
+			return nil
+		}
+	}
+
+	if isValidDpdkVmbusDriver(driver) {
+		if err := bindVmbusDeviceDpdkDriver(v.UUID, driver, prepare); err != nil {
+			return fmt.Errorf("bindVmbusDeviceDriver: %s", err.Error())
+		}
+	} else {
+		if err := bindVmbusDeviceDriver(v.UUID, driver, prepare); err != nil {
+			return fmt.Errorf("bindVmbusDeviceDriver: %s", err.Error())
+		}
+	}
+
+	// update Driver
+	v.Driver = driver
+
+	return nil
+}
+
+func (v *vmbusDevice) Unbind() error {
+	current, err := GetCurrentVmbusDriver(v.UUID)
+	if err != nil {
+		return err
+	}
+	if current == "" {
+		// NOTE: if no current vmbus driver, don't unbind it and don't return error
+		return nil
+	}
+	return unbindVmbusDeviceDriver(v.UUID, current)
+}
+
+func (v *vmbusDevice) CurrentDriver() (string, error) {
+	return GetCurrentVmbusDriver(v.UUID)
+}
+
+func (v *vmbusDevice) Probe() error {
+	return ErrNotProbe
+}
+
+func (v *vmbusDevice) ID() string {
+	return v.UUID
+}
+
+func (v *vmbusDevice) String() string {
+	return vmbusDeviceStringer.With(v.UUID, v.Driver)
+}
+
+// GetCurrentVmbusDriver update the current driver device bound to.
+func GetCurrentVmbusDriver(uuid string) (string, error) {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "find", PathSysVmbusDrivers, "-type", "l", "-iname", uuid)
+	if err != nil {
+		return "", fmt.Errorf("Cmd Execute find failed: %s", err.Error())
+	}
+
+	matches := rVmbusDriver.FindSubmatch(output)
+	if len(matches) >= 2 {
+		return string(matches[1]), nil
+	}
+
+	return "", nil
+}
+
+func bindVmbusDeviceDriver(devUUID, driver string, prepares ...prepareFunc) error {
+	for _, prepare := range prepares {
+		if prepare == nil {
+			continue
+		}
+		if err := prepare(); err != nil {
+			return err
+		}
+	}
+	return writeToTargetWithData(pathSysVmbusDriversBind.With(driver), os.O_WRONLY, 0200, devUUID)
+}
+
+func bindVmbusDeviceDpdkDriver(devUUID, driver string, prepares ...prepareFunc) error {
+	// NOTE: ignore error of write vmbus driver new_id
+	writeToTargetWithData(pathSysVmbusDriversNewID.With(driver), os.O_WRONLY, 0200, newNetUUID())
+	return bindVmbusDeviceDriver(devUUID, driver, prepares...)
+}
+
+func unbindVmbusDeviceDriver(devUUID, driver string) error {
+	return writeToTargetWithData(pathSysVmbusDriversUnbind.With(driver), os.O_WRONLY, 0200, devUUID)
+}
+
+func isValidDpdkVmbusDriver(driver string) bool {
+	for _, dpdkDriver := range DpdkVmbusDrivers {
+		if driver == dpdkDriver {
+			return true
+		}
+	}
+	return false
+}
+
+// newNetUUID returns a valid net UUID
+// FIXME: Dummy implement, always return same result - see
+// https://doc.dpdk.org/guides/nics/netvsc.html#installation NET_UUID
+func newNetUUID() string {
+	return "f8615163-df3e-46c5-913f-f2d2f965ed0e"
+}
+
+// bindVmbusDeviceDriverKernelGreaterThan418 is only available on linux kernel >= 4.18
+// XXX: see https://doc.dpdk.org/guides/nics/netvsc.html#installation
+func bindVmbusDeviceDriverKernelGreaterThan418(devUUID, driver string) error {
+	// driverctl -b vmbus set-override $DEV_UUID uio_hv_generic
+	_, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "driverctl", "-b", "vmbus", "set-override", devUUID, driver)
+	if err != nil {
+		return fmt.Errorf("Cmd Execute driverctl failed: %s", err.Error())
+	}
+	return nil
+}
diff --git a/examples/.gitignore b/examples/.gitignore
index 5946c6e8..c60b8caa 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -10,3 +10,4 @@ gtpu
 pingReplay
 timer
 netlink
+devbind
diff --git a/examples/Makefile b/examples/Makefile
index 83f55d77..27e2f94b 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -6,7 +6,7 @@ PATH_TO_MK = ../mk
 IMAGENAME = nff-go-examples
 EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
 			  createPacket sendFixedPktsNumber gtpu pingReplay \
-			  netlink gopacketParserExample
+			  netlink gopacketParserExample devbind
 SUBDIRS = nat tutorial antiddos demo fileReadWrite firewall forwarding
 
 .PHONY: dpi nffPktgen
diff --git a/examples/devbind.go b/examples/devbind.go
new file mode 100644
index 00000000..baca580b
--- /dev/null
+++ b/examples/devbind.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"log"
+	"flag"
+
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+	"github.com/intel-go/nff-go/devices"
+)
+
+// Example that shows how to bind a driver to a NIC
+func main() {
+	nic := flag.String("nic", "enp0s9", "network interface to run DPDK")
+	bind := flag.String("bind", "igb_uio", "dpdk uio driver")
+	flag.Parse()
+
+	device, err := devices.New(*nic)
+	flow.CheckFatal(err)
+
+	driver, err := device.CurrentDriver()
+	if err != nil {
+		log.Println("Failed to get CurrentDriver:", err)
+		return
+	}
+
+	defer func() {
+		flow.SystemStop()
+
+		// Re-Bind to original driver
+		device.Bind(driver)
+	}()
+
+	// Bind to new user specified driver
+	device.Bind(*bind)
+
+	config := &flow.Config{
+		HWTXChecksum: true,
+	}
+	flow.SystemInit(config)
+
+	var port uint16 = 0
+
+	mainFlow, err := flow.SetReceiver(port)
+	if err != nil {
+		log.Println("Failed to SetReceiver:", err)
+		return
+	}
+
+	err = flow.SetHandlerDrop(mainFlow, handler, nil)
+	if err != nil {
+		log.Println("Failed to SetHandlerDrop:", err)
+		return
+	}
+
+	err = flow.SetSender(mainFlow, port)
+	if err != nil {
+		log.Println("Failed to SetSender:", err)
+		return
+	}
+
+	err = flow.SystemStart()
+	if err != nil {
+		log.Println("Failed to SystemStart:", err)
+		return
+	}
+}
+
+func handler(*packet.Packet, flow.UserContext) bool {
+	return true
+}

From 1e38919e490b0eacadcffa9ebcc0d968f8d64ee1 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Fri, 14 Sep 2018 20:14:34 +0000
Subject: [PATCH 41/50] Implemented IPv6 NAT with DHCPv6, ND, GRPC and port
 forwarding

- Implemented Neighbor Discovery responce to neighbor solicitation message
- Implemented Neighbor Solicitation and Advertisement messages for IPv6
- Implemented port forwarding to IPv6 addresses
- Implemented IPv6 packets forwarding to KNI interfaces
- Implemented DHCPv6
- Added missing IPv6 support to GRPC configuration
- Fixes for DHCPv6 and KNI interopration
- Set local addresses on KNI interfaces when they are received from DHCP
  or set by GRPC.
- If KNI is present, all forwarded traffic directed at local IPs is
  directed to KNI now, so port forwarding with zero address is kinda
  obsolete now.
- Fixed checksum and icmpv6 tests
---
 common/common.go                              |  11 +-
 examples/nat/arp.go                           |  86 ++++
 examples/nat/cksum.go                         |  48 +-
 examples/nat/client/client.go                 |  25 +-
 examples/nat/config.go                        | 284 +++++++++--
 examples/nat/dhcp.go                          |  68 +--
 examples/nat/dhcp6.go                         | 456 +++++++++++++++++
 examples/nat/grpc.go                          |  29 +-
 examples/nat/icmp.go                          |  95 ++++
 examples/nat/main/Makefile                    |   2 +-
 examples/nat/main/config-dhcp.json            |  24 +-
 examples/nat/main/config-kni-dhcp.json        |  32 ++
 examples/nat/main/config-kni.json             |  32 ++
 examples/nat/main/config.json                 |  24 +-
 examples/nat/neigh.go                         |  88 ++++
 examples/nat/portalloc.go                     |  44 +-
 examples/nat/translation.go                   | 471 ++++++++----------
 examples/nat/updatecfg/updatecfg.proto        |   3 +
 examples/nat/util.go                          | 185 ++++++-
 examples/nffPktgen/generator/generator.go     |   2 +-
 mk/leaf.mk                                    |   2 +-
 packet/acl_internal_test.go                   |  10 +-
 packet/checksum.go                            |  16 +-
 packet/checksum_test.go                       |  23 +-
 packet/icmp6.go                               | 223 +++++++++
 packet/packet.go                              |  26 +-
 packet/packet_test.go                         |  24 +-
 packet/utils_for_test.go                      |  26 +
 test/stability/testCksum/testCksum.go         |   4 +-
 .../testCksumCommon/testCksumCommon.go        |   6 +-
 30 files changed, 1927 insertions(+), 442 deletions(-)
 create mode 100644 examples/nat/arp.go
 create mode 100644 examples/nat/dhcp6.go
 create mode 100644 examples/nat/icmp.go
 create mode 100644 examples/nat/neigh.go
 create mode 100644 packet/icmp6.go

diff --git a/common/common.go b/common/common.go
index f3f5923a..39cbb967 100644
--- a/common/common.go
+++ b/common/common.go
@@ -48,13 +48,18 @@ const (
 	IPNumber     = 0x04
 	TCPNumber    = 0x06
 	UDPNumber    = 0x11
-	NoNextHeader = 0x3B
+	ICMPv6Number = 0x3a
+	NoNextHeader = 0x3b
 )
 
 // Supported ICMP Types
 const (
-	ICMPTypeEchoRequest  uint8 = 8
-	ICMPTypeEchoResponse uint8 = 0
+	ICMPTypeEchoRequest         uint8 = 8
+	ICMPTypeEchoResponse        uint8 = 0
+	ICMPv6TypeEchoRequest       uint8 = 128
+	ICMPv6TypeEchoResponse      uint8 = 129
+	ICMPv6NeighborSolicitation  uint8 = 135
+	ICMPv6NeighborAdvertisement uint8 = 136
 )
 
 // These constants keep length of supported headers in bytes.
diff --git a/examples/nat/arp.go b/examples/nat/arp.go
new file mode 100644
index 00000000..28ed36c9
--- /dev/null
+++ b/examples/nat/arp.go
@@ -0,0 +1,86 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
+)
+
+func (port *ipPort) handleARP(pkt *packet.Packet) uint {
+	arp := pkt.GetARPNoCheck()
+
+	if packet.SwapBytesUint16(arp.Operation) != packet.ARPRequest {
+		if packet.SwapBytesUint16(arp.Operation) == packet.ARPReply {
+			ipv4 := packet.SwapBytesUint32(packet.ArrayToIPv4(arp.SPA))
+			port.arpTable.Store(ipv4, arp.SHA)
+		}
+		if port.KNIName != "" {
+			return dirKNI
+		}
+		return dirDROP
+	}
+
+	// If there is a KNI interface, direct all ARP traffic to it
+	if port.KNIName != "" {
+		return dirKNI
+	}
+
+	// Check that someone is asking about MAC of my IP address and HW
+	// address is blank in request
+	if packet.BytesToIPv4(arp.TPA[0], arp.TPA[1], arp.TPA[2], arp.TPA[3]) != packet.SwapBytesUint32(port.Subnet.Addr) {
+		println("Warning! Got an ARP packet with target IPv4 address", StringIPv4Array(arp.TPA),
+			"different from IPv4 address on interface. Should be", StringIPv4Int(port.Subnet.Addr),
+			". ARP request ignored.")
+		return dirDROP
+	}
+	if arp.THA != [common.EtherAddrLen]byte{} {
+		println("Warning! Got an ARP packet with non-zero MAC address", StringMAC(arp.THA),
+			". ARP request ignored.")
+		return dirDROP
+	}
+
+	// Prepare an answer to this request
+	answerPacket, err := packet.NewPacket()
+	if err != nil {
+		common.LogFatal(common.Debug, err)
+	}
+
+	packet.InitARPReplyPacket(answerPacket, port.SrcMACAddress, arp.SHA, packet.ArrayToIPv4(arp.TPA), packet.ArrayToIPv4(arp.SPA))
+	vlan := pkt.GetVLAN()
+	if vlan != nil {
+		answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
+	}
+
+	port.dumpPacket(answerPacket, dirSEND)
+	answerPacket.SendPacket(port.Index)
+
+	return dirDROP
+}
+
+func (port *ipPort) getMACForIPv4(ip uint32) (macAddress, bool) {
+	v, found := port.arpTable.Load(ip)
+	if found {
+		return macAddress(v.([common.EtherAddrLen]byte)), true
+	}
+	port.sendARPRequest(ip)
+	return macAddress{}, false
+}
+
+func (port *ipPort) sendARPRequest(ip uint32) {
+	requestPacket, err := packet.NewPacket()
+	if err != nil {
+		common.LogFatal(common.Debug, err)
+	}
+
+	packet.InitARPRequestPacket(requestPacket, port.SrcMACAddress,
+		packet.SwapBytesUint32(port.Subnet.Addr), packet.SwapBytesUint32(ip))
+	if port.Vlan != 0 {
+		requestPacket.AddVLANTag(port.Vlan)
+	}
+
+	port.dumpPacket(requestPacket, dirSEND)
+	requestPacket.SendPacket(port.Index)
+}
diff --git a/examples/nat/cksum.go b/examples/nat/cksum.go
index f138d0b7..538aafca 100644
--- a/examples/nat/cksum.go
+++ b/examples/nat/cksum.go
@@ -64,8 +64,54 @@ func setIPv4ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum boo
 			l3.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(l3))
 		}
 		l4 := pkt.GetICMPNoCheck()
-		l4.Cksum = 0
+		l4.Cksum = 0 // Need to zero l4.Cksum or otherwise it is included into calculation
 		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(l3, l4,
 			unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.ICMPLen)))
 	}
 }
+
+func setIPv6UDPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
+	if calculateChecksum {
+		l3 := pkt.GetIPv6NoCheck()
+		l4 := pkt.GetUDPNoCheck()
+		if hWTXChecksum {
+			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv6UDPCksum(l3, l4))
+			l2len := uint32(common.EtherLen)
+			if pkt.Ether.EtherType == common.SwapVLANNumber {
+				l2len += common.VLANLen
+			}
+			pkt.SetTXIPv6UDPOLFlags(l2len, common.IPv6Len)
+		} else {
+			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv6UDPChecksum(l3, l4,
+				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+uintptr(common.UDPLen))))
+		}
+	}
+}
+
+func setIPv6TCPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
+	if calculateChecksum {
+		l3 := pkt.GetIPv6NoCheck()
+		l4 := pkt.GetTCPNoCheck()
+		if hWTXChecksum {
+			l4.Cksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv6TCPCksum(l3))
+			l2len := uint32(common.EtherLen)
+			if pkt.Ether.EtherType == common.SwapVLANNumber {
+				l2len += common.VLANLen
+			}
+			pkt.SetTXIPv6TCPOLFlags(l2len, common.IPv6Len)
+		} else {
+			l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(l3, l4,
+				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.TCPMinLen)))
+		}
+	}
+}
+
+func setIPv6ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
+	if calculateChecksum {
+		l3 := pkt.GetIPv6NoCheck()
+
+		l4 := pkt.GetICMPNoCheck()
+		l4.Cksum = 0 // Need to zero l4.Cksum or otherwise it is included into calculation
+		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(l3, l4))
+	}
+}
diff --git a/examples/nat/client/client.go b/examples/nat/client/client.go
index 8597df8c..44733f6f 100644
--- a/examples/nat/client/client.go
+++ b/examples/nat/client/client.go
@@ -77,7 +77,7 @@ func (acra *addresChangeRequestArray) String() string {
 }
 
 func (acra *addresChangeRequestArray) Set(value string) error {
-	parts := strings.Split(value, ":")
+	parts := strings.Split(value, ",")
 	if len(parts) != 2 {
 		return fmt.Errorf("Bad port index and subnet address specified \"%s\"", value)
 	}
@@ -90,16 +90,13 @@ func (acra *addresChangeRequestArray) Set(value string) error {
 	if err != nil {
 		return err
 	}
-	if ip.To4() == nil {
-		return fmt.Errorf("Only IPv4 addresses are supported yet: %s", parts[1])
-	}
 	ones, _ := ipnet.Mask.Size()
 
 	*acra = append(*acra, &upd.InterfaceAddressChangeRequest{
 		InterfaceId: uint32(index),
 		PortSubnet: &upd.Subnet{
 			Address: &upd.IPAddress{
-				Address: ip.To4(),
+				Address: ip,
 			},
 			MaskBitsNumber: uint32(ones),
 		},
@@ -116,7 +113,7 @@ func (pfra *portForwardRequestArray) String() string {
 }
 
 func (pfra *portForwardRequestArray) Set(value string) error {
-	parts := strings.Split(value, ":")
+	parts := strings.Split(value, ",")
 	if len(parts) != 6 {
 		return fmt.Errorf("Bad port forwarding specification \"%s\"", value)
 	}
@@ -148,9 +145,6 @@ func (pfra *portForwardRequestArray) Set(value string) error {
 	if ip == nil {
 		return fmt.Errorf("Bad IP address specified \"%s\"", parts[4])
 	}
-	if ip.To4() == nil {
-		return fmt.Errorf("Only IPv4 addresses are supported yet: %s", parts[1])
-	}
 
 	tport, err := strconv.ParseUint(parts[5], 10, 16)
 	if err != nil {
@@ -163,7 +157,7 @@ func (pfra *portForwardRequestArray) Set(value string) error {
 		Port: &upd.ForwardedPort{
 			SourcePortNumber: uint32(sport),
 			TargetAddress: &upd.IPAddress{
-				Address: ip.To4(),
+				Address: ip,
 			},
 			TargetPortNumber: uint32(tport),
 			Protocol:         upd.Protocol(proto),
@@ -174,7 +168,7 @@ func (pfra *portForwardRequestArray) Set(value string) error {
 
 func main() {
 	flag.Usage = func() {
-		fmt.Printf(`Usage: client [-a server:port] [-d {+|-}{d|t|k}] [-s index:subnet] [-p {TCP|UDP}:port number:target IP address:target port]
+		fmt.Printf(`Usage: client [-a server:port] [-d {+|-}{d|t|k}] [-s index:subnet] [-p {+|-},{TCP|UDP|TCP6|UDP6},port number,target IP address,target port]
 
 Client sends GRPS requests to NAT server controlling packets trace dump,
 ports subnet adresses and forwarded ports. Multiple requests of the same
@@ -192,11 +186,12 @@ e.g. +d or -t or +k:
     t means to trace translated (normally sent) packets,
     k means to trace packets that were sent to KNI interface.`)
 	flag.Var(&addresChangeRequests, "s", `Control network interface subnet in a form of index:subnet,
-e.g. 1:192.168.5.1/24. Port index is DPDK port number. Subnet
-is given in form of port IP address and prefix bits.`)
+e.g. 1,192.168.5.1/24 or 1,fd16::1/128. Port index is DPDK port
+number. Subnet is given in form of port IP address and prefix bits.`)
 	flag.Var(&portForwardRequests, "p", `Control TCP and UDP port forwarding in a form of
-+/-:index:protocol:source port:target IP address:target port, e.g.
-+:1:TCP:2222:192.168.5.7:22 or -:0:TCP:22:0.0.0.0:0. If target
++/-,index,protocol,source port,target IP address,target port, e.g.
++,1,TCP,2222,192.168.5.7,22 or -,0,TCP,22,0.0.0.0,0 or
++,1,TCP6,2222,fd14::3,22 or -,0,TCP6,22,::,0. If target
 address is zero, it means that port is forwarded to corresponding
 network port KNI interface. Port forwarding to a non-zero
 target address (not to a KNI interface) is possible only for
diff --git a/examples/nat/config.go b/examples/nat/config.go
index 1aabc420..6d9d90dd 100644
--- a/examples/nat/config.go
+++ b/examples/nat/config.go
@@ -39,23 +39,45 @@ const (
 	portReuseTimeout  time.Duration = 1 * time.Second
 )
 
+var (
+	zeroIPv6Addr = [common.IPv6AddrLen]uint8{}
+)
+
 type hostPort struct {
-	Addr uint32
-	Port uint16
+	Addr4 uint32
+	Addr6 [common.IPv6AddrLen]uint8
+	Port  uint16
+	ipv6  bool
 }
 
-type protocolId uint8
+type protocolId struct {
+	id   uint8
+	ipv6 bool
+}
 
 type forwardedPort struct {
-	Port         uint16     `json:"port"`
-	Destination  hostPort   `json:"destination"`
-	Protocol     protocolId `json:"protocol"`
-	forwardToKNI bool
+	Port        uint16     `json:"port"`
+	Destination hostPort   `json:"destination"`
+	Protocol    protocolId `json:"protocol"`
 }
 
 var protocolIdLookup map[string]protocolId = map[string]protocolId{
-	"TCP": common.TCPNumber,
-	"UDP": common.UDPNumber,
+	"TCP": protocolId{
+		id:   common.TCPNumber,
+		ipv6: false,
+	},
+	"UDP": protocolId{
+		id:   common.UDPNumber,
+		ipv6: false,
+	},
+	"TCP6": protocolId{
+		id:   common.TCPNumber,
+		ipv6: true,
+	},
+	"UDP6": protocolId{
+		id:   common.UDPNumber,
+		ipv6: true,
+	},
 }
 
 func (out *protocolId) UnmarshalJSON(b []byte) error {
@@ -81,7 +103,11 @@ type ipv4Subnet struct {
 }
 
 func (fp *forwardedPort) String() string {
-	return fmt.Sprintf("Port:%d, Destination:%+v, Protocol: %d", fp.Port, packet.IPv4ToString(fp.Destination.Addr), fp.Protocol)
+	return fmt.Sprintf("Port:%d, Destination IPv4: %v, Destination IPv6: %v, Protocol: %d",
+		fp.Port,
+		packet.IPv4ToString(fp.Destination.Addr4),
+		packet.IPv6ToString(fp.Destination.Addr6),
+		fp.Protocol)
 }
 
 func (subnet *ipv4Subnet) String() string {
@@ -104,29 +130,67 @@ func (subnet *ipv4Subnet) checkAddrWithingSubnet(addr uint32) bool {
 	return addr&subnet.Mask == subnet.Addr&subnet.Mask
 }
 
+type ipv6Subnet struct {
+	Addr            [common.IPv6AddrLen]uint8
+	multicastAddr   [common.IPv6AddrLen]uint8
+	Mask            [common.IPv6AddrLen]uint8
+	llAddr          [common.IPv6AddrLen]uint8
+	llMulticastAddr [common.IPv6AddrLen]uint8
+	addressAcquired bool
+	ds              dhcpv6State
+}
+
+func (subnet *ipv6Subnet) String() string {
+	if subnet.addressAcquired {
+		// Count most significant set bits
+		i := 0
+		for ; i <= 128; i++ {
+			mask := uint8(1) << uint(7-(i&7))
+			if i == 128 || subnet.Mask[i>>3]&mask == 0 {
+				break
+			}
+		}
+		return packet.IPv6ToString(subnet.Addr) + "/" + strconv.Itoa(i)
+	}
+	return "DHCP address not acquired"
+}
+
+func (subnet *ipv6Subnet) andMask(addr [common.IPv6AddrLen]uint8) [common.IPv6AddrLen]uint8 {
+	var result [common.IPv6AddrLen]uint8
+	for i := range addr {
+		result[i] = addr[i] & subnet.Mask[i]
+	}
+	return result
+}
+
+func (subnet *ipv6Subnet) checkAddrWithingSubnet(addr [common.IPv6AddrLen]uint8) bool {
+	return subnet.andMask(addr) == subnet.andMask(subnet.Addr)
+}
+
 type macAddress [common.EtherAddrLen]uint8
 
 type portMapEntry struct {
 	lastused             time.Time
-	addr                 uint32
 	finCount             uint8
 	terminationDirection terminationDirection
 	static               bool
 }
 
 // Type describing a network port
-type ipv4Port struct {
+type ipPort struct {
 	Index         uint16          `json:"index"`
 	Subnet        ipv4Subnet      `json:"subnet"`
+	Subnet6       ipv6Subnet      `json:"subnet6"`
 	Vlan          uint16          `json:"vlan-tag"`
 	KNIName       string          `json:"kni-name"`
 	ForwardPorts  []forwardedPort `json:"forward-ports"`
 	SrcMACAddress macAddress
 	Type          interfaceType
 	// Pointer to an opposite port in a pair
-	opposite *ipv4Port
+	opposite *ipPort
 	// Map of allocated IP ports on public interface
-	portmap [][]portMapEntry
+	portmap  [][]portMapEntry
+	portmap6 [][]portMapEntry
 	// Main lookup table which contains entries for packets coming at this port
 	translationTable []*sync.Map
 	// ARP lookup table
@@ -138,8 +202,8 @@ type ipv4Port struct {
 
 // Config for one port pair.
 type portPair struct {
-	PrivatePort ipv4Port `json:"private-port"`
-	PublicPort  ipv4Port `json:"public-port"`
+	PrivatePort ipPort `json:"private-port"`
+	PublicPort  ipPort `json:"public-port"`
 	// Synchronization point for lookup table modifications
 	mutex sync.Mutex
 	// Port that was allocated last
@@ -230,6 +294,42 @@ func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
 	return errors.New("Failed to parse address " + s)
 }
 
+// UnmarshalJSON parses ipv 4 subnet details.
+func (out *ipv6Subnet) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	if s == "dhcp" {
+		out.Addr = [common.IPv6AddrLen]uint8{}
+		out.Mask = [common.IPv6AddrLen]uint8{}
+		out.addressAcquired = false
+		return nil
+	}
+
+	if ip, ipnet, err := net.ParseCIDR(s); err == nil {
+		if ip.To16() == nil {
+			return fmt.Errorf("Bad IPv6 address: %s", s)
+		}
+		copy(out.Addr[:], ip.To16())
+		copy(out.Mask[:], ipnet.Mask)
+		out.addressAcquired = true
+		return nil
+	}
+
+	if ip := net.ParseIP(s); ip != nil {
+		if ip.To16() == nil {
+			return fmt.Errorf("Bad IPv6 address: %s", s)
+		}
+		copy(out.Addr[:], ip.To16())
+		out.Mask = [common.IPv6AddrLen]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+		out.addressAcquired = true
+		return nil
+	}
+	return errors.New("Failed to parse address " + s)
+}
+
 // UnmarshalJSON parses ipv4 host:port string. Port may be omitted and
 // is set to zero in this case.
 func (out *hostPort) UnmarshalJSON(b []byte) error {
@@ -245,11 +345,16 @@ func (out *hostPort) UnmarshalJSON(b []byte) error {
 
 	ipArray := net.ParseIP(hostStr)
 	if ipArray == nil {
-		return errors.New("Bad IPv4 address specified: " + hostStr)
+		return errors.New("Bad IP address specified: " + hostStr)
 	}
-	out.Addr, err = convertIPv4(ipArray.To4())
+	out.Addr4, err = convertIPv4(ipArray.To4())
 	if err != nil {
-		return err
+		ipv6addr := ipArray.To16()
+		if ipv6addr == nil {
+			return err
+		}
+		copy(out.Addr6[:], ipv6addr)
+		out.ipv6 = true
 	}
 
 	if portStr != "" {
@@ -339,16 +444,26 @@ func ReadConfig(fileName string) error {
 	return nil
 }
 
-func (port *ipv4Port) checkPortForwarding(fp *forwardedPort) error {
-	if fp.Destination.Addr == 0 {
+func (port *ipPort) checkPortForwarding(fp *forwardedPort) error {
+	if fp.Destination.ipv6 != fp.Protocol.ipv6 {
+		return fmt.Errorf("Port forwarding protocol should be TCP or UDP for IPv4 addresses and TCP6 or UDP6 for IPv6 addresses")
+	}
+
+	var isAddrZero bool
+	if fp.Destination.ipv6 {
+		isAddrZero = fp.Destination.Addr6 == zeroIPv6Addr
+	} else {
+		isAddrZero = fp.Destination.Addr4 == 0
+	}
+
+	if isAddrZero {
 		if port.KNIName == "" {
 			return errors.New("Port with index " +
 				strconv.Itoa(int(port.Index)) +
-				" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0")
+				" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0 or [::]")
 		}
-		fp.forwardToKNI = true
 		if fp.Destination.Port != fp.Port {
-			return errors.New("When address 0.0.0.0 is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
+			return errors.New("When address 0.0.0.0 or [::] is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
 				strconv.Itoa(int(fp.Port)) + " and " +
 				strconv.Itoa(int(fp.Destination.Port)))
 		}
@@ -357,12 +472,23 @@ func (port *ipv4Port) checkPortForwarding(fp *forwardedPort) error {
 		if port.Type == iPRIVATE {
 			return errors.New("Only KNI port forwarding is allowed on private port. All translated connections from private to public network can be initiated without any forwarding rules.")
 		}
-		if !port.opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr) {
-			return errors.New("Destination address " +
-				packet.IPv4ToString(fp.Destination.Addr) +
-				" should be within subnet " +
-				port.opposite.Subnet.String())
+
+		if fp.Destination.ipv6 {
+			if !port.opposite.Subnet6.checkAddrWithingSubnet(fp.Destination.Addr6) {
+				return errors.New("Destination address " +
+					packet.IPv6ToString(fp.Destination.Addr6) +
+					" should be within subnet " +
+					port.opposite.Subnet6.String())
+			}
+		} else {
+			if !port.opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr4) {
+				return errors.New("Destination address " +
+					packet.IPv4ToString(fp.Destination.Addr4) +
+					" should be within subnet " +
+					port.opposite.Subnet.String())
+			}
 		}
+
 		if fp.Destination.Port == 0 {
 			fp.Destination.Port = fp.Port
 		}
@@ -376,46 +502,93 @@ func (pp *portPair) initLocalMACs() {
 	pp.PrivatePort.SrcMACAddress = flow.GetPortMACAddress(pp.PrivatePort.Index)
 }
 
-func (port *ipv4Port) allocatePublicPortPortMap() {
-	port.portmap = make([][]portMapEntry, common.UDPNumber+1)
+func (port *ipPort) initIPv6LLAddresses() {
+	packet.CalculateIPv6LinkLocalAddrForMAC(&port.Subnet6.llAddr, port.SrcMACAddress)
+	println("Configured link local address", packet.IPv6ToString(port.Subnet6.llAddr), "for port", port.Index)
+	packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.llMulticastAddr, port.Subnet6.llAddr)
+	println("Configured link local multicast address", packet.IPv6ToString(port.Subnet6.llMulticastAddr), "for port", port.Index)
+	if port.Subnet6.Addr != zeroIPv6Addr {
+		packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
+		println("Configured multicast address", packet.IPv6ToString(port.Subnet6.multicastAddr), "for port", port.Index)
+	}
+}
+
+func (port *ipPort) allocatePublicPortPortMap() {
+	port.portmap = make([][]portMapEntry, 256)
 	port.portmap[common.ICMPNumber] = make([]portMapEntry, portEnd)
 	port.portmap[common.TCPNumber] = make([]portMapEntry, portEnd)
 	port.portmap[common.UDPNumber] = make([]portMapEntry, portEnd)
+	port.portmap6 = make([][]portMapEntry, 256)
+	port.portmap6[common.TCPNumber] = make([]portMapEntry, portEnd)
+	port.portmap6[common.UDPNumber] = make([]portMapEntry, portEnd)
+	port.portmap6[common.ICMPv6Number] = make([]portMapEntry, portEnd)
 }
 
-func (port *ipv4Port) allocateLookupMap() {
-	port.translationTable = make([]*sync.Map, common.UDPNumber+1)
+func (port *ipPort) allocateLookupMap() {
+	port.translationTable = make([]*sync.Map, 256)
 	for i := range port.translationTable {
 		port.translationTable[i] = new(sync.Map)
 	}
 }
 
-func (port *ipv4Port) initPublicPortPortForwardingEntries() {
+func (port *ipPort) initPortPortForwardingEntries() {
 	// Initialize port forwarding rules on public interface
 	for i := range port.ForwardPorts {
 		port.enableStaticPortForward(&port.ForwardPorts[i])
 	}
 }
 
-func (port *ipv4Port) enableStaticPortForward(fp *forwardedPort) {
-	keyEntry := Tuple{
-		addr: port.Subnet.Addr,
-		port: fp.Port,
-	}
-	valEntry := Tuple{
-		addr: fp.Destination.Addr,
-		port: fp.Destination.Port,
-	}
-	port.translationTable[fp.Protocol].Store(keyEntry, valEntry)
-	if fp.Destination.Addr != 0 {
-		port.opposite.translationTable[fp.Protocol].Store(valEntry, keyEntry)
+func (port *ipPort) enableStaticPortForward(fp *forwardedPort) {
+	if fp.Protocol.ipv6 {
+		keyEntry := Tuple6{
+			addr: port.Subnet6.Addr,
+			port: fp.Port,
+		}
+		valEntry := Tuple6{
+			addr: fp.Destination.Addr6,
+			port: fp.Destination.Port,
+		}
+		port.translationTable[fp.Protocol.id].Store(keyEntry, valEntry)
+		if fp.Destination.Addr6 != zeroIPv6Addr {
+			port.opposite.translationTable[fp.Protocol.id].Store(valEntry, keyEntry)
+		}
+		if port.Type == iPUBLIC {
+			port.getPortmap(fp.Protocol.ipv6, fp.Protocol.id)[fp.Port] = portMapEntry{
+				lastused:             time.Now(),
+				finCount:             0,
+				terminationDirection: 0,
+				static:               true,
+			}
+		}
+	} else {
+		keyEntry := Tuple{
+			addr: port.Subnet.Addr,
+			port: fp.Port,
+		}
+		valEntry := Tuple{
+			addr: fp.Destination.Addr4,
+			port: fp.Destination.Port,
+		}
+		port.translationTable[fp.Protocol.id].Store(keyEntry, valEntry)
+		if fp.Destination.Addr4 != 0 {
+			port.opposite.translationTable[fp.Protocol.id].Store(valEntry, keyEntry)
+		}
+		if port.Type == iPUBLIC {
+			port.getPortmap(fp.Protocol.ipv6, fp.Protocol.id)[fp.Port] = portMapEntry{
+				lastused:             time.Now(),
+				finCount:             0,
+				terminationDirection: 0,
+				static:               true,
+			}
+		}
 	}
-	port.portmap[fp.Protocol][fp.Port] = portMapEntry{
-		lastused:             time.Now(),
-		addr:                 fp.Destination.Addr,
-		finCount:             0,
-		terminationDirection: 0,
-		static:               true,
+}
+
+func (port *ipPort) getPortmap(ipv6 bool, protocol uint8) []portMapEntry {
+	if ipv6 {
+		return port.portmap6[protocol]
+	} else {
+		return port.portmap[protocol]
 	}
 }
 
@@ -426,11 +599,14 @@ func InitFlows() {
 
 		// Init port pairs state
 		pp.initLocalMACs()
+		pp.PrivatePort.initIPv6LLAddresses()
+		pp.PublicPort.initIPv6LLAddresses()
 		pp.PrivatePort.allocateLookupMap()
 		pp.PublicPort.allocateLookupMap()
 		pp.PublicPort.allocatePublicPortPortMap()
 		pp.lastport = portStart
-		pp.PublicPort.initPublicPortPortForwardingEntries()
+		pp.PrivatePort.initPortPortForwardingEntries()
+		pp.PublicPort.initPortPortForwardingEntries()
 
 		// Handler context with handler index
 		context := new(pairIndex)
@@ -514,7 +690,7 @@ func CheckHWOffloading() bool {
 	return flow.CheckHWCapability(flow.HWTXChecksumCapability, ports)
 }
 
-func (c *Config) getPortAndPairByID(portId uint32) (*ipv4Port, *portPair) {
+func (c *Config) getPortAndPairByID(portId uint32) (*ipPort, *portPair) {
 	for i := range c.PortPairs {
 		pp := &c.PortPairs[i]
 		if uint32(pp.PublicPort.Index) == portId {
diff --git a/examples/nat/dhcp.go b/examples/nat/dhcp.go
index 98667e00..3a30eec5 100644
--- a/examples/nat/dhcp.go
+++ b/examples/nat/dhcp.go
@@ -12,8 +12,6 @@ import (
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 
-	"github.com/vishvananda/netlink"
-
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/packet"
 )
@@ -25,8 +23,8 @@ type dhcpState struct {
 
 const (
 	requestInterval = 10 * time.Second
-	DHCPSrcPort     = 68
-	DHCPDstPort     = 67
+	DHCPServerPort  = 67
+	DHCPClientPort  = 68
 	BroadcastIPv4   = uint32(0xffffffff)
 )
 
@@ -80,9 +78,17 @@ func sendDHCPRequests() {
 			if !pp.PublicPort.Subnet.addressAcquired {
 				pp.PublicPort.sendDHCPDiscoverRequest()
 			}
+			if !pp.PublicPort.Subnet6.addressAcquired {
+				pp.PublicPort.setLinkLocalIPv6KNIAddress(pp.PublicPort.Subnet6.llAddr, SingleIPMask)
+				pp.PublicPort.sendDHCPv6SolicitRequest()
+			}
 			if !pp.PrivatePort.Subnet.addressAcquired {
 				pp.PrivatePort.sendDHCPDiscoverRequest()
 			}
+			if !pp.PrivatePort.Subnet6.addressAcquired {
+				pp.PrivatePort.setLinkLocalIPv6KNIAddress(pp.PrivatePort.Subnet6.llAddr, SingleIPMask)
+				pp.PrivatePort.sendDHCPv6SolicitRequest()
+			}
 		}
 		time.Sleep(requestInterval)
 	}
@@ -97,7 +103,7 @@ func getDHCPOption(dhcp *layers.DHCPv4, optionType layers.DHCPOpt) *layers.DHCPO
 	return nil
 }
 
-func (port *ipv4Port) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, options []layers.DHCPOption) {
+func (port *ipPort) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, options []layers.DHCPOption) {
 	hwa := make([]byte, common.EtherAddrLen)
 	copy(hwa, port.SrcMACAddress[:])
 
@@ -127,18 +133,25 @@ func (port *ipv4Port) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, op
 	}
 	payloadBuffer := buf.Bytes()
 	packet.InitEmptyIPv4UDPPacket(pkt, uint(len(payloadBuffer)))
-	if port.Vlan != 0 {
-		pkt.AddVLANTag(port.Vlan)
-	}
-	payload, _ := pkt.GetPacketPayload()
-	copy(payload, payloadBuffer)
 
+	// Fill up L2
 	pkt.Ether.SAddr = port.SrcMACAddress
 	pkt.Ether.DAddr = BroadcastMAC
+
+	// Fill up L3
 	pkt.GetIPv4NoCheck().SrcAddr = uint32(0)
 	pkt.GetIPv4NoCheck().DstAddr = BroadcastIPv4
-	pkt.GetUDPNoCheck().SrcPort = packet.SwapBytesUint16(DHCPSrcPort)
-	pkt.GetUDPNoCheck().DstPort = packet.SwapBytesUint16(DHCPDstPort)
+
+	// Fill up L4
+	pkt.GetUDPNoCheck().SrcPort = packet.SwapBytesUint16(DHCPClientPort)
+	pkt.GetUDPNoCheck().DstPort = packet.SwapBytesUint16(DHCPServerPort)
+
+	payload, _ := pkt.GetPacketPayload()
+	copy(payload, payloadBuffer)
+
+	if port.Vlan != 0 {
+		pkt.AddVLANTag(port.Vlan)
+	}
 
 	setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
 	port.dumpPacket(pkt, dirSEND)
@@ -147,26 +160,26 @@ func (port *ipv4Port) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, op
 	port.Subnet.ds.lastDHCPPacketTypeSent = packetType
 }
 
-func (port *ipv4Port) sendDHCPDiscoverRequest() {
+func (port *ipPort) sendDHCPDiscoverRequest() {
 	port.Subnet.ds.dhcpTransactionId = rnd.Uint32()
 	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeDiscover, dhcpOptions)
 }
 
-func (port *ipv4Port) sendDHCPRequestRequest(serverIP, clientIP []byte) {
+func (port *ipPort) sendDHCPRequestRequest(serverIP, clientIP []byte) {
 	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeRequest, append(dhcpOptions,
 		layers.NewDHCPOption(layers.DHCPOptServerID, serverIP),
 		layers.NewDHCPOption(layers.DHCPOptRequestIP, clientIP)))
 }
 
-func (port *ipv4Port) handleDHCP(pkt *packet.Packet) bool {
+func (port *ipPort) handleDHCP(pkt *packet.Packet) bool {
 	if port.Subnet.addressAcquired {
 		// Port already has address, ignore this traffic
 		return false
 	}
 
 	// Check that this is DHCP offer or acknowledgement traffic
-	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPSrcPort) ||
-		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPDstPort) {
+	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPClientPort) ||
+		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPServerPort) {
 		return false
 	}
 
@@ -201,11 +214,11 @@ func (port *ipv4Port) handleDHCP(pkt *packet.Packet) bool {
 	return true
 }
 
-func (port *ipv4Port) handleDHCPOffer(pkt *packet.Packet, dhcp *layers.DHCPv4) {
+func (port *ipPort) handleDHCPOffer(pkt *packet.Packet, dhcp *layers.DHCPv4) {
 	port.sendDHCPRequestRequest(dhcp.NextServerIP, dhcp.YourClientIP)
 }
 
-func (port *ipv4Port) handleDHCPAck(pkt *packet.Packet, dhcp *layers.DHCPv4) {
+func (port *ipPort) handleDHCPAck(pkt *packet.Packet, dhcp *layers.DHCPv4) {
 	maskOption := getDHCPOption(dhcp, layers.DHCPOptSubnetMask)
 	if maskOption == nil {
 		println("Warning! Received a DHCP response without subnet mask! Trying again with discover request.")
@@ -216,21 +229,8 @@ func (port *ipv4Port) handleDHCPAck(pkt *packet.Packet, dhcp *layers.DHCPv4) {
 	port.Subnet.Addr, _ = convertIPv4(dhcp.YourClientIP.To4())
 	port.Subnet.Mask, _ = convertIPv4(maskOption.Data)
 	port.Subnet.addressAcquired = true
-	println("Successfully acquired IP address:", port.Subnet.String())
+	println("Successfully acquired IP address:", port.Subnet.String(), "on port", port.Index)
 
 	// Set address on KNI interface if present
-	if port.KNIName != "" {
-		myKNI, err := netlink.LinkByName(port.KNIName)
-		if err != nil {
-			println("Failed to get KNI interface", port.KNIName, ":", err)
-			return
-		}
-		addr := &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   dhcp.YourClientIP,
-				Mask: maskOption.Data,
-			},
-		}
-		netlink.AddrAdd(myKNI, addr)
-	}
+	port.setLinkLocalIPv4KNIAddress(port.Subnet.Addr, port.Subnet.Mask)
 }
diff --git a/examples/nat/dhcp6.go b/examples/nat/dhcp6.go
new file mode 100644
index 00000000..ce123f70
--- /dev/null
+++ b/examples/nat/dhcp6.go
@@ -0,0 +1,456 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"encoding/binary"
+	"errors"
+	"net"
+	"strings"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
+)
+
+type dhcpv6State struct {
+	lastDHCPv6PacketTypeSent layers.DHCPv6MsgType
+	dhcpv6TransactionId      [3]byte
+}
+
+const (
+	DHCPv6ClientPort = 546
+	DHCPv6ServerPort = 547
+)
+
+var (
+	BroadcastIPv6 = [common.IPv6AddrLen]uint8{
+		0xff, 0x02, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00,
+		0x00, 0x01, 0x00, 0x02,
+	}
+	SingleIPMask = [common.IPv6AddrLen]uint8{
+		0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff,
+	}
+	SingleIPNetMask = net.CIDRMask(128, 128)
+	hardwareTypeId  = []byte{0, 3}
+)
+
+func getDHCPv6Option(options layers.DHCPv6Options, optionType layers.DHCPv6Opt) *layers.DHCPv6Option {
+	for i := range options {
+		if options[i].Code == optionType {
+			return &options[i]
+		}
+	}
+	return nil
+}
+
+func (port *ipPort) composeAndSendDHCPv6Packet(packetType layers.DHCPv6MsgType, options []layers.DHCPv6Option) {
+	dhcpv6 := &layers.DHCPv6{
+		MsgType: packetType,
+	}
+	copy(dhcpv6.TransactionID, port.Subnet6.ds.dhcpv6TransactionId[:])
+
+	dhcpv6.Options = append(dhcpv6.Options, options...)
+	// Add client ID and FQDN options
+	clientID := &layers.DHCPv6DUID{
+		Type:         layers.DHCPv6DUIDTypeLL,
+		HardwareType: hardwareTypeId,
+	}
+	clientID.LinkLayerAddress = make([]byte, len(port.SrcMACAddress))
+	copy(clientID.LinkLayerAddress, port.SrcMACAddress[:])
+	fqdn := DHCPv6FQDN{
+		DomainName: Natconfig.HostName,
+	}
+	dhcpv6.Options = append(dhcpv6.Options,
+		layers.NewDHCPv6Option(layers.DHCPv6OptClientID, clientID.Encode()),
+		layers.NewDHCPv6Option(DHCPv6OptFQDNOptionCode, fqdn.Encode()))
+
+	buf := gopacket.NewSerializeBuffer()
+	opts := gopacket.SerializeOptions{
+		FixLengths:       true,
+		ComputeChecksums: true,
+	}
+	err := gopacket.SerializeLayers(buf, opts, dhcpv6)
+	if err != nil {
+		common.LogFatal(common.No, err)
+	}
+
+	// Convert gopacket data structure into NFF-Go packet and send it
+	pkt, err := packet.NewPacket()
+	if err != nil {
+		println(err)
+	}
+	payloadBuffer := buf.Bytes()
+	packet.InitEmptyIPv6UDPPacket(pkt, uint(len(payloadBuffer)))
+
+	// Fill up L2
+	pkt.Ether.SAddr = port.SrcMACAddress
+	packet.CalculateIPv6BroadcastMACForDstMulticastIP(&pkt.Ether.DAddr, BroadcastIPv6)
+
+	// Fill up L3
+	ipv6 := pkt.GetIPv6NoCheck()
+	ipv6.SrcAddr = port.Subnet6.llAddr
+	ipv6.DstAddr = BroadcastIPv6
+	ipv6.HopLimits = 1
+
+	// Fill up L4
+	udp := pkt.GetUDPNoCheck()
+	udp.SrcPort = packet.SwapBytesUint16(DHCPv6ClientPort)
+	udp.DstPort = packet.SwapBytesUint16(DHCPv6ServerPort)
+
+	// Fill up L7
+	payload, _ := pkt.GetPacketPayload()
+	copy(payload, payloadBuffer)
+
+	if port.Vlan != 0 {
+		pkt.AddVLANTag(port.Vlan)
+	}
+
+	setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+	port.dumpPacket(pkt, dirSEND)
+	pkt.SendPacket(port.Index)
+
+	port.Subnet6.ds.lastDHCPv6PacketTypeSent = packetType
+}
+
+func (port *ipPort) checkDHCPv6ServerAnswerAndGetIANA(dhcpv6 *layers.DHCPv6) *layers.DHCPv6Option {
+	statusOption := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptStatusCode)
+	if statusOption == nil {
+		println("Warning! Received a DHCPv6 reply without status! Trying again with discover request.")
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return nil
+	}
+
+	status := DHCPv6ServerStatusCode{}
+	err := status.DecodeFromBytes(statusOption.Data)
+	if err != nil {
+		println("Warning! Bad reply from server. Cannot decode status option: ", err)
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return nil
+	}
+
+	if layers.DHCPv6StatusCode(status.StatusCode) != layers.DHCPv6StatusCodeSuccess {
+		println("Warning! Server returned status", layers.DHCPv6StatusCode(status.StatusCode).String(), status.StatusMessage)
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return nil
+	}
+
+	// Ignore multiple IA_NA options in DHCP server reply. Use the first one.
+	ianaOption := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptIANA)
+	if ianaOption == nil {
+		println("Warning! Received a DHCPv6 reply without IANA! Trying again with discover request.")
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return nil
+	}
+
+	return ianaOption
+}
+
+func (port *ipPort) sendDHCPv6SolicitRequest() {
+	// Create new transaction ID
+	port.Subnet6.ds.dhcpv6TransactionId = [3]byte{
+		uint8(rnd.Uint32()),
+		uint8(rnd.Uint32()),
+		uint8(rnd.Uint32()),
+	}
+	iana := DHCPv6IANA{
+		IAID:    rnd.Uint32(),
+		Options: layers.DHCPv6Options{},
+	}
+	port.composeAndSendDHCPv6Packet(layers.DHCPv6MsgTypeSolicit,
+		[]layers.DHCPv6Option{layers.NewDHCPv6Option(layers.DHCPv6OptIANA, iana.Encode())})
+}
+
+func (port *ipPort) handleDHCPv6Advertise(pkt *packet.Packet, dhcpv6 *layers.DHCPv6) {
+	// Send request packet
+	ianaOption := port.checkDHCPv6ServerAnswerAndGetIANA(dhcpv6)
+	if ianaOption == nil {
+		println("Warning! No IANA option in server Advertise packet!")
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return
+	}
+	options := []layers.DHCPv6Option{*ianaOption}
+	serverID := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptServerID)
+	if serverID != nil {
+		options = append(options, *serverID)
+	}
+	port.composeAndSendDHCPv6Packet(layers.DHCPv6MsgTypeRequest, options)
+}
+
+func (port *ipPort) handleDHCPv6(pkt *packet.Packet) bool {
+	if port.Subnet6.addressAcquired {
+		// Port already has address, ignore this traffic
+		return false
+	}
+
+	// Check that this is DHCP offer or acknowledgement traffic
+	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPv6ClientPort) ||
+		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPv6ServerPort) {
+		return false
+	}
+
+	var dhcpv6 layers.DHCPv6
+	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeDHCPv6, &dhcpv6)
+	payload, _ := pkt.GetPacketPayload()
+	decoded := []gopacket.LayerType{}
+	err := parser.DecodeLayers(payload, &decoded)
+
+	if err != nil || len(decoded) != 1 || decoded[0] != layers.LayerTypeDHCPv6 {
+		println("Warning! Failed to parse DHCPv6 packet", err)
+		return false
+	}
+
+	if port.Subnet6.ds.lastDHCPv6PacketTypeSent == layers.DHCPv6MsgTypeSolicit && dhcpv6.MsgType == layers.DHCPv6MsgTypeAdverstise {
+		port.handleDHCPv6Advertise(pkt, &dhcpv6)
+	} else if port.Subnet6.ds.lastDHCPv6PacketTypeSent == layers.DHCPv6MsgTypeRequest && dhcpv6.MsgType == layers.DHCPv6MsgTypeReply {
+		port.handleDHCPv6Reply(pkt, &dhcpv6)
+	} else {
+		println("Warning! Received some bad response from DHCPv6 server", dhcpv6.MsgType.String())
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+	}
+	return true
+}
+
+func (port *ipPort) handleDHCPv6Reply(pkt *packet.Packet, dhcpv6 *layers.DHCPv6) {
+	ianaOption := port.checkDHCPv6ServerAnswerAndGetIANA(dhcpv6)
+	if ianaOption == nil {
+		println("Warning! No IANA option in server Reply packet!")
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return
+	}
+
+	var iana DHCPv6IANA
+	err := iana.DecodeFromBytes(ianaOption.Data)
+	if err != nil {
+		println("Warning! Bad reply from server. Cannot decode IANA option: ", err)
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return
+	}
+
+	// Ignore multiple addresses in DHCP server reply. Use the first one.
+	addressOption := getDHCPv6Option(iana.Options, layers.DHCPv6OptIAAddr)
+	if addressOption == nil {
+		println("Warning! Received a DHCPv6 reply with IANA which contains no address! Trying again with discover request.")
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return
+	}
+
+	var ia DHCPv6IAAddress
+	err = ia.DecodeFromBytes(addressOption.Data)
+	if err != nil {
+		println("Warning! Bad reply from server. Cannot decode IA Address option: ", err)
+		port.Subnet6.addressAcquired = false
+		port.Subnet6.ds = dhcpv6State{}
+		return
+	}
+
+	copy(port.Subnet6.Addr[:], ia.Address.To16())
+	packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
+	port.Subnet6.Mask = SingleIPMask
+	port.Subnet6.addressAcquired = true
+	println("Successfully acquired IP address:", port.Subnet6.String(), "on port", port.Index)
+
+	// Set address on KNI interface if present
+	port.setLinkLocalIPv6KNIAddress(port.Subnet6.Addr, port.Subnet6.Mask)
+}
+
+type DHCPv6FQDNFlags byte
+
+const (
+	DHCPv6OptFQDNOptionCode                layers.DHCPv6Opt = 39
+	DHCPv6FQDNOptionServerUpdateForwardDNS DHCPv6FQDNFlags  = 1
+	DHCPv6FQDNOptionServerOverride         DHCPv6FQDNFlags  = 2
+	DHCPv6FQDNOptionServerNoDNSUpdate      DHCPv6FQDNFlags  = 4
+)
+
+// FQDN option encoded according to RFC 4704
+type DHCPv6FQDN struct {
+	Flags      DHCPv6FQDNFlags
+	DomainName string
+}
+
+func (fqdn *DHCPv6FQDN) Encode() []byte {
+	encoded := []byte{
+		byte(fqdn.Flags),
+	}
+	names := strings.Split(fqdn.DomainName, ".")
+	for i := range names {
+		b := []byte(names[i])
+		encoded = append(encoded, byte(len(b)))
+		encoded = append(encoded, b...)
+	}
+	encoded = append(encoded, byte(0))
+	return encoded
+}
+
+func (fqdn *DHCPv6FQDN) DecodeFromBytes(data []byte) error {
+	datalength := len(data)
+	if datalength < 2 {
+		return errors.New("Not enough bytes to decode: " + string(len(data)))
+	}
+
+	fqdn.Flags = DHCPv6FQDNFlags(data[0])
+
+	index := 2
+	fqdn.DomainName = ""
+	length := int(data[index-1])
+	for length != 0 {
+		if datalength-index < length+1 {
+			return errors.New("Option encoded incorrectly, not enough bytes to decode")
+		}
+		fqdn.DomainName += string(data[index : index+length])
+		index += length + 1
+		length = int(data[index-1])
+	}
+	return nil
+}
+
+// Server status code option
+type DHCPv6ServerStatusCode struct {
+	StatusCode    uint16
+	StatusMessage string
+}
+
+func (sc *DHCPv6ServerStatusCode) Encode() []byte {
+	messageBytes := []byte(sc.StatusMessage)
+	length := 2 + len(messageBytes)
+	data := make([]byte, length)
+	binary.BigEndian.PutUint16(data[0:2], uint16(sc.StatusCode))
+	copy(data[2:], messageBytes)
+	return data
+}
+
+func (sc *DHCPv6ServerStatusCode) DecodeFromBytes(data []byte) error {
+	if len(data) < 2 {
+		return errors.New("Not enough bytes to decode: " + string(len(data)))
+	}
+
+	sc.StatusCode = binary.BigEndian.Uint16(data[:2])
+	sc.StatusMessage = string(data[2:])
+	return nil
+}
+
+// Returns length of data portion of option (excluding IANA code 3 and
+// option length)
+func OptionsLen(options layers.DHCPv6Options) int {
+	n := 0
+	for _, o := range options {
+		n += int(o.Length) + 4
+	}
+	return n
+}
+
+// Identity Association for Non-temporary Addresses Option
+type DHCPv6IANA struct {
+	IAID    uint32
+	T1, T2  uint32
+	Options layers.DHCPv6Options
+}
+
+func (iana *DHCPv6IANA) Encode() []byte {
+	data := make([]byte, 12+OptionsLen(iana.Options))
+	binary.BigEndian.PutUint32(data[0:4], iana.IAID)
+	binary.BigEndian.PutUint32(data[4:8], iana.T1)
+	binary.BigEndian.PutUint32(data[8:12], iana.T2)
+	offset := 12
+
+	for _, o := range iana.Options {
+		// TODO: use (*DHCPv6Option) encode here
+		binary.BigEndian.PutUint16(data[offset:offset+2], uint16(o.Code))
+		binary.BigEndian.PutUint16(data[offset+2:offset+4], o.Length)
+		copy(data[offset+4:], o.Data)
+		offset += int(o.Length) + 4
+	}
+	return data
+}
+
+func (iana *DHCPv6IANA) DecodeFromBytes(data []byte) error {
+	if len(data) < 12 {
+		return errors.New("Not enough bytes to decode: " + string(len(data)))
+	}
+
+	iana.IAID = binary.BigEndian.Uint32(data[:4])
+	iana.T1 = binary.BigEndian.Uint32(data[4:8])
+	iana.T2 = binary.BigEndian.Uint32(data[8:12])
+	iana.Options = iana.Options[:0]
+	offset := 12
+
+	stop := len(data)
+	for offset < stop {
+		// TODO: use (*DHCPv6Option) decode here
+		o := layers.DHCPv6Option{}
+		o.Code = layers.DHCPv6Opt(binary.BigEndian.Uint16(data[offset : offset+2]))
+		o.Length = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+		o.Data = data[offset+4 : offset+4+int(o.Length)]
+		iana.Options = append(iana.Options, o)
+		offset += int(o.Length) + 4
+	}
+	return nil
+}
+
+// IA Address Option
+type DHCPv6IAAddress struct {
+	Address           net.IP
+	PreferredLifetime uint32
+	ValidLifetime     uint32
+	Options           layers.DHCPv6Options
+}
+
+func (ia *DHCPv6IAAddress) Encode() []byte {
+	data := make([]byte, 16+4+4+OptionsLen(ia.Options))
+	copy(data[:16], ia.Address.To16())
+	binary.BigEndian.PutUint32(data[16:20], ia.PreferredLifetime)
+	binary.BigEndian.PutUint32(data[20:24], ia.ValidLifetime)
+	offset := 24
+
+	for _, o := range ia.Options {
+		// TODO: use (*DHCPv6Option) encode here
+		binary.BigEndian.PutUint16(data[offset:offset+2], uint16(o.Code))
+		binary.BigEndian.PutUint16(data[offset+2:offset+4], o.Length)
+		copy(data[offset+4:], o.Data)
+		offset += int(o.Length) + 4
+	}
+	return data
+}
+
+func (ia *DHCPv6IAAddress) DecodeFromBytes(data []byte) error {
+	if len(data) < 24 {
+		return errors.New("Not enough bytes to decode: " + string(len(data)))
+	}
+
+	ia.Address = net.IP(data[:16])
+	ia.PreferredLifetime = binary.BigEndian.Uint32(data[16:20])
+	ia.ValidLifetime = binary.BigEndian.Uint32(data[20:24])
+	ia.Options = ia.Options[:0]
+	offset := 24
+
+	stop := len(data)
+	for offset < stop {
+		// TODO: use (*DHCPv6Option) decode here
+		o := layers.DHCPv6Option{}
+		o.Code = layers.DHCPv6Opt(binary.BigEndian.Uint16(data[offset : offset+2]))
+		o.Length = binary.BigEndian.Uint16(data[offset+2 : offset+4])
+		o.Data = data[offset+4 : offset+4+int(o.Length)]
+		ia.Options = append(ia.Options, o)
+		offset += int(o.Length) + 4
+	}
+	return nil
+}
diff --git a/examples/nat/grpc.go b/examples/nat/grpc.go
index 3eb58302..600b0bfe 100644
--- a/examples/nat/grpc.go
+++ b/examples/nat/grpc.go
@@ -15,6 +15,7 @@ import (
 	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
 
 	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
 )
 
 const (
@@ -60,14 +61,30 @@ func (s *server) ChangeInterfaceAddress(ctx context.Context, in *upd.InterfaceAd
 	if port == nil {
 		return nil, fmt.Errorf("Interface with ID %d not found", portId)
 	}
-	subnet, err := convertSubnet(in.GetPortSubnet())
+	subnet4, subnet6, err := convertSubnet(in.GetPortSubnet())
 	if err != nil {
 		return nil, err
 	}
-	port.Subnet = *subnet
+	var str string
+	if subnet4 != nil {
+		port.Subnet.Addr = subnet4.Addr
+		port.Subnet.Mask = subnet4.Mask
+		port.Subnet.addressAcquired = true
+		port.setLinkLocalIPv4KNIAddress(port.Subnet.Addr, port.Subnet.Mask)
+		str = port.Subnet.String()
+	}
+	if subnet6 != nil {
+		port.Subnet6.Addr = subnet6.Addr
+		port.Subnet6.Mask = subnet6.Mask
+		port.Subnet6.addressAcquired = true
+		port.setLinkLocalIPv6KNIAddress(port.Subnet6.Addr, port.Subnet6.Mask)
+		packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
+		port.setLinkLocalIPv6KNIAddress(port.Subnet6.llAddr, SingleIPMask)
+		str = port.Subnet6.String()
+	}
 
 	return &upd.Reply{
-		Msg: fmt.Sprintf("Successfully set port %d subnet to %s", portId, subnet.String()),
+		Msg: fmt.Sprintf("Successfully set port %d subnet to %s", portId, str),
 	}, nil
 }
 
@@ -88,7 +105,11 @@ func (s *server) ChangePortForwarding(ctx context.Context, in *upd.PortForwardin
 	}
 
 	pp.mutex.Lock()
-	pp.deleteOldConnection(uint8(fp.Protocol), int(fp.Port))
+	if port.Type == iPUBLIC {
+		pp.deleteOldConnection(fp.Protocol.ipv6, fp.Protocol.id, int(fp.Port))
+	} else {
+		port.deletePortForwardingEntry(fp.Protocol.ipv6, fp.Protocol.id, int(fp.Port))
+	}
 	if in.GetEnableForwarding() {
 		port.enableStaticPortForward(fp)
 	}
diff --git a/examples/nat/icmp.go b/examples/nat/icmp.go
new file mode 100644
index 00000000..25404435
--- /dev/null
+++ b/examples/nat/icmp.go
@@ -0,0 +1,95 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"time"
+
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
+)
+
+func (port *ipPort) handleICMP(protocol uint8, pkt *packet.Packet, key interface{}) uint {
+	// Check that received ICMP packet is addressed at this host. If
+	// not, packet should be translated
+	var requestCode uint8
+	var packetSentToUs bool
+	var packetSentToMulticast bool
+	if protocol == common.ICMPNumber {
+		if packet.SwapBytesUint32(pkt.GetIPv4NoCheck().DstAddr) == port.Subnet.Addr {
+			packetSentToUs = true
+		}
+		requestCode = common.ICMPTypeEchoRequest
+	} else {
+		// If message is not targeted at NAT host, it is subject of
+		// address translation
+		ipv6 := pkt.GetIPv6NoCheck()
+		if ipv6.DstAddr == port.Subnet6.Addr ||
+			ipv6.DstAddr == port.Subnet6.llAddr {
+			packetSentToUs = true
+		} else if ipv6.DstAddr == port.Subnet6.multicastAddr ||
+			ipv6.DstAddr == port.Subnet6.llMulticastAddr {
+			packetSentToMulticast = true
+		}
+		requestCode = common.ICMPv6TypeEchoRequest
+	}
+
+	// Check IPv6 Neighbor Discovery first. If packet is handled, it
+	// returns DROP or KNI, otherwise continue to process it.
+	if packetSentToMulticast || (packetSentToUs && protocol == common.ICMPv6Number) {
+		dir := port.handleIPv6NeighborDiscovery(pkt)
+		if dir != dirSEND {
+			return dir
+		}
+	}
+
+	icmp := pkt.GetICMPNoCheck()
+
+	// If there is KNI interface, direct all ICMP traffic which
+	// doesn't have an active translation entry. It may happen only
+	// for public->private translation when all packets are directed
+	// to NAT public interface IP, so port.portmap exists because port
+	// is public.
+	if packetSentToUs && port.KNIName != "" {
+		if key != nil {
+			_, ok := port.translationTable[protocol].Load(key)
+			if !ok || time.Since(port.getPortmap(protocol == common.ICMPv6Number, protocol)[icmp.Identifier].lastused) > connectionTimeout {
+				return dirKNI
+			}
+		}
+	}
+
+	// If packet sent to us, check that it is ICMP echo request. NAT
+	// does not support any other messages yet, so process them in
+	// normal way. Maybe these are packets which should be passed
+	// through translation.
+	if !packetSentToUs || icmp.Type != requestCode || icmp.Code != 0 {
+		return dirSEND
+	}
+
+	// Return a packet back to sender
+	answerPacket, err := packet.NewPacket()
+	if err != nil {
+		common.LogFatal(common.Debug, err)
+	}
+	packet.GeneratePacketFromByte(answerPacket, pkt.GetRawPacketBytes())
+
+	answerPacket.ParseL3CheckVLAN()
+	if protocol == common.ICMPNumber {
+		swapAddrIPv4(answerPacket)
+		answerPacket.ParseL4ForIPv4()
+		(answerPacket.GetICMPNoCheck()).Type = common.ICMPTypeEchoResponse
+		setIPv4ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
+	} else {
+		swapAddrIPv6(answerPacket)
+		answerPacket.ParseL4ForIPv6()
+		(answerPacket.GetICMPNoCheck()).Type = common.ICMPv6TypeEchoResponse
+		setIPv6ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
+	}
+
+	port.dumpPacket(answerPacket, dirSEND)
+	answerPacket.SendPacket(port.Index)
+	return dirDROP
+}
diff --git a/examples/nat/main/Makefile b/examples/nat/main/Makefile
index 7ac82cbc..502f4a47 100644
--- a/examples/nat/main/Makefile
+++ b/examples/nat/main/Makefile
@@ -6,7 +6,7 @@ PATH_TO_MK = ../../../mk
 IMAGENAME = nff-go-nat
 EXECUTABLES = nat
 
-nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../grpc.go ../dhcp.go ../updatecfg/updatecfg.pb.go
+nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../arp.go ../icmp.go ../grpc.go ../dhcp.go ../neigh.go ../updatecfg/updatecfg.pb.go
 
 ../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
 	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
diff --git a/examples/nat/main/config-dhcp.json b/examples/nat/main/config-dhcp.json
index 65c44392..24cd0913 100644
--- a/examples/nat/main/config-dhcp.json
+++ b/examples/nat/main/config-dhcp.json
@@ -4,31 +4,53 @@
         {
             "private-port": {
                 "index": 0,
-                "subnet": "dhcp"
+                "subnet": "dhcp",
+                "subnet6": "dhcp"
             },
             "public-port": {
                 "index": 1,
                 "subnet": "dhcp",
+                "subnet6": "dhcp",
                 "forward-ports": [
                     {
                         "port": 8080,
                         "destination": "192.168.14.3:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8080,
+                        "destination": "[fd14::3]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2222,
                         "destination": "192.168.14.3:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 2222,
+                        "destination": "[fd14::3]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8081,
                         "destination": "192.168.14.4:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8081,
+                        "destination": "[fd14::4]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2223,
                         "destination": "192.168.14.4:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "[fd14::4]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             }
diff --git a/examples/nat/main/config-kni-dhcp.json b/examples/nat/main/config-kni-dhcp.json
index aad31117..9a156ba3 100644
--- a/examples/nat/main/config-kni-dhcp.json
+++ b/examples/nat/main/config-kni-dhcp.json
@@ -5,18 +5,25 @@
             "private-port": {
                 "index": 0,
                 "subnet": "dhcp",
+                "subnet6": "dhcp",
                 "kni-name": "priv0",
                 "forward-ports": [
                     {
                         "port": 22,
                         "destination": "0.0.0.0:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 22,
+                        "destination": "[::]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             },
             "public-port": {
                 "index": 1,
                 "subnet": "dhcp",
+                "subnet6": "dhcp",
                 "kni-name": "pub1",
                 "forward-ports": [
                     {
@@ -24,25 +31,50 @@
                         "destination": "0.0.0.0:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 22,
+                        "destination": "[::]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8080,
                         "destination": "192.168.14.3:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8080,
+                        "destination": "[fd14::3]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2222,
                         "destination": "192.168.14.3:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 2222,
+                        "destination": "[fd14::3]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8081,
                         "destination": "192.168.14.4:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8081,
+                        "destination": "[fd14::4]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2223,
                         "destination": "192.168.14.4:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "[fd14::4]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             }
diff --git a/examples/nat/main/config-kni.json b/examples/nat/main/config-kni.json
index e5ad5272..0a9676c7 100644
--- a/examples/nat/main/config-kni.json
+++ b/examples/nat/main/config-kni.json
@@ -4,18 +4,25 @@
             "private-port": {
                 "index": 0,
                 "subnet": "192.168.14.1/24",
+                "subnet6": "fd14::1/64",
                 "kni-name": "priv0",
                 "forward-ports": [
                     {
                         "port": 22,
                         "destination": "0.0.0.0:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 22,
+                        "destination": "[::]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             },
             "public-port": {
                 "index": 1,
                 "subnet": "192.168.16.1/24",
+                "subnet6": "fd16::1/64",
                 "kni-name": "pub1",
                 "forward-ports": [
                     {
@@ -23,25 +30,50 @@
                         "destination": "0.0.0.0:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 22,
+                        "destination": "[::]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8080,
                         "destination": "192.168.14.3:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8080,
+                        "destination": "[fd14::3]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2222,
                         "destination": "192.168.14.3:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 2222,
+                        "destination": "[fd14::3]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8081,
                         "destination": "192.168.14.4:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8081,
+                        "destination": "[fd14::4]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2223,
                         "destination": "192.168.14.4:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "[fd14::4]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             }
diff --git a/examples/nat/main/config.json b/examples/nat/main/config.json
index 427586c1..d95cc1f7 100644
--- a/examples/nat/main/config.json
+++ b/examples/nat/main/config.json
@@ -3,31 +3,53 @@
         {
             "private-port": {
                 "index": 0,
-                "subnet": "192.168.14.1/24"
+                "subnet": "192.168.14.1/24",
+                "subnet6": "fd14::1/64"
             },
             "public-port": {
                 "index": 1,
                 "subnet": "192.168.16.1/24",
+                "subnet6": "fd16::1/64",
                 "forward-ports": [
                     {
                         "port": 8080,
                         "destination": "192.168.14.3:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8080,
+                        "destination": "[fd14::3]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2222,
                         "destination": "192.168.14.3:22",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 2222,
+                        "destination": "[fd14::3]:22",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 8081,
                         "destination": "192.168.14.4:80",
                         "protocol": "TCP"
                     },
+                    {
+                        "port": 8081,
+                        "destination": "[fd14::4]:80",
+                        "protocol": "TCP6"
+                    },
                     {
                         "port": 2223,
                         "destination": "192.168.14.4:22",
                         "protocol": "TCP"
+                    },
+                    {
+                        "port": 2223,
+                        "destination": "[fd14::4]:22",
+                        "protocol": "TCP6"
                     }
                 ]
             }
diff --git a/examples/nat/neigh.go b/examples/nat/neigh.go
new file mode 100644
index 00000000..70b4991d
--- /dev/null
+++ b/examples/nat/neigh.go
@@ -0,0 +1,88 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package nat
+
+import (
+	"github.com/intel-go/nff-go/common"
+	"github.com/intel-go/nff-go/packet"
+)
+
+func (port *ipPort) handleIPv6NeighborDiscovery(pkt *packet.Packet) uint {
+	icmp := pkt.GetICMPNoCheck()
+	if icmp.Type == common.ICMPv6NeighborSolicitation {
+		// If there is KNI interface, forward all of this here
+		if port.KNIName != "" {
+			return dirKNI
+		}
+		pkt.ParseL7(common.ICMPv6Number)
+		msg := pkt.GetICMPv6NeighborSolicitationMessage()
+		if msg.TargetAddr != port.Subnet6.Addr && msg.TargetAddr != port.Subnet6.llAddr {
+			return dirDROP
+		}
+		option := pkt.GetICMPv6NDSourceLinkLayerAddressOption(packet.ICMPv6NeighborSolicitationMessageSize)
+		if option != nil && option.Type == packet.ICMPv6NDSourceLinkLayerAddress {
+			answerPacket, err := packet.NewPacket()
+			if err != nil {
+				common.LogFatal(common.Debug, err)
+			}
+
+			packet.InitICMPv6NeighborAdvertisementPacket(answerPacket, port.SrcMACAddress, option.LinkLayerAddress, msg.TargetAddr, pkt.GetIPv6NoCheck().SrcAddr)
+
+			vlan := pkt.GetVLAN()
+			if vlan != nil {
+				answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
+			}
+
+			setIPv6ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
+			port.dumpPacket(answerPacket, dirSEND)
+			answerPacket.SendPacket(port.Index)
+		}
+	} else if icmp.Type == common.ICMPv6NeighborAdvertisement {
+		pkt.ParseL7(common.ICMPv6Number)
+		msg := pkt.GetICMPv6NeighborAdvertisementMessage()
+		option := pkt.GetICMPv6NDTargetLinkLayerAddressOption(packet.ICMPv6NeighborAdvertisementMessageSize)
+		if option != nil && option.Type == packet.ICMPv6NDTargetLinkLayerAddress {
+			port.arpTable.Store(msg.TargetAddr, option.LinkLayerAddress)
+		}
+
+		if port.KNIName != "" {
+			return dirKNI
+		}
+	} else {
+		if port.KNIName != "" {
+			return dirKNI
+		}
+		return dirSEND
+	}
+
+	return dirDROP
+}
+
+func (port *ipPort) getMACForIPv6(ip [common.IPv6AddrLen]uint8) (macAddress, bool) {
+	v, found := port.arpTable.Load(ip)
+	if found {
+		return macAddress(v.([common.EtherAddrLen]byte)), true
+	}
+	port.sendNDNeighborSolicitationRequest(ip)
+	return macAddress{}, false
+}
+
+func (port *ipPort) sendNDNeighborSolicitationRequest(ip [common.IPv6AddrLen]uint8) {
+	requestPacket, err := packet.NewPacket()
+	if err != nil {
+		common.LogFatal(common.Debug, err)
+	}
+
+	packet.InitICMPv6NeighborSolicitationPacket(requestPacket, port.SrcMACAddress,
+		port.Subnet6.Addr, ip)
+
+	if port.Vlan != 0 {
+		requestPacket.AddVLANTag(port.Vlan)
+	}
+
+	setIPv6ICMPChecksum(requestPacket, !NoCalculateChecksum, !NoHWTXChecksum)
+	port.dumpPacket(requestPacket, dirSEND)
+	requestPacket.SendPacket(port.Index)
+}
diff --git a/examples/nat/portalloc.go b/examples/nat/portalloc.go
index e21a2860..74d3ae58 100644
--- a/examples/nat/portalloc.go
+++ b/examples/nat/portalloc.go
@@ -28,14 +28,16 @@ func (dir terminationDirection) String() string {
 	}
 }
 
-func (pp *portPair) deleteOldConnection(protocol uint8, port int) {
+func (port *ipPort) deletePortForwardingEntry(ipv6 bool, protocol uint8, portNumber int) {
+	key := port.makePortAddrTuple(ipv6, uint16(portNumber))
+	port.translationTable[protocol].Delete(key)
+}
+
+func (pp *portPair) deleteOldConnection(ipv6 bool, protocol uint8, port int) {
 	pubTable := pp.PublicPort.translationTable[protocol]
-	pm := pp.PublicPort.portmap[protocol]
+	pm := pp.getPublicPortPortmap(ipv6, protocol)
 
-	pub2priKey := Tuple{
-		addr: pm[port].addr,
-		port: uint16(port),
-	}
+	pub2priKey := pp.PublicPort.makePortAddrTuple(ipv6, uint16(port))
 	pri2pubKey, found := pubTable.Load(pub2priKey)
 
 	if found {
@@ -47,13 +49,13 @@ func (pp *portPair) deleteOldConnection(protocol uint8, port int) {
 
 // This function currently is not thread safe and should be executed
 // under a global lock
-func (pp *portPair) allocNewPort(protocol uint8) (int, error) {
-	pm := pp.PublicPort.portmap[protocol]
+func (pp *portPair) allocNewPort(ipv6 bool, protocol uint8) (int, error) {
+	pm := pp.getPublicPortPortmap(ipv6, protocol)
 	for {
 		for p := pp.lastport; p < portEnd; p++ {
 			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
 				pp.lastport = p
-				pp.deleteOldConnection(protocol, p)
+				pp.deleteOldConnection(ipv6, protocol, p)
 				return p, nil
 			}
 		}
@@ -61,10 +63,32 @@ func (pp *portPair) allocNewPort(protocol uint8) (int, error) {
 		for p := portStart; p < pp.lastport; p++ {
 			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
 				pp.lastport = p
-				pp.deleteOldConnection(protocol, p)
+				pp.deleteOldConnection(ipv6, protocol, p)
 				return p, nil
 			}
 		}
 		return 0, errors.New("WARNING! All ports are allocated! Trying again")
 	}
 }
+
+func (pp *portPair) getPublicPortPortmap(ipv6 bool, protocol uint8) []portMapEntry {
+	if ipv6 {
+		return pp.PublicPort.portmap6[protocol]
+	} else {
+		return pp.PublicPort.portmap[protocol]
+	}
+}
+
+func (port *ipPort) makePortAddrTuple(ipv6 bool, portNumber uint16) interface{} {
+	if ipv6 {
+		return Tuple6{
+			addr: port.Subnet6.Addr,
+			port: uint16(portNumber),
+		}
+	} else {
+		return Tuple{
+			addr: port.Subnet.Addr,
+			port: uint16(portNumber),
+		}
+	}
+}
diff --git a/examples/nat/translation.go b/examples/nat/translation.go
index 53e5ab4f..f04ba5a9 100644
--- a/examples/nat/translation.go
+++ b/examples/nat/translation.go
@@ -18,35 +18,50 @@ type Tuple struct {
 	port uint16
 }
 
-func (pp *portPair) allocateNewEgressConnection(protocol uint8, privEntry *Tuple) (Tuple, error) {
+type Tuple6 struct {
+	addr [common.IPv6AddrLen]uint8
+	port uint16
+}
+
+func (pp *portPair) allocateNewEgressConnection(ipv6 bool, protocol uint8, privEntry interface{}) (uint32, [common.IPv6AddrLen]uint8, uint16, error) {
 	pp.mutex.Lock()
 
-	port, err := pp.allocNewPort(protocol)
+	port, err := pp.allocNewPort(ipv6, protocol)
 	if err != nil {
 		pp.mutex.Unlock()
-		return Tuple{}, err
+		return 0, [common.IPv6AddrLen]uint8{}, 0, err
 	}
 
-	publicAddr := pp.PublicPort.Subnet.Addr
-	pubEntry := Tuple{
-		addr: publicAddr,
-		port: uint16(port),
+	var pubEntry interface{}
+	var v4addr uint32
+	var v6addr [common.IPv6AddrLen]uint8
+	if ipv6 {
+		v6addr = pp.PublicPort.Subnet6.Addr
+		pubEntry = Tuple6{
+			addr: v6addr,
+			port: uint16(port),
+		}
+	} else {
+		v4addr = pp.PublicPort.Subnet.Addr
+		pubEntry = Tuple{
+			addr: v4addr,
+			port: uint16(port),
+		}
 	}
 
-	pp.PublicPort.portmap[protocol][port] = portMapEntry{
+	pp.PublicPort.getPortmap(ipv6, protocol)[port] = portMapEntry{
 		lastused:             time.Now(),
-		addr:                 publicAddr,
 		finCount:             0,
 		terminationDirection: 0,
 		static:               false,
 	}
 
 	// Add lookup entries for packet translation
-	pp.PublicPort.translationTable[protocol].Store(pubEntry, *privEntry)
-	pp.PrivatePort.translationTable[protocol].Store(*privEntry, pubEntry)
+	pp.PublicPort.translationTable[protocol].Store(pubEntry, privEntry)
+	pp.PrivatePort.translationTable[protocol].Store(privEntry, pubEntry)
 
 	pp.mutex.Unlock()
-	return pubEntry, nil
+	return v4addr, v6addr, uint16(port), nil
 }
 
 // PublicToPrivateTranslation does ingress translation.
@@ -58,33 +73,57 @@ func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 	port.dumpPacket(pkt, dirSEND)
 
 	// Parse packet type and address
-	dir, pktVLAN, pktIPv4 := port.parsePacketAndCheckARP(pkt)
-	if pktIPv4 == nil {
+	dir, pktVLAN, pktIPv4, pktIPv6 := port.parsePacketAndCheckARP(pkt)
+	if pktIPv4 == nil && pktIPv6 == nil {
 		return dir
 	}
 
+	protocol, pktTCP, pktUDP, pktICMP, _, DstPort := ParseAllKnownL4(pkt, pktIPv4, pktIPv6)
+	if protocol == 0 {
+		// Only TCP, UDP and ICMP are supported now, all other protocols are ignored
+		port.dumpPacket(pkt, dirDROP)
+		return dirDROP
+	}
+	portNumber := DstPort
 	// Create a lookup key from packet destination address and port
-	pktTCP, pktUDP, pktICMP := pkt.ParseAllKnownL4ForIPv4()
-	protocol := pktIPv4.NextProtoID
-	pub2priKey, dir := port.generateLookupKeyFromDstAndHandleICMP(pkt, pktIPv4, pktTCP, pktUDP, pktICMP)
-	if pub2priKey == nil {
-		return dir
+	pub2priKey := generateLookupKeyFromDstAddr(pkt, pktIPv4, pktIPv6, portNumber)
+	// Check for ICMP traffic first
+	if pktICMP != nil {
+		dir := port.handleICMP(protocol, pkt, pub2priKey)
+		if dir != dirSEND {
+			port.dumpPacket(pkt, dir)
+			return dir
+		}
 	}
-
+	ipv6 := pktIPv6 != nil
 	// Check for DHCP traffic. We need to get an address if it not set yet
 	if pktUDP != nil {
-		if port.handleDHCP(pkt) {
+		var handled bool
+		if ipv6 {
+			handled = port.handleDHCPv6(pkt)
+		} else {
+			handled = port.handleDHCP(pkt)
+		}
+		if handled {
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
 	}
 
 	// Do lookup
-	v, found := port.translationTable[protocol].Load(*pub2priKey)
+	v, found := port.translationTable[protocol].Load(pub2priKey)
+	kniPresent := port.KNIName != ""
 
 	if !found {
 		// Store new local network entry in ARP cache
-		port.arpTable.Store(pktIPv4.SrcAddr, pkt.Ether.SAddr)
+		var addressAcquired bool
+		if ipv6 {
+			port.arpTable.Store(pktIPv6.SrcAddr, pkt.Ether.SAddr)
+			addressAcquired = port.Subnet6.addressAcquired
+		} else {
+			port.arpTable.Store(pktIPv4.SrcAddr, pkt.Ether.SAddr)
+			addressAcquired = port.Subnet.addressAcquired
+		}
 
 		// For ingress connections packets are allowed only if a
 		// connection has been previosly established with a egress
@@ -92,7 +131,7 @@ func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 		// incoming packet is ignored unless there is a KNI
 		// interface. If KNI is present and its IP address is known,
 		// traffic is directed there.
-		if port.KNIName != "" && port.Subnet.addressAcquired {
+		if kniPresent && addressAcquired {
 			dir = dirKNI
 		} else {
 			dir = dirDROP
@@ -100,40 +139,53 @@ func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 		port.dumpPacket(pkt, dir)
 		return dir
 	}
-	value := v.(Tuple)
+	v4addr, v6addr, newPort, zeroAddr := getAddrFromTuple(v, ipv6)
 
+	portmap := port.getPortmap(ipv6, protocol)
 	// Check whether connection is too old
-	if port.portmap[protocol][pub2priKey.port].static || time.Since(port.portmap[protocol][pub2priKey.port].lastused) <= connectionTimeout {
-		port.portmap[protocol][pub2priKey.port].lastused = time.Now()
+	if portmap[portNumber].static || time.Since(portmap[portNumber].lastused) <= connectionTimeout {
+		portmap[portNumber].lastused = time.Now()
 	} else {
 		// There was no transfer on this port for too long
 		// time. We don't allow it any more
 		pp.mutex.Lock()
-		pp.deleteOldConnection(protocol, int(pub2priKey.port))
+		pp.deleteOldConnection(pktIPv6 != nil, protocol, int(portNumber))
 		pp.mutex.Unlock()
 		port.dumpPacket(pkt, dirDROP)
 		return dirDROP
 	}
 
-	if value.addr != 0 {
+	if !zeroAddr {
 		// Check whether TCP connection could be reused
-		if pktTCP != nil && !pp.PublicPort.portmap[protocol][value.port].static {
-			pp.checkTCPTermination(pktTCP, int(pub2priKey.port), pub2pri)
+		if pktTCP != nil && !portmap[portNumber].static {
+			pp.checkTCPTermination(ipv6, pktTCP, int(portNumber), pub2pri)
 		}
 
-		// Do packet translation
-		mac, found := port.opposite.getMACForIP(value.addr)
+		// Find corresponding MAC address
+		var mac macAddress
+		var found bool
+		if ipv6 {
+			mac, found = port.opposite.getMACForIPv6(v6addr)
+		} else {
+			mac, found = port.opposite.getMACForIPv4(v4addr)
+		}
 		if !found {
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
+
+		// Do packet translation
 		pkt.Ether.DAddr = mac
 		pkt.Ether.SAddr = port.SrcMACAddress
 		if pktVLAN != nil {
 			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
 		}
-		pktIPv4.DstAddr = packet.SwapBytesUint32(value.addr)
-		setPacketDstPort(pkt, value.port, pktTCP, pktUDP, pktICMP)
+		if ipv6 {
+			pktIPv6.DstAddr = v6addr
+		} else {
+			pktIPv4.DstAddr = packet.SwapBytesUint32(v4addr)
+		}
+		setPacketDstPort(pkt, ipv6, newPort, pktTCP, pktUDP, pktICMP)
 
 		port.dumpPacket(pkt, dirSEND)
 		return dirSEND
@@ -152,81 +204,135 @@ func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 	port.dumpPacket(pkt, dirSEND)
 
 	// Parse packet type and address
-	dir, pktVLAN, pktIPv4 := port.parsePacketAndCheckARP(pkt)
-	if pktIPv4 == nil {
+	dir, pktVLAN, pktIPv4, pktIPv6 := port.parsePacketAndCheckARP(pkt)
+	if pktIPv4 == nil && pktIPv6 == nil {
 		return dir
 	}
 
+	protocol, pktTCP, pktUDP, pktICMP, SrcPort, _ := ParseAllKnownL4(pkt, pktIPv4, pktIPv6)
+	if protocol == 0 {
+		// Only TCP, UDP and ICMP are supported now, all other protocols are ignored
+		port.dumpPacket(pkt, dirDROP)
+		return dirDROP
+	}
+	portNumber := SrcPort
 	// Create a lookup key from packet source address and port
-	pktTCP, pktUDP, pktICMP := pkt.ParseAllKnownL4ForIPv4()
-	protocol := pktIPv4.NextProtoID
-	pri2pubKey, dir := port.generateLookupKeyFromSrcAndHandleICMP(pkt, pktIPv4, pktTCP, pktUDP, pktICMP)
-	if pri2pubKey == nil {
-		return dir
+	pri2pubKey, saddr := generateLookupKeyFromSrcAddr(pkt, pktIPv4, pktIPv6, portNumber)
+	// Check for ICMP traffic first
+	if pktICMP != nil {
+		dir := port.handleICMP(protocol, pkt, pri2pubKey)
+		if dir != dirSEND {
+			port.dumpPacket(pkt, dir)
+			return dir
+		}
 	}
-
+	ipv6 := pktIPv6 != nil
 	// Check for DHCP traffic. We need to get an address if it not set yet
 	if pktUDP != nil {
-		if port.handleDHCP(pkt) {
+		var handled bool
+		if ipv6 {
+			handled = port.handleDHCPv6(pkt)
+		} else {
+			handled = port.handleDHCP(pkt)
+		}
+		if handled {
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
 	}
 
+	kniPresent := port.KNIName != ""
+	var addressAcquired bool
+	var packetSentToUs bool
+	if ipv6 {
+		addressAcquired = port.Subnet6.addressAcquired
+		packetSentToUs = port.Subnet6.Addr == pktIPv6.DstAddr ||
+			port.Subnet6.llAddr == pktIPv6.DstAddr ||
+			port.Subnet6.multicastAddr == pktIPv6.DstAddr ||
+			port.Subnet6.llMulticastAddr == pktIPv6.DstAddr
+	} else {
+		addressAcquired = port.Subnet.addressAcquired
+		packetSentToUs = port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr)
+	}
+
 	// If traffic is directed at private interface IP and KNI is
 	// present, this traffic is directed to KNI
-	if port.KNIName != "" && port.Subnet.addressAcquired && port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr) {
+	if kniPresent && addressAcquired && packetSentToUs {
 		port.dumpPacket(pkt, dirKNI)
 		return dirKNI
 	}
 
 	// Do lookup
-	var value Tuple
-	v, found := port.translationTable[protocol].Load(*pri2pubKey)
+	v, found := port.translationTable[protocol].Load(pri2pubKey)
+
+	var v4addr uint32
+	var v6addr [common.IPv6AddrLen]uint8
+	var newPort uint16
+	var zeroAddr bool
 
 	if !found {
 		var err error
 		// Store new local network entry in ARP cache
-		port.arpTable.Store(pri2pubKey.addr, pkt.Ether.SAddr)
+		port.arpTable.Store(saddr, pkt.Ether.SAddr)
 
-		if !port.Subnet.addressAcquired || !port.opposite.Subnet.addressAcquired {
+		var publicAddressAcquired bool
+		if ipv6 {
+			publicAddressAcquired = port.opposite.Subnet6.addressAcquired
+		} else {
+			publicAddressAcquired = port.opposite.Subnet.addressAcquired
+		}
+
+		if !addressAcquired || !publicAddressAcquired {
 			// No packets are allowed yet because ports address is not
 			// known yet
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
 		// Allocate new connection from private to public network
-		value, err = pp.allocateNewEgressConnection(protocol, pri2pubKey)
+		v4addr, v6addr, newPort, err = pp.allocateNewEgressConnection(pktIPv6 != nil, protocol, pri2pubKey)
 
 		if err != nil {
 			println("Warning! Failed to allocate new connection", err)
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
+		zeroAddr = false
 	} else {
-		value = v.(Tuple)
-		pp.PublicPort.portmap[protocol][value.port].lastused = time.Now()
+		v4addr, v6addr, newPort, zeroAddr = getAddrFromTuple(v, ipv6)
+		pp.PublicPort.getPortmap(ipv6, protocol)[newPort].lastused = time.Now()
 	}
 
-	if value.addr != 0 {
+	if !zeroAddr {
 		// Check whether TCP connection could be reused
-		if pktTCP != nil && !pp.PublicPort.portmap[protocol][value.port].static {
-			pp.checkTCPTermination(pktTCP, int(value.port), pri2pub)
+		if pktTCP != nil && !pp.PublicPort.getPortmap(ipv6, protocol)[newPort].static {
+			pp.checkTCPTermination(ipv6, pktTCP, int(newPort), pri2pub)
 		}
 
-		// Do packet translation
-		mac, found := port.opposite.getMACForIP(packet.SwapBytesUint32(pktIPv4.DstAddr))
+		// Find corresponding MAC address
+		var mac macAddress
+		var found bool
+		if pktIPv6 != nil {
+			mac, found = port.opposite.getMACForIPv6(pktIPv6.DstAddr)
+		} else {
+			mac, found = port.opposite.getMACForIPv4(packet.SwapBytesUint32(pktIPv4.DstAddr))
+		}
 		if !found {
 			port.dumpPacket(pkt, dirDROP)
 			return dirDROP
 		}
+
+		// Do packet translation
 		pkt.Ether.DAddr = mac
 		pkt.Ether.SAddr = port.SrcMACAddress
 		if pktVLAN != nil {
 			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
 		}
-		pktIPv4.SrcAddr = packet.SwapBytesUint32(value.addr)
-		setPacketSrcPort(pkt, value.port, pktTCP, pktUDP, pktICMP)
+		if ipv6 {
+			pktIPv6.SrcAddr = v6addr
+		} else {
+			pktIPv4.SrcAddr = packet.SwapBytesUint32(v4addr)
+		}
+		setPacketSrcPort(pkt, ipv6, newPort, pktTCP, pktUDP, pktICMP)
 
 		port.dumpPacket(pkt, dirSEND)
 		return dirSEND
@@ -237,86 +343,39 @@ func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
 }
 
 // Used to generate key in public to private translation
-func (port *ipv4Port) generateLookupKeyFromDstAndHandleICMP(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) (*Tuple, uint) {
-	key := Tuple{
-		addr: packet.SwapBytesUint32(pktIPv4.DstAddr),
-	}
-
-	// Parse packet destination port
-	if pktTCP != nil {
-		key.port = packet.SwapBytesUint16(pktTCP.DstPort)
-	} else if pktUDP != nil {
-		key.port = packet.SwapBytesUint16(pktUDP.DstPort)
-	} else if pktICMP != nil {
-		// Check if this ICMP packet destination is NAT itself. If
-		// yes, reply back with ICMP and stop packet processing.
-		key.port = packet.SwapBytesUint16(pktICMP.Identifier)
-		dir := port.handleICMP(pkt, &key)
-		if dir != dirSEND {
-			return nil, dir
+func generateLookupKeyFromDstAddr(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr, port uint16) interface{} {
+	if pktIPv4 != nil {
+		key := Tuple{
+			addr: packet.SwapBytesUint32(pktIPv4.DstAddr),
+			port: port,
 		}
+		return key
 	} else {
-		port.dumpPacket(pkt, dirDROP)
-		return nil, dirDROP
-	}
-	return &key, dirSEND
-}
-
-// Used to generate key in private to public translation
-func (port *ipv4Port) generateLookupKeyFromSrcAndHandleICMP(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) (*Tuple, uint) {
-	key := Tuple{
-		addr: packet.SwapBytesUint32(pktIPv4.SrcAddr),
-	}
-
-	// Parse packet source port
-	if pktTCP != nil {
-		key.port = packet.SwapBytesUint16(pktTCP.SrcPort)
-	} else if pktUDP != nil {
-		key.port = packet.SwapBytesUint16(pktUDP.SrcPort)
-	} else if pktICMP != nil {
-		// Check if this ICMP packet destination is NAT itself. If
-		// yes, reply back with ICMP and stop packet processing or
-		// direct to KNI if KNI is present.
-		dir := port.handleICMP(pkt, nil)
-		if dir != dirSEND {
-			return nil, dir
+		return Tuple6{
+			addr: pktIPv6.DstAddr,
+			port: port,
 		}
-		key.port = packet.SwapBytesUint16(pktICMP.Identifier)
-	} else {
-		port.dumpPacket(pkt, dirDROP)
-		return nil, dirDROP
-	}
-	return &key, dirSEND
-}
-
-func setPacketDstPort(pkt *packet.Packet, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
-	if pktTCP != nil {
-		pktTCP.DstPort = packet.SwapBytesUint16(port)
-		setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	} else if pktUDP != nil {
-		pktUDP.DstPort = packet.SwapBytesUint16(port)
-		setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(port)
-		setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
 	}
 }
 
-func setPacketSrcPort(pkt *packet.Packet, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
-	if pktTCP != nil {
-		pktTCP.SrcPort = packet.SwapBytesUint16(port)
-		setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	} else if pktUDP != nil {
-		pktUDP.SrcPort = packet.SwapBytesUint16(port)
-		setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+// Used to generate key in private to public translation
+func generateLookupKeyFromSrcAddr(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr, port uint16) (interface{}, interface{}) {
+	if pktIPv4 != nil {
+		saddr := packet.SwapBytesUint32(pktIPv4.SrcAddr)
+		return Tuple{
+			addr: saddr,
+			port: port,
+		}, saddr
 	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(port)
-		setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		return Tuple6{
+			addr: pktIPv6.SrcAddr,
+			port: port,
+		}, pktIPv6.SrcAddr
 	}
 }
 
 // Simple check for FIN or RST in TCP
-func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir terminationDirection) {
+func (pp *portPair) checkTCPTermination(ipv6 bool, hdr *packet.TCPHdr, port int, dir terminationDirection) {
 	if hdr.TCPFlags&common.TCPFlagFin != 0 {
 		// First check for FIN
 		pp.mutex.Lock()
@@ -333,7 +392,7 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 	} else if hdr.TCPFlags&common.TCPFlagRst != 0 {
 		// RST means that connection is terminated immediately
 		pp.mutex.Lock()
-		pp.deleteOldConnection(common.TCPNumber, port)
+		pp.deleteOldConnection(ipv6, common.TCPNumber, port)
 		pp.mutex.Unlock()
 	} else if hdr.TCPFlags&common.TCPFlagAck != 0 {
 		// Check for ACK last so that if there is also FIN,
@@ -343,7 +402,7 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 
 		pme := &pp.PublicPort.portmap[common.TCPNumber][port]
 		if pme.finCount == 2 {
-			pp.deleteOldConnection(common.TCPNumber, port)
+			pp.deleteOldConnection(ipv6, common.TCPNumber, port)
 			// Set some time while port cannot be used before
 			// connection timeout is reached
 			pme.lastused = time.Now().Add(time.Duration(portReuseTimeout - connectionTimeout))
@@ -353,144 +412,32 @@ func (pp *portPair) checkTCPTermination(hdr *packet.TCPHdr, port int, dir termin
 	}
 }
 
-func (port *ipv4Port) parsePacketAndCheckARP(pkt *packet.Packet) (dir uint, vhdr *packet.VLANHdr, iphdr *packet.IPv4Hdr) {
+func (port *ipPort) parsePacketAndCheckARP(pkt *packet.Packet) (dir uint, vlanhdr *packet.VLANHdr, ipv4hdr *packet.IPv4Hdr, ipv6hdr *packet.IPv6Hdr) {
 	pktVLAN := pkt.ParseL3CheckVLAN()
 	pktIPv4 := pkt.GetIPv4CheckVLAN()
 	if pktIPv4 == nil {
-		arp := pkt.GetARPCheckVLAN()
-		if arp != nil {
-			dir := port.handleARP(pkt)
-			port.dumpPacket(pkt, dir)
-			return dir, pktVLAN, nil
-		}
-		// We don't currently support anything except for IPv4 and ARP
-		port.dumpPacket(pkt, dirDROP)
-		return dirDROP, pktVLAN, nil
-	}
-	return dirSEND, pktVLAN, pktIPv4
-}
-
-func (port *ipv4Port) handleARP(pkt *packet.Packet) uint {
-	arp := pkt.GetARPNoCheck()
-
-	if packet.SwapBytesUint16(arp.Operation) != packet.ARPRequest {
-		if packet.SwapBytesUint16(arp.Operation) == packet.ARPReply {
-			ipv4 := packet.SwapBytesUint32(packet.ArrayToIPv4(arp.SPA))
-			port.arpTable.Store(ipv4, arp.SHA)
-		}
-		if port.KNIName != "" {
-			return dirKNI
-		}
-		return dirDROP
-	}
-
-	// If there is a KNI interface, direct all ARP traffic to it
-	if port.KNIName != "" {
-		return dirKNI
-	}
-
-	// Check that someone is asking about MAC of my IP address and HW
-	// address is blank in request
-	if packet.BytesToIPv4(arp.TPA[0], arp.TPA[1], arp.TPA[2], arp.TPA[3]) != packet.SwapBytesUint32(port.Subnet.Addr) {
-		println("Warning! Got an ARP packet with target IPv4 address", StringIPv4Array(arp.TPA),
-			"different from IPv4 address on interface. Should be", StringIPv4Int(port.Subnet.Addr),
-			". ARP request ignored.")
-		return dirDROP
-	}
-	if arp.THA != [common.EtherAddrLen]byte{} {
-		println("Warning! Got an ARP packet with non-zero MAC address", StringMAC(arp.THA),
-			". ARP request ignored.")
-		return dirDROP
-	}
-
-	// Prepare an answer to this request
-	answerPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-
-	packet.InitARPReplyPacket(answerPacket, port.SrcMACAddress, arp.SHA, packet.ArrayToIPv4(arp.TPA), packet.ArrayToIPv4(arp.SPA))
-	vlan := pkt.GetVLAN()
-	if vlan != nil {
-		answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
-	}
-
-	port.dumpPacket(answerPacket, dirSEND)
-	answerPacket.SendPacket(port.Index)
-
-	return dirDROP
-}
-
-func (port *ipv4Port) getMACForIP(ip uint32) (macAddress, bool) {
-	v, found := port.arpTable.Load(ip)
-	if found {
-		return macAddress(v.([common.EtherAddrLen]byte)), true
-	}
-	port.sendARPRequest(ip)
-	return macAddress{}, false
-}
-
-func (port *ipv4Port) sendARPRequest(ip uint32) {
-	// Prepare an answer to this request
-	requestPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-
-	packet.InitARPRequestPacket(requestPacket, port.SrcMACAddress,
-		packet.SwapBytesUint32(port.Subnet.Addr), packet.SwapBytesUint32(ip))
-	if port.Vlan != 0 {
-		requestPacket.AddVLANTag(port.Vlan)
-	}
-
-	port.dumpPacket(requestPacket, dirSEND)
-	requestPacket.SendPacket(port.Index)
-}
-
-func (port *ipv4Port) handleICMP(pkt *packet.Packet, key *Tuple) uint {
-	ipv4 := pkt.GetIPv4NoCheck()
-
-	// Check that received ICMP packet is addressed at this host. If
-	// not, packet should be translated
-	if packet.SwapBytesUint32(ipv4.DstAddr) != port.Subnet.Addr {
-		return dirSEND
-	}
-
-	icmp := pkt.GetICMPNoCheck()
-
-	// If there is KNI interface, direct all ICMP traffic which
-	// doesn't have an active translation entry
-	if port.KNIName != "" {
-		if key != nil {
-			_, ok := port.translationTable[common.ICMPNumber].Load(*key)
-			if !ok || time.Since(port.portmap[common.ICMPNumber][key.port].lastused) > connectionTimeout {
-				return dirKNI
+		pktIPv6 := pkt.GetIPv6CheckVLAN()
+		if pktIPv6 == nil {
+			arp := pkt.GetARPCheckVLAN()
+			if arp != nil {
+				dir := port.handleARP(pkt)
+				port.dumpPacket(pkt, dir)
+				return dir, pktVLAN, nil, nil
 			}
+			port.dumpPacket(pkt, dirDROP)
+			return dirDROP, pktVLAN, nil, nil
 		}
+		return dirSEND, pktVLAN, nil, pktIPv6
 	}
+	return dirSEND, pktVLAN, pktIPv4, nil
+}
 
-	// Check that received ICMP packet is echo request packet. We
-	// don't support any other messages yet, so process them in normal
-	// NAT way. Maybe these are packets which should be passed through
-	// translation.
-	if icmp.Type != common.ICMPTypeEchoRequest || icmp.Code != 0 {
-		return dirSEND
-	}
-
-	// Return a packet back to sender
-	answerPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
+func getAddrFromTuple(v interface{}, ipv6 bool) (uint32, [common.IPv6AddrLen]uint8, uint16, bool) {
+	if ipv6 {
+		value := v.(Tuple6)
+		return 0, value.addr, value.port, value.addr == [common.IPv6AddrLen]uint8{}
+	} else {
+		value := v.(Tuple)
+		return value.addr, [common.IPv6AddrLen]uint8{}, value.port, value.addr == 0
 	}
-	packet.GeneratePacketFromByte(answerPacket, pkt.GetRawPacketBytes())
-
-	answerPacket.ParseL3CheckVLAN()
-	swapAddrIPv4(answerPacket)
-	answerPacket.ParseL4ForIPv4()
-	(answerPacket.GetICMPNoCheck()).Type = common.ICMPTypeEchoResponse
-	setIPv4ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
-
-	port.dumpPacket(answerPacket, dirSEND)
-	answerPacket.SendPacket(port.Index)
-	return dirDROP
 }
diff --git a/examples/nat/updatecfg/updatecfg.proto b/examples/nat/updatecfg/updatecfg.proto
index 6a0f9d8b..3884b9d6 100644
--- a/examples/nat/updatecfg/updatecfg.proto
+++ b/examples/nat/updatecfg/updatecfg.proto
@@ -30,6 +30,9 @@ enum Protocol {
   UNKNOWN = 0;
   TCP = 0x06;
   UDP = 0x11;
+  IPv6_Flag = 0x10000;
+  TCP6 = 0x10006;
+  UDP6 = 0x10011;
 }
 
 message IPAddress {
diff --git a/examples/nat/util.go b/examples/nat/util.go
index e36cf3e0..324711be 100644
--- a/examples/nat/util.go
+++ b/examples/nat/util.go
@@ -7,8 +7,11 @@ package nat
 import (
 	"fmt"
 	"log"
+	"net"
 	"os"
 
+	"github.com/vishvananda/netlink"
+
 	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
 
 	"github.com/intel-go/nff-go/common"
@@ -47,7 +50,14 @@ func swapAddrIPv4(pkt *packet.Packet) {
 	ipv4.SrcAddr, ipv4.DstAddr = ipv4.DstAddr, ipv4.SrcAddr
 }
 
-func (port *ipv4Port) startTrace(dir uint) *os.File {
+func swapAddrIPv6(pkt *packet.Packet) {
+	ipv6 := pkt.GetIPv6NoCheck()
+
+	pkt.Ether.SAddr, pkt.Ether.DAddr = pkt.Ether.DAddr, pkt.Ether.SAddr
+	ipv6.SrcAddr, ipv6.DstAddr = ipv6.DstAddr, ipv6.SrcAddr
+}
+
+func (port *ipPort) startTrace(dir uint) *os.File {
 	dumpNameLookup := [dirKNI + 1]string{
 		"drop",
 		"dump",
@@ -64,7 +74,7 @@ func (port *ipv4Port) startTrace(dir uint) *os.File {
 	return file
 }
 
-func (port *ipv4Port) dumpPacket(pkt *packet.Packet, dir uint) {
+func (port *ipPort) dumpPacket(pkt *packet.Packet, dir uint) {
 	if DumpEnabled[dir] {
 		port.dumpsync[dir].Lock()
 		if port.fdump[dir] == nil {
@@ -79,7 +89,7 @@ func (port *ipv4Port) dumpPacket(pkt *packet.Packet, dir uint) {
 	}
 }
 
-func (port *ipv4Port) closePortTraces() {
+func (port *ipPort) closePortTraces() {
 	for _, f := range port.fdump {
 		if f != nil {
 			f.Close()
@@ -95,33 +105,182 @@ func CloseAllDumpFiles() {
 	}
 }
 
-func convertSubnet(s *upd.Subnet) (*ipv4Subnet, error) {
-	addr, err := convertIPv4(s.GetAddress().GetAddress())
+func convertSubnet(s *upd.Subnet) (*ipv4Subnet, *ipv6Subnet, error) {
+	a := s.GetAddress().GetAddress()
+	addr, err := convertIPv4(a)
 	if err != nil {
-		return nil, err
+		if net.IP(a).To16() == nil {
+			return nil, nil, err
+		}
+		ret := ipv6Subnet{}
+		copy(ret.Addr[:], a)
+		copy(ret.Mask[:], net.CIDRMask(int(s.GetMaskBitsNumber()), 128))
+		return nil, &ret, nil
 	}
 
 	return &ipv4Subnet{
 		Addr: addr,
 		Mask: uint32(0xffffffff) << (32 - s.GetMaskBitsNumber()),
-	}, nil
+	}, nil, nil
 }
 
 func convertForwardedPort(p *upd.ForwardedPort) (*forwardedPort, error) {
-	addr, err := convertIPv4(p.GetTargetAddress().GetAddress())
+	bytes := p.GetTargetAddress().GetAddress()
+	addr, err := convertIPv4(bytes)
+	var addr6 [common.IPv6AddrLen]uint8
+	var ipv6 bool
 	if err != nil {
-		return nil, err
+		if len(bytes) == common.IPv6AddrLen {
+			copy(addr6[:], bytes)
+			ipv6 = true
+		} else {
+			return nil, err
+		}
 	}
-	if p.GetProtocol() != common.TCPNumber && p.GetProtocol() != common.UDPNumber {
+	if uint8(p.GetProtocol()) != common.TCPNumber &&
+		uint8(p.GetProtocol()) != common.UDPNumber &&
+		p.GetProtocol() != (common.TCPNumber|upd.Protocol_IPv6_Flag) &&
+		p.GetProtocol() != (common.UDPNumber|upd.Protocol_IPv6_Flag) {
 		return nil, fmt.Errorf("Bad protocol identifier %d", p.GetProtocol())
 	}
 
 	return &forwardedPort{
 		Port: uint16(p.GetSourcePortNumber()),
 		Destination: hostPort{
-			Addr: addr,
-			Port: uint16(p.GetTargetPortNumber()),
+			Addr4: addr,
+			Addr6: addr6,
+			Port:  uint16(p.GetTargetPortNumber()),
+			ipv6:  ipv6,
+		},
+		Protocol: protocolId{
+			id:   uint8(p.GetProtocol() &^ upd.Protocol_IPv6_Flag),
+			ipv6: p.GetProtocol()&upd.Protocol_IPv6_Flag != 0,
 		},
-		Protocol: protocolId(p.GetProtocol()),
 	}, nil
 }
+
+func setPacketDstPort(pkt *packet.Packet, ipv6 bool, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
+	if pktTCP != nil {
+		pktTCP.DstPort = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	} else if pktUDP != nil {
+		pktUDP.DstPort = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	} else {
+		pktICMP.Identifier = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	}
+}
+
+func setPacketSrcPort(pkt *packet.Packet, ipv6 bool, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
+	if pktTCP != nil {
+		pktTCP.SrcPort = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	} else if pktUDP != nil {
+		pktUDP.SrcPort = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	} else {
+		pktICMP.Identifier = packet.SwapBytesUint16(port)
+		if ipv6 {
+			setIPv6ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		} else {
+			setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
+		}
+	}
+}
+
+func ParseAllKnownL4(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr) (uint8, *packet.TCPHdr, *packet.UDPHdr, *packet.ICMPHdr, uint16, uint16) {
+	var protocol uint8
+
+	if pktIPv4 != nil {
+		protocol = pktIPv4.NextProtoID
+		pkt.ParseL4ForIPv4()
+	} else {
+		protocol = pktIPv6.Proto
+		pkt.ParseL4ForIPv6()
+	}
+
+	switch protocol {
+	case common.TCPNumber:
+		pktTCP := (*packet.TCPHdr)(pkt.L4)
+		return protocol, pktTCP, nil, nil, packet.SwapBytesUint16(pktTCP.SrcPort), packet.SwapBytesUint16(pktTCP.DstPort)
+	case common.UDPNumber:
+		pktUDP := (*packet.UDPHdr)(pkt.L4)
+		return protocol, nil, pktUDP, nil, packet.SwapBytesUint16(pktUDP.SrcPort), packet.SwapBytesUint16(pktUDP.DstPort)
+	case common.ICMPNumber:
+		pktICMP := (*packet.ICMPHdr)(pkt.L4)
+		return protocol, nil, nil, pktICMP, packet.SwapBytesUint16(pktICMP.Identifier), packet.SwapBytesUint16(pktICMP.Identifier)
+	case common.ICMPv6Number:
+		pktICMP := (*packet.ICMPHdr)(pkt.L4)
+		return protocol, nil, nil, pktICMP, packet.SwapBytesUint16(pktICMP.Identifier), packet.SwapBytesUint16(pktICMP.Identifier)
+	default:
+		return 0, nil, nil, nil, 0, 0
+	}
+}
+
+func (port *ipPort) setLinkLocalIPv4KNIAddress(ipv4addr, mask uint32) {
+	if port.KNIName != "" {
+		myKNI, err := netlink.LinkByName(port.KNIName)
+		if err != nil {
+			fmt.Println("Failed to get KNI interface", port.KNIName, ":", err)
+			return
+		}
+		a := packet.IPv4ToBytes(ipv4addr)
+		m := packet.IPv4ToBytes(mask)
+		addr := &netlink.Addr{
+			IPNet: &net.IPNet{
+				IP:   net.IPv4(a[3], a[2], a[1], a[0]),
+				Mask: net.IPv4Mask(m[3], m[2], m[1], m[0]),
+			},
+		}
+		fmt.Println("Setting address", addr)
+		err = netlink.AddrAdd(myKNI, addr)
+		if err != nil {
+			fmt.Println("Failed to set interface", port.KNIName, "address", addr, ":")
+		} else {
+			fmt.Println("Set address", addr, "on KNI interface", port.KNIName)
+		}
+	}
+}
+
+func (port *ipPort) setLinkLocalIPv6KNIAddress(ipv6addr, mask [common.IPv6AddrLen]uint8) {
+	if port.KNIName != "" {
+		myKNI, err := netlink.LinkByName(port.KNIName)
+		if err != nil {
+			fmt.Println("Failed to get KNI interface", port.KNIName, ":", err)
+			return
+		}
+		addr := &netlink.Addr{
+			IPNet: &net.IPNet{
+				IP:   ipv6addr[:],
+				Mask: mask[:],
+			},
+		}
+		err = netlink.AddrAdd(myKNI, addr)
+		if err != nil {
+			fmt.Println("Failed to set interface", port.KNIName, "address", addr, ":", err)
+		} else {
+			fmt.Println("Set address", addr, "on KNI interface", port.KNIName)
+		}
+	}
+}
diff --git a/examples/nffPktgen/generator/generator.go b/examples/nffPktgen/generator/generator.go
index d416e219..2bcf5995 100644
--- a/examples/nffPktgen/generator/generator.go
+++ b/examples/nffPktgen/generator/generator.go
@@ -445,7 +445,7 @@ func generateICMPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(pktIP, pktICMP, pkt.Data))
 	} else if l3.Version == 6 {
 		pktIP := (*packet.IPv6Hdr)(pkt.L3)
-		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pktIP, pktICMP, pkt.Data))
+		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pktIP, pktICMP))
 	}
 }
 
diff --git a/mk/leaf.mk b/mk/leaf.mk
index 9443300d..72e4f59a 100644
--- a/mk/leaf.mk
+++ b/mk/leaf.mk
@@ -12,7 +12,7 @@ include $(PATH_TO_MK)/include.mk
 
 ifdef NFF_GO_DEBUG
 # Flags to build Go files without optimizations
-export GO_COMPILE_FLAGS += -gcflags '-N -l'
+export GO_COMPILE_FLAGS += -gcflags=all='-N -l'
 endif
 
 $(EXECUTABLES) : % : %.go
diff --git a/packet/acl_internal_test.go b/packet/acl_internal_test.go
index f64b3bd3..68375344 100644
--- a/packet/acl_internal_test.go
+++ b/packet/acl_internal_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -108,7 +108,7 @@ var rulesL3Ctxt = rawL3RuleTestCtxt{
 				[16]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 			},
 		},
-		{"dead::beaf/16",
+		{"dead::beef/16",
 			gtAddr6{
 				[16]uint8{0xde, 0xad, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 				[16]uint8{0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
@@ -128,9 +128,9 @@ var rulesL3Ctxt = rawL3RuleTestCtxt{
 				[16]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
 			},
 		},
-		{"dead::beaf/128",
+		{"dead::beef/128",
 			gtAddr6{
-				[16]uint8{0xde, 0xad, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbe, 0xaf},
+				[16]uint8{0xde, 0xad, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbe, 0xef},
 				[16]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
 			},
 		},
@@ -1033,7 +1033,7 @@ func TestInternal_l3ACL_l4ACL_packetIPv6_ICMP(t *testing.T) {
 			{id: 0, mask: 0, ok: true},
 			{id: 6, mask: 0xff, ok: false},
 			{id: 17, mask: 0xff, ok: false},
-			{id: 1, mask: 0xff, ok: true}, // ICMP
+			{id: 58, mask: 0xff, ok: true}, // ICMPv6
 		},
 		srcRange: []portRange{
 			{min: 0, max: 65535, valid: false, ok: true},
diff --git a/packet/checksum.go b/packet/checksum.go
index f81cdd7e..f77f64b6 100644
--- a/packet/checksum.go
+++ b/packet/checksum.go
@@ -132,6 +132,7 @@ func SetHWOffloadingHdrChecksum(p *Packet) {
 		} else if icmp != nil {
 			// calculate full software checksum for icmp, cause there is no
 			// hardware calculation for icmp packets.
+			p.GetICMPForIPv4().Cksum = 0
 			p.GetICMPForIPv4().Cksum = SwapBytesUint16(CalculateIPv4ICMPChecksum(ipv4, icmp,
 				unsafe.Pointer(uintptr(unsafe.Pointer(icmp))+ICMPLen)))
 		}
@@ -144,8 +145,8 @@ func SetHWOffloadingHdrChecksum(p *Packet) {
 		} else if icmp != nil {
 			// calculate full software checksum for icmp, cause there is no
 			// hardware calculation for icmp packets.
-			p.GetICMPForIPv6().Cksum = SwapBytesUint16(CalculateIPv6ICMPChecksum(ipv6, icmp,
-				unsafe.Pointer(uintptr(unsafe.Pointer(icmp))+ICMPLen)))
+			p.GetICMPForIPv6().Cksum = 0
+			p.GetICMPForIPv6().Cksum = SwapBytesUint16(CalculateIPv6ICMPChecksum(ipv6, icmp))
 		}
 	}
 }
@@ -324,12 +325,13 @@ func CalculateIPv4ICMPChecksum(hdr *IPv4Hdr, icmp *ICMPHdr, data unsafe.Pointer)
 
 // CalculateIPv6ICMPChecksum calculates ICMP checksum in case if L3
 // protocol is IPv6.
-func CalculateIPv6ICMPChecksum(hdr *IPv6Hdr, icmp *ICMPHdr, data unsafe.Pointer) uint16 {
-	dataLength := SwapBytesUint16(hdr.PayloadLen) - ICMPLen
+func CalculateIPv6ICMPChecksum(hdr *IPv6Hdr, icmp *ICMPHdr) uint16 {
+	data := unsafe.Pointer(icmp)
+	dataLength := SwapBytesUint16(hdr.PayloadLen)
 
-	sum := uint32(uint16(icmp.Type)<<8|uint16(icmp.Code)) +
-		uint32(SwapBytesUint16(icmp.Identifier)) +
-		uint32(SwapBytesUint16(icmp.SeqNum)) +
+	sum := calculateIPv6AddrChecksum(hdr) +
+		uint32(dataLength) +
+		uint32(hdr.Proto) +
 		calculateDataChecksum(unsafe.Pointer(data), int(dataLength), 0)
 
 	return ^reduceChecksum(sum)
diff --git a/packet/checksum_test.go b/packet/checksum_test.go
index bc450c44..495863f4 100644
--- a/packet/checksum_test.go
+++ b/packet/checksum_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -20,13 +20,13 @@ type Payload struct {
 
 // ground truth values (wireshark)
 var (
-	wantIPv4     uint16 = 0x726c
+	wantIPv4     uint16 = 0x326c
 	wantIPv4TCP  uint16 = 0x8e7f
 	wantIPv6TCP  uint16 = 0x7abd
 	wantIPv4UDP  uint16 = 0xddd8
 	wantIPv6UDP  uint16 = 0xca16
 	wantIPv4ICMP uint16 = 0x61e4
-	wantIPv6ICMP uint16 = 0x61e4
+	wantIPv6ICMP uint16 = 0x057b
 )
 
 /*
@@ -41,11 +41,13 @@ func TestCalculateIPv4Checksum(t *testing.T) {
 	pkt := getPacket()
 	InitEmptyIPv4TCPPacket(pkt, payloadSizeLocal)
 	initIPv4AddrsLocal(pkt)
+	pkt.GetTCPNoCheck().Cksum = 0
 
 	want := wantIPv4
 	got := CalculateIPv4Checksum(pkt.GetIPv4())
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv4Checksum", pkt)
 	}
 }
 
@@ -60,9 +62,11 @@ func TestCalculateIPv4TCPChecksum(t *testing.T) {
 	pkt.GetIPv4().HdrChecksum = SwapBytesUint16(ipcksum)
 
 	want := wantIPv4TCP
+	pkt.GetTCPNoCheck().Cksum = 0
 	got := CalculateIPv4TCPChecksum(pkt.GetIPv4(), pkt.GetTCPForIPv4(), pkt.Data)
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv4TCPChecksum", pkt)
 	}
 }
 
@@ -74,9 +78,11 @@ func TestCalculateIPv6TCPChecksum(t *testing.T) {
 	initData(pkt)
 
 	want := wantIPv6TCP
+	pkt.GetTCPNoCheck().Cksum = 0
 	got := CalculateIPv6TCPChecksum(pkt.GetIPv6(), pkt.GetTCPForIPv6(), pkt.Data)
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv6TCPChecksum", pkt)
 	}
 }
 
@@ -91,9 +97,11 @@ func TestCalculateIPv4UDPChecksum(t *testing.T) {
 	pkt.GetIPv4().HdrChecksum = SwapBytesUint16(ipcksum)
 
 	want := wantIPv4UDP
+	pkt.GetUDPNoCheck().DgramCksum = 0
 	got := CalculateIPv4UDPChecksum(pkt.GetIPv4(), pkt.GetUDPForIPv4(), pkt.Data)
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv4UDPChecksum", pkt)
 	}
 }
 
@@ -105,9 +113,11 @@ func TestCalculateIPv6UDPChecksum(t *testing.T) {
 	initData(pkt)
 
 	want := wantIPv6UDP
+	pkt.GetUDPNoCheck().DgramCksum = 0
 	got := CalculateIPv6UDPChecksum(pkt.GetIPv6(), pkt.GetUDPForIPv6(), pkt.Data)
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv6UDPChecksum", pkt)
 	}
 }
 
@@ -122,10 +132,12 @@ func TestCalculateIPv4ICMPChecksum(t *testing.T) {
 	pkt.GetIPv4().HdrChecksum = SwapBytesUint16(ipcksum)
 
 	want := wantIPv4ICMP
+	pkt.GetICMPNoCheck().Cksum = 0
 	got := CalculateIPv4ICMPChecksum(pkt.GetIPv4(), pkt.GetICMPForIPv4(), pkt.Data)
 
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv4ICMPChecksum", pkt)
 	}
 }
 
@@ -137,10 +149,12 @@ func TestCalculateIPv6ICMPChecksum(t *testing.T) {
 	initICMP(pkt.GetICMPForIPv6())
 
 	want := wantIPv6ICMP
-	got := CalculateIPv6ICMPChecksum(pkt.GetIPv6(), pkt.GetICMPForIPv6(), pkt.Data)
+	pkt.GetICMPNoCheck().Cksum = 0
+	got := CalculateIPv6ICMPChecksum(pkt.GetIPv6(), pkt.GetICMPNoCheck())
 
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
+		dumpPacketToPcap("TestCalculateIPv6ICMPChecksum", pkt)
 	}
 }
 
@@ -148,6 +162,7 @@ func initIPv4AddrsLocal(pkt *Packet) {
 	ipv4 := pkt.GetIPv4()
 	ipv4.SrcAddr = binary.LittleEndian.Uint32(net.ParseIP("131.151.32.21").To4())
 	ipv4.DstAddr = binary.LittleEndian.Uint32(net.ParseIP("131.151.32.129").To4())
+	ipv4.HdrChecksum = 0
 }
 
 func initIPv6AddrsLocal(pkt *Packet) {
diff --git a/packet/icmp6.go b/packet/icmp6.go
new file mode 100644
index 00000000..7b7db94e
--- /dev/null
+++ b/packet/icmp6.go
@@ -0,0 +1,223 @@
+// Copyright 2018 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packet
+
+import (
+	"unsafe"
+
+	"github.com/intel-go/nff-go/common"
+)
+
+const (
+	ICMPv6NDSourceLinkLayerAddress uint8 = 1
+	ICMPv6NDTargetLinkLayerAddress uint8 = 2
+	ICMPv6NDPrefixInformation      uint8 = 3
+	ICMPv6NDRedirectedHeader       uint8 = 4
+	ICMPv6NDMTU                    uint8 = 5
+
+	ICMPv6RNDouterFlag    uint16 = 0x8000
+	ICMPv6NDSolicitedFlag uint16 = 0x4000
+	ICMPv6NDOverrideFlag  uint16 = 0x2000
+
+	ICMPv6NDMessageOptionUnitSize = 8
+)
+
+var (
+	ipv6LinkLocalPrefix          = []uint8{0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
+	ipv6LinkLocalMulticastPrefix = []uint8{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff}
+	ipv6EtherMulticastPrefix     = []uint8{0x33, 0x33}
+
+	ICMPv6NeighborSolicitationMessageSize    uint = uint(unsafe.Sizeof(ICMPv6NeighborSolicitationMessage{}))
+	ICMPv6NeighborAdvertisementMessageSize   uint = uint(unsafe.Sizeof(ICMPv6NeighborAdvertisementMessage{}))
+	ICMPv6NDSourceLinkLayerAddressOptionSize uint = uint(unsafe.Sizeof(ICMPv6NDSourceLinkLayerAddressOption{}))
+	ICMPv6NDTargetLinkLayerAddressOptionSize uint = uint(unsafe.Sizeof(ICMPv6NDTargetLinkLayerAddressOption{}))
+)
+
+type ICMPv6NDSourceLinkLayerAddressOption struct {
+	Type             uint8
+	Length           uint8
+	LinkLayerAddress [common.EtherAddrLen]uint8
+}
+
+type ICMPv6NDTargetLinkLayerAddressOption struct {
+	Type             uint8
+	Length           uint8
+	LinkLayerAddress [common.EtherAddrLen]uint8
+}
+
+type ICMPv6NDPrefixInformationOption struct {
+	Type              uint8
+	Length            uint8
+	PrefixLength      uint8
+	LAFlags           uint8
+	ValidLifetime     uint32
+	PreferredLifetime uint32
+	Reserved2         uint32
+	Prefix            [common.IPv6AddrLen]uint8
+}
+
+type ICMPv6NDRedirectedHeaderOption struct {
+	Type     uint8
+	Length   uint8
+	Reserved uint32
+}
+
+type ICMPv6NDMTUOption struct {
+	Type   uint8
+	Length uint8
+	MTU    uint32
+}
+
+type ICMPv6NeighborSolicitationMessage struct {
+	TargetAddr [common.IPv6AddrLen]uint8
+}
+
+type ICMPv6NeighborAdvertisementMessage struct {
+	TargetAddr [common.IPv6AddrLen]uint8
+}
+
+// GetICMPv6NeighborSolicitationMessage returns pointer to ICMPv6
+// Neighbor Solicitation message buffer. It should be called after
+// packet.Data field is initialized with ParseL7 or ParseData calls.
+func (packet *Packet) GetICMPv6NeighborSolicitationMessage() *ICMPv6NeighborSolicitationMessage {
+	return (*ICMPv6NeighborSolicitationMessage)(packet.Data)
+}
+
+// GetICMPv6NeighborAdvertisementMessage returns pointer to ICMPv6
+// Neighbor Solicitation message buffer. It should be called after
+// packet.Data field is initialized with ParseL7 or ParseData calls.
+func (packet *Packet) GetICMPv6NeighborAdvertisementMessage() *ICMPv6NeighborAdvertisementMessage {
+	return (*ICMPv6NeighborAdvertisementMessage)(packet.Data)
+}
+
+// checkEnoughSpace returns true if there are more than space bytes in
+// packet and false if there are less than or equal bytes than space.
+func (packet *Packet) checkEnoughSpace(space uint) bool {
+	pktStartAddr := packet.StartAtOffset(0)
+	hdrsLen := int64(uintptr(packet.Data) - uintptr(pktStartAddr))
+	packetLength := int64(packet.GetPacketSegmentLen())
+	dataLength := packetLength - hdrsLen - int64(space)
+	return dataLength > 0
+}
+
+// GetICMPv6NDSourceLinkLayerAddressOption returns Neighbor Discovery
+// Source Link Layer option for an ICMPv6 message packet following a
+// message of length msgLength. If packet is not long enough to
+// contain this option, nil is returned.
+func (packet *Packet) GetICMPv6NDSourceLinkLayerAddressOption(msgLength uint) *ICMPv6NDSourceLinkLayerAddressOption {
+	if packet.checkEnoughSpace(msgLength) {
+		return (*ICMPv6NDSourceLinkLayerAddressOption)(unsafe.Pointer(uintptr(packet.Data) + uintptr(msgLength)))
+	}
+	return nil
+}
+
+// GetICMPv6NDTargetLinkLayerAddressOption returns Neighbor Discovery
+// Target Link Layer option for an ICMPv6 message packet following a
+// message of length msgLength. If packet is not long enough to
+// contain this option, nil is returned.
+func (packet *Packet) GetICMPv6NDTargetLinkLayerAddressOption(msgLength uint) *ICMPv6NDSourceLinkLayerAddressOption {
+	if packet.checkEnoughSpace(msgLength) {
+		return (*ICMPv6NDSourceLinkLayerAddressOption)(unsafe.Pointer(uintptr(packet.Data) + uintptr(msgLength)))
+	}
+	return nil
+}
+
+// CalculateIPv6LinkLocalAddrForMAC generates IPv6 link local address
+// based on interface MAC address.
+func CalculateIPv6LinkLocalAddrForMAC(llAddr *[common.IPv6AddrLen]uint8, mac [common.EtherAddrLen]uint8) {
+	copy((*llAddr)[:], ipv6LinkLocalPrefix)
+	(*llAddr)[8] = mac[0] ^ 0x02
+	(*llAddr)[9] = mac[1]
+	(*llAddr)[10] = mac[2]
+	(*llAddr)[11] = 0xff
+	(*llAddr)[12] = 0xfe
+	(*llAddr)[13] = mac[3]
+	(*llAddr)[14] = mac[4]
+	(*llAddr)[15] = mac[5]
+}
+
+// CalculateIPv6MulticastAddrForDstIP generates IPv6 multicast address
+// that other hosts use to solicit its MAC address. This address is
+// used as destination for all Neighbor Solicitation ICMPv6 messages
+// and NAT should answer packets coming to it.
+func CalculateIPv6MulticastAddrForDstIP(muticastAddr *[common.IPv6AddrLen]uint8, dstIP [common.IPv6AddrLen]uint8) {
+	copy((*muticastAddr)[:], ipv6LinkLocalMulticastPrefix)
+	(*muticastAddr)[13] = dstIP[13]
+	(*muticastAddr)[14] = dstIP[14]
+	(*muticastAddr)[15] = dstIP[15]
+}
+
+func CalculateIPv6BroadcastMACForDstMulticastIP(dstMAC *[common.EtherAddrLen]uint8, dstIP [common.IPv6AddrLen]uint8) {
+	copy((*dstMAC)[:], ipv6EtherMulticastPrefix)
+	(*dstMAC)[2] = dstIP[12]
+	(*dstMAC)[3] = dstIP[13]
+	(*dstMAC)[4] = dstIP[14]
+	(*dstMAC)[5] = dstIP[15]
+}
+
+// InitICMPv6NeighborSolicitationPacket allocates and initializes
+// ICMPv6 Neighbor Solicitation request message packet with source MAC
+// and IPv6 address and target IPv6 address.
+func InitICMPv6NeighborSolicitationPacket(packet *Packet, srcMAC [common.EtherAddrLen]uint8, srcIP, dstIP [common.IPv6AddrLen]uint8) {
+	InitEmptyIPv6ICMPPacket(packet, ICMPv6NeighborSolicitationMessageSize+ICMPv6NDSourceLinkLayerAddressOptionSize)
+
+	var targetMulticastAddr [common.IPv6AddrLen]uint8
+	CalculateIPv6MulticastAddrForDstIP(&targetMulticastAddr, dstIP)
+
+	// Fill up L2
+	CalculateIPv6BroadcastMACForDstMulticastIP(&packet.Ether.DAddr, targetMulticastAddr)
+	packet.Ether.SAddr = srcMAC
+
+	// Fill up L3
+	ipv6 := packet.GetIPv6NoCheck()
+	ipv6.DstAddr = targetMulticastAddr
+	ipv6.SrcAddr = srcIP
+
+	// Fill up L4
+	icmp := packet.GetICMPNoCheck()
+	icmp.Type = common.ICMPv6NeighborSolicitation
+	icmp.Identifier = 0
+	icmp.SeqNum = 0
+
+	// Fill up L7
+	packet.ParseL7(common.ICMPv6Number)
+	msg := packet.GetICMPv6NeighborSolicitationMessage()
+	msg.TargetAddr = dstIP
+	option := packet.GetICMPv6NDSourceLinkLayerAddressOption(ICMPv6NeighborSolicitationMessageSize)
+	option.Type = ICMPv6NDSourceLinkLayerAddress
+	option.Length = uint8(ICMPv6NDSourceLinkLayerAddressOptionSize / ICMPv6NDMessageOptionUnitSize)
+	option.LinkLayerAddress = srcMAC
+}
+
+// InitICMPv6NeighborAdvertisementPacket allocates and initializes
+// ICMPv6 Neighbor Advertisement answer message packet with source MAC
+// and IPv6 address and target IPv6 address.
+func InitICMPv6NeighborAdvertisementPacket(packet *Packet, srcMAC, dstMAC [common.EtherAddrLen]uint8, srcIP, dstIP [common.IPv6AddrLen]uint8) {
+	InitEmptyIPv6ICMPPacket(packet, ICMPv6NeighborAdvertisementMessageSize+ICMPv6NDTargetLinkLayerAddressOptionSize)
+
+	// Fill up L2
+	packet.Ether.DAddr = dstMAC
+	packet.Ether.SAddr = srcMAC
+
+	// Fill up L3
+	ipv6 := packet.GetIPv6NoCheck()
+	ipv6.DstAddr = dstIP
+	ipv6.SrcAddr = srcIP
+
+	// Fill up L4
+	icmp := packet.GetICMPNoCheck()
+	icmp.Type = common.ICMPv6NeighborAdvertisement
+	icmp.Identifier = SwapBytesUint16(ICMPv6NDSolicitedFlag | ICMPv6NDOverrideFlag)
+	icmp.SeqNum = 0
+
+	// Fill up L7
+	packet.ParseL7(common.ICMPv6Number)
+	msg := packet.GetICMPv6NeighborAdvertisementMessage()
+	msg.TargetAddr = srcIP
+	option := packet.GetICMPv6NDTargetLinkLayerAddressOption(ICMPv6NeighborAdvertisementMessageSize)
+	option.Type = ICMPv6NDTargetLinkLayerAddress
+	option.Length = uint8(ICMPv6NDTargetLinkLayerAddressOptionSize / ICMPv6NDMessageOptionUnitSize)
+	option.LinkLayerAddress = srcMAC
+}
diff --git a/packet/packet.go b/packet/packet.go
index 21f4c50b..90f4a72c 100644
--- a/packet/packet.go
+++ b/packet/packet.go
@@ -142,13 +142,19 @@ type IPv6Hdr struct {
 	DstAddr    [IPv6AddrLen]uint8 // IP address of destination host(s)
 }
 
+func IPv6ToString(addr [IPv6AddrLen]uint8) string {
+	return fmt.Sprintf("[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]",
+		addr[0], addr[1], addr[2], addr[3],
+		addr[4], addr[5], addr[6], addr[7],
+		addr[8], addr[9], addr[10], addr[11],
+		addr[12], addr[13], addr[14], addr[15])
+}
+
 func (hdr *IPv6Hdr) String() string {
-	r0 := "    L3 protocol: IPv6\n"
-	s := hdr.SrcAddr
-	r1 := fmt.Sprintf("    IPv6 Source: %02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n", s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10], s[11], s[12], s[13], s[14], s[15])
-	d := hdr.DstAddr
-	r2 := fmt.Sprintf("    IPv6 Destination %02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n", d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15])
-	return r0 + r1 + r2
+	return fmt.Sprintf(`    L3 protocol: IPv6
+    IPv6 Source: %s
+    IPv6 Destination %s
+`, IPv6ToString(hdr.SrcAddr), IPv6ToString(hdr.DstAddr))
 }
 
 // TCPHdr L4 header from DPDK: lib/librte_net/rte_tcp.h
@@ -353,7 +359,7 @@ func (packet *Packet) GetICMPNoCheck() *ICMPHdr {
 // GetICMPForIPv6 ensures if L4 type is ICMP and cast L4 pointer to *ICMPHdr type.
 // L3 supposed to be parsed before and of IPv6 type.
 func (packet *Packet) GetICMPForIPv6() *ICMPHdr {
-	if packet.GetIPv6NoCheck().Proto == ICMPNumber {
+	if packet.GetIPv6NoCheck().Proto == ICMPv6Number {
 		return (*ICMPHdr)(packet.L4)
 	}
 	return nil
@@ -406,6 +412,8 @@ func (packet *Packet) ParseL7(protocol uint) {
 	case UDPNumber:
 		packet.Data = unsafe.Pointer(uintptr(packet.L4) + uintptr(UDPLen))
 	case ICMPNumber:
+		fallthrough
+	case ICMPv6Number:
 		packet.Data = unsafe.Pointer(uintptr(packet.L4) + uintptr(ICMPLen))
 	}
 }
@@ -500,12 +508,14 @@ func fillIPv4Default(packet *Packet, plLen uint16, nextProto uint8) {
 	packet.GetIPv4NoCheck().VersionIhl = IPv4VersionIhl
 	packet.GetIPv4NoCheck().TotalLength = SwapBytesUint16(plLen)
 	packet.GetIPv4NoCheck().NextProtoID = nextProto
+	packet.GetIPv4NoCheck().TimeToLive = 64
 }
 
 func fillIPv6Default(packet *Packet, totalLen uint16, nextProto uint8) {
 	packet.GetIPv6NoCheck().PayloadLen = SwapBytesUint16(totalLen)
 	packet.GetIPv6NoCheck().VtcFlow = IPv6VtcFlow
 	packet.GetIPv6NoCheck().Proto = nextProto
+	packet.GetIPv6NoCheck().HopLimits = 255
 }
 
 // InitEmptyIPv4Packet initializes input packet with preallocated plSize of bytes for payload
@@ -692,7 +702,7 @@ func InitEmptyIPv6ICMPPacket(packet *Packet, plSize uint) bool {
 	packet.Data = unsafe.Pointer(uintptr(packet.unparsed()) + IPv6Len + ICMPLen)
 
 	packet.ParseL3()
-	fillIPv6Default(packet, uint16(ICMPLen+plSize), ICMPNumber)
+	fillIPv6Default(packet, uint16(ICMPLen+plSize), ICMPv6Number)
 	packet.ParseL4ForIPv6()
 	return true
 }
diff --git a/packet/packet_test.go b/packet/packet_test.go
index 53877a2c..4b11b21e 100644
--- a/packet/packet_test.go
+++ b/packet/packet_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -423,13 +423,13 @@ var (
 
 	gtLineEther = "0011223344550111213141510000"
 
-	gtLineIPv4    = "00000000000000000000000008004500001c00000000003b00007f00000180090905ffdd0000bbaa0000"
-	gtLineIPv4TCP = "0000000000000000000000000800450000300000000000060000000000000000000004d2162e00000000000000005000000000000000ffdd0000bbaa0000"
-	gtLineIPv4UDP = "0000000000000000000000000800450000240000000000110000000000000000000004d2162e00100000ffdd0000bbaa0000"
+	gtLineIPv4    = "00000000000000000000000008004500001c00000000403b00007f00000180090905ffdd0000bbaa0000"
+	gtLineIPv4TCP = "0000000000000000000000000800450000300000000040060000000000000000000004d2162e00000000000000005000000000000000ffdd0000bbaa0000"
+	gtLineIPv4UDP = "0000000000000000000000000800450000240000000040110000000000000000000004d2162e00100000ffdd0000bbaa0000"
 
-	gtLineIPv6    = "00000000000000000000000086dd6000000000083b00dead000000000000000000000000beaf00000000000000000000000000000000ffdd0000bbaa0000"
-	gtLineIPv6TCP = "00000000000000000000000086dd6000000000140600000000000000000000000000000000000000000000000000000000000000000004d2162e00000000000000005000000000000000ffdd0000bbaa0000"
-	gtLineIPv6UDP = "00000000000000000000000086dd6000000000101100000000000000000000000000000000000000000000000000000000000000000004d2162e00100000ffdd0000bbaa0000"
+	gtLineIPv6    = "00000000000000000000000086dd6000000000083bffdead000000000000000000000000beef00000000000000000000000000000000ffdd0000bbaa0000"
+	gtLineIPv6TCP = "00000000000000000000000086dd60000000001406ff000000000000000000000000000000000000000000000000000000000000000004d2162e00000000000000005000000000000000ffdd0000bbaa0000"
+	gtLineIPv6UDP = "00000000000000000000000086dd60000000001011ff000000000000000000000000000000000000000000000000000000000000000004d2162e00100000ffdd0000bbaa0000"
 )
 
 func TestInitEmptyPacket(t *testing.T) {
@@ -481,7 +481,7 @@ func TestInitEmptyIPv6Packet(t *testing.T) {
 	// Create empty packet, set IPv6 header fields
 	pkt := getPacket()
 	InitEmptyIPv6Packet(pkt, testPlSize)
-	pkt.GetIPv6().SrcAddr = [16]uint8{0xde, 0xad, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbe, 0xaf}
+	pkt.GetIPv6().SrcAddr = [16]uint8{0xde, 0xad, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbe, 0xef}
 	ptrData := (*Packetdata)(pkt.Data)
 	ptrData.F1 = 0xddff
 	ptrData.F2 = 0xaabb
@@ -616,10 +616,10 @@ func TestGetPacketPayload(t *testing.T) {
 			status:  true,
 		},
 		{
-			name:    "ICMPv6 (unsupported)",
-			header:  "3333ffc8e5c80018ded027d786dd6000000000183aff00000000000000000000000000000000ff0200000000000000000001ffc8e5c887008de800000000fe8000000000000085edbc2edfc8e5c8",
-			payload: "",
-			status:  false,
+			name:    "IPv6-ICMPv6",
+			header:  "3333ffc8e5c80018ded027d786dd6000000000183aff00000000000000000000000000000000ff0200000000000000000001ffc8e5c887008de800000000",
+			payload: "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
+			status:  true,
 		},
 		{
 			name:    "IPv4-SCTP (unsupported)",
diff --git a/packet/utils_for_test.go b/packet/utils_for_test.go
index 8839747b..bb8c5e70 100644
--- a/packet/utils_for_test.go
+++ b/packet/utils_for_test.go
@@ -4,8 +4,10 @@ package packet
 
 import (
 	"encoding/binary"
+	"fmt"
 	"log"
 	"net"
+	"os"
 
 	"github.com/intel-go/nff-go/common"
 	"github.com/intel-go/nff-go/low"
@@ -14,6 +16,7 @@ import (
 // isInit is common for all tests
 var isInit bool
 
+const testDevelopmentMode = false
 const payloadSize = 100
 
 func tInitDPDK() {
@@ -130,3 +133,26 @@ func getPacket() *Packet {
 	}
 	return pkt
 }
+
+func dumpPacketToPcap(fileName string, pkt *Packet) {
+	if !testDevelopmentMode {
+		return
+	}
+
+	file, err := os.Create(fileName + ".pcap")
+	if err != nil {
+		fmt.Println(err)
+	}
+	err = WritePcapGlobalHdr(file)
+	if err != nil {
+		fmt.Println(err)
+	}
+	err = pkt.WritePcapOnePacket(file)
+	if err != nil {
+		fmt.Println(err)
+	}
+	err = file.Close()
+	if err != nil {
+		fmt.Println(err)
+	}
+}
diff --git a/test/stability/testCksum/testCksum.go b/test/stability/testCksum/testCksum.go
index eaccc8c9..f08cefef 100644
--- a/test/stability/testCksum/testCksum.go
+++ b/test/stability/testCksum/testCksum.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -588,7 +588,7 @@ func generateIPv6ICMP(emptyPacket *packet.Packet, rnd *rand.Rand) {
 	if !hwol {
 		pIPv6 := emptyPacket.GetIPv6()
 		pICMP := emptyPacket.GetICMPForIPv6()
-		pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, emptyPacket.Data))
+		pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP))
 	}
 }
 
diff --git a/test/stability/testCksum/testCksumCommon/testCksumCommon.go b/test/stability/testCksum/testCksumCommon/testCksumCommon.go
index 67d00191..71bb84ea 100644
--- a/test/stability/testCksum/testCksumCommon/testCksumCommon.go
+++ b/test/stability/testCksum/testCksumCommon/testCksumCommon.go
@@ -1,4 +1,4 @@
-// Copyright 2017 Intel Corporation.
+// Copyright 2017-2018 Intel Corporation.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -98,7 +98,7 @@ func CheckPacketChecksums(p *packet.Packet) bool {
 			}
 		} else if pIPv6.Proto == common.ICMPNumber {
 			pICMP := p.GetICMPForIPv6()
-			csum := packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, p.Data)
+			csum := packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP)
 			if packet.SwapBytesUint16(pICMP.Cksum) != csum {
 				println("IPv6 ICMP checksum mismatch", packet.SwapBytesUint16(pICMP.Cksum), "should be", csum)
 			} else {
@@ -143,7 +143,7 @@ func CalculateChecksum(p *packet.Packet) {
 			pTCP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(pIPv6, pTCP, p.Data))
 		} else if pIPv6.Proto == common.ICMPNumber {
 			pICMP := p.GetICMPForIPv6()
-			pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, p.Data))
+			pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP))
 		} else {
 			println("Unknown IPv6 protocol number", pIPv6.Proto)
 			println("TEST FAILED")

From d76f0babfec894cca1c0ad3377cf95e042ccb58d Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Mon, 1 Oct 2018 19:32:37 +0000
Subject: [PATCH 42/50] Fixed CI tests on ICMPv6 checksums

Changed ICMPv6 checksum calculation to be consistent with other checksum
calculation functions. Now it is not necessary to zero out checksum
field in the header before calling checksum calculation.
---
 examples/nat/cksum.go                            |  5 ++---
 examples/nffPktgen/generator/generator.go        |  2 +-
 packet/checksum.go                               | 16 +++++++++++-----
 packet/checksum_test.go                          |  3 +--
 test/stability/testCksum/testCksum.go            |  2 +-
 .../testCksum/testCksumCommon/testCksumCommon.go |  8 ++++----
 6 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/examples/nat/cksum.go b/examples/nat/cksum.go
index 538aafca..b712d46c 100644
--- a/examples/nat/cksum.go
+++ b/examples/nat/cksum.go
@@ -64,7 +64,6 @@ func setIPv4ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum boo
 			l3.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(l3))
 		}
 		l4 := pkt.GetICMPNoCheck()
-		l4.Cksum = 0 // Need to zero l4.Cksum or otherwise it is included into calculation
 		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(l3, l4,
 			unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.ICMPLen)))
 	}
@@ -111,7 +110,7 @@ func setIPv6ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum boo
 		l3 := pkt.GetIPv6NoCheck()
 
 		l4 := pkt.GetICMPNoCheck()
-		l4.Cksum = 0 // Need to zero l4.Cksum or otherwise it is included into calculation
-		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(l3, l4))
+		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(l3, l4,
+			unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.ICMPLen)))
 	}
 }
diff --git a/examples/nffPktgen/generator/generator.go b/examples/nffPktgen/generator/generator.go
index 2bcf5995..d416e219 100644
--- a/examples/nffPktgen/generator/generator.go
+++ b/examples/nffPktgen/generator/generator.go
@@ -445,7 +445,7 @@ func generateICMPIP(pkt *packet.Packet, config *PacketConfig, rnd *rand.Rand) {
 		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(pktIP, pktICMP, pkt.Data))
 	} else if l3.Version == 6 {
 		pktIP := (*packet.IPv6Hdr)(pkt.L3)
-		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pktIP, pktICMP))
+		pktICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pktIP, pktICMP, pkt.Data))
 	}
 }
 
diff --git a/packet/checksum.go b/packet/checksum.go
index f77f64b6..b1ec78c3 100644
--- a/packet/checksum.go
+++ b/packet/checksum.go
@@ -146,7 +146,8 @@ func SetHWOffloadingHdrChecksum(p *Packet) {
 			// calculate full software checksum for icmp, cause there is no
 			// hardware calculation for icmp packets.
 			p.GetICMPForIPv6().Cksum = 0
-			p.GetICMPForIPv6().Cksum = SwapBytesUint16(CalculateIPv6ICMPChecksum(ipv6, icmp))
+			p.GetICMPForIPv6().Cksum = SwapBytesUint16(CalculateIPv6ICMPChecksum(ipv6, icmp,
+				unsafe.Pointer(uintptr(unsafe.Pointer(icmp))+ICMPLen)))
 		}
 	}
 }
@@ -325,14 +326,19 @@ func CalculateIPv4ICMPChecksum(hdr *IPv4Hdr, icmp *ICMPHdr, data unsafe.Pointer)
 
 // CalculateIPv6ICMPChecksum calculates ICMP checksum in case if L3
 // protocol is IPv6.
-func CalculateIPv6ICMPChecksum(hdr *IPv6Hdr, icmp *ICMPHdr) uint16 {
-	data := unsafe.Pointer(icmp)
+func CalculateIPv6ICMPChecksum(hdr *IPv6Hdr, icmp *ICMPHdr, data unsafe.Pointer) uint16 {
 	dataLength := SwapBytesUint16(hdr.PayloadLen)
 
-	sum := calculateIPv6AddrChecksum(hdr) +
+	// ICMP payload
+	sum := calculateDataChecksum(data, int(dataLength-ICMPLen), 0)
+
+	sum += calculateIPv6AddrChecksum(hdr) + // IPv6 Header
 		uint32(dataLength) +
 		uint32(hdr.Proto) +
-		calculateDataChecksum(unsafe.Pointer(data), int(dataLength), 0)
+		// ICMP header excluding checksum
+		uint32(uint16(icmp.Type)<<8|uint16(icmp.Code)) +
+		uint32(SwapBytesUint16(icmp.Identifier)) +
+		uint32(SwapBytesUint16(icmp.SeqNum))
 
 	return ^reduceChecksum(sum)
 }
diff --git a/packet/checksum_test.go b/packet/checksum_test.go
index 495863f4..042751b5 100644
--- a/packet/checksum_test.go
+++ b/packet/checksum_test.go
@@ -132,7 +132,6 @@ func TestCalculateIPv4ICMPChecksum(t *testing.T) {
 	pkt.GetIPv4().HdrChecksum = SwapBytesUint16(ipcksum)
 
 	want := wantIPv4ICMP
-	pkt.GetICMPNoCheck().Cksum = 0
 	got := CalculateIPv4ICMPChecksum(pkt.GetIPv4(), pkt.GetICMPForIPv4(), pkt.Data)
 
 	if got != want {
@@ -150,7 +149,7 @@ func TestCalculateIPv6ICMPChecksum(t *testing.T) {
 
 	want := wantIPv6ICMP
 	pkt.GetICMPNoCheck().Cksum = 0
-	got := CalculateIPv6ICMPChecksum(pkt.GetIPv6(), pkt.GetICMPNoCheck())
+	got := CalculateIPv6ICMPChecksum(pkt.GetIPv6(), pkt.GetICMPNoCheck(), pkt.Data)
 
 	if got != want {
 		t.Errorf("Incorrect result:\ngot: %x, \nwant: %x\n\n", got, want)
diff --git a/test/stability/testCksum/testCksum.go b/test/stability/testCksum/testCksum.go
index f08cefef..0c96702a 100644
--- a/test/stability/testCksum/testCksum.go
+++ b/test/stability/testCksum/testCksum.go
@@ -588,7 +588,7 @@ func generateIPv6ICMP(emptyPacket *packet.Packet, rnd *rand.Rand) {
 	if !hwol {
 		pIPv6 := emptyPacket.GetIPv6()
 		pICMP := emptyPacket.GetICMPForIPv6()
-		pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP))
+		pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, emptyPacket.Data))
 	}
 }
 
diff --git a/test/stability/testCksum/testCksumCommon/testCksumCommon.go b/test/stability/testCksum/testCksumCommon/testCksumCommon.go
index 71bb84ea..7d11d0bd 100644
--- a/test/stability/testCksum/testCksumCommon/testCksumCommon.go
+++ b/test/stability/testCksum/testCksumCommon/testCksumCommon.go
@@ -96,9 +96,9 @@ func CheckPacketChecksums(p *packet.Packet) bool {
 			} else {
 				status = true
 			}
-		} else if pIPv6.Proto == common.ICMPNumber {
+		} else if pIPv6.Proto == common.ICMPv6Number {
 			pICMP := p.GetICMPForIPv6()
-			csum := packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP)
+			csum := packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, p.Data)
 			if packet.SwapBytesUint16(pICMP.Cksum) != csum {
 				println("IPv6 ICMP checksum mismatch", packet.SwapBytesUint16(pICMP.Cksum), "should be", csum)
 			} else {
@@ -141,9 +141,9 @@ func CalculateChecksum(p *packet.Packet) {
 		} else if pIPv6.Proto == common.TCPNumber {
 			pTCP := p.GetTCPForIPv6()
 			pTCP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(pIPv6, pTCP, p.Data))
-		} else if pIPv6.Proto == common.ICMPNumber {
+		} else if pIPv6.Proto == common.ICMPv6Number {
 			pICMP := p.GetICMPForIPv6()
-			pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP))
+			pICMP.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(pIPv6, pICMP, p.Data))
 		} else {
 			println("Unknown IPv6 protocol number", pIPv6.Proto)
 			println("TEST FAILED")

From 2dbef45f4750db0b4430968201d9c1a0d91e3503 Mon Sep 17 00:00:00 2001
From: Lehner Florian <dev@der-flo.net>
Date: Tue, 2 Oct 2018 19:51:57 +0200
Subject: [PATCH 43/50] Add functions for MPLS

Signed-off-by: Lehner Florian <dev@der-flo.net>
---
 packet/mpls.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packet/mpls.go b/packet/mpls.go
index 2097a71b..97fca5c6 100644
--- a/packet/mpls.go
+++ b/packet/mpls.go
@@ -31,6 +31,21 @@ func (hdr *MPLSHdr) SetMPLSLabel(tag uint32) {
 	hdr.mpls = SwapBytesUint32((SwapBytesUint32(hdr.mpls) & 0xfff) | (tag << 12))
 }
 
+// GetMPLSTC returns the Traffic Class (formerly known as EXP).
+func (hdr *MPLSHdr) GetMPLSTC() uint32 {
+	return (SwapBytesUint32(hdr.mpls) >> 9) & 0x00000007
+}
+
+// GetMPLSS returns the Bottom-Of-Stack value
+func (hdr *MPLSHdr) GetMPLSS() uint32 {
+	return (SwapBytesUint32(hdr.mpls) >> 8) & 1
+}
+
+// GetMPLSTTL returns the Time-to-Live value
+func (hdr *MPLSHdr) GetMPLSTTL() uint32 {
+	return SwapBytesUint32(hdr.mpls) & 0x000000ff
+}
+
 // GetMPLS returns MPLS header pointer if it is present in the packet.
 func (packet *Packet) GetMPLS() *MPLSHdr {
 	// MPLS shouldn't be used with VLAN tags, so we don't check any VLAN tags here

From 5d11a74102973e9baa8ab090b385b03a6864bf19 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Tue, 2 Oct 2018 19:32:10 +0000
Subject: [PATCH 44/50] Removed NAT example because it is moved to nff-go-nat
 repository

---
 .travis.yml                                   |   1 -
 examples/Makefile                             |   2 +-
 examples/nat/Makefile                         |   8 -
 examples/nat/arp.go                           |  86 --
 examples/nat/cksum.go                         | 116 ---
 examples/nat/client/.gitignore                |   1 -
 examples/nat/client/Makefile                  |  19 -
 examples/nat/client/client.go                 | 235 -----
 examples/nat/config.go                        | 704 --------------
 examples/nat/dhcp.go                          | 236 -----
 examples/nat/dhcp6.go                         | 456 ---------
 examples/nat/grpc.go                          | 121 ---
 examples/nat/icmp.go                          |  95 --
 examples/nat/main/.gitignore                  |   1 -
 examples/nat/main/Dockerfile                  |  16 -
 examples/nat/main/Makefile                    |  14 -
 examples/nat/main/config-dhcp.json            |  59 --
 examples/nat/main/config-kni-dhcp.json        |  83 --
 examples/nat/main/config-kni.json             |  82 --
 examples/nat/main/config-vlan.json            |  16 -
 examples/nat/main/config.json                 |  58 --
 examples/nat/main/config2ports.json           |  26 -
 examples/nat/main/nat.go                      |  76 --
 examples/nat/neigh.go                         |  88 --
 examples/nat/portalloc.go                     |  94 --
 examples/nat/translation.go                   | 443 ---------
 examples/nat/updatecfg/.gitignore             |   1 -
 examples/nat/updatecfg/updatecfg.proto        |  67 --
 examples/nat/util.go                          | 286 ------
 scripts/get-depends.sh                        |   6 -
 .../main/nat/perf-nat-linux-vlan.json         | 100 --
 test/framework/main/nat/perf-nat-linux.json   | 100 --
 test/framework/main/nat/perf-nat-vlan.json    | 142 ---
 test/framework/main/nat/perf-nat.json         | 142 ---
 .../main/nat/stability-nat-vlan.json          |  86 --
 test/framework/main/nat/stability-nat.json    |  86 --
 test/framework/main/perf.json                 | 912 ------------------
 37 files changed, 1 insertion(+), 5063 deletions(-)
 delete mode 100644 examples/nat/Makefile
 delete mode 100644 examples/nat/arp.go
 delete mode 100644 examples/nat/cksum.go
 delete mode 100644 examples/nat/client/.gitignore
 delete mode 100644 examples/nat/client/Makefile
 delete mode 100644 examples/nat/client/client.go
 delete mode 100644 examples/nat/config.go
 delete mode 100644 examples/nat/dhcp.go
 delete mode 100644 examples/nat/dhcp6.go
 delete mode 100644 examples/nat/grpc.go
 delete mode 100644 examples/nat/icmp.go
 delete mode 100644 examples/nat/main/.gitignore
 delete mode 100644 examples/nat/main/Dockerfile
 delete mode 100644 examples/nat/main/Makefile
 delete mode 100644 examples/nat/main/config-dhcp.json
 delete mode 100644 examples/nat/main/config-kni-dhcp.json
 delete mode 100644 examples/nat/main/config-kni.json
 delete mode 100644 examples/nat/main/config-vlan.json
 delete mode 100644 examples/nat/main/config.json
 delete mode 100644 examples/nat/main/config2ports.json
 delete mode 100644 examples/nat/main/nat.go
 delete mode 100644 examples/nat/neigh.go
 delete mode 100644 examples/nat/portalloc.go
 delete mode 100644 examples/nat/translation.go
 delete mode 100644 examples/nat/updatecfg/.gitignore
 delete mode 100644 examples/nat/updatecfg/updatecfg.proto
 delete mode 100644 examples/nat/util.go
 delete mode 100644 test/framework/main/nat/perf-nat-linux-vlan.json
 delete mode 100644 test/framework/main/nat/perf-nat-linux.json
 delete mode 100644 test/framework/main/nat/perf-nat-vlan.json
 delete mode 100644 test/framework/main/nat/perf-nat.json
 delete mode 100644 test/framework/main/nat/stability-nat-vlan.json
 delete mode 100644 test/framework/main/nat/stability-nat.json

diff --git a/.travis.yml b/.travis.yml
index 5c9d4cee..0ee9041f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,6 @@ before_script:
 
 script:
   - docker exec -i test-nff-go ./scripts/get-depends.sh
-  - docker exec -i test-nff-go apt-get install -y protobuf-compiler
   - docker exec -i test-nff-go make
   # Build standalone examples
   - docker exec -i test-nff-go bash -c "cd examples && make gopacketParserExample && cd .."
diff --git a/examples/Makefile b/examples/Makefile
index 27e2f94b..495aa05f 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -7,7 +7,7 @@ IMAGENAME = nff-go-examples
 EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
 			  createPacket sendFixedPktsNumber gtpu pingReplay \
 			  netlink gopacketParserExample devbind
-SUBDIRS = nat tutorial antiddos demo fileReadWrite firewall forwarding
+SUBDIRS = tutorial antiddos demo fileReadWrite firewall forwarding
 
 .PHONY: dpi nffPktgen
 dpi:
diff --git a/examples/nat/Makefile b/examples/nat/Makefile
deleted file mode 100644
index 4a03d043..00000000
--- a/examples/nat/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Intel Corporation.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-PATH_TO_MK = ../../mk
-SUBDIRS = main client
-
-include $(PATH_TO_MK)/intermediate.mk
diff --git a/examples/nat/arp.go b/examples/nat/arp.go
deleted file mode 100644
index 28ed36c9..00000000
--- a/examples/nat/arp.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-func (port *ipPort) handleARP(pkt *packet.Packet) uint {
-	arp := pkt.GetARPNoCheck()
-
-	if packet.SwapBytesUint16(arp.Operation) != packet.ARPRequest {
-		if packet.SwapBytesUint16(arp.Operation) == packet.ARPReply {
-			ipv4 := packet.SwapBytesUint32(packet.ArrayToIPv4(arp.SPA))
-			port.arpTable.Store(ipv4, arp.SHA)
-		}
-		if port.KNIName != "" {
-			return dirKNI
-		}
-		return dirDROP
-	}
-
-	// If there is a KNI interface, direct all ARP traffic to it
-	if port.KNIName != "" {
-		return dirKNI
-	}
-
-	// Check that someone is asking about MAC of my IP address and HW
-	// address is blank in request
-	if packet.BytesToIPv4(arp.TPA[0], arp.TPA[1], arp.TPA[2], arp.TPA[3]) != packet.SwapBytesUint32(port.Subnet.Addr) {
-		println("Warning! Got an ARP packet with target IPv4 address", StringIPv4Array(arp.TPA),
-			"different from IPv4 address on interface. Should be", StringIPv4Int(port.Subnet.Addr),
-			". ARP request ignored.")
-		return dirDROP
-	}
-	if arp.THA != [common.EtherAddrLen]byte{} {
-		println("Warning! Got an ARP packet with non-zero MAC address", StringMAC(arp.THA),
-			". ARP request ignored.")
-		return dirDROP
-	}
-
-	// Prepare an answer to this request
-	answerPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-
-	packet.InitARPReplyPacket(answerPacket, port.SrcMACAddress, arp.SHA, packet.ArrayToIPv4(arp.TPA), packet.ArrayToIPv4(arp.SPA))
-	vlan := pkt.GetVLAN()
-	if vlan != nil {
-		answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
-	}
-
-	port.dumpPacket(answerPacket, dirSEND)
-	answerPacket.SendPacket(port.Index)
-
-	return dirDROP
-}
-
-func (port *ipPort) getMACForIPv4(ip uint32) (macAddress, bool) {
-	v, found := port.arpTable.Load(ip)
-	if found {
-		return macAddress(v.([common.EtherAddrLen]byte)), true
-	}
-	port.sendARPRequest(ip)
-	return macAddress{}, false
-}
-
-func (port *ipPort) sendARPRequest(ip uint32) {
-	requestPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-
-	packet.InitARPRequestPacket(requestPacket, port.SrcMACAddress,
-		packet.SwapBytesUint32(port.Subnet.Addr), packet.SwapBytesUint32(ip))
-	if port.Vlan != 0 {
-		requestPacket.AddVLANTag(port.Vlan)
-	}
-
-	port.dumpPacket(requestPacket, dirSEND)
-	requestPacket.SendPacket(port.Index)
-}
diff --git a/examples/nat/cksum.go b/examples/nat/cksum.go
deleted file mode 100644
index b712d46c..00000000
--- a/examples/nat/cksum.go
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2017 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-	"unsafe"
-)
-
-func setIPv4UDPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv4NoCheck()
-		l4 := pkt.GetUDPNoCheck()
-		if hWTXChecksum {
-			l3.HdrChecksum = 0
-			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv4UDPCksum(l3, l4))
-			l2len := uint32(common.EtherLen)
-			if pkt.Ether.EtherType == common.SwapVLANNumber {
-				l2len += common.VLANLen
-			}
-			pkt.SetTXIPv4UDPOLFlags(l2len, common.IPv4MinLen)
-		} else {
-			l3.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(l3))
-			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv4UDPChecksum(l3, l4,
-				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+uintptr(common.UDPLen))))
-		}
-	}
-}
-
-func setIPv4TCPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv4NoCheck()
-		l4 := pkt.GetTCPNoCheck()
-		if hWTXChecksum {
-			l3.HdrChecksum = 0
-			l4.Cksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv4TCPCksum(l3))
-			l2len := uint32(common.EtherLen)
-			if pkt.Ether.EtherType == common.SwapVLANNumber {
-				l2len += common.VLANLen
-			}
-			pkt.SetTXIPv4TCPOLFlags(l2len, common.IPv4MinLen)
-		} else {
-			l3.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(l3))
-			l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4TCPChecksum(l3, l4,
-				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.TCPMinLen)))
-		}
-	}
-}
-
-func setIPv4ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv4NoCheck()
-		if hWTXChecksum {
-			l3.HdrChecksum = 0
-			l2len := uint32(common.EtherLen)
-			if pkt.Ether.EtherType == common.SwapVLANNumber {
-				l2len += common.VLANLen
-			}
-			pkt.SetTXIPv4OLFlags(l2len, common.IPv4MinLen)
-		} else {
-			l3.HdrChecksum = packet.SwapBytesUint16(packet.CalculateIPv4Checksum(l3))
-		}
-		l4 := pkt.GetICMPNoCheck()
-		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv4ICMPChecksum(l3, l4,
-			unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.ICMPLen)))
-	}
-}
-
-func setIPv6UDPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv6NoCheck()
-		l4 := pkt.GetUDPNoCheck()
-		if hWTXChecksum {
-			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv6UDPCksum(l3, l4))
-			l2len := uint32(common.EtherLen)
-			if pkt.Ether.EtherType == common.SwapVLANNumber {
-				l2len += common.VLANLen
-			}
-			pkt.SetTXIPv6UDPOLFlags(l2len, common.IPv6Len)
-		} else {
-			l4.DgramCksum = packet.SwapBytesUint16(packet.CalculateIPv6UDPChecksum(l3, l4,
-				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+uintptr(common.UDPLen))))
-		}
-	}
-}
-
-func setIPv6TCPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv6NoCheck()
-		l4 := pkt.GetTCPNoCheck()
-		if hWTXChecksum {
-			l4.Cksum = packet.SwapBytesUint16(packet.CalculatePseudoHdrIPv6TCPCksum(l3))
-			l2len := uint32(common.EtherLen)
-			if pkt.Ether.EtherType == common.SwapVLANNumber {
-				l2len += common.VLANLen
-			}
-			pkt.SetTXIPv6TCPOLFlags(l2len, common.IPv6Len)
-		} else {
-			l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6TCPChecksum(l3, l4,
-				unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.TCPMinLen)))
-		}
-	}
-}
-
-func setIPv6ICMPChecksum(pkt *packet.Packet, calculateChecksum, hWTXChecksum bool) {
-	if calculateChecksum {
-		l3 := pkt.GetIPv6NoCheck()
-
-		l4 := pkt.GetICMPNoCheck()
-		l4.Cksum = packet.SwapBytesUint16(packet.CalculateIPv6ICMPChecksum(l3, l4,
-			unsafe.Pointer(uintptr(unsafe.Pointer(l4))+common.ICMPLen)))
-	}
-}
diff --git a/examples/nat/client/.gitignore b/examples/nat/client/.gitignore
deleted file mode 100644
index b051c6c5..00000000
--- a/examples/nat/client/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-client
diff --git a/examples/nat/client/Makefile b/examples/nat/client/Makefile
deleted file mode 100644
index dd68f254..00000000
--- a/examples/nat/client/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2018 Intel Corporation.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-% : %.go
-	go build $<
-
-all: client ../updatecfg/updatecfg.pb.go
-
-../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
-	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
-
-clean:
-	-rm client
-
-images: all
-deploy: all
-clean-images: clean
-cleanall: clean
diff --git a/examples/nat/client/client.go b/examples/nat/client/client.go
deleted file mode 100644
index 44733f6f..00000000
--- a/examples/nat/client/client.go
+++ /dev/null
@@ -1,235 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"net"
-	"strconv"
-	"strings"
-	"time"
-
-	"golang.org/x/net/context"
-	"google.golang.org/grpc"
-
-	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
-)
-
-type dumpRequestArray []*upd.DumpControlRequest
-type addresChangeRequestArray []*upd.InterfaceAddressChangeRequest
-type portForwardRequestArray []*upd.PortForwardingChangeRequest
-
-var (
-	dumpRequests         dumpRequestArray
-	addresChangeRequests addresChangeRequestArray
-	portForwardRequests  portForwardRequestArray
-)
-
-func (dra *dumpRequestArray) String() string {
-	res := ""
-	for _, r := range *dra {
-		res += r.String() + "\n"
-	}
-	return res
-}
-
-func (dra *dumpRequestArray) Set(value string) error {
-	req, ok := map[string]upd.DumpControlRequest{
-		"+d": upd.DumpControlRequest{
-			EnableTrace: true,
-			TraceType:   upd.TraceType_DUMP_DROP,
-		},
-		"-d": upd.DumpControlRequest{
-			EnableTrace: false,
-			TraceType:   upd.TraceType_DUMP_DROP,
-		},
-		"+t": upd.DumpControlRequest{
-			EnableTrace: true,
-			TraceType:   upd.TraceType_DUMP_TRANSLATE,
-		},
-		"-t": upd.DumpControlRequest{
-			EnableTrace: false,
-			TraceType:   upd.TraceType_DUMP_TRANSLATE,
-		},
-		"+k": upd.DumpControlRequest{
-			EnableTrace: true,
-			TraceType:   upd.TraceType_DUMP_KNI,
-		},
-		"-k": upd.DumpControlRequest{
-			EnableTrace: false,
-			TraceType:   upd.TraceType_DUMP_KNI,
-		},
-	}[value]
-
-	if !ok {
-		return fmt.Errorf("Bad dump control specification \"%s\"", value)
-	}
-	*dra = append(*dra, &req)
-	return nil
-}
-
-func (acra *addresChangeRequestArray) String() string {
-	res := ""
-	for _, r := range *acra {
-		res += r.String() + "\n"
-	}
-	return res
-}
-
-func (acra *addresChangeRequestArray) Set(value string) error {
-	parts := strings.Split(value, ",")
-	if len(parts) != 2 {
-		return fmt.Errorf("Bad port index and subnet address specified \"%s\"", value)
-	}
-	index, err := strconv.ParseUint(parts[0], 10, 32)
-	if err != nil {
-		return err
-	}
-
-	ip, ipnet, err := net.ParseCIDR(parts[1])
-	if err != nil {
-		return err
-	}
-	ones, _ := ipnet.Mask.Size()
-
-	*acra = append(*acra, &upd.InterfaceAddressChangeRequest{
-		InterfaceId: uint32(index),
-		PortSubnet: &upd.Subnet{
-			Address: &upd.IPAddress{
-				Address: ip,
-			},
-			MaskBitsNumber: uint32(ones),
-		},
-	})
-	return nil
-}
-
-func (pfra *portForwardRequestArray) String() string {
-	res := ""
-	for _, r := range *pfra {
-		res += r.String() + "\n"
-	}
-	return res
-}
-
-func (pfra *portForwardRequestArray) Set(value string) error {
-	parts := strings.Split(value, ",")
-	if len(parts) != 6 {
-		return fmt.Errorf("Bad port forwarding specification \"%s\"", value)
-	}
-
-	enable, ok := map[string]bool{
-		"+": true,
-		"-": false,
-	}[parts[0]]
-	if !ok {
-		return fmt.Errorf("Bad port forwarding enable sign string \"%s\"", parts[0])
-	}
-
-	index, err := strconv.ParseUint(parts[1], 10, 32)
-	if err != nil {
-		return err
-	}
-
-	proto, ok := upd.Protocol_value[parts[2]]
-	if !ok || proto == int32(upd.Protocol_UNKNOWN) {
-		return fmt.Errorf("Bad protocol specified \"%s\"", parts[0])
-	}
-
-	sport, err := strconv.ParseUint(parts[3], 10, 16)
-	if err != nil {
-		return err
-	}
-
-	ip := net.ParseIP(parts[4])
-	if ip == nil {
-		return fmt.Errorf("Bad IP address specified \"%s\"", parts[4])
-	}
-
-	tport, err := strconv.ParseUint(parts[5], 10, 16)
-	if err != nil {
-		return err
-	}
-
-	*pfra = append(*pfra, &upd.PortForwardingChangeRequest{
-		EnableForwarding: enable,
-		InterfaceId:      uint32(index),
-		Port: &upd.ForwardedPort{
-			SourcePortNumber: uint32(sport),
-			TargetAddress: &upd.IPAddress{
-				Address: ip,
-			},
-			TargetPortNumber: uint32(tport),
-			Protocol:         upd.Protocol(proto),
-		},
-	})
-	return nil
-}
-
-func main() {
-	flag.Usage = func() {
-		fmt.Printf(`Usage: client [-a server:port] [-d {+|-}{d|t|k}] [-s index:subnet] [-p {+|-},{TCP|UDP|TCP6|UDP6},port number,target IP address,target port]
-
-Client sends GRPS requests to NAT server controlling packets trace dump,
-ports subnet adresses and forwarded ports. Multiple requests of the same
-type are allowed and are processed in the following order: all dump, all
-subnet, all port forwarding requests.
-
-`)
-		flag.PrintDefaults()
-	}
-	address := flag.String("a", "localhost:60602", "Specifies server address")
-	flag.Var(&dumpRequests, "d", `Control dump trace output in a form of +/- and letter,
-e.g. +d or -t or +k:
-    + and - mean to enable or disable corresponding trace,
-    d means to trace dropped packets,
-    t means to trace translated (normally sent) packets,
-    k means to trace packets that were sent to KNI interface.`)
-	flag.Var(&addresChangeRequests, "s", `Control network interface subnet in a form of index:subnet,
-e.g. 1,192.168.5.1/24 or 1,fd16::1/128. Port index is DPDK port
-number. Subnet is given in form of port IP address and prefix bits.`)
-	flag.Var(&portForwardRequests, "p", `Control TCP and UDP port forwarding in a form of
-+/-,index,protocol,source port,target IP address,target port, e.g.
-+,1,TCP,2222,192.168.5.7,22 or -,0,TCP,22,0.0.0.0,0 or
-+,1,TCP6,2222,fd14::3,22 or -,0,TCP6,22,::,0. If target
-address is zero, it means that port is forwarded to corresponding
-network port KNI interface. Port forwarding to a non-zero
-target address (not to a KNI interface) is possible only for
-public network port.`)
-	flag.Parse()
-
-	// Set up a connection to the server.
-	conn, err := grpc.Dial(*address, grpc.WithInsecure())
-	if err != nil {
-		log.Fatalf("did not connect: %v", err)
-	}
-	defer conn.Close()
-	c := upd.NewUpdaterClient(conn)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	for _, r := range dumpRequests {
-		reply, err := c.ControlDump(ctx, r)
-		if err != nil {
-			log.Fatalf("could not update: %v", err)
-		}
-		log.Printf("update successful: \"%s\"", reply.String())
-	}
-
-	for _, r := range addresChangeRequests {
-		reply, err := c.ChangeInterfaceAddress(ctx, r)
-		if err != nil {
-			log.Fatalf("could not update: %v", err)
-		}
-		log.Printf("update successful: \"%s\"", reply.String())
-	}
-
-	for _, r := range portForwardRequests {
-		reply, err := c.ChangePortForwarding(ctx, r)
-		if err != nil {
-			log.Fatalf("could not update: %v", err)
-		}
-		log.Printf("update successful: \"%s\"", reply.String())
-	}
-}
diff --git a/examples/nat/config.go b/examples/nat/config.go
deleted file mode 100644
index 6d9d90dd..00000000
--- a/examples/nat/config.go
+++ /dev/null
@@ -1,704 +0,0 @@
-// Copyright 2017-2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net"
-	"os"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/flow"
-	"github.com/intel-go/nff-go/packet"
-
-	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
-)
-
-type terminationDirection uint8
-type interfaceType int
-
-const (
-	pri2pub terminationDirection = 0x0f
-	pub2pri terminationDirection = 0xf0
-
-	iPUBLIC  interfaceType = 0
-	iPRIVATE interfaceType = 1
-
-	dirDROP = uint(upd.TraceType_DUMP_DROP)
-	dirSEND = uint(upd.TraceType_DUMP_TRANSLATE)
-	dirKNI  = uint(upd.TraceType_DUMP_KNI)
-
-	connectionTimeout time.Duration = 1 * time.Minute
-	portReuseTimeout  time.Duration = 1 * time.Second
-)
-
-var (
-	zeroIPv6Addr = [common.IPv6AddrLen]uint8{}
-)
-
-type hostPort struct {
-	Addr4 uint32
-	Addr6 [common.IPv6AddrLen]uint8
-	Port  uint16
-	ipv6  bool
-}
-
-type protocolId struct {
-	id   uint8
-	ipv6 bool
-}
-
-type forwardedPort struct {
-	Port        uint16     `json:"port"`
-	Destination hostPort   `json:"destination"`
-	Protocol    protocolId `json:"protocol"`
-}
-
-var protocolIdLookup map[string]protocolId = map[string]protocolId{
-	"TCP": protocolId{
-		id:   common.TCPNumber,
-		ipv6: false,
-	},
-	"UDP": protocolId{
-		id:   common.UDPNumber,
-		ipv6: false,
-	},
-	"TCP6": protocolId{
-		id:   common.TCPNumber,
-		ipv6: true,
-	},
-	"UDP6": protocolId{
-		id:   common.UDPNumber,
-		ipv6: true,
-	},
-}
-
-func (out *protocolId) UnmarshalJSON(b []byte) error {
-	var s string
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	result, ok := protocolIdLookup[s]
-	if !ok {
-		return errors.New("Bad protocol name: " + s)
-	}
-
-	*out = result
-	return nil
-}
-
-type ipv4Subnet struct {
-	Addr            uint32
-	Mask            uint32
-	addressAcquired bool
-	ds              dhcpState
-}
-
-func (fp *forwardedPort) String() string {
-	return fmt.Sprintf("Port:%d, Destination IPv4: %v, Destination IPv6: %v, Protocol: %d",
-		fp.Port,
-		packet.IPv4ToString(fp.Destination.Addr4),
-		packet.IPv6ToString(fp.Destination.Addr6),
-		fp.Protocol)
-}
-
-func (subnet *ipv4Subnet) String() string {
-	if subnet.addressAcquired {
-		// Count most significant set bits
-		mask := uint32(1) << 31
-		i := 0
-		for ; i <= 32; i++ {
-			if subnet.Mask&mask == 0 {
-				break
-			}
-			mask >>= 1
-		}
-		return packet.IPv4ToString(packet.SwapBytesUint32(subnet.Addr)) + "/" + strconv.Itoa(i)
-	}
-	return "DHCP address not acquired"
-}
-
-func (subnet *ipv4Subnet) checkAddrWithingSubnet(addr uint32) bool {
-	return addr&subnet.Mask == subnet.Addr&subnet.Mask
-}
-
-type ipv6Subnet struct {
-	Addr            [common.IPv6AddrLen]uint8
-	multicastAddr   [common.IPv6AddrLen]uint8
-	Mask            [common.IPv6AddrLen]uint8
-	llAddr          [common.IPv6AddrLen]uint8
-	llMulticastAddr [common.IPv6AddrLen]uint8
-	addressAcquired bool
-	ds              dhcpv6State
-}
-
-func (subnet *ipv6Subnet) String() string {
-	if subnet.addressAcquired {
-		// Count most significant set bits
-		i := 0
-		for ; i <= 128; i++ {
-			mask := uint8(1) << uint(7-(i&7))
-			if i == 128 || subnet.Mask[i>>3]&mask == 0 {
-				break
-			}
-		}
-		return packet.IPv6ToString(subnet.Addr) + "/" + strconv.Itoa(i)
-	}
-	return "DHCP address not acquired"
-}
-
-func (subnet *ipv6Subnet) andMask(addr [common.IPv6AddrLen]uint8) [common.IPv6AddrLen]uint8 {
-	var result [common.IPv6AddrLen]uint8
-	for i := range addr {
-		result[i] = addr[i] & subnet.Mask[i]
-	}
-	return result
-}
-
-func (subnet *ipv6Subnet) checkAddrWithingSubnet(addr [common.IPv6AddrLen]uint8) bool {
-	return subnet.andMask(addr) == subnet.andMask(subnet.Addr)
-}
-
-type macAddress [common.EtherAddrLen]uint8
-
-type portMapEntry struct {
-	lastused             time.Time
-	finCount             uint8
-	terminationDirection terminationDirection
-	static               bool
-}
-
-// Type describing a network port
-type ipPort struct {
-	Index         uint16          `json:"index"`
-	Subnet        ipv4Subnet      `json:"subnet"`
-	Subnet6       ipv6Subnet      `json:"subnet6"`
-	Vlan          uint16          `json:"vlan-tag"`
-	KNIName       string          `json:"kni-name"`
-	ForwardPorts  []forwardedPort `json:"forward-ports"`
-	SrcMACAddress macAddress
-	Type          interfaceType
-	// Pointer to an opposite port in a pair
-	opposite *ipPort
-	// Map of allocated IP ports on public interface
-	portmap  [][]portMapEntry
-	portmap6 [][]portMapEntry
-	// Main lookup table which contains entries for packets coming at this port
-	translationTable []*sync.Map
-	// ARP lookup table
-	arpTable sync.Map
-	// Debug dump stuff
-	fdump    [dirKNI + 1]*os.File
-	dumpsync [dirKNI + 1]sync.Mutex
-}
-
-// Config for one port pair.
-type portPair struct {
-	PrivatePort ipPort `json:"private-port"`
-	PublicPort  ipPort `json:"public-port"`
-	// Synchronization point for lookup table modifications
-	mutex sync.Mutex
-	// Port that was allocated last
-	lastport int
-}
-
-// Config for NAT.
-type Config struct {
-	HostName  string     `json:"host-name"`
-	PortPairs []portPair `json:"port-pairs"`
-}
-
-// Type used to pass handler index to translation functions.
-type pairIndex struct {
-	index int
-}
-
-var (
-	// Natconfig is a config file.
-	Natconfig *Config
-	// CalculateChecksum is a flag whether checksums should be
-	// calculated for modified packets.
-	NoCalculateChecksum bool
-	// HWTXChecksum is a flag whether checksums calculation should be
-	// offloaded to HW.
-	NoHWTXChecksum bool
-	NeedKNI        bool
-	NeedDHCP       bool
-
-	// Debug variables
-	DumpEnabled [dirKNI + 1]bool
-)
-
-func (pi pairIndex) Copy() interface{} {
-	return pairIndex{
-		index: pi.index,
-	}
-}
-
-func (pi pairIndex) Delete() {
-}
-
-func convertIPv4(in []byte) (uint32, error) {
-	if in == nil || len(in) > 4 {
-		return 0, fmt.Errorf("Only IPv4 addresses are supported now while your address has %d bytes", len(in))
-	}
-
-	addr := (uint32(in[0]) << 24) | (uint32(in[1]) << 16) |
-		(uint32(in[2]) << 8) | uint32(in[3])
-
-	return addr, nil
-}
-
-// UnmarshalJSON parses ipv 4 subnet details.
-func (out *ipv4Subnet) UnmarshalJSON(b []byte) error {
-	var s string
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	if s == "dhcp" {
-		out.Addr = uint32(0)
-		out.Mask = uint32(0)
-		out.addressAcquired = false
-		return nil
-	}
-
-	if ip, ipnet, err := net.ParseCIDR(s); err == nil {
-		if out.Addr, err = convertIPv4(ip.To4()); err != nil {
-			return err
-		}
-		if out.Mask, err = convertIPv4(ipnet.Mask); err != nil {
-			return err
-		}
-		out.addressAcquired = true
-		return nil
-	}
-
-	if ip := net.ParseIP(s); ip != nil {
-		var err error
-		if out.Addr, err = convertIPv4(ip.To4()); err != nil {
-			return err
-		}
-		out.Mask = 0xffffffff
-		out.addressAcquired = true
-		return nil
-	}
-	return errors.New("Failed to parse address " + s)
-}
-
-// UnmarshalJSON parses ipv 4 subnet details.
-func (out *ipv6Subnet) UnmarshalJSON(b []byte) error {
-	var s string
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	if s == "dhcp" {
-		out.Addr = [common.IPv6AddrLen]uint8{}
-		out.Mask = [common.IPv6AddrLen]uint8{}
-		out.addressAcquired = false
-		return nil
-	}
-
-	if ip, ipnet, err := net.ParseCIDR(s); err == nil {
-		if ip.To16() == nil {
-			return fmt.Errorf("Bad IPv6 address: %s", s)
-		}
-		copy(out.Addr[:], ip.To16())
-		copy(out.Mask[:], ipnet.Mask)
-		out.addressAcquired = true
-		return nil
-	}
-
-	if ip := net.ParseIP(s); ip != nil {
-		if ip.To16() == nil {
-			return fmt.Errorf("Bad IPv6 address: %s", s)
-		}
-		copy(out.Addr[:], ip.To16())
-		out.Mask = [common.IPv6AddrLen]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
-		out.addressAcquired = true
-		return nil
-	}
-	return errors.New("Failed to parse address " + s)
-}
-
-// UnmarshalJSON parses ipv4 host:port string. Port may be omitted and
-// is set to zero in this case.
-func (out *hostPort) UnmarshalJSON(b []byte) error {
-	var s string
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	hostStr, portStr, err := net.SplitHostPort(s)
-	if err != nil {
-		return err
-	}
-
-	ipArray := net.ParseIP(hostStr)
-	if ipArray == nil {
-		return errors.New("Bad IP address specified: " + hostStr)
-	}
-	out.Addr4, err = convertIPv4(ipArray.To4())
-	if err != nil {
-		ipv6addr := ipArray.To16()
-		if ipv6addr == nil {
-			return err
-		}
-		copy(out.Addr6[:], ipv6addr)
-		out.ipv6 = true
-	}
-
-	if portStr != "" {
-		port, err := strconv.ParseInt(portStr, 10, 32)
-		if err != nil {
-			return err
-		}
-		out.Port = uint16(port)
-	} else {
-		out.Port = 0
-	}
-
-	return nil
-}
-
-// UnmarshalJSON parses MAC address.
-func (out *macAddress) UnmarshalJSON(b []byte) error {
-	var s string
-	if err := json.Unmarshal(b, &s); err != nil {
-		return err
-	}
-
-	hw, err := net.ParseMAC(s)
-	if err != nil {
-		return err
-	}
-
-	copy(out[:], hw)
-	return nil
-}
-
-// ReadConfig function reads and parses config file
-func ReadConfig(fileName string) error {
-	file, err := os.Open(fileName)
-	if err != nil {
-		return err
-	}
-	decoder := json.NewDecoder(file)
-
-	err = decoder.Decode(&Natconfig)
-	if err != nil {
-		return err
-	}
-
-	for i := range Natconfig.PortPairs {
-		pp := &Natconfig.PortPairs[i]
-
-		pp.PrivatePort.Type = iPRIVATE
-		pp.PublicPort.Type = iPUBLIC
-		pp.PublicPort.opposite = &pp.PrivatePort
-		pp.PrivatePort.opposite = &pp.PublicPort
-
-		if pp.PrivatePort.Vlan == 0 && pp.PublicPort.Vlan != 0 {
-			return errors.New("Private port with index " +
-				strconv.Itoa(int(pp.PrivatePort.Index)) +
-				" has zero vlan tag while public port with index " +
-				strconv.Itoa(int(pp.PublicPort.Index)) +
-				" has non-zero vlan tag. Transition between VLAN-enabled and VLAN-disabled networks is not supported yet.")
-		} else if pp.PrivatePort.Vlan != 0 && pp.PublicPort.Vlan == 0 {
-			return errors.New("Private port with index " +
-				strconv.Itoa(int(pp.PrivatePort.Index)) +
-				" has non-zero vlan tag while public port with index " +
-				strconv.Itoa(int(pp.PublicPort.Index)) +
-				" has zero vlan tag. Transition between VLAN-enabled and VLAN-disabled networks is not supported yet.")
-		}
-
-		port := &pp.PrivatePort
-		for pi := 0; pi < 2; pi++ {
-			if !port.Subnet.addressAcquired {
-				if Natconfig.HostName == "" {
-					return fmt.Errorf("DHCP option for port %d requires that you set host-name configuration option", port.Index)
-				}
-				NeedDHCP = true
-			}
-
-			for fpi := range port.ForwardPorts {
-				fp := &port.ForwardPorts[fpi]
-				err := port.checkPortForwarding(fp)
-				if err != nil {
-					return err
-				}
-			}
-			port = &pp.PublicPort
-		}
-	}
-
-	return nil
-}
-
-func (port *ipPort) checkPortForwarding(fp *forwardedPort) error {
-	if fp.Destination.ipv6 != fp.Protocol.ipv6 {
-		return fmt.Errorf("Port forwarding protocol should be TCP or UDP for IPv4 addresses and TCP6 or UDP6 for IPv6 addresses")
-	}
-
-	var isAddrZero bool
-	if fp.Destination.ipv6 {
-		isAddrZero = fp.Destination.Addr6 == zeroIPv6Addr
-	} else {
-		isAddrZero = fp.Destination.Addr4 == 0
-	}
-
-	if isAddrZero {
-		if port.KNIName == "" {
-			return errors.New("Port with index " +
-				strconv.Itoa(int(port.Index)) +
-				" should have \"kni-name\" setting if you want to forward packets to KNI address 0.0.0.0 or [::]")
-		}
-		if fp.Destination.Port != fp.Port {
-			return errors.New("When address 0.0.0.0 or [::] is specified, it means that packets are forwarded to KNI interface. In this case destination port should be equal to forwarded port. You have different values: " +
-				strconv.Itoa(int(fp.Port)) + " and " +
-				strconv.Itoa(int(fp.Destination.Port)))
-		}
-		NeedKNI = true
-	} else {
-		if port.Type == iPRIVATE {
-			return errors.New("Only KNI port forwarding is allowed on private port. All translated connections from private to public network can be initiated without any forwarding rules.")
-		}
-
-		if fp.Destination.ipv6 {
-			if !port.opposite.Subnet6.checkAddrWithingSubnet(fp.Destination.Addr6) {
-				return errors.New("Destination address " +
-					packet.IPv6ToString(fp.Destination.Addr6) +
-					" should be within subnet " +
-					port.opposite.Subnet6.String())
-			}
-		} else {
-			if !port.opposite.Subnet.checkAddrWithingSubnet(fp.Destination.Addr4) {
-				return errors.New("Destination address " +
-					packet.IPv4ToString(fp.Destination.Addr4) +
-					" should be within subnet " +
-					port.opposite.Subnet.String())
-			}
-		}
-
-		if fp.Destination.Port == 0 {
-			fp.Destination.Port = fp.Port
-		}
-	}
-	return nil
-}
-
-// Reads MAC addresses for local interfaces into pair ports.
-func (pp *portPair) initLocalMACs() {
-	pp.PublicPort.SrcMACAddress = flow.GetPortMACAddress(pp.PublicPort.Index)
-	pp.PrivatePort.SrcMACAddress = flow.GetPortMACAddress(pp.PrivatePort.Index)
-}
-
-func (port *ipPort) initIPv6LLAddresses() {
-	packet.CalculateIPv6LinkLocalAddrForMAC(&port.Subnet6.llAddr, port.SrcMACAddress)
-	println("Configured link local address", packet.IPv6ToString(port.Subnet6.llAddr), "for port", port.Index)
-	packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.llMulticastAddr, port.Subnet6.llAddr)
-	println("Configured link local multicast address", packet.IPv6ToString(port.Subnet6.llMulticastAddr), "for port", port.Index)
-	if port.Subnet6.Addr != zeroIPv6Addr {
-		packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
-		println("Configured multicast address", packet.IPv6ToString(port.Subnet6.multicastAddr), "for port", port.Index)
-	}
-}
-
-func (port *ipPort) allocatePublicPortPortMap() {
-	port.portmap = make([][]portMapEntry, 256)
-	port.portmap[common.ICMPNumber] = make([]portMapEntry, portEnd)
-	port.portmap[common.TCPNumber] = make([]portMapEntry, portEnd)
-	port.portmap[common.UDPNumber] = make([]portMapEntry, portEnd)
-	port.portmap6 = make([][]portMapEntry, 256)
-	port.portmap6[common.TCPNumber] = make([]portMapEntry, portEnd)
-	port.portmap6[common.UDPNumber] = make([]portMapEntry, portEnd)
-	port.portmap6[common.ICMPv6Number] = make([]portMapEntry, portEnd)
-}
-
-func (port *ipPort) allocateLookupMap() {
-	port.translationTable = make([]*sync.Map, 256)
-	for i := range port.translationTable {
-		port.translationTable[i] = new(sync.Map)
-	}
-}
-
-func (port *ipPort) initPortPortForwardingEntries() {
-	// Initialize port forwarding rules on public interface
-	for i := range port.ForwardPorts {
-		port.enableStaticPortForward(&port.ForwardPorts[i])
-	}
-}
-
-func (port *ipPort) enableStaticPortForward(fp *forwardedPort) {
-	if fp.Protocol.ipv6 {
-		keyEntry := Tuple6{
-			addr: port.Subnet6.Addr,
-			port: fp.Port,
-		}
-		valEntry := Tuple6{
-			addr: fp.Destination.Addr6,
-			port: fp.Destination.Port,
-		}
-		port.translationTable[fp.Protocol.id].Store(keyEntry, valEntry)
-		if fp.Destination.Addr6 != zeroIPv6Addr {
-			port.opposite.translationTable[fp.Protocol.id].Store(valEntry, keyEntry)
-		}
-		if port.Type == iPUBLIC {
-			port.getPortmap(fp.Protocol.ipv6, fp.Protocol.id)[fp.Port] = portMapEntry{
-				lastused:             time.Now(),
-				finCount:             0,
-				terminationDirection: 0,
-				static:               true,
-			}
-		}
-	} else {
-		keyEntry := Tuple{
-			addr: port.Subnet.Addr,
-			port: fp.Port,
-		}
-		valEntry := Tuple{
-			addr: fp.Destination.Addr4,
-			port: fp.Destination.Port,
-		}
-		port.translationTable[fp.Protocol.id].Store(keyEntry, valEntry)
-		if fp.Destination.Addr4 != 0 {
-			port.opposite.translationTable[fp.Protocol.id].Store(valEntry, keyEntry)
-		}
-		if port.Type == iPUBLIC {
-			port.getPortmap(fp.Protocol.ipv6, fp.Protocol.id)[fp.Port] = portMapEntry{
-				lastused:             time.Now(),
-				finCount:             0,
-				terminationDirection: 0,
-				static:               true,
-			}
-		}
-	}
-}
-
-func (port *ipPort) getPortmap(ipv6 bool, protocol uint8) []portMapEntry {
-	if ipv6 {
-		return port.portmap6[protocol]
-	} else {
-		return port.portmap[protocol]
-	}
-}
-
-// InitFlows initializes flow graph for all interface pairs.
-func InitFlows() {
-	for i := range Natconfig.PortPairs {
-		pp := &Natconfig.PortPairs[i]
-
-		// Init port pairs state
-		pp.initLocalMACs()
-		pp.PrivatePort.initIPv6LLAddresses()
-		pp.PublicPort.initIPv6LLAddresses()
-		pp.PrivatePort.allocateLookupMap()
-		pp.PublicPort.allocateLookupMap()
-		pp.PublicPort.allocatePublicPortPortMap()
-		pp.lastport = portStart
-		pp.PrivatePort.initPortPortForwardingEntries()
-		pp.PublicPort.initPortPortForwardingEntries()
-
-		// Handler context with handler index
-		context := new(pairIndex)
-		context.index = i
-
-		var fromPubKNI, fromPrivKNI, toPub, toPriv *flow.Flow
-		var pubKNI, privKNI *flow.Kni
-		var outsPub = uint(2)
-		var outsPriv = uint(2)
-
-		// Initialize public to private flow
-		publicToPrivate, err := flow.SetReceiver(pp.PublicPort.Index)
-		flow.CheckFatal(err)
-		if pp.PublicPort.KNIName != "" {
-			outsPub = 3
-		}
-		pubTranslationOut, err := flow.SetSplitter(publicToPrivate, PublicToPrivateTranslation, outsPub, context)
-		flow.CheckFatal(err)
-		flow.CheckFatal(flow.SetStopper(pubTranslationOut[dirDROP]))
-
-		// Initialize public KNI interface if requested
-		if pp.PublicPort.KNIName != "" {
-			pubKNI, err = flow.CreateKniDevice(pp.PublicPort.Index, pp.PublicPort.KNIName)
-			flow.CheckFatal(err)
-			flow.CheckFatal(flow.SetSenderKNI(pubTranslationOut[dirKNI], pubKNI))
-			fromPubKNI = flow.SetReceiverKNI(pubKNI)
-		}
-
-		// Initialize private to public flow
-		privateToPublic, err := flow.SetReceiver(pp.PrivatePort.Index)
-		flow.CheckFatal(err)
-		if pp.PrivatePort.KNIName != "" {
-			outsPriv = 3
-		}
-		privTranslationOut, err := flow.SetSplitter(privateToPublic, PrivateToPublicTranslation, outsPriv, context)
-		flow.CheckFatal(err)
-		flow.CheckFatal(flow.SetStopper(privTranslationOut[dirDROP]))
-
-		// Initialize private KNI interface if requested
-		if pp.PrivatePort.KNIName != "" {
-			privKNI, err = flow.CreateKniDevice(pp.PrivatePort.Index, pp.PrivatePort.KNIName)
-			flow.CheckFatal(err)
-			flow.CheckFatal(flow.SetSenderKNI(privTranslationOut[dirKNI], privKNI))
-			fromPrivKNI = flow.SetReceiverKNI(privKNI)
-		}
-
-		// Merge traffic coming from public KNI with translated
-		// traffic from private side
-		if fromPubKNI != nil {
-			toPub, err = flow.SetMerger(fromPubKNI, privTranslationOut[dirSEND])
-			flow.CheckFatal(err)
-		} else {
-			toPub = privTranslationOut[dirSEND]
-		}
-
-		// Merge traffic coming from private KNI with translated
-		// traffic from public side
-		if fromPrivKNI != nil {
-			toPriv, err = flow.SetMerger(fromPrivKNI, pubTranslationOut[dirSEND])
-			flow.CheckFatal(err)
-		} else {
-			toPriv = pubTranslationOut[dirSEND]
-		}
-
-		// Set senders to output packets
-		err = flow.SetSender(toPriv, pp.PrivatePort.Index)
-		flow.CheckFatal(err)
-		err = flow.SetSender(toPub, pp.PublicPort.Index)
-		flow.CheckFatal(err)
-	}
-}
-
-func CheckHWOffloading() bool {
-	ports := []uint16{}
-
-	for i := range Natconfig.PortPairs {
-		pp := &Natconfig.PortPairs[i]
-		ports = append(ports, pp.PublicPort.Index, pp.PrivatePort.Index)
-	}
-
-	return flow.CheckHWCapability(flow.HWTXChecksumCapability, ports)
-}
-
-func (c *Config) getPortAndPairByID(portId uint32) (*ipPort, *portPair) {
-	for i := range c.PortPairs {
-		pp := &c.PortPairs[i]
-		if uint32(pp.PublicPort.Index) == portId {
-			return &pp.PublicPort, pp
-		}
-		if uint32(pp.PrivatePort.Index) == portId {
-			return &pp.PrivatePort, pp
-		}
-	}
-	return nil, nil
-}
diff --git a/examples/nat/dhcp.go b/examples/nat/dhcp.go
deleted file mode 100644
index 3a30eec5..00000000
--- a/examples/nat/dhcp.go
+++ /dev/null
@@ -1,236 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"math/rand"
-	"net"
-	"time"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-type dhcpState struct {
-	lastDHCPPacketTypeSent layers.DHCPMsgType
-	dhcpTransactionId      uint32
-}
-
-const (
-	requestInterval = 10 * time.Second
-	DHCPServerPort  = 67
-	DHCPClientPort  = 68
-	BroadcastIPv4   = uint32(0xffffffff)
-)
-
-var (
-	rnd          = rand.New(rand.NewSource(time.Now().UnixNano()))
-	BroadcastMAC = [common.EtherAddrLen]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
-	dhcpOptions  = []layers.DHCPOption{
-		layers.NewDHCPOption(layers.DHCPOptParamsRequest,
-			[]byte{
-				byte(layers.DHCPOptSubnetMask),
-				byte(layers.DHCPOptBroadcastAddr),
-				byte(layers.DHCPOptTimeOffset),
-				byte(layers.DHCPOptRouter),
-				byte(layers.DHCPOptDomainName),
-				byte(layers.DHCPOptDNS),
-				byte(layers.DHCPOptDomainSearch),
-				byte(layers.DHCPOptHostname),
-				byte(layers.DHCPOptInterfaceMTU),
-			},
-		),
-	}
-	dhcpRequestPacket = layers.DHCPv4{
-		Operation:    layers.DHCPOpRequest,
-		HardwareType: layers.LinkTypeEthernet,
-		ClientIP: net.IP{
-			0, 0, 0, 0,
-		},
-		YourClientIP: net.IP{
-			0, 0, 0, 0,
-		},
-		NextServerIP: net.IP{
-			0, 0, 0, 0,
-		},
-		RelayAgentIP: net.IP{
-			0, 0, 0, 0,
-		},
-	}
-)
-
-func StartDHCPClient() {
-	go func() {
-		sendDHCPRequests()
-	}()
-}
-
-func sendDHCPRequests() {
-	// Endless loop of sending DHCP requests
-	for {
-		for i := range Natconfig.PortPairs {
-			pp := &Natconfig.PortPairs[i]
-			if !pp.PublicPort.Subnet.addressAcquired {
-				pp.PublicPort.sendDHCPDiscoverRequest()
-			}
-			if !pp.PublicPort.Subnet6.addressAcquired {
-				pp.PublicPort.setLinkLocalIPv6KNIAddress(pp.PublicPort.Subnet6.llAddr, SingleIPMask)
-				pp.PublicPort.sendDHCPv6SolicitRequest()
-			}
-			if !pp.PrivatePort.Subnet.addressAcquired {
-				pp.PrivatePort.sendDHCPDiscoverRequest()
-			}
-			if !pp.PrivatePort.Subnet6.addressAcquired {
-				pp.PrivatePort.setLinkLocalIPv6KNIAddress(pp.PrivatePort.Subnet6.llAddr, SingleIPMask)
-				pp.PrivatePort.sendDHCPv6SolicitRequest()
-			}
-		}
-		time.Sleep(requestInterval)
-	}
-}
-
-func getDHCPOption(dhcp *layers.DHCPv4, optionType layers.DHCPOpt) *layers.DHCPOption {
-	for i := range dhcp.Options {
-		if dhcp.Options[i].Type == optionType {
-			return &dhcp.Options[i]
-		}
-	}
-	return nil
-}
-
-func (port *ipPort) composeAndSendDHCPPacket(packetType layers.DHCPMsgType, options []layers.DHCPOption) {
-	hwa := make([]byte, common.EtherAddrLen)
-	copy(hwa, port.SrcMACAddress[:])
-
-	buf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-
-	// Make local copy for modifications
-	dhcp := dhcpRequestPacket
-	dhcp.Xid = port.Subnet.ds.dhcpTransactionId
-	dhcp.ClientHWAddr = hwa
-	options = append(options,
-		layers.NewDHCPOption(layers.DHCPOptMessageType, []byte{byte(packetType)}),
-		layers.NewDHCPOption(layers.DHCPOptHostname, []byte(Natconfig.HostName)))
-	dhcp.Options = options
-	err := gopacket.SerializeLayers(buf, opts, &dhcp)
-	if err != nil {
-		common.LogFatal(common.No, err)
-	}
-
-	// Convert gopacket data structure into NFF-Go packet and send it
-	pkt, err := packet.NewPacket()
-	if err != nil {
-		println(err)
-	}
-	payloadBuffer := buf.Bytes()
-	packet.InitEmptyIPv4UDPPacket(pkt, uint(len(payloadBuffer)))
-
-	// Fill up L2
-	pkt.Ether.SAddr = port.SrcMACAddress
-	pkt.Ether.DAddr = BroadcastMAC
-
-	// Fill up L3
-	pkt.GetIPv4NoCheck().SrcAddr = uint32(0)
-	pkt.GetIPv4NoCheck().DstAddr = BroadcastIPv4
-
-	// Fill up L4
-	pkt.GetUDPNoCheck().SrcPort = packet.SwapBytesUint16(DHCPClientPort)
-	pkt.GetUDPNoCheck().DstPort = packet.SwapBytesUint16(DHCPServerPort)
-
-	payload, _ := pkt.GetPacketPayload()
-	copy(payload, payloadBuffer)
-
-	if port.Vlan != 0 {
-		pkt.AddVLANTag(port.Vlan)
-	}
-
-	setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	port.dumpPacket(pkt, dirSEND)
-	pkt.SendPacket(port.Index)
-
-	port.Subnet.ds.lastDHCPPacketTypeSent = packetType
-}
-
-func (port *ipPort) sendDHCPDiscoverRequest() {
-	port.Subnet.ds.dhcpTransactionId = rnd.Uint32()
-	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeDiscover, dhcpOptions)
-}
-
-func (port *ipPort) sendDHCPRequestRequest(serverIP, clientIP []byte) {
-	port.composeAndSendDHCPPacket(layers.DHCPMsgTypeRequest, append(dhcpOptions,
-		layers.NewDHCPOption(layers.DHCPOptServerID, serverIP),
-		layers.NewDHCPOption(layers.DHCPOptRequestIP, clientIP)))
-}
-
-func (port *ipPort) handleDHCP(pkt *packet.Packet) bool {
-	if port.Subnet.addressAcquired {
-		// Port already has address, ignore this traffic
-		return false
-	}
-
-	// Check that this is DHCP offer or acknowledgement traffic
-	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPClientPort) ||
-		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPServerPort) {
-		return false
-	}
-
-	var dhcp layers.DHCPv4
-	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeDHCPv4, &dhcp)
-	payload, _ := pkt.GetPacketPayload()
-	decoded := []gopacket.LayerType{}
-	err := parser.DecodeLayers(payload, &decoded)
-
-	if err != nil || len(decoded) != 1 || decoded[0] != layers.LayerTypeDHCPv4 {
-		println("Warning! Failed to parse DHCP packet", err)
-		return false
-	}
-
-	dhcpMessageType := getDHCPOption(&dhcp, layers.DHCPOptMessageType)
-	if dhcpMessageType == nil {
-		println("Warning! DHCP packet without message type received")
-		return false
-	}
-
-	if port.Subnet.ds.lastDHCPPacketTypeSent == layers.DHCPMsgTypeDiscover &&
-		dhcpMessageType.Data[0] == byte(layers.DHCPMsgTypeOffer) {
-		port.handleDHCPOffer(pkt, &dhcp)
-	} else if port.Subnet.ds.lastDHCPPacketTypeSent == layers.DHCPMsgTypeRequest &&
-		dhcpMessageType.Data[0] == byte(layers.DHCPMsgTypeAck) {
-		port.handleDHCPAck(pkt, &dhcp)
-	} else {
-		println("Warning! Received some bad response from DHCP server. Trying again with discover request.")
-		port.Subnet.addressAcquired = false
-		port.Subnet.ds = dhcpState{}
-	}
-	return true
-}
-
-func (port *ipPort) handleDHCPOffer(pkt *packet.Packet, dhcp *layers.DHCPv4) {
-	port.sendDHCPRequestRequest(dhcp.NextServerIP, dhcp.YourClientIP)
-}
-
-func (port *ipPort) handleDHCPAck(pkt *packet.Packet, dhcp *layers.DHCPv4) {
-	maskOption := getDHCPOption(dhcp, layers.DHCPOptSubnetMask)
-	if maskOption == nil {
-		println("Warning! Received a DHCP response without subnet mask! Trying again with discover request.")
-		port.Subnet.addressAcquired = false
-		port.Subnet.ds = dhcpState{}
-		return
-	}
-	port.Subnet.Addr, _ = convertIPv4(dhcp.YourClientIP.To4())
-	port.Subnet.Mask, _ = convertIPv4(maskOption.Data)
-	port.Subnet.addressAcquired = true
-	println("Successfully acquired IP address:", port.Subnet.String(), "on port", port.Index)
-
-	// Set address on KNI interface if present
-	port.setLinkLocalIPv4KNIAddress(port.Subnet.Addr, port.Subnet.Mask)
-}
diff --git a/examples/nat/dhcp6.go b/examples/nat/dhcp6.go
deleted file mode 100644
index ce123f70..00000000
--- a/examples/nat/dhcp6.go
+++ /dev/null
@@ -1,456 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"encoding/binary"
-	"errors"
-	"net"
-	"strings"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-type dhcpv6State struct {
-	lastDHCPv6PacketTypeSent layers.DHCPv6MsgType
-	dhcpv6TransactionId      [3]byte
-}
-
-const (
-	DHCPv6ClientPort = 546
-	DHCPv6ServerPort = 547
-)
-
-var (
-	BroadcastIPv6 = [common.IPv6AddrLen]uint8{
-		0xff, 0x02, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x00, 0x00, 0x00,
-		0x00, 0x01, 0x00, 0x02,
-	}
-	SingleIPMask = [common.IPv6AddrLen]uint8{
-		0xff, 0xff, 0xff, 0xff,
-		0xff, 0xff, 0xff, 0xff,
-		0xff, 0xff, 0xff, 0xff,
-		0xff, 0xff, 0xff, 0xff,
-	}
-	SingleIPNetMask = net.CIDRMask(128, 128)
-	hardwareTypeId  = []byte{0, 3}
-)
-
-func getDHCPv6Option(options layers.DHCPv6Options, optionType layers.DHCPv6Opt) *layers.DHCPv6Option {
-	for i := range options {
-		if options[i].Code == optionType {
-			return &options[i]
-		}
-	}
-	return nil
-}
-
-func (port *ipPort) composeAndSendDHCPv6Packet(packetType layers.DHCPv6MsgType, options []layers.DHCPv6Option) {
-	dhcpv6 := &layers.DHCPv6{
-		MsgType: packetType,
-	}
-	copy(dhcpv6.TransactionID, port.Subnet6.ds.dhcpv6TransactionId[:])
-
-	dhcpv6.Options = append(dhcpv6.Options, options...)
-	// Add client ID and FQDN options
-	clientID := &layers.DHCPv6DUID{
-		Type:         layers.DHCPv6DUIDTypeLL,
-		HardwareType: hardwareTypeId,
-	}
-	clientID.LinkLayerAddress = make([]byte, len(port.SrcMACAddress))
-	copy(clientID.LinkLayerAddress, port.SrcMACAddress[:])
-	fqdn := DHCPv6FQDN{
-		DomainName: Natconfig.HostName,
-	}
-	dhcpv6.Options = append(dhcpv6.Options,
-		layers.NewDHCPv6Option(layers.DHCPv6OptClientID, clientID.Encode()),
-		layers.NewDHCPv6Option(DHCPv6OptFQDNOptionCode, fqdn.Encode()))
-
-	buf := gopacket.NewSerializeBuffer()
-	opts := gopacket.SerializeOptions{
-		FixLengths:       true,
-		ComputeChecksums: true,
-	}
-	err := gopacket.SerializeLayers(buf, opts, dhcpv6)
-	if err != nil {
-		common.LogFatal(common.No, err)
-	}
-
-	// Convert gopacket data structure into NFF-Go packet and send it
-	pkt, err := packet.NewPacket()
-	if err != nil {
-		println(err)
-	}
-	payloadBuffer := buf.Bytes()
-	packet.InitEmptyIPv6UDPPacket(pkt, uint(len(payloadBuffer)))
-
-	// Fill up L2
-	pkt.Ether.SAddr = port.SrcMACAddress
-	packet.CalculateIPv6BroadcastMACForDstMulticastIP(&pkt.Ether.DAddr, BroadcastIPv6)
-
-	// Fill up L3
-	ipv6 := pkt.GetIPv6NoCheck()
-	ipv6.SrcAddr = port.Subnet6.llAddr
-	ipv6.DstAddr = BroadcastIPv6
-	ipv6.HopLimits = 1
-
-	// Fill up L4
-	udp := pkt.GetUDPNoCheck()
-	udp.SrcPort = packet.SwapBytesUint16(DHCPv6ClientPort)
-	udp.DstPort = packet.SwapBytesUint16(DHCPv6ServerPort)
-
-	// Fill up L7
-	payload, _ := pkt.GetPacketPayload()
-	copy(payload, payloadBuffer)
-
-	if port.Vlan != 0 {
-		pkt.AddVLANTag(port.Vlan)
-	}
-
-	setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-	port.dumpPacket(pkt, dirSEND)
-	pkt.SendPacket(port.Index)
-
-	port.Subnet6.ds.lastDHCPv6PacketTypeSent = packetType
-}
-
-func (port *ipPort) checkDHCPv6ServerAnswerAndGetIANA(dhcpv6 *layers.DHCPv6) *layers.DHCPv6Option {
-	statusOption := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptStatusCode)
-	if statusOption == nil {
-		println("Warning! Received a DHCPv6 reply without status! Trying again with discover request.")
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return nil
-	}
-
-	status := DHCPv6ServerStatusCode{}
-	err := status.DecodeFromBytes(statusOption.Data)
-	if err != nil {
-		println("Warning! Bad reply from server. Cannot decode status option: ", err)
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return nil
-	}
-
-	if layers.DHCPv6StatusCode(status.StatusCode) != layers.DHCPv6StatusCodeSuccess {
-		println("Warning! Server returned status", layers.DHCPv6StatusCode(status.StatusCode).String(), status.StatusMessage)
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return nil
-	}
-
-	// Ignore multiple IA_NA options in DHCP server reply. Use the first one.
-	ianaOption := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptIANA)
-	if ianaOption == nil {
-		println("Warning! Received a DHCPv6 reply without IANA! Trying again with discover request.")
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return nil
-	}
-
-	return ianaOption
-}
-
-func (port *ipPort) sendDHCPv6SolicitRequest() {
-	// Create new transaction ID
-	port.Subnet6.ds.dhcpv6TransactionId = [3]byte{
-		uint8(rnd.Uint32()),
-		uint8(rnd.Uint32()),
-		uint8(rnd.Uint32()),
-	}
-	iana := DHCPv6IANA{
-		IAID:    rnd.Uint32(),
-		Options: layers.DHCPv6Options{},
-	}
-	port.composeAndSendDHCPv6Packet(layers.DHCPv6MsgTypeSolicit,
-		[]layers.DHCPv6Option{layers.NewDHCPv6Option(layers.DHCPv6OptIANA, iana.Encode())})
-}
-
-func (port *ipPort) handleDHCPv6Advertise(pkt *packet.Packet, dhcpv6 *layers.DHCPv6) {
-	// Send request packet
-	ianaOption := port.checkDHCPv6ServerAnswerAndGetIANA(dhcpv6)
-	if ianaOption == nil {
-		println("Warning! No IANA option in server Advertise packet!")
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return
-	}
-	options := []layers.DHCPv6Option{*ianaOption}
-	serverID := getDHCPv6Option(dhcpv6.Options, layers.DHCPv6OptServerID)
-	if serverID != nil {
-		options = append(options, *serverID)
-	}
-	port.composeAndSendDHCPv6Packet(layers.DHCPv6MsgTypeRequest, options)
-}
-
-func (port *ipPort) handleDHCPv6(pkt *packet.Packet) bool {
-	if port.Subnet6.addressAcquired {
-		// Port already has address, ignore this traffic
-		return false
-	}
-
-	// Check that this is DHCP offer or acknowledgement traffic
-	if pkt.GetUDPNoCheck().DstPort != packet.SwapBytesUint16(DHCPv6ClientPort) ||
-		pkt.GetUDPNoCheck().SrcPort != packet.SwapBytesUint16(DHCPv6ServerPort) {
-		return false
-	}
-
-	var dhcpv6 layers.DHCPv6
-	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeDHCPv6, &dhcpv6)
-	payload, _ := pkt.GetPacketPayload()
-	decoded := []gopacket.LayerType{}
-	err := parser.DecodeLayers(payload, &decoded)
-
-	if err != nil || len(decoded) != 1 || decoded[0] != layers.LayerTypeDHCPv6 {
-		println("Warning! Failed to parse DHCPv6 packet", err)
-		return false
-	}
-
-	if port.Subnet6.ds.lastDHCPv6PacketTypeSent == layers.DHCPv6MsgTypeSolicit && dhcpv6.MsgType == layers.DHCPv6MsgTypeAdverstise {
-		port.handleDHCPv6Advertise(pkt, &dhcpv6)
-	} else if port.Subnet6.ds.lastDHCPv6PacketTypeSent == layers.DHCPv6MsgTypeRequest && dhcpv6.MsgType == layers.DHCPv6MsgTypeReply {
-		port.handleDHCPv6Reply(pkt, &dhcpv6)
-	} else {
-		println("Warning! Received some bad response from DHCPv6 server", dhcpv6.MsgType.String())
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-	}
-	return true
-}
-
-func (port *ipPort) handleDHCPv6Reply(pkt *packet.Packet, dhcpv6 *layers.DHCPv6) {
-	ianaOption := port.checkDHCPv6ServerAnswerAndGetIANA(dhcpv6)
-	if ianaOption == nil {
-		println("Warning! No IANA option in server Reply packet!")
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return
-	}
-
-	var iana DHCPv6IANA
-	err := iana.DecodeFromBytes(ianaOption.Data)
-	if err != nil {
-		println("Warning! Bad reply from server. Cannot decode IANA option: ", err)
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return
-	}
-
-	// Ignore multiple addresses in DHCP server reply. Use the first one.
-	addressOption := getDHCPv6Option(iana.Options, layers.DHCPv6OptIAAddr)
-	if addressOption == nil {
-		println("Warning! Received a DHCPv6 reply with IANA which contains no address! Trying again with discover request.")
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return
-	}
-
-	var ia DHCPv6IAAddress
-	err = ia.DecodeFromBytes(addressOption.Data)
-	if err != nil {
-		println("Warning! Bad reply from server. Cannot decode IA Address option: ", err)
-		port.Subnet6.addressAcquired = false
-		port.Subnet6.ds = dhcpv6State{}
-		return
-	}
-
-	copy(port.Subnet6.Addr[:], ia.Address.To16())
-	packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
-	port.Subnet6.Mask = SingleIPMask
-	port.Subnet6.addressAcquired = true
-	println("Successfully acquired IP address:", port.Subnet6.String(), "on port", port.Index)
-
-	// Set address on KNI interface if present
-	port.setLinkLocalIPv6KNIAddress(port.Subnet6.Addr, port.Subnet6.Mask)
-}
-
-type DHCPv6FQDNFlags byte
-
-const (
-	DHCPv6OptFQDNOptionCode                layers.DHCPv6Opt = 39
-	DHCPv6FQDNOptionServerUpdateForwardDNS DHCPv6FQDNFlags  = 1
-	DHCPv6FQDNOptionServerOverride         DHCPv6FQDNFlags  = 2
-	DHCPv6FQDNOptionServerNoDNSUpdate      DHCPv6FQDNFlags  = 4
-)
-
-// FQDN option encoded according to RFC 4704
-type DHCPv6FQDN struct {
-	Flags      DHCPv6FQDNFlags
-	DomainName string
-}
-
-func (fqdn *DHCPv6FQDN) Encode() []byte {
-	encoded := []byte{
-		byte(fqdn.Flags),
-	}
-	names := strings.Split(fqdn.DomainName, ".")
-	for i := range names {
-		b := []byte(names[i])
-		encoded = append(encoded, byte(len(b)))
-		encoded = append(encoded, b...)
-	}
-	encoded = append(encoded, byte(0))
-	return encoded
-}
-
-func (fqdn *DHCPv6FQDN) DecodeFromBytes(data []byte) error {
-	datalength := len(data)
-	if datalength < 2 {
-		return errors.New("Not enough bytes to decode: " + string(len(data)))
-	}
-
-	fqdn.Flags = DHCPv6FQDNFlags(data[0])
-
-	index := 2
-	fqdn.DomainName = ""
-	length := int(data[index-1])
-	for length != 0 {
-		if datalength-index < length+1 {
-			return errors.New("Option encoded incorrectly, not enough bytes to decode")
-		}
-		fqdn.DomainName += string(data[index : index+length])
-		index += length + 1
-		length = int(data[index-1])
-	}
-	return nil
-}
-
-// Server status code option
-type DHCPv6ServerStatusCode struct {
-	StatusCode    uint16
-	StatusMessage string
-}
-
-func (sc *DHCPv6ServerStatusCode) Encode() []byte {
-	messageBytes := []byte(sc.StatusMessage)
-	length := 2 + len(messageBytes)
-	data := make([]byte, length)
-	binary.BigEndian.PutUint16(data[0:2], uint16(sc.StatusCode))
-	copy(data[2:], messageBytes)
-	return data
-}
-
-func (sc *DHCPv6ServerStatusCode) DecodeFromBytes(data []byte) error {
-	if len(data) < 2 {
-		return errors.New("Not enough bytes to decode: " + string(len(data)))
-	}
-
-	sc.StatusCode = binary.BigEndian.Uint16(data[:2])
-	sc.StatusMessage = string(data[2:])
-	return nil
-}
-
-// Returns length of data portion of option (excluding IANA code 3 and
-// option length)
-func OptionsLen(options layers.DHCPv6Options) int {
-	n := 0
-	for _, o := range options {
-		n += int(o.Length) + 4
-	}
-	return n
-}
-
-// Identity Association for Non-temporary Addresses Option
-type DHCPv6IANA struct {
-	IAID    uint32
-	T1, T2  uint32
-	Options layers.DHCPv6Options
-}
-
-func (iana *DHCPv6IANA) Encode() []byte {
-	data := make([]byte, 12+OptionsLen(iana.Options))
-	binary.BigEndian.PutUint32(data[0:4], iana.IAID)
-	binary.BigEndian.PutUint32(data[4:8], iana.T1)
-	binary.BigEndian.PutUint32(data[8:12], iana.T2)
-	offset := 12
-
-	for _, o := range iana.Options {
-		// TODO: use (*DHCPv6Option) encode here
-		binary.BigEndian.PutUint16(data[offset:offset+2], uint16(o.Code))
-		binary.BigEndian.PutUint16(data[offset+2:offset+4], o.Length)
-		copy(data[offset+4:], o.Data)
-		offset += int(o.Length) + 4
-	}
-	return data
-}
-
-func (iana *DHCPv6IANA) DecodeFromBytes(data []byte) error {
-	if len(data) < 12 {
-		return errors.New("Not enough bytes to decode: " + string(len(data)))
-	}
-
-	iana.IAID = binary.BigEndian.Uint32(data[:4])
-	iana.T1 = binary.BigEndian.Uint32(data[4:8])
-	iana.T2 = binary.BigEndian.Uint32(data[8:12])
-	iana.Options = iana.Options[:0]
-	offset := 12
-
-	stop := len(data)
-	for offset < stop {
-		// TODO: use (*DHCPv6Option) decode here
-		o := layers.DHCPv6Option{}
-		o.Code = layers.DHCPv6Opt(binary.BigEndian.Uint16(data[offset : offset+2]))
-		o.Length = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-		o.Data = data[offset+4 : offset+4+int(o.Length)]
-		iana.Options = append(iana.Options, o)
-		offset += int(o.Length) + 4
-	}
-	return nil
-}
-
-// IA Address Option
-type DHCPv6IAAddress struct {
-	Address           net.IP
-	PreferredLifetime uint32
-	ValidLifetime     uint32
-	Options           layers.DHCPv6Options
-}
-
-func (ia *DHCPv6IAAddress) Encode() []byte {
-	data := make([]byte, 16+4+4+OptionsLen(ia.Options))
-	copy(data[:16], ia.Address.To16())
-	binary.BigEndian.PutUint32(data[16:20], ia.PreferredLifetime)
-	binary.BigEndian.PutUint32(data[20:24], ia.ValidLifetime)
-	offset := 24
-
-	for _, o := range ia.Options {
-		// TODO: use (*DHCPv6Option) encode here
-		binary.BigEndian.PutUint16(data[offset:offset+2], uint16(o.Code))
-		binary.BigEndian.PutUint16(data[offset+2:offset+4], o.Length)
-		copy(data[offset+4:], o.Data)
-		offset += int(o.Length) + 4
-	}
-	return data
-}
-
-func (ia *DHCPv6IAAddress) DecodeFromBytes(data []byte) error {
-	if len(data) < 24 {
-		return errors.New("Not enough bytes to decode: " + string(len(data)))
-	}
-
-	ia.Address = net.IP(data[:16])
-	ia.PreferredLifetime = binary.BigEndian.Uint32(data[16:20])
-	ia.ValidLifetime = binary.BigEndian.Uint32(data[20:24])
-	ia.Options = ia.Options[:0]
-	offset := 24
-
-	stop := len(data)
-	for offset < stop {
-		// TODO: use (*DHCPv6Option) decode here
-		o := layers.DHCPv6Option{}
-		o.Code = layers.DHCPv6Opt(binary.BigEndian.Uint16(data[offset : offset+2]))
-		o.Length = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-		o.Data = data[offset+4 : offset+4+int(o.Length)]
-		ia.Options = append(ia.Options, o)
-		offset += int(o.Length) + 4
-	}
-	return nil
-}
diff --git a/examples/nat/grpc.go b/examples/nat/grpc.go
deleted file mode 100644
index 600b0bfe..00000000
--- a/examples/nat/grpc.go
+++ /dev/null
@@ -1,121 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"fmt"
-	"net"
-
-	"golang.org/x/net/context"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/reflection"
-
-	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-const (
-	GRPCServerPort = ":60602"
-)
-
-type server struct{}
-
-func StartGRPCServer() error {
-	lis, err := net.Listen("tcp", GRPCServerPort)
-	if err != nil {
-		return err
-	}
-	s := grpc.NewServer()
-	upd.RegisterUpdaterServer(s, &server{})
-	// Register reflection service on gRPC server.
-	reflection.Register(s)
-
-	go func() {
-		if err := s.Serve(lis); err != nil {
-			common.LogWarning(common.Initialization, "Error while serving GRPC requests:", err)
-		}
-	}()
-	return nil
-}
-
-func (s *server) ControlDump(ctx context.Context, in *upd.DumpControlRequest) (*upd.Reply, error) {
-	enable := in.GetEnableTrace()
-	dumpType := in.GetTraceType()
-	if dumpType < upd.TraceType_DUMP_DROP || dumpType > upd.TraceType_DUMP_KNI {
-		return nil, fmt.Errorf("Bad value of dump type: %d", dumpType)
-	}
-	DumpEnabled[dumpType] = enable
-
-	return &upd.Reply{
-		Msg: "Success",
-	}, nil
-}
-
-func (s *server) ChangeInterfaceAddress(ctx context.Context, in *upd.InterfaceAddressChangeRequest) (*upd.Reply, error) {
-	portId := in.GetInterfaceId()
-	port, _ := Natconfig.getPortAndPairByID(portId)
-	if port == nil {
-		return nil, fmt.Errorf("Interface with ID %d not found", portId)
-	}
-	subnet4, subnet6, err := convertSubnet(in.GetPortSubnet())
-	if err != nil {
-		return nil, err
-	}
-	var str string
-	if subnet4 != nil {
-		port.Subnet.Addr = subnet4.Addr
-		port.Subnet.Mask = subnet4.Mask
-		port.Subnet.addressAcquired = true
-		port.setLinkLocalIPv4KNIAddress(port.Subnet.Addr, port.Subnet.Mask)
-		str = port.Subnet.String()
-	}
-	if subnet6 != nil {
-		port.Subnet6.Addr = subnet6.Addr
-		port.Subnet6.Mask = subnet6.Mask
-		port.Subnet6.addressAcquired = true
-		port.setLinkLocalIPv6KNIAddress(port.Subnet6.Addr, port.Subnet6.Mask)
-		packet.CalculateIPv6MulticastAddrForDstIP(&port.Subnet6.multicastAddr, port.Subnet6.Addr)
-		port.setLinkLocalIPv6KNIAddress(port.Subnet6.llAddr, SingleIPMask)
-		str = port.Subnet6.String()
-	}
-
-	return &upd.Reply{
-		Msg: fmt.Sprintf("Successfully set port %d subnet to %s", portId, str),
-	}, nil
-}
-
-func (s *server) ChangePortForwarding(ctx context.Context, in *upd.PortForwardingChangeRequest) (*upd.Reply, error) {
-	portId := in.GetInterfaceId()
-	port, pp := Natconfig.getPortAndPairByID(portId)
-	if port == nil {
-		return nil, fmt.Errorf("Interface with ID %d not found", portId)
-	}
-
-	fp, err := convertForwardedPort(in.GetPort())
-	if err != nil {
-		return nil, err
-	}
-	err = port.checkPortForwarding(fp)
-	if err != nil {
-		return nil, err
-	}
-
-	pp.mutex.Lock()
-	if port.Type == iPUBLIC {
-		pp.deleteOldConnection(fp.Protocol.ipv6, fp.Protocol.id, int(fp.Port))
-	} else {
-		port.deletePortForwardingEntry(fp.Protocol.ipv6, fp.Protocol.id, int(fp.Port))
-	}
-	if in.GetEnableForwarding() {
-		port.enableStaticPortForward(fp)
-	}
-	pp.mutex.Unlock()
-
-	return &upd.Reply{
-		Msg: "Success",
-	}, nil
-}
diff --git a/examples/nat/icmp.go b/examples/nat/icmp.go
deleted file mode 100644
index 25404435..00000000
--- a/examples/nat/icmp.go
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"time"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-func (port *ipPort) handleICMP(protocol uint8, pkt *packet.Packet, key interface{}) uint {
-	// Check that received ICMP packet is addressed at this host. If
-	// not, packet should be translated
-	var requestCode uint8
-	var packetSentToUs bool
-	var packetSentToMulticast bool
-	if protocol == common.ICMPNumber {
-		if packet.SwapBytesUint32(pkt.GetIPv4NoCheck().DstAddr) == port.Subnet.Addr {
-			packetSentToUs = true
-		}
-		requestCode = common.ICMPTypeEchoRequest
-	} else {
-		// If message is not targeted at NAT host, it is subject of
-		// address translation
-		ipv6 := pkt.GetIPv6NoCheck()
-		if ipv6.DstAddr == port.Subnet6.Addr ||
-			ipv6.DstAddr == port.Subnet6.llAddr {
-			packetSentToUs = true
-		} else if ipv6.DstAddr == port.Subnet6.multicastAddr ||
-			ipv6.DstAddr == port.Subnet6.llMulticastAddr {
-			packetSentToMulticast = true
-		}
-		requestCode = common.ICMPv6TypeEchoRequest
-	}
-
-	// Check IPv6 Neighbor Discovery first. If packet is handled, it
-	// returns DROP or KNI, otherwise continue to process it.
-	if packetSentToMulticast || (packetSentToUs && protocol == common.ICMPv6Number) {
-		dir := port.handleIPv6NeighborDiscovery(pkt)
-		if dir != dirSEND {
-			return dir
-		}
-	}
-
-	icmp := pkt.GetICMPNoCheck()
-
-	// If there is KNI interface, direct all ICMP traffic which
-	// doesn't have an active translation entry. It may happen only
-	// for public->private translation when all packets are directed
-	// to NAT public interface IP, so port.portmap exists because port
-	// is public.
-	if packetSentToUs && port.KNIName != "" {
-		if key != nil {
-			_, ok := port.translationTable[protocol].Load(key)
-			if !ok || time.Since(port.getPortmap(protocol == common.ICMPv6Number, protocol)[icmp.Identifier].lastused) > connectionTimeout {
-				return dirKNI
-			}
-		}
-	}
-
-	// If packet sent to us, check that it is ICMP echo request. NAT
-	// does not support any other messages yet, so process them in
-	// normal way. Maybe these are packets which should be passed
-	// through translation.
-	if !packetSentToUs || icmp.Type != requestCode || icmp.Code != 0 {
-		return dirSEND
-	}
-
-	// Return a packet back to sender
-	answerPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-	packet.GeneratePacketFromByte(answerPacket, pkt.GetRawPacketBytes())
-
-	answerPacket.ParseL3CheckVLAN()
-	if protocol == common.ICMPNumber {
-		swapAddrIPv4(answerPacket)
-		answerPacket.ParseL4ForIPv4()
-		(answerPacket.GetICMPNoCheck()).Type = common.ICMPTypeEchoResponse
-		setIPv4ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
-	} else {
-		swapAddrIPv6(answerPacket)
-		answerPacket.ParseL4ForIPv6()
-		(answerPacket.GetICMPNoCheck()).Type = common.ICMPv6TypeEchoResponse
-		setIPv6ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
-	}
-
-	port.dumpPacket(answerPacket, dirSEND)
-	answerPacket.SendPacket(port.Index)
-	return dirDROP
-}
diff --git a/examples/nat/main/.gitignore b/examples/nat/main/.gitignore
deleted file mode 100644
index 911b1372..00000000
--- a/examples/nat/main/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-nat
diff --git a/examples/nat/main/Dockerfile b/examples/nat/main/Dockerfile
deleted file mode 100644
index 8edd3be6..00000000
--- a/examples/nat/main/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright 2017 Intel Corporation.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-ARG USER_NAME
-FROM ${USER_NAME}/nff-go-base
-
-LABEL RUN docker run -it --privileged -v /sys/bus/pci/drivers:/sys/bus/pci/drivers -v /sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages -v /sys/devices/system/node:/sys/devices/system/node -v /dev:/dev --name NAME -e NAME=NAME -e IMAGE=IMAGE IMAGE
-
-RUN dnf -y install iputils httpd wget; dnf clean all
-
-WORKDIR /workdir
-COPY nat .
-COPY config.json .
-COPY config2ports.json .
-COPY config-vlan.json .
diff --git a/examples/nat/main/Makefile b/examples/nat/main/Makefile
deleted file mode 100644
index 502f4a47..00000000
--- a/examples/nat/main/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2017-2018 Intel Corporation.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-PATH_TO_MK = ../../../mk
-IMAGENAME = nff-go-nat
-EXECUTABLES = nat
-
-nat: ../config.go ../translation.go ../portalloc.go ../cksum.go ../util.go ../arp.go ../icmp.go ../grpc.go ../dhcp.go ../neigh.go ../updatecfg/updatecfg.pb.go
-
-../updatecfg/updatecfg.pb.go: ../updatecfg/updatecfg.proto
-	protoc -I ../updatecfg --go_out=plugins=grpc:../updatecfg ../updatecfg/updatecfg.proto
-
-include $(PATH_TO_MK)/leaf.mk
diff --git a/examples/nat/main/config-dhcp.json b/examples/nat/main/config-dhcp.json
deleted file mode 100644
index 24cd0913..00000000
--- a/examples/nat/main/config-dhcp.json
+++ /dev/null
@@ -1,59 +0,0 @@
-{
-    "host-name": "nat",
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 0,
-                "subnet": "dhcp",
-                "subnet6": "dhcp"
-            },
-            "public-port": {
-                "index": 1,
-                "subnet": "dhcp",
-                "subnet6": "dhcp",
-                "forward-ports": [
-                    {
-                        "port": 8080,
-                        "destination": "192.168.14.3:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "[fd14::3]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "192.168.14.3:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "[fd14::3]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "192.168.14.4:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "[fd14::4]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "192.168.14.4:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "[fd14::4]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/config-kni-dhcp.json b/examples/nat/main/config-kni-dhcp.json
deleted file mode 100644
index 9a156ba3..00000000
--- a/examples/nat/main/config-kni-dhcp.json
+++ /dev/null
@@ -1,83 +0,0 @@
-{
-    "host-name": "nat",
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 0,
-                "subnet": "dhcp",
-                "subnet6": "dhcp",
-                "kni-name": "priv0",
-                "forward-ports": [
-                    {
-                        "port": 22,
-                        "destination": "0.0.0.0:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 22,
-                        "destination": "[::]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            },
-            "public-port": {
-                "index": 1,
-                "subnet": "dhcp",
-                "subnet6": "dhcp",
-                "kni-name": "pub1",
-                "forward-ports": [
-                    {
-                        "port": 22,
-                        "destination": "0.0.0.0:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 22,
-                        "destination": "[::]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "192.168.14.3:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "[fd14::3]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "192.168.14.3:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "[fd14::3]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "192.168.14.4:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "[fd14::4]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "192.168.14.4:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "[fd14::4]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/config-kni.json b/examples/nat/main/config-kni.json
deleted file mode 100644
index 0a9676c7..00000000
--- a/examples/nat/main/config-kni.json
+++ /dev/null
@@ -1,82 +0,0 @@
-{
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 0,
-                "subnet": "192.168.14.1/24",
-                "subnet6": "fd14::1/64",
-                "kni-name": "priv0",
-                "forward-ports": [
-                    {
-                        "port": 22,
-                        "destination": "0.0.0.0:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 22,
-                        "destination": "[::]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            },
-            "public-port": {
-                "index": 1,
-                "subnet": "192.168.16.1/24",
-                "subnet6": "fd16::1/64",
-                "kni-name": "pub1",
-                "forward-ports": [
-                    {
-                        "port": 22,
-                        "destination": "0.0.0.0:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 22,
-                        "destination": "[::]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "192.168.14.3:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "[fd14::3]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "192.168.14.3:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "[fd14::3]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "192.168.14.4:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "[fd14::4]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "192.168.14.4:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "[fd14::4]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/config-vlan.json b/examples/nat/main/config-vlan.json
deleted file mode 100644
index f4f55e26..00000000
--- a/examples/nat/main/config-vlan.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 0,
-                "subnet": "192.168.114.1/24",
-                "vlan-tag": 114
-            },
-            "public-port": {
-                "index": 1,
-                "subnet": "192.168.116.1/24",
-                "vlan-tag": 116
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/config.json b/examples/nat/main/config.json
deleted file mode 100644
index d95cc1f7..00000000
--- a/examples/nat/main/config.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 0,
-                "subnet": "192.168.14.1/24",
-                "subnet6": "fd14::1/64"
-            },
-            "public-port": {
-                "index": 1,
-                "subnet": "192.168.16.1/24",
-                "subnet6": "fd16::1/64",
-                "forward-ports": [
-                    {
-                        "port": 8080,
-                        "destination": "192.168.14.3:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8080,
-                        "destination": "[fd14::3]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "192.168.14.3:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2222,
-                        "destination": "[fd14::3]:22",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "192.168.14.4:80",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 8081,
-                        "destination": "[fd14::4]:80",
-                        "protocol": "TCP6"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "192.168.14.4:22",
-                        "protocol": "TCP"
-                    },
-                    {
-                        "port": 2223,
-                        "destination": "[fd14::4]:22",
-                        "protocol": "TCP6"
-                    }
-                ]
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/config2ports.json b/examples/nat/main/config2ports.json
deleted file mode 100644
index 68dbb398..00000000
--- a/examples/nat/main/config2ports.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-    "port-pairs": [
-        {
-            "private-port": {
-                "index": 2,
-                "subnet": "192.168.14.1/24"
-            },
-            "public-port": {
-                "index": 3,
-                "subnet": "192.168.16.1/24"
-            }
-        },
-        {
-            "private-port": {
-                "index": 4,
-                "subnet": "192.168.44.1/24",
-                "vlan-tag": 144
-            },
-            "public-port": {
-                "index": 5,
-                "subnet": "192.168.46.1/24",
-                "vlan-tag": 146
-            }
-        }
-    ]
-}
diff --git a/examples/nat/main/nat.go b/examples/nat/main/nat.go
deleted file mode 100644
index 790c3be4..00000000
--- a/examples/nat/main/nat.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2017 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-	"os/signal"
-
-	"github.com/intel-go/nff-go/examples/nat"
-	"github.com/intel-go/nff-go/flow"
-)
-
-func main() {
-	// Parse arguments
-	cores := flag.String("cores", "", "Specify CPU cores to use")
-	configFile := flag.String("config", "config.json", "Specify config file name")
-	flag.BoolVar(&nat.NoCalculateChecksum, "nocsum", false, "Specify whether to calculate checksums in modified packets")
-	flag.BoolVar(&nat.NoHWTXChecksum, "nohwcsum", false, "Specify whether to use hardware offloading for checksums calculation (requires -csum)")
-	noscheduler := flag.Bool("no-scheduler", false, "disable scheduler")
-	dpdkLogLevel := flag.String("dpdk", "--log-level=0", "Passes an arbitrary argument to dpdk EAL")
-	flag.Parse()
-
-	// Set up reaction to SIGINT (Ctrl-C)
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-
-	// Read config
-	flow.CheckFatal(nat.ReadConfig(*configFile))
-
-	// Init NFF-GO system at 16 available cores
-	nffgoconfig := flow.Config{
-		CPUList:          *cores,
-		HWTXChecksum:     !nat.NoHWTXChecksum,
-		DPDKArgs:         []string{*dpdkLogLevel},
-		DisableScheduler: *noscheduler,
-		NeedKNI:          nat.NeedKNI,
-	}
-
-	flow.CheckFatal(flow.SystemInit(&nffgoconfig))
-
-	offloadingAvailable := nat.CheckHWOffloading()
-	if !nat.NoHWTXChecksum && !offloadingAvailable {
-		println("Warning! Requested hardware offloading is not available on all ports. Falling back to software checksum calculation.")
-		nat.NoHWTXChecksum = true
-		flow.SetUseHWCapability(flow.HWTXChecksumCapability, false)
-	}
-
-	// Initialize flows and necessary state
-	nat.InitFlows()
-
-	// Start GRPC server
-	flow.CheckFatal(nat.StartGRPCServer())
-
-	// Perform all network initialization so that DHCP client could
-	// start sending packets
-	flow.CheckFatal(flow.SystemInitPortsAndMemory())
-
-	// Start DHCP client
-	if nat.NeedDHCP {
-		nat.StartDHCPClient()
-	}
-
-	// Start flow scheduler
-	go func() {
-		flow.CheckFatal(flow.SystemStartScheduler())
-	}()
-
-	// Wait for interrupt
-	sig := <-c
-	fmt.Printf("Received signal %v\n", sig)
-	nat.CloseAllDumpFiles()
-}
diff --git a/examples/nat/neigh.go b/examples/nat/neigh.go
deleted file mode 100644
index 70b4991d..00000000
--- a/examples/nat/neigh.go
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-func (port *ipPort) handleIPv6NeighborDiscovery(pkt *packet.Packet) uint {
-	icmp := pkt.GetICMPNoCheck()
-	if icmp.Type == common.ICMPv6NeighborSolicitation {
-		// If there is KNI interface, forward all of this here
-		if port.KNIName != "" {
-			return dirKNI
-		}
-		pkt.ParseL7(common.ICMPv6Number)
-		msg := pkt.GetICMPv6NeighborSolicitationMessage()
-		if msg.TargetAddr != port.Subnet6.Addr && msg.TargetAddr != port.Subnet6.llAddr {
-			return dirDROP
-		}
-		option := pkt.GetICMPv6NDSourceLinkLayerAddressOption(packet.ICMPv6NeighborSolicitationMessageSize)
-		if option != nil && option.Type == packet.ICMPv6NDSourceLinkLayerAddress {
-			answerPacket, err := packet.NewPacket()
-			if err != nil {
-				common.LogFatal(common.Debug, err)
-			}
-
-			packet.InitICMPv6NeighborAdvertisementPacket(answerPacket, port.SrcMACAddress, option.LinkLayerAddress, msg.TargetAddr, pkt.GetIPv6NoCheck().SrcAddr)
-
-			vlan := pkt.GetVLAN()
-			if vlan != nil {
-				answerPacket.AddVLANTag(packet.SwapBytesUint16(vlan.TCI))
-			}
-
-			setIPv6ICMPChecksum(answerPacket, !NoCalculateChecksum, !NoHWTXChecksum)
-			port.dumpPacket(answerPacket, dirSEND)
-			answerPacket.SendPacket(port.Index)
-		}
-	} else if icmp.Type == common.ICMPv6NeighborAdvertisement {
-		pkt.ParseL7(common.ICMPv6Number)
-		msg := pkt.GetICMPv6NeighborAdvertisementMessage()
-		option := pkt.GetICMPv6NDTargetLinkLayerAddressOption(packet.ICMPv6NeighborAdvertisementMessageSize)
-		if option != nil && option.Type == packet.ICMPv6NDTargetLinkLayerAddress {
-			port.arpTable.Store(msg.TargetAddr, option.LinkLayerAddress)
-		}
-
-		if port.KNIName != "" {
-			return dirKNI
-		}
-	} else {
-		if port.KNIName != "" {
-			return dirKNI
-		}
-		return dirSEND
-	}
-
-	return dirDROP
-}
-
-func (port *ipPort) getMACForIPv6(ip [common.IPv6AddrLen]uint8) (macAddress, bool) {
-	v, found := port.arpTable.Load(ip)
-	if found {
-		return macAddress(v.([common.EtherAddrLen]byte)), true
-	}
-	port.sendNDNeighborSolicitationRequest(ip)
-	return macAddress{}, false
-}
-
-func (port *ipPort) sendNDNeighborSolicitationRequest(ip [common.IPv6AddrLen]uint8) {
-	requestPacket, err := packet.NewPacket()
-	if err != nil {
-		common.LogFatal(common.Debug, err)
-	}
-
-	packet.InitICMPv6NeighborSolicitationPacket(requestPacket, port.SrcMACAddress,
-		port.Subnet6.Addr, ip)
-
-	if port.Vlan != 0 {
-		requestPacket.AddVLANTag(port.Vlan)
-	}
-
-	setIPv6ICMPChecksum(requestPacket, !NoCalculateChecksum, !NoHWTXChecksum)
-	port.dumpPacket(requestPacket, dirSEND)
-	requestPacket.SendPacket(port.Index)
-}
diff --git a/examples/nat/portalloc.go b/examples/nat/portalloc.go
deleted file mode 100644
index 74d3ae58..00000000
--- a/examples/nat/portalloc.go
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2017 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"errors"
-	"strconv"
-	"time"
-)
-
-const (
-	numPubAddr = 1
-
-	portStart = 1024
-	portEnd   = 65500
-	numPorts  = portEnd - portStart
-)
-
-func (dir terminationDirection) String() string {
-	if dir == pub2pri {
-		return "pub2pri"
-	} else if dir == pri2pub {
-		return "pri2pub"
-	} else {
-		return "unknown(" + strconv.Itoa(int(dir)) + ")"
-	}
-}
-
-func (port *ipPort) deletePortForwardingEntry(ipv6 bool, protocol uint8, portNumber int) {
-	key := port.makePortAddrTuple(ipv6, uint16(portNumber))
-	port.translationTable[protocol].Delete(key)
-}
-
-func (pp *portPair) deleteOldConnection(ipv6 bool, protocol uint8, port int) {
-	pubTable := pp.PublicPort.translationTable[protocol]
-	pm := pp.getPublicPortPortmap(ipv6, protocol)
-
-	pub2priKey := pp.PublicPort.makePortAddrTuple(ipv6, uint16(port))
-	pri2pubKey, found := pubTable.Load(pub2priKey)
-
-	if found {
-		pp.PrivatePort.translationTable[protocol].Delete(pri2pubKey)
-		pubTable.Delete(pub2priKey)
-	}
-	pm[port] = portMapEntry{}
-}
-
-// This function currently is not thread safe and should be executed
-// under a global lock
-func (pp *portPair) allocNewPort(ipv6 bool, protocol uint8) (int, error) {
-	pm := pp.getPublicPortPortmap(ipv6, protocol)
-	for {
-		for p := pp.lastport; p < portEnd; p++ {
-			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
-				pp.lastport = p
-				pp.deleteOldConnection(ipv6, protocol, p)
-				return p, nil
-			}
-		}
-
-		for p := portStart; p < pp.lastport; p++ {
-			if !pm[p].static && time.Since(pm[p].lastused) > connectionTimeout {
-				pp.lastport = p
-				pp.deleteOldConnection(ipv6, protocol, p)
-				return p, nil
-			}
-		}
-		return 0, errors.New("WARNING! All ports are allocated! Trying again")
-	}
-}
-
-func (pp *portPair) getPublicPortPortmap(ipv6 bool, protocol uint8) []portMapEntry {
-	if ipv6 {
-		return pp.PublicPort.portmap6[protocol]
-	} else {
-		return pp.PublicPort.portmap[protocol]
-	}
-}
-
-func (port *ipPort) makePortAddrTuple(ipv6 bool, portNumber uint16) interface{} {
-	if ipv6 {
-		return Tuple6{
-			addr: port.Subnet6.Addr,
-			port: uint16(portNumber),
-		}
-	} else {
-		return Tuple{
-			addr: port.Subnet.Addr,
-			port: uint16(portNumber),
-		}
-	}
-}
diff --git a/examples/nat/translation.go b/examples/nat/translation.go
deleted file mode 100644
index f04ba5a9..00000000
--- a/examples/nat/translation.go
+++ /dev/null
@@ -1,443 +0,0 @@
-// Copyright 2017-2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"time"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/flow"
-	"github.com/intel-go/nff-go/packet"
-)
-
-// Tuple is a pair of address and port.
-type Tuple struct {
-	addr uint32
-	port uint16
-}
-
-type Tuple6 struct {
-	addr [common.IPv6AddrLen]uint8
-	port uint16
-}
-
-func (pp *portPair) allocateNewEgressConnection(ipv6 bool, protocol uint8, privEntry interface{}) (uint32, [common.IPv6AddrLen]uint8, uint16, error) {
-	pp.mutex.Lock()
-
-	port, err := pp.allocNewPort(ipv6, protocol)
-	if err != nil {
-		pp.mutex.Unlock()
-		return 0, [common.IPv6AddrLen]uint8{}, 0, err
-	}
-
-	var pubEntry interface{}
-	var v4addr uint32
-	var v6addr [common.IPv6AddrLen]uint8
-	if ipv6 {
-		v6addr = pp.PublicPort.Subnet6.Addr
-		pubEntry = Tuple6{
-			addr: v6addr,
-			port: uint16(port),
-		}
-	} else {
-		v4addr = pp.PublicPort.Subnet.Addr
-		pubEntry = Tuple{
-			addr: v4addr,
-			port: uint16(port),
-		}
-	}
-
-	pp.PublicPort.getPortmap(ipv6, protocol)[port] = portMapEntry{
-		lastused:             time.Now(),
-		finCount:             0,
-		terminationDirection: 0,
-		static:               false,
-	}
-
-	// Add lookup entries for packet translation
-	pp.PublicPort.translationTable[protocol].Store(pubEntry, privEntry)
-	pp.PrivatePort.translationTable[protocol].Store(privEntry, pubEntry)
-
-	pp.mutex.Unlock()
-	return v4addr, v6addr, uint16(port), nil
-}
-
-// PublicToPrivateTranslation does ingress translation.
-func PublicToPrivateTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
-	pi := ctx.(pairIndex)
-	pp := &Natconfig.PortPairs[pi.index]
-	port := &pp.PublicPort
-
-	port.dumpPacket(pkt, dirSEND)
-
-	// Parse packet type and address
-	dir, pktVLAN, pktIPv4, pktIPv6 := port.parsePacketAndCheckARP(pkt)
-	if pktIPv4 == nil && pktIPv6 == nil {
-		return dir
-	}
-
-	protocol, pktTCP, pktUDP, pktICMP, _, DstPort := ParseAllKnownL4(pkt, pktIPv4, pktIPv6)
-	if protocol == 0 {
-		// Only TCP, UDP and ICMP are supported now, all other protocols are ignored
-		port.dumpPacket(pkt, dirDROP)
-		return dirDROP
-	}
-	portNumber := DstPort
-	// Create a lookup key from packet destination address and port
-	pub2priKey := generateLookupKeyFromDstAddr(pkt, pktIPv4, pktIPv6, portNumber)
-	// Check for ICMP traffic first
-	if pktICMP != nil {
-		dir := port.handleICMP(protocol, pkt, pub2priKey)
-		if dir != dirSEND {
-			port.dumpPacket(pkt, dir)
-			return dir
-		}
-	}
-	ipv6 := pktIPv6 != nil
-	// Check for DHCP traffic. We need to get an address if it not set yet
-	if pktUDP != nil {
-		var handled bool
-		if ipv6 {
-			handled = port.handleDHCPv6(pkt)
-		} else {
-			handled = port.handleDHCP(pkt)
-		}
-		if handled {
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-	}
-
-	// Do lookup
-	v, found := port.translationTable[protocol].Load(pub2priKey)
-	kniPresent := port.KNIName != ""
-
-	if !found {
-		// Store new local network entry in ARP cache
-		var addressAcquired bool
-		if ipv6 {
-			port.arpTable.Store(pktIPv6.SrcAddr, pkt.Ether.SAddr)
-			addressAcquired = port.Subnet6.addressAcquired
-		} else {
-			port.arpTable.Store(pktIPv4.SrcAddr, pkt.Ether.SAddr)
-			addressAcquired = port.Subnet.addressAcquired
-		}
-
-		// For ingress connections packets are allowed only if a
-		// connection has been previosly established with a egress
-		// (private to public) packet. So if lookup fails, this
-		// incoming packet is ignored unless there is a KNI
-		// interface. If KNI is present and its IP address is known,
-		// traffic is directed there.
-		if kniPresent && addressAcquired {
-			dir = dirKNI
-		} else {
-			dir = dirDROP
-		}
-		port.dumpPacket(pkt, dir)
-		return dir
-	}
-	v4addr, v6addr, newPort, zeroAddr := getAddrFromTuple(v, ipv6)
-
-	portmap := port.getPortmap(ipv6, protocol)
-	// Check whether connection is too old
-	if portmap[portNumber].static || time.Since(portmap[portNumber].lastused) <= connectionTimeout {
-		portmap[portNumber].lastused = time.Now()
-	} else {
-		// There was no transfer on this port for too long
-		// time. We don't allow it any more
-		pp.mutex.Lock()
-		pp.deleteOldConnection(pktIPv6 != nil, protocol, int(portNumber))
-		pp.mutex.Unlock()
-		port.dumpPacket(pkt, dirDROP)
-		return dirDROP
-	}
-
-	if !zeroAddr {
-		// Check whether TCP connection could be reused
-		if pktTCP != nil && !portmap[portNumber].static {
-			pp.checkTCPTermination(ipv6, pktTCP, int(portNumber), pub2pri)
-		}
-
-		// Find corresponding MAC address
-		var mac macAddress
-		var found bool
-		if ipv6 {
-			mac, found = port.opposite.getMACForIPv6(v6addr)
-		} else {
-			mac, found = port.opposite.getMACForIPv4(v4addr)
-		}
-		if !found {
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-
-		// Do packet translation
-		pkt.Ether.DAddr = mac
-		pkt.Ether.SAddr = port.SrcMACAddress
-		if pktVLAN != nil {
-			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
-		}
-		if ipv6 {
-			pktIPv6.DstAddr = v6addr
-		} else {
-			pktIPv4.DstAddr = packet.SwapBytesUint32(v4addr)
-		}
-		setPacketDstPort(pkt, ipv6, newPort, pktTCP, pktUDP, pktICMP)
-
-		port.dumpPacket(pkt, dirSEND)
-		return dirSEND
-	} else {
-		port.dumpPacket(pkt, dirKNI)
-		return dirKNI
-	}
-}
-
-// PrivateToPublicTranslation does egress translation.
-func PrivateToPublicTranslation(pkt *packet.Packet, ctx flow.UserContext) uint {
-	pi := ctx.(pairIndex)
-	pp := &Natconfig.PortPairs[pi.index]
-	port := &pp.PrivatePort
-
-	port.dumpPacket(pkt, dirSEND)
-
-	// Parse packet type and address
-	dir, pktVLAN, pktIPv4, pktIPv6 := port.parsePacketAndCheckARP(pkt)
-	if pktIPv4 == nil && pktIPv6 == nil {
-		return dir
-	}
-
-	protocol, pktTCP, pktUDP, pktICMP, SrcPort, _ := ParseAllKnownL4(pkt, pktIPv4, pktIPv6)
-	if protocol == 0 {
-		// Only TCP, UDP and ICMP are supported now, all other protocols are ignored
-		port.dumpPacket(pkt, dirDROP)
-		return dirDROP
-	}
-	portNumber := SrcPort
-	// Create a lookup key from packet source address and port
-	pri2pubKey, saddr := generateLookupKeyFromSrcAddr(pkt, pktIPv4, pktIPv6, portNumber)
-	// Check for ICMP traffic first
-	if pktICMP != nil {
-		dir := port.handleICMP(protocol, pkt, pri2pubKey)
-		if dir != dirSEND {
-			port.dumpPacket(pkt, dir)
-			return dir
-		}
-	}
-	ipv6 := pktIPv6 != nil
-	// Check for DHCP traffic. We need to get an address if it not set yet
-	if pktUDP != nil {
-		var handled bool
-		if ipv6 {
-			handled = port.handleDHCPv6(pkt)
-		} else {
-			handled = port.handleDHCP(pkt)
-		}
-		if handled {
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-	}
-
-	kniPresent := port.KNIName != ""
-	var addressAcquired bool
-	var packetSentToUs bool
-	if ipv6 {
-		addressAcquired = port.Subnet6.addressAcquired
-		packetSentToUs = port.Subnet6.Addr == pktIPv6.DstAddr ||
-			port.Subnet6.llAddr == pktIPv6.DstAddr ||
-			port.Subnet6.multicastAddr == pktIPv6.DstAddr ||
-			port.Subnet6.llMulticastAddr == pktIPv6.DstAddr
-	} else {
-		addressAcquired = port.Subnet.addressAcquired
-		packetSentToUs = port.Subnet.Addr == packet.SwapBytesUint32(pktIPv4.DstAddr)
-	}
-
-	// If traffic is directed at private interface IP and KNI is
-	// present, this traffic is directed to KNI
-	if kniPresent && addressAcquired && packetSentToUs {
-		port.dumpPacket(pkt, dirKNI)
-		return dirKNI
-	}
-
-	// Do lookup
-	v, found := port.translationTable[protocol].Load(pri2pubKey)
-
-	var v4addr uint32
-	var v6addr [common.IPv6AddrLen]uint8
-	var newPort uint16
-	var zeroAddr bool
-
-	if !found {
-		var err error
-		// Store new local network entry in ARP cache
-		port.arpTable.Store(saddr, pkt.Ether.SAddr)
-
-		var publicAddressAcquired bool
-		if ipv6 {
-			publicAddressAcquired = port.opposite.Subnet6.addressAcquired
-		} else {
-			publicAddressAcquired = port.opposite.Subnet.addressAcquired
-		}
-
-		if !addressAcquired || !publicAddressAcquired {
-			// No packets are allowed yet because ports address is not
-			// known yet
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-		// Allocate new connection from private to public network
-		v4addr, v6addr, newPort, err = pp.allocateNewEgressConnection(pktIPv6 != nil, protocol, pri2pubKey)
-
-		if err != nil {
-			println("Warning! Failed to allocate new connection", err)
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-		zeroAddr = false
-	} else {
-		v4addr, v6addr, newPort, zeroAddr = getAddrFromTuple(v, ipv6)
-		pp.PublicPort.getPortmap(ipv6, protocol)[newPort].lastused = time.Now()
-	}
-
-	if !zeroAddr {
-		// Check whether TCP connection could be reused
-		if pktTCP != nil && !pp.PublicPort.getPortmap(ipv6, protocol)[newPort].static {
-			pp.checkTCPTermination(ipv6, pktTCP, int(newPort), pri2pub)
-		}
-
-		// Find corresponding MAC address
-		var mac macAddress
-		var found bool
-		if pktIPv6 != nil {
-			mac, found = port.opposite.getMACForIPv6(pktIPv6.DstAddr)
-		} else {
-			mac, found = port.opposite.getMACForIPv4(packet.SwapBytesUint32(pktIPv4.DstAddr))
-		}
-		if !found {
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP
-		}
-
-		// Do packet translation
-		pkt.Ether.DAddr = mac
-		pkt.Ether.SAddr = port.SrcMACAddress
-		if pktVLAN != nil {
-			pktVLAN.SetVLANTagIdentifier(port.opposite.Vlan)
-		}
-		if ipv6 {
-			pktIPv6.SrcAddr = v6addr
-		} else {
-			pktIPv4.SrcAddr = packet.SwapBytesUint32(v4addr)
-		}
-		setPacketSrcPort(pkt, ipv6, newPort, pktTCP, pktUDP, pktICMP)
-
-		port.dumpPacket(pkt, dirSEND)
-		return dirSEND
-	} else {
-		port.dumpPacket(pkt, dirKNI)
-		return dirKNI
-	}
-}
-
-// Used to generate key in public to private translation
-func generateLookupKeyFromDstAddr(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr, port uint16) interface{} {
-	if pktIPv4 != nil {
-		key := Tuple{
-			addr: packet.SwapBytesUint32(pktIPv4.DstAddr),
-			port: port,
-		}
-		return key
-	} else {
-		return Tuple6{
-			addr: pktIPv6.DstAddr,
-			port: port,
-		}
-	}
-}
-
-// Used to generate key in private to public translation
-func generateLookupKeyFromSrcAddr(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr, port uint16) (interface{}, interface{}) {
-	if pktIPv4 != nil {
-		saddr := packet.SwapBytesUint32(pktIPv4.SrcAddr)
-		return Tuple{
-			addr: saddr,
-			port: port,
-		}, saddr
-	} else {
-		return Tuple6{
-			addr: pktIPv6.SrcAddr,
-			port: port,
-		}, pktIPv6.SrcAddr
-	}
-}
-
-// Simple check for FIN or RST in TCP
-func (pp *portPair) checkTCPTermination(ipv6 bool, hdr *packet.TCPHdr, port int, dir terminationDirection) {
-	if hdr.TCPFlags&common.TCPFlagFin != 0 {
-		// First check for FIN
-		pp.mutex.Lock()
-
-		pme := &pp.PublicPort.portmap[common.TCPNumber][port]
-		if pme.finCount == 0 {
-			pme.finCount = 1
-			pme.terminationDirection = dir
-		} else if pme.finCount == 1 && pme.terminationDirection == ^dir {
-			pme.finCount = 2
-		}
-
-		pp.mutex.Unlock()
-	} else if hdr.TCPFlags&common.TCPFlagRst != 0 {
-		// RST means that connection is terminated immediately
-		pp.mutex.Lock()
-		pp.deleteOldConnection(ipv6, common.TCPNumber, port)
-		pp.mutex.Unlock()
-	} else if hdr.TCPFlags&common.TCPFlagAck != 0 {
-		// Check for ACK last so that if there is also FIN,
-		// termination doesn't happen. Last ACK should come without
-		// FIN
-		pp.mutex.Lock()
-
-		pme := &pp.PublicPort.portmap[common.TCPNumber][port]
-		if pme.finCount == 2 {
-			pp.deleteOldConnection(ipv6, common.TCPNumber, port)
-			// Set some time while port cannot be used before
-			// connection timeout is reached
-			pme.lastused = time.Now().Add(time.Duration(portReuseTimeout - connectionTimeout))
-		}
-
-		pp.mutex.Unlock()
-	}
-}
-
-func (port *ipPort) parsePacketAndCheckARP(pkt *packet.Packet) (dir uint, vlanhdr *packet.VLANHdr, ipv4hdr *packet.IPv4Hdr, ipv6hdr *packet.IPv6Hdr) {
-	pktVLAN := pkt.ParseL3CheckVLAN()
-	pktIPv4 := pkt.GetIPv4CheckVLAN()
-	if pktIPv4 == nil {
-		pktIPv6 := pkt.GetIPv6CheckVLAN()
-		if pktIPv6 == nil {
-			arp := pkt.GetARPCheckVLAN()
-			if arp != nil {
-				dir := port.handleARP(pkt)
-				port.dumpPacket(pkt, dir)
-				return dir, pktVLAN, nil, nil
-			}
-			port.dumpPacket(pkt, dirDROP)
-			return dirDROP, pktVLAN, nil, nil
-		}
-		return dirSEND, pktVLAN, nil, pktIPv6
-	}
-	return dirSEND, pktVLAN, pktIPv4, nil
-}
-
-func getAddrFromTuple(v interface{}, ipv6 bool) (uint32, [common.IPv6AddrLen]uint8, uint16, bool) {
-	if ipv6 {
-		value := v.(Tuple6)
-		return 0, value.addr, value.port, value.addr == [common.IPv6AddrLen]uint8{}
-	} else {
-		value := v.(Tuple)
-		return value.addr, [common.IPv6AddrLen]uint8{}, value.port, value.addr == 0
-	}
-}
diff --git a/examples/nat/updatecfg/.gitignore b/examples/nat/updatecfg/.gitignore
deleted file mode 100644
index fa6b8979..00000000
--- a/examples/nat/updatecfg/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-updatecfg.pb.go
diff --git a/examples/nat/updatecfg/updatecfg.proto b/examples/nat/updatecfg/updatecfg.proto
deleted file mode 100644
index 3884b9d6..00000000
--- a/examples/nat/updatecfg/updatecfg.proto
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-syntax = "proto3";
-
-option java_multiple_files = true;
-option java_outer_classname = "UpdateNatCfg";
-
-package updatecfg;
-
-service Updater {
-  rpc ControlDump (DumpControlRequest) returns (Reply) {}
-  rpc ChangeInterfaceAddress (InterfaceAddressChangeRequest) returns (Reply) {}
-  rpc ChangePortForwarding (PortForwardingChangeRequest) returns (Reply) {}
-}
-
-enum TraceType {
-  DUMP_DROP = 0;
-  DUMP_TRANSLATE = 1;
-  DUMP_KNI = 2;
-}
-
-message DumpControlRequest {
-  bool enable_trace = 1;
-  TraceType trace_type = 2;
-}
-
-enum Protocol {
-  UNKNOWN = 0;
-  TCP = 0x06;
-  UDP = 0x11;
-  IPv6_Flag = 0x10000;
-  TCP6 = 0x10006;
-  UDP6 = 0x10011;
-}
-
-message IPAddress {
-  bytes address = 1;
-}
-
-message Subnet {
-  IPAddress address = 1;
-  uint32 mask_bits_number = 2;
-}
-
-message InterfaceAddressChangeRequest {
-  uint32 interface_id = 1;
-  Subnet port_subnet = 2;
-}
-
-message ForwardedPort {
-  uint32 source_port_number = 1;
-  IPAddress target_address = 2;
-  uint32 target_port_number = 3;
-  Protocol protocol = 4;
-}
-
-message PortForwardingChangeRequest {
-  bool enable_forwarding = 1;
-  uint32 interface_id = 2;
-  ForwardedPort port = 3;
-}
-
-message Reply {
-  string msg = 2;
-}
diff --git a/examples/nat/util.go b/examples/nat/util.go
deleted file mode 100644
index 324711be..00000000
--- a/examples/nat/util.go
+++ /dev/null
@@ -1,286 +0,0 @@
-// Copyright 2017-2018 Intel Corporation.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package nat
-
-import (
-	"fmt"
-	"log"
-	"net"
-	"os"
-
-	"github.com/vishvananda/netlink"
-
-	upd "github.com/intel-go/nff-go/examples/nat/updatecfg"
-
-	"github.com/intel-go/nff-go/common"
-	"github.com/intel-go/nff-go/packet"
-)
-
-func (t *Tuple) String() string {
-	return fmt.Sprintf("addr = %d.%d.%d.%d:%d",
-		(t.addr>>24)&0xff,
-		(t.addr>>16)&0xff,
-		(t.addr>>8)&0xff,
-		t.addr&0xff,
-		t.port)
-}
-
-func StringIPv4Int(addr uint32) string {
-	return fmt.Sprintf("%d.%d.%d.%d",
-		(addr>>24)&0xff,
-		(addr>>16)&0xff,
-		(addr>>8)&0xff,
-		addr&0xff)
-}
-
-func StringIPv4Array(addr [common.IPv4AddrLen]uint8) string {
-	return fmt.Sprintf("%d.%d.%d.%d", addr[0], addr[1], addr[2], addr[3])
-}
-
-func StringMAC(mac [common.EtherAddrLen]uint8) string {
-	return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5])
-}
-
-func swapAddrIPv4(pkt *packet.Packet) {
-	ipv4 := pkt.GetIPv4NoCheck()
-
-	pkt.Ether.SAddr, pkt.Ether.DAddr = pkt.Ether.DAddr, pkt.Ether.SAddr
-	ipv4.SrcAddr, ipv4.DstAddr = ipv4.DstAddr, ipv4.SrcAddr
-}
-
-func swapAddrIPv6(pkt *packet.Packet) {
-	ipv6 := pkt.GetIPv6NoCheck()
-
-	pkt.Ether.SAddr, pkt.Ether.DAddr = pkt.Ether.DAddr, pkt.Ether.SAddr
-	ipv6.SrcAddr, ipv6.DstAddr = ipv6.DstAddr, ipv6.SrcAddr
-}
-
-func (port *ipPort) startTrace(dir uint) *os.File {
-	dumpNameLookup := [dirKNI + 1]string{
-		"drop",
-		"dump",
-		"kni",
-	}
-
-	fname := fmt.Sprintf("%s-%d-%s.pcap", dumpNameLookup[dir], port.Index, packet.MACToString(port.SrcMACAddress))
-
-	file, err := os.Create(fname)
-	if err != nil {
-		log.Fatal(err)
-	}
-	packet.WritePcapGlobalHdr(file)
-	return file
-}
-
-func (port *ipPort) dumpPacket(pkt *packet.Packet, dir uint) {
-	if DumpEnabled[dir] {
-		port.dumpsync[dir].Lock()
-		if port.fdump[dir] == nil {
-			port.fdump[dir] = port.startTrace(dir)
-		}
-
-		err := pkt.WritePcapOnePacket(port.fdump[dir])
-		if err != nil {
-			log.Fatal(err)
-		}
-		port.dumpsync[dir].Unlock()
-	}
-}
-
-func (port *ipPort) closePortTraces() {
-	for _, f := range port.fdump {
-		if f != nil {
-			f.Close()
-		}
-	}
-}
-
-// CloseAllDumpFiles closes all debug dump files.
-func CloseAllDumpFiles() {
-	for i := range Natconfig.PortPairs {
-		Natconfig.PortPairs[i].PrivatePort.closePortTraces()
-		Natconfig.PortPairs[i].PublicPort.closePortTraces()
-	}
-}
-
-func convertSubnet(s *upd.Subnet) (*ipv4Subnet, *ipv6Subnet, error) {
-	a := s.GetAddress().GetAddress()
-	addr, err := convertIPv4(a)
-	if err != nil {
-		if net.IP(a).To16() == nil {
-			return nil, nil, err
-		}
-		ret := ipv6Subnet{}
-		copy(ret.Addr[:], a)
-		copy(ret.Mask[:], net.CIDRMask(int(s.GetMaskBitsNumber()), 128))
-		return nil, &ret, nil
-	}
-
-	return &ipv4Subnet{
-		Addr: addr,
-		Mask: uint32(0xffffffff) << (32 - s.GetMaskBitsNumber()),
-	}, nil, nil
-}
-
-func convertForwardedPort(p *upd.ForwardedPort) (*forwardedPort, error) {
-	bytes := p.GetTargetAddress().GetAddress()
-	addr, err := convertIPv4(bytes)
-	var addr6 [common.IPv6AddrLen]uint8
-	var ipv6 bool
-	if err != nil {
-		if len(bytes) == common.IPv6AddrLen {
-			copy(addr6[:], bytes)
-			ipv6 = true
-		} else {
-			return nil, err
-		}
-	}
-	if uint8(p.GetProtocol()) != common.TCPNumber &&
-		uint8(p.GetProtocol()) != common.UDPNumber &&
-		p.GetProtocol() != (common.TCPNumber|upd.Protocol_IPv6_Flag) &&
-		p.GetProtocol() != (common.UDPNumber|upd.Protocol_IPv6_Flag) {
-		return nil, fmt.Errorf("Bad protocol identifier %d", p.GetProtocol())
-	}
-
-	return &forwardedPort{
-		Port: uint16(p.GetSourcePortNumber()),
-		Destination: hostPort{
-			Addr4: addr,
-			Addr6: addr6,
-			Port:  uint16(p.GetTargetPortNumber()),
-			ipv6:  ipv6,
-		},
-		Protocol: protocolId{
-			id:   uint8(p.GetProtocol() &^ upd.Protocol_IPv6_Flag),
-			ipv6: p.GetProtocol()&upd.Protocol_IPv6_Flag != 0,
-		},
-	}, nil
-}
-
-func setPacketDstPort(pkt *packet.Packet, ipv6 bool, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
-	if pktTCP != nil {
-		pktTCP.DstPort = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	} else if pktUDP != nil {
-		pktUDP.DstPort = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	}
-}
-
-func setPacketSrcPort(pkt *packet.Packet, ipv6 bool, port uint16, pktTCP *packet.TCPHdr, pktUDP *packet.UDPHdr, pktICMP *packet.ICMPHdr) {
-	if pktTCP != nil {
-		pktTCP.SrcPort = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4TCPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	} else if pktUDP != nil {
-		pktUDP.SrcPort = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4UDPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	} else {
-		pktICMP.Identifier = packet.SwapBytesUint16(port)
-		if ipv6 {
-			setIPv6ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		} else {
-			setIPv4ICMPChecksum(pkt, !NoCalculateChecksum, !NoHWTXChecksum)
-		}
-	}
-}
-
-func ParseAllKnownL4(pkt *packet.Packet, pktIPv4 *packet.IPv4Hdr, pktIPv6 *packet.IPv6Hdr) (uint8, *packet.TCPHdr, *packet.UDPHdr, *packet.ICMPHdr, uint16, uint16) {
-	var protocol uint8
-
-	if pktIPv4 != nil {
-		protocol = pktIPv4.NextProtoID
-		pkt.ParseL4ForIPv4()
-	} else {
-		protocol = pktIPv6.Proto
-		pkt.ParseL4ForIPv6()
-	}
-
-	switch protocol {
-	case common.TCPNumber:
-		pktTCP := (*packet.TCPHdr)(pkt.L4)
-		return protocol, pktTCP, nil, nil, packet.SwapBytesUint16(pktTCP.SrcPort), packet.SwapBytesUint16(pktTCP.DstPort)
-	case common.UDPNumber:
-		pktUDP := (*packet.UDPHdr)(pkt.L4)
-		return protocol, nil, pktUDP, nil, packet.SwapBytesUint16(pktUDP.SrcPort), packet.SwapBytesUint16(pktUDP.DstPort)
-	case common.ICMPNumber:
-		pktICMP := (*packet.ICMPHdr)(pkt.L4)
-		return protocol, nil, nil, pktICMP, packet.SwapBytesUint16(pktICMP.Identifier), packet.SwapBytesUint16(pktICMP.Identifier)
-	case common.ICMPv6Number:
-		pktICMP := (*packet.ICMPHdr)(pkt.L4)
-		return protocol, nil, nil, pktICMP, packet.SwapBytesUint16(pktICMP.Identifier), packet.SwapBytesUint16(pktICMP.Identifier)
-	default:
-		return 0, nil, nil, nil, 0, 0
-	}
-}
-
-func (port *ipPort) setLinkLocalIPv4KNIAddress(ipv4addr, mask uint32) {
-	if port.KNIName != "" {
-		myKNI, err := netlink.LinkByName(port.KNIName)
-		if err != nil {
-			fmt.Println("Failed to get KNI interface", port.KNIName, ":", err)
-			return
-		}
-		a := packet.IPv4ToBytes(ipv4addr)
-		m := packet.IPv4ToBytes(mask)
-		addr := &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   net.IPv4(a[3], a[2], a[1], a[0]),
-				Mask: net.IPv4Mask(m[3], m[2], m[1], m[0]),
-			},
-		}
-		fmt.Println("Setting address", addr)
-		err = netlink.AddrAdd(myKNI, addr)
-		if err != nil {
-			fmt.Println("Failed to set interface", port.KNIName, "address", addr, ":")
-		} else {
-			fmt.Println("Set address", addr, "on KNI interface", port.KNIName)
-		}
-	}
-}
-
-func (port *ipPort) setLinkLocalIPv6KNIAddress(ipv6addr, mask [common.IPv6AddrLen]uint8) {
-	if port.KNIName != "" {
-		myKNI, err := netlink.LinkByName(port.KNIName)
-		if err != nil {
-			fmt.Println("Failed to get KNI interface", port.KNIName, ":", err)
-			return
-		}
-		addr := &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   ipv6addr[:],
-				Mask: mask[:],
-			},
-		}
-		err = netlink.AddrAdd(myKNI, addr)
-		if err != nil {
-			fmt.Println("Failed to set interface", port.KNIName, "address", addr, ":", err)
-		} else {
-			fmt.Println("Set address", addr, "on KNI interface", port.KNIName)
-		}
-	}
-}
diff --git a/scripts/get-depends.sh b/scripts/get-depends.sh
index 1d43abc3..125bffea 100755
--- a/scripts/get-depends.sh
+++ b/scripts/get-depends.sh
@@ -10,9 +10,3 @@ go get -v golang.org/x/tools/cmd/stringer
 go get -v github.com/vishvananda/netlink
 go get -v github.com/google/gopacket
 
-# GRPC support for NAT example
-go get -v google.golang.org/grpc
-go get -v github.com/golang/protobuf/protoc-gen-go
-if ! command -v protoc &> /dev/null; then
-    echo You should install protobuf compiler package, e.g. \"sudo dnf install protobuf-compiler\" or \"sudo apt-get install protobuf-compiler\"
-fi
diff --git a/test/framework/main/nat/perf-nat-linux-vlan.json b/test/framework/main/nat/perf-nat-linux-vlan.json
deleted file mode 100644
index ea0d4257..00000000
--- a/test/framework/main/nat/perf-nat-linux-vlan.json
+++ /dev/null
@@ -1,100 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "LinuxNAT-VLAN-10k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 10000 http://192.168.116.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-VLAN-100k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 1000 http://192.168.116.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-VLAN-1M-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 1000 http://192.168.116.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-VLAN-10k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 10000 http://192.168.116.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-VLAN-100k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 1000 http://192.168.116.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-VLAN-1M-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 1000 http://192.168.116.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/nat/perf-nat-linux.json b/test/framework/main/nat/perf-nat-linux.json
deleted file mode 100644
index 063f562b..00000000
--- a/test/framework/main/nat/perf-nat-linux.json
+++ /dev/null
@@ -1,100 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "LinuxNAT-10k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 10000 http://192.168.16.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-100k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 1000 http://192.168.16.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-1M-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 1 -n 1000 http://192.168.16.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-10k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 10000 http://192.168.16.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-100k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 1000 http://192.168.16.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "LinuxNAT-1M-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "ab -c 10 -n 1000 http://192.168.16.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/nat/perf-nat-vlan.json b/test/framework/main/nat/perf-nat-vlan.json
deleted file mode 100644
index 6d2eebfc..00000000
--- a/test/framework/main/nat/perf-nat-vlan.json
+++ /dev/null
@@ -1,142 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "NFFGoNAT-VLAN-10k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 10000 http://192.168.116.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-VLAN-100k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 1000 http://192.168.116.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-VLAN-1M-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 1000 http://192.168.116.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-VLAN-10k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 10000 http://192.168.116.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-VLAN-100k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 1000 http://192.168.116.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-VLAN-1M-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 1000 http://192.168.116.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/nat/perf-nat.json b/test/framework/main/nat/perf-nat.json
deleted file mode 100644
index 04ceb355..00000000
--- a/test/framework/main/nat/perf-nat.json
+++ /dev/null
@@ -1,142 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "NFFGoNAT-10k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 10000 http://192.168.16.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-100k-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 1000 http://192.168.16.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-1M-1c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 1 -n 1000 http://192.168.16.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-10k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 10000 http://192.168.16.2/10k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-100k-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 1000 http://192.168.16.2/100k.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NFFGoNAT-1M-10c",
-            "test-time": 90000000000,
-            "test-type": "TestTypeApacheBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppApacheBenchmark",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 20; ab -c 10 -n 1000 http://192.168.16.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/nat/stability-nat-vlan.json b/test/framework/main/nat/stability-nat-vlan.json
deleted file mode 100644
index 03326cf8..00000000
--- a/test/framework/main/nat/stability-nat-vlan.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "PingNAT-VLANFromPrivate",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; ping -c 5 192.168.114.1 && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "PingNAT-VLANFromPublic",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sleep", "60"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; ping -c 5 192.168.116.1 && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NAT-VLANDownloadFile",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; wget --no-proxy http://192.168.116.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config-vlan.json"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/nat/stability-nat.json b/test/framework/main/nat/stability-nat.json
deleted file mode 100644
index 39771368..00000000
--- a/test/framework/main/nat/stability-nat.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
-    "docker-config": {
-        "request-timeout": 10000000000,
-        "docker-client-version": "1.24",
-        "privileged": true,
-        "map-volumes": [
-            "/sys/bus/pci/drivers:/sys/bus/pci/drivers",
-            "/sys/kernel/mm/hugepages:/sys/kernel/mm/hugepages",
-            "/sys/devices/system/node:/sys/devices/system/node",
-            "/dev:/dev"
-        ],
-        "pktgen-port": 22022
-    },
-    "tests": [
-        {
-            "name": "PingNATFromPrivate",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; ping -c 5 192.168.14.1 && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "PingNATFromPublic",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sleep", "60"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; ping -c 5 192.168.16.1 && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                }
-            ]
-        },
-        {
-            "name": "NATDownloadFile",
-            "test-time": 60000000000,
-            "test-type": "TestTypeScenario",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "sh", "-c", "sleep 10; wget --no-proxy http://192.168.16.2/1m.bin && echo TEST PASSED || echo TEST FAILED"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "-config", "config.json"
-                    ]
-                }
-            ]
-        }
-    ]
-}
diff --git a/test/framework/main/perf.json b/test/framework/main/perf.json
index 8fd33213..a460a3be 100644
--- a/test/framework/main/perf.json
+++ b/test/framework/main/perf.json
@@ -2899,918 +2899,6 @@
                 "measure-after": 40000000000,
                 "measure-for": 25000000000
             }
-        },
-        {
-            "name": "nat_fixed_ports_off_64",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_on_64",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_off_128",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_on_128",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_off_256",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_on_256",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_off_1518",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_fixed_ports_on_1518",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"0\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_off_64",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_on_64",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 64);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_off_128",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_on_128",
-            "test-time": 97000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 128);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 60000000000,
-                "measure-for": 35000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_off_256",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_on_256",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 256);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_off_1518",
-            "test-time": 32000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat", "--no-scheduler"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 15000000000,
-                "measure-for": 15000000000
-            }
-        },
-        {
-            "name": "nat_nonfixed_ports_on_1518",
-            "test-time": 67000000000,
-            "test-type": "TestTypeBenchmark",
-            "test-apps": [
-                {
-                    "image-name": "nff-go-nat",
-                    "app-type": "TestAppGo",
-                    "exec-cmd": [
-                        "./nat"
-                    ]
-                },
-                {
-                    "image-name": "nff-go-pktgen",
-                    "app-type": "TestAppPktgen",
-                    "exec-cmd": [
-                        "./pktgen", "-c", "PKTGENCOREMASK", "-n", "4", "--", "-P", "-m", "[1-2:3-4].0, [5-6:7-8].1", "-G"
-                    ]
-                }
-            ],
-            "benchmarking-settings": {
-                "pktgen-startup-commands": [
-                    "pktgen.range.src_ip(\"0\", \"start\", \"192.168.1.2\");",
-                    "pktgen.range.src_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"0\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.dst_ip(\"0\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"0\", \"start\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"min\", \"1234\");",
-                    "pktgen.range.src_port(\"0\", \"max\", \"65535\");",
-                    "pktgen.range.src_port(\"0\", \"inc\", \"1\");",
-                    "pktgen.range.dst_port(\"0\", \"start\", \"80\");",
-                    "pktgen.range.dst_port(\"0\", \"inc\", \"0\");",
-
-                    "pktgen.range.src_ip(\"1\", \"start\", \"10.1.1.2\");",
-                    "pktgen.range.src_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.dst_ip(\"1\", \"start\", \"10.1.1.1\");",
-                    "pktgen.range.dst_ip(\"1\", \"inc\", \"0.0.0.0\");",
-                    "pktgen.range.src_port(\"1\", \"start\", \"80\");",
-                    "pktgen.range.src_port(\"1\", \"inc\", \"0\");",
-                    "pktgen.range.dst_port(\"1\", \"start\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"min\", \"1024\");",
-                    "pktgen.range.dst_port(\"1\", \"max\", \"65325\");",
-                    "pktgen.range.dst_port(\"1\", \"inc\", \"1\");",
-
-                    "pktgen.range.pkt_size(\"0\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"0\", \"inc\", 0);",
-                    "pktgen.range.pkt_size(\"1\", \"start\", 1518);",
-                    "pktgen.range.pkt_size(\"1\", \"inc\", 0);",
-
-                    "pktgen.range.ip_proto(\"0-1\", \"udp\");",
-
-                    "pktgen.set_range(\"0-1\", \"on\");",
-                    "pktgen.start(0);",
-                    "pktgen.start(1);"
-                ],
-                "measure-after": 40000000000,
-                "measure-for": 25000000000
-            }
         }
     ]
 }

From 876be97973fa69af8c55cb9717a38e5224309aca Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Tue, 2 Oct 2018 23:06:42 +0000
Subject: [PATCH 45/50] Added go.mod for dependency tracking

---
 go.mod | 13 +++++++++++++
 go.sum | 27 +++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 go.mod
 create mode 100644 go.sum

diff --git a/go.mod b/go.mod
new file mode 100644
index 00000000..afaca5be
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,13 @@
+module github.com/intel-go/nff-go
+
+require (
+	github.com/docker/distribution v2.6.2+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-units v0.3.3 // indirect
+	github.com/flier/gohs v1.0.0
+	github.com/google/gopacket v1.1.15 // indirect
+	github.com/pkg/errors v0.8.0
+	golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 // indirect
+	golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e // indirect
+)
diff --git a/go.sum b/go.sum
new file mode 100644
index 00000000..102d3d12
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,27 @@
+github.com/docker/distribution v2.6.2+incompatible h1:4FI6af79dfCS/CYb+RRtkSHw3q1L/bnDjG1PcPZtQhM=
+github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.13.1 h1:5VBhsO6ckUxB0A8CE5LlUJdXzik9cbEbBTQ/ggeml7M=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/flier/gohs v1.0.0 h1:Q0mmufGWTigzKb140WmJ0+k3EGAf335Qgv/pz5SOPvU=
+github.com/flier/gohs v1.0.0/go.mod h1:Jlg6A1xXSMhPorF74/LkYHkCHZ87Txi8CqIHHyIKgKg=
+github.com/google/gopacket v1.1.14/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw=
+github.com/google/gopacket v1.1.15 h1:M6W3hwQXo5rq1wyhRByGhqOw0m9p+HWtUJ3Bj4/fT6E=
+github.com/google/gopacket v1.1.15/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw=
+github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 h1:dgd4x4kJt7G4k4m93AYLzM8Ni6h2qLTfh9n9vXJT3/0=
+golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e h1:x8cnE8uLkl6ATwMpvL/N/wYBk/53BdeePq1JaYt1zuo=
+golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 1311c6a36f23494cfd328013fe25c4198e89f44b Mon Sep 17 00:00:00 2001
From: Lehner Florian <dev@der-flo.net>
Date: Wed, 3 Oct 2018 19:59:46 +0200
Subject: [PATCH 46/50] Use new MPLS functions

Signed-off-by: Lehner Florian <dev@der-flo.net>
---
 packet/mpls.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packet/mpls.go b/packet/mpls.go
index 97fca5c6..2202b499 100644
--- a/packet/mpls.go
+++ b/packet/mpls.go
@@ -17,8 +17,7 @@ type MPLSHdr struct {
 
 func (hdr *MPLSHdr) String() string {
 	return fmt.Sprintf(`MPLS: Label: %d, EXP: %d, S: %d TTL: %d`,
-		SwapBytesUint32(hdr.mpls)>>12, (SwapBytesUint32(hdr.mpls)>>9)&0x00000007,
-		(SwapBytesUint32(hdr.mpls)>>8)&1, SwapBytesUint32(hdr.mpls)&0x000000ff)
+		hdr.GetMPLSLabel(), hdr.GetMPLSTC(), hdr.GetMPLSS(), hdr.GetMPLSTTL())
 }
 
 // GetMPLSLabel returns Label (20 first bits of MPLS header).

From 811008e341e1429fd33b7e46e48472daa97345d4 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Fri, 10 Aug 2018 18:14:41 +0000
Subject: [PATCH 47/50] Upgrade DPDK to version 18.08

---
 low/low.go          |  2 +-
 low/low.h           | 33 ++++++++++++++++++++-------------
 mk/include.mk       |  4 ++--
 vagrant/Vagrantfile |  2 +-
 vagrant/scripts.sh  |  2 +-
 5 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/low/low.go b/low/low.go
index 97da29d5..837b6fa1 100644
--- a/low/low.go
+++ b/low/low.go
@@ -13,7 +13,7 @@ package low
 // it increases executable size and build time.
 
 /*
-#cgo LDFLAGS: -lrte_distributor -lrte_reorder -lrte_kni -lrte_pipeline -lrte_table -lrte_port -lrte_timer -lrte_jobstats -lrte_lpm -lrte_power -lrte_acl -lrte_meter -lrte_sched -lrte_vhost -lrte_ip_frag -lrte_cfgfile -Wl,--whole-archive -Wl,--start-group -lrte_kvargs -lrte_mbuf -lrte_hash -lrte_ethdev -lrte_mempool -lrte_ring -lrte_mempool_ring -lrte_eal -lrte_cmdline -lrte_net -lrte_bus_pci -lrte_pci -lrte_bus_vdev -lrte_pmd_bond -lrte_pmd_vmxnet3_uio -lrte_pmd_virtio -lrte_pmd_cxgbe -lrte_pmd_enic -lrte_pmd_i40e -lrte_pmd_fm10k -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ena -lrte_pmd_ring -lrte_pmd_af_packet -lrte_pmd_null -Wl,--end-group -Wl,--no-whole-archive -lrt -lm -ldl -lnuma
+#cgo LDFLAGS: -lrte_distributor -lrte_reorder -lrte_kni -lrte_pipeline -lrte_table -lrte_port -lrte_timer -lrte_jobstats -lrte_lpm -lrte_power -lrte_acl -lrte_meter -lrte_sched -lrte_vhost -lrte_ip_frag -lrte_cfgfile -Wl,--whole-archive -Wl,--start-group -lrte_kvargs -lrte_mbuf -lrte_hash -lrte_ethdev -lrte_mempool -lrte_ring -lrte_mempool_ring -lrte_eal -lrte_cmdline -lrte_net -lrte_bus_pci -lrte_pci -lrte_bus_vdev -lrte_timer -lrte_pmd_bond -lrte_pmd_vmxnet3_uio -lrte_pmd_virtio -lrte_pmd_cxgbe -lrte_pmd_enic -lrte_pmd_i40e -lrte_pmd_fm10k -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ena -lrte_pmd_ring -lrte_pmd_af_packet -lrte_pmd_null -Wl,--end-group -Wl,--no-whole-archive -lrt -lm -ldl -lnuma
 #include "low.h"
 */
 import "C"
diff --git a/low/low.h b/low/low.h
index 24b63674..b8020399 100644
--- a/low/low.h
+++ b/low/low.h
@@ -108,6 +108,9 @@ void setAffinity(int coreId) {
 
 int create_kni(uint16_t port, uint32_t core, char *name, struct rte_mempool *mbuf_pool) {
 	struct rte_eth_dev_info dev_info;
+	const struct rte_pci_device *pci_dev;
+	const struct rte_bus *bus = NULL;
+
 	memset(&dev_info, 0, sizeof(dev_info));
 	rte_eth_dev_info_get(port, &dev_info);
 
@@ -117,9 +120,13 @@ int create_kni(uint16_t port, uint32_t core, char *name, struct rte_mempool *mbu
 	conf_default.core_id = core; // Core ID to bind kernel thread on
 	conf_default.group_id = port;
 	conf_default.mbuf_size = 2048;
-	if (dev_info.pci_dev) {
-		conf_default.addr = dev_info.pci_dev->addr;
-		conf_default.id = dev_info.pci_dev->id;
+	if (dev_info.device) {
+		bus = rte_bus_find_by_device(dev_info.device);
+	}
+	if (bus && !strcmp(bus->name, "pci")) {
+		pci_dev = RTE_DEV_TO_PCI(dev_info.device);
+		conf_default.addr = pci_dev->addr;
+		conf_default.id = pci_dev->id;
 	}
 	conf_default.force_bind = 1; // Flag to bind kernel thread
 	rte_eth_macaddr_get(port, (struct ether_addr *)&conf_default.mac_addr);
@@ -179,13 +186,18 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 		return -1;
 
 	struct rte_eth_conf port_conf_default = {
-	    .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN,
-			.mq_mode = ETH_MQ_RX_RSS    },
-	    .txmode = { .mq_mode = ETH_MQ_TX_NONE, },
-	    .rx_adv_conf.rss_conf.rss_key = NULL,
-	    .rx_adv_conf.rss_conf.rss_hf = dev_info.flow_type_rss_offloads
+		.rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN,
+					.mq_mode = ETH_MQ_RX_RSS    },
+		.txmode = { .mq_mode = ETH_MQ_TX_NONE, },
+		.rx_adv_conf.rss_conf.rss_key = NULL,
+		.rx_adv_conf.rss_conf.rss_hf = dev_info.flow_type_rss_offloads
 	};
 
+	if (hwtxchecksum) {
+        /* Enable everything that is supported by hardware */
+        port_conf_default.txmode.offloads = dev_info.tx_offload_capa;
+	}
+
 	/* Configure the Ethernet device. */
 	retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf_default);
 	if (retval != 0)
@@ -199,11 +211,6 @@ int port_init(uint16_t port, bool willReceive, uint16_t sendQueuesNumber, struct
 			return retval;
 	}
 
-	if (hwtxchecksum) {
-		/* Default TX settings are to disable offload operations, need to fix it */
-		dev_info.default_txconf.txq_flags = 0;
-	}
-
 	/* Allocate and set up TX queues per Ethernet port. */
 	for (q = 0; q < tx_rings; q++) {
 		retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE,
diff --git a/mk/include.mk b/mk/include.mk
index 0ff530ed..4afdeef0 100644
--- a/mk/include.mk
+++ b/mk/include.mk
@@ -6,12 +6,12 @@
 
 PROJECT_ROOT := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..)
 
-DPDK_VERSION = 18.02
+DPDK_VERSION = 18.08
 DPDK_DIR = dpdk-$(DPDK_VERSION)
 ifndef DPDK_URL
 DPDK_URL=http://fast.dpdk.org/rel/dpdk-$(DPDK_VERSION).tar.xz
 endif
-PKTGEN_VERSION=3.4.9
+PKTGEN_VERSION=3.5.2
 PKTGEN_DIR=pktgen-dpdk-pktgen-$(PKTGEN_VERSION)
 ifndef PKTGEN_URL
 PKTGEN_URL=http://git.dpdk.org/apps/pktgen-dpdk/snapshot/pktgen-dpdk-pktgen-$(PKTGEN_VERSION).tar.xz
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 012531db..87c109a8 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -80,7 +80,7 @@ echo Reassigning "${syscon}" interface to system name
 sudo nmcli c mod "${syscon}" connection.id 'System connection'
 
 echo Unpacking Go language into /opt
-(cd /opt; sudo sh -c 'curl -L -s https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz | tar zx')
+(cd /opt; sudo sh -c 'curl -L -s https://dl.google.com/go/go1.11.1.linux-amd64.tar.gz | tar zx')
 mkdir go
 chmod +x ~/scripts.sh
 . ~/scripts.sh
diff --git a/vagrant/scripts.sh b/vagrant/scripts.sh
index 1c0a0104..5b10ff00 100644
--- a/vagrant/scripts.sh
+++ b/vagrant/scripts.sh
@@ -1,4 +1,4 @@
-export DPDK_VERSION=18.02
+export DPDK_VERSION=18.08
 export GOPATH="$HOME"/go
 export GOROOT=/opt/go
 export NFF_GO="$GOPATH"/src/github.com/intel-go/nff-go

From 2fa67321802c13b98b900ed248e7d72ee8466e15 Mon Sep 17 00:00:00 2001
From: Areg Melik-Adamyan <aregm@users.noreply.github.com>
Date: Wed, 3 Oct 2018 14:21:31 -0500
Subject: [PATCH 48/50] Update README.md

#478  fixed.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 16e720ad..cecdeec2 100644
--- a/README.md
+++ b/README.md
@@ -205,7 +205,7 @@ downloaded automatically.
 
 If you want to contribute to NFF-Go, check our [Contributing
 guide](https://github.com/intel-go/yanff/blob/master/CONTRIBUTING.md). We also
-recommend checking the 'janitorial' bugs in our list of open issues; these bugs
+recommend checking the bugs with 'help-wanted' or 'easyfix' in our list of open issues; these bugs
 can be solved without an extensive knowledge of NFF-Go. We would love to help
 you start contributing.
 

From 433e61879bc03ff3391a75bce24b8f0493d75746 Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Thu, 4 Oct 2018 14:58:53 +0000
Subject: [PATCH 49/50] Updated go.mod files with current versions

---
 go.mod | 3 +++
 go.sum | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/go.mod b/go.mod
index afaca5be..30106fe9 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,9 @@ require (
 	github.com/flier/gohs v1.0.0
 	github.com/google/gopacket v1.1.15 // indirect
 	github.com/pkg/errors v0.8.0
+	github.com/vishvananda/netlink v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
 	golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 // indirect
+	golang.org/x/sys v0.0.0-20181004145325-8469e314837c // indirect
 	golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e // indirect
 )
diff --git a/go.sum b/go.sum
index 102d3d12..8fcbd2f8 100644
--- a/go.sum
+++ b/go.sum
@@ -21,7 +21,13 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
+github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
+github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
+github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3 h1:dgd4x4kJt7G4k4m93AYLzM8Ni6h2qLTfh9n9vXJT3/0=
 golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20181004145325-8469e314837c h1:SJ7JoQNVl3mC7EWkkONgBWgCno8LcABIJwFMkWBC+EY=
+golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e h1:x8cnE8uLkl6ATwMpvL/N/wYBk/53BdeePq1JaYt1zuo=
 golang.org/x/tools v0.0.0-20181002223833-cd09f19c2f7e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From f7001c2cb87a6d15493d857a63a11b03c4228bea Mon Sep 17 00:00:00 2001
From: Gregory Shimansky <gregory.shimansky@intel.com>
Date: Thu, 4 Oct 2018 14:05:31 -0500
Subject: [PATCH 50/50] Added proxy to dnf.conf config file

---
 nff-go-base/Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nff-go-base/Makefile b/nff-go-base/Makefile
index 22c0515c..5ea9b2ee 100644
--- a/nff-go-base/Makefile
+++ b/nff-go-base/Makefile
@@ -13,6 +13,8 @@ Dockerfile: Makefile
 	if [ -n '${http_proxy}' ]; then					\
 		echo 'ENV http_proxy ${http_proxy}' >> Dockerfile;	\
 		echo 'ENV https_proxy ${http_proxy}' >> Dockerfile;	\
+		echo 'RUN echo proxy=${http_proxy} >> /etc/dnf/dnf.conf' >> Dockerfile;	\
+		echo 'RUN echo proxy=${http_proxy} >> /etc/dnf/dnf.conf' >> Dockerfile;	\
 	fi
 	echo 'RUN dnf -y install numactl-libs.x86_64; dnf clean all' >> Dockerfile
 	echo 'CMD ["/bin/bash"]' >> Dockerfile