Skip to content

Commit

Permalink
Merge pull request #4 from Webcampak/develop
Browse files Browse the repository at this point in the history 
Preparation for release v3.0.1
  • Loading branch information
@Fgerthoffert
Fgerthoffert authored Nov 20, 2016
2 parents 0d19d2e + 4673576 commit df4f1ba
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 103 deletions.
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ psutil
Pillow
jsonschema #https://github.com/Julian/jsonschema
Phidgets
rrdtool
rrdtool==0.1.6
tabulate
2 changes: 1 addition & 1 deletion webcampak/core/wpakFileUtils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def sizeof_fmt(num, suffix='B'):
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
else:
return num
return "n/a"

# Function: SecondsSinceLastCapture
# Description; This function is used to return the number of seconds since last picture captured from a source
Expand Down
18 changes: 7 additions & 11 deletions webcampak/core/wpakReportsDaily.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License along with Webcampak.
# If not, see http://www.gnu.org/licenses/

from __future__ import division
import os
import time
import gettext
Expand Down Expand Up @@ -128,7 +129,7 @@ def run(self):
self.log.info("reportsDaily.run(): " + _("Source disk usage is: %(sourceDiskUsage)s") % {
'sourceDiskUsage': str(sourceDiskUsage)})
if sourceQuota == None or int(sourceQuota) == 0:
sourceDiskPercentUsed = None
sourceDiskPercentUsed = "n/a"
else:
sourceDiskPercentUsed = int(int(sourceDiskUsage) / int(sourceQuota) * 100)

Expand Down Expand Up @@ -245,10 +246,10 @@ def prepareEmailReportContent(self, currentSource, currentReportDay, currentSour
'currentSource': str(currentSource), 'currentReportDay': str(currentReportDay)})
reportDatetime = datetime.strptime(str(currentReportDay) + "120000", "%Y%m%d%H%M%S")
table = [ \
["Source Name", self.dbUtils.getSourceName(currentSource)] \
, ["Source ID", currentSource] \
["Source Name (ID)", self.dbUtils.getSourceName(currentSource) + " (" + str(currentSource) + ")"] \
, ["Report Day", reportDatetime.strftime("%B %d, %Y")] \
, ["Report Creation", self.timeUtils.getCurrentDate().strftime("%B %d, %Y")] \
, ["Captured JPG | RAW", str(currentSourceReport['capture']['JPG']['count']) + " | " + str(currentSourceReport['capture']['RAW']['count'])] \
]
dailyReportContentSource = tabulate(table, tablefmt="fancy_grid") + "\n"
if currentSourceReport['source']['quota'] == None:
Expand All @@ -264,7 +265,8 @@ def prepareEmailReportContent(self, currentSource, currentReportDay, currentSour
currentSourceReport['source']['percentUsed']] \
, ["Total Disk", fileUtils.sizeof_fmt(currentSourceReport['disk']['used']),
fileUtils.sizeof_fmt(currentSourceReport['disk']['total']),
fileUtils.sizeof_fmt(currentSourceReport['disk']['free']), currentSourceReport['disk']['percentUsed']] \
fileUtils.sizeof_fmt(currentSourceReport['disk']['free']),
currentSourceReport['disk']['percentUsed']] \
]
dailyReportContentSource = dailyReportContentSource + tabulate(table, headers, tablefmt="fancy_grid") + "\n"

Expand Down Expand Up @@ -299,13 +301,7 @@ def prepareEmailReportContent(self, currentSource, currentReportDay, currentSour
] \
]
dailyReportContentSource = dailyReportContentSource + tabulate(table, headers, tablefmt="fancy_grid") + "\n"
else:
headers = ["", "Captured"]
table = [ \
["JPG", currentSourceReport['capture']['JPG']['count']] \
, ["RAW", currentSourceReport['capture']['RAW']['count']] \
]
dailyReportContentSource = dailyReportContentSource + tabulate(table, headers, tablefmt="fancy_grid") + "\n"

return dailyReportContentSource

def generateReport(self, currentSource, currentReportDay, sourceSchedule):
Expand Down
175 changes: 88 additions & 87 deletions webcampak/core/wpakStatsRrd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,93 +151,94 @@ def run(self):
sensors = self.getSensorsFromFile(currentCaptureFile)
sensorsDay = self.getSensorDayFromFile(currentCaptureFile)
captureInterval = self.getCaptureIntervalFromFile(currentCaptureFile)
self.log.info("statsrrd.run(): " + _("Capture interval: %(captureInterval)s seconds") % {'captureInterval': captureInterval})

# {"scriptEndDate": "2016-09-29T04:59:43.956108+02:00", "totalCaptureSize": 10945697, "captureSuccess": true, "scriptRuntime": 2531, "storedRawSize": 0
# , "scriptStartDate": "2016-09-29T04:59:41.424850+02:00", "processedPicturesCount": 1, "storedJpgSize": 10945697, "captureDate": "2016-09-29T04:59:41.472365+02:00"
# , "sensors": {"789275965fe98d9ad9275648a21b095982d673a189f5cb3fad8155f9": {"type": "Temperature", "legend": "Outside Tempe abcd", "value": 25.8, "valueRaw": 1601}, "574eb9c9ee7e0bbe610a7aab0e359864fdd7810d113edee1da80a5af": {"type": "Temperature", "legend": "Inside Temperature", "value": 25.8, "valueRaw": 1601}, "fbdde0c3fe0b6aecc5f1027262ec79813f2cf77c9361642c4a7d57a3": {"type": "Luminosity", "legend": "Humidity", "value": 460.8, "valueRaw": 1887}}}

for currentSensor in sensors:
if os.path.isfile(self.dirCurrentSourcePictures + sensorsDay + "/sensor-" + currentSensor + ".rrd") == False or processedCpt <= 1:
self.log.info("statsrrd.run(): " + _("Currently processing Sensor: %(currentSensor)s") % {
'currentSensor': currentSensor})

ValueTable = {}
SensorLegend = "UNAVAILABLE"
SensorColor = "#FF0000"
for line in open(currentCaptureFile).readlines():
try:
currentCaptureLine = json.loads(line)
except Exception:
self.log.error("statsrrd.run(): Unable to decode JSON line: " + line)
break
sensorDate = dateutil.parser.parse(currentCaptureLine['date'])
currentTimestamp = int(time.mktime(sensorDate.timetuple()))
ValueTable[currentTimestamp] = "NaN"
if 'sensors' in currentCaptureLine:
if currentCaptureLine['sensors'] != None:
if currentSensor in currentCaptureLine['sensors']:
ValueTable[currentTimestamp] = currentCaptureLine['sensors'][currentSensor][
'value']
SensorLegend = currentCaptureLine['sensors'][currentSensor]['legend']
if 'color' in currentCaptureLine['sensors'][currentSensor]:
SensorColor = currentCaptureLine['sensors'][currentSensor]['color']

ValueTableKeys = ValueTable.keys()
ValueTableKeys.sort()

self.log.info("statsrrd.run(): " + _("Preparing the RRD base file: %(SensorRRDFile)s") % {
'SensorRRDFile': str(
self.dirCurrentSourcePictures + sensorsDay + "/sensor-" + currentSensor + ".rrd")})

rrdstart = str(int(min(ValueTableKeys)))
# rrdstart = str(int(min(ValueTableKeys)))
rrdend = str(max(ValueTableKeys))
ret = rrdtool.create(
str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".rrd") \
, "--step", str(captureInterval) \
, "--start" \
, rrdstart \
, "DS:GRAPHAREA:GAUGE:600:U:U" \
, "RRA:AVERAGE:0.5:1:" + str(len(ValueTable)))

for i in xrange(len(ValueTableKeys)):
self.log.info("statsrrd.run(): " + _(
"Adding Timestamp: %(currentTimestamp)s - Value: %(currentValue)s") % {
'currentTimestamp': str(ValueTableKeys[i]),
'currentValue': str(ValueTable[ValueTableKeys[i]]),})
if i == 0:
currentTimestamp = int(ValueTableKeys[i])
else:
currentTimestamp = currentTimestamp + captureInterval
ret = rrdtool.update(str(self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd"),
str(int(currentTimestamp)) + ':' + str(
ValueTable[ValueTableKeys[i]]))

ret = rrdtool.graph(
str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".png") \
, "--start" \
, rrdstart \
, "--end" \
, rrdend \
, "--vertical-label=" + str(SensorLegend) \
, "DEF:GRAPHAREA=" + str(self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd") + ":GRAPHAREA:AVERAGE" \
, "AREA:GRAPHAREA" + str(SensorColor) + ":" + str(SensorLegend))

self.log.info("statsrrd.run(): " + _("PNG Graph created: %(pnggraph)s") % {'pnggraph': str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".png")})

if self.configSource.getConfig('cfgftpphidgetserverid') != "":
currentTime = self.timeUtils.getCurrentSourceTime(self.configSource)
self.transferUtils.transferFile(currentTime, self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd", "sensor-" + currentSensor + ".rrd",
self.configSource.getConfig('cfgftpphidgetserverid'),
self.configSource.getConfig('cfgftpphidgetserverretry'))
self.transferUtils.transferFile(currentTime, self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".png", "sensor-" + currentSensor + ".png",
self.configSource.getConfig('cfgftpphidgetserverid'),
self.configSource.getConfig('cfgftpphidgetserverretry'))
if captureInterval != None:
self.log.info("statsrrd.run(): " + _("Capture interval: %(captureInterval)s seconds") % {'captureInterval': captureInterval})

# {"scriptEndDate": "2016-09-29T04:59:43.956108+02:00", "totalCaptureSize": 10945697, "captureSuccess": true, "scriptRuntime": 2531, "storedRawSize": 0
# , "scriptStartDate": "2016-09-29T04:59:41.424850+02:00", "processedPicturesCount": 1, "storedJpgSize": 10945697, "captureDate": "2016-09-29T04:59:41.472365+02:00"
# , "sensors": {"789275965fe98d9ad9275648a21b095982d673a189f5cb3fad8155f9": {"type": "Temperature", "legend": "Outside Tempe abcd", "value": 25.8, "valueRaw": 1601}, "574eb9c9ee7e0bbe610a7aab0e359864fdd7810d113edee1da80a5af": {"type": "Temperature", "legend": "Inside Temperature", "value": 25.8, "valueRaw": 1601}, "fbdde0c3fe0b6aecc5f1027262ec79813f2cf77c9361642c4a7d57a3": {"type": "Luminosity", "legend": "Humidity", "value": 460.8, "valueRaw": 1887}}}

for currentSensor in sensors:
if os.path.isfile(self.dirCurrentSourcePictures + sensorsDay + "/sensor-" + currentSensor + ".rrd") == False or processedCpt <= 1:
self.log.info("statsrrd.run(): " + _("Currently processing Sensor: %(currentSensor)s") % {
'currentSensor': currentSensor})

ValueTable = {}
SensorLegend = "UNAVAILABLE"
SensorColor = "#FF0000"
for line in open(currentCaptureFile).readlines():
try:
currentCaptureLine = json.loads(line)
except Exception:
self.log.error("statsrrd.run(): Unable to decode JSON line: " + line)
break
sensorDate = dateutil.parser.parse(currentCaptureLine['date'])
currentTimestamp = int(time.mktime(sensorDate.timetuple()))
ValueTable[currentTimestamp] = "NaN"
if 'sensors' in currentCaptureLine:
if currentCaptureLine['sensors'] != None:
if currentSensor in currentCaptureLine['sensors']:
ValueTable[currentTimestamp] = currentCaptureLine['sensors'][currentSensor][
'value']
SensorLegend = currentCaptureLine['sensors'][currentSensor]['legend']
if 'color' in currentCaptureLine['sensors'][currentSensor]:
SensorColor = currentCaptureLine['sensors'][currentSensor]['color']

ValueTableKeys = ValueTable.keys()
ValueTableKeys.sort()

self.log.info("statsrrd.run(): " + _("Preparing the RRD base file: %(SensorRRDFile)s") % {
'SensorRRDFile': str(
self.dirCurrentSourcePictures + sensorsDay + "/sensor-" + currentSensor + ".rrd")})

rrdstart = str(int(min(ValueTableKeys)))
# rrdstart = str(int(min(ValueTableKeys)))
rrdend = str(max(ValueTableKeys))
ret = rrdtool.create(
str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".rrd") \
, "--step", str(captureInterval) \
, "--start" \
, rrdstart \
, "DS:GRAPHAREA:GAUGE:600:U:U" \
, "RRA:AVERAGE:0.5:1:" + str(len(ValueTable)))

for i in xrange(len(ValueTableKeys)):
self.log.info("statsrrd.run(): " + _(
"Adding Timestamp: %(currentTimestamp)s - Value: %(currentValue)s") % {
'currentTimestamp': str(ValueTableKeys[i]),
'currentValue': str(ValueTable[ValueTableKeys[i]]),})
if i == 0:
currentTimestamp = int(ValueTableKeys[i])
else:
currentTimestamp = currentTimestamp + captureInterval
ret = rrdtool.update(str(self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd"),
str(int(currentTimestamp)) + ':' + str(
ValueTable[ValueTableKeys[i]]))

ret = rrdtool.graph(
str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".png") \
, "--start" \
, rrdstart \
, "--end" \
, rrdend \
, "--vertical-label=" + str(SensorLegend) \
, "DEF:GRAPHAREA=" + str(self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd") + ":GRAPHAREA:AVERAGE" \
, "AREA:GRAPHAREA" + str(SensorColor) + ":" + str(SensorLegend))

self.log.info("statsrrd.run(): " + _("PNG Graph created: %(pnggraph)s") % {'pnggraph': str(self.dirCurrentSourcePictures + str(sensorsDay) + "/sensor-" + currentSensor + ".png")})

if self.configSource.getConfig('cfgftpphidgetserverid') != "":
currentTime = self.timeUtils.getCurrentSourceTime(self.configSource)
self.transferUtils.transferFile(currentTime, self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".rrd", "sensor-" + currentSensor + ".rrd",
self.configSource.getConfig('cfgftpphidgetserverid'),
self.configSource.getConfig('cfgftpphidgetserverretry'))
self.transferUtils.transferFile(currentTime, self.dirCurrentSourcePictures + str(
sensorsDay) + "/sensor-" + currentSensor + ".png", "sensor-" + currentSensor + ".png",
self.configSource.getConfig('cfgftpphidgetserverid'),
self.configSource.getConfig('cfgftpphidgetserverretry'))

else:
self.log.info(
Expand Down
6 changes: 3 additions & 3 deletions webcampak/core/wpakXferStart.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def __init__(self, log, appConfig, config_dir, threadUUID):
self.dirLogs = self.configPaths.getConfig('parameters')['dir_logs']
self.dirLocale = self.configPaths.getConfig('parameters')['dir_locale']
self.dirLocaleMessage = self.configPaths.getConfig('parameters')['dir_locale_message']
self.dirSyncReports = self.configPaths.getConfig('parameters')['dir_sync-reports']

self.setupLog()

Expand Down Expand Up @@ -246,9 +247,8 @@ def moveThreadFileAfterTransfer(self, jobJsonContent, firstThreadFile):
self.log.info("(" + str(
os.getpid()) + ")xferStart.moveThreadFileAfterTransfer(): Transfer has been successful, storing completed file")
if jobJsonContent['job'].has_key('sync-report'):
destinationFilePath = self.dirSources + "/source" + str(
jobJsonContent['job']['filesourceid']) + "/resources/sync-reports/" + \
os.path.splitext(jobJsonContent['job']['sync-report']['filename'])[0] + "/"
#destinationFilePath = self.dirSources + "/source" + str(jobJsonContent['job']['filesourceid']) + "/resources/sync-reports/" + os.path.splitext(jobJsonContent['job']['sync-report']['filename'])[0] + "/"
destinationFilePath = self.dirSyncReports + "/completed/" + os.path.splitext(jobJsonContent['job']['sync-report']['filename'])[0] + "/"
else:
destinationFilePath = self.dirSources + "/source" + str(
jobJsonContent['job']['filesourceid']) + "/resources/xfer/" + os.path.basename(firstThreadFile)[
Expand Down

0 comments on commit df4f1ba

Please sign in to comment.