addstatic

#!/bin/bash
# addstatic
# KD4Z
# Version:  4.53
# Part of the user.db creation step.  Far more advanced than just
# including ham-digital data.
# Ths script can merge in static DMR ids as part of the process.
#
# Optional static.users file requires a format that mimics the exact field layout of user.bin
# must contain seven fields, separated by commas
# (that means 6 commas per line!)
# leave field blank if no data is needed for a given field
#
# Example:
#  privateID1,call1,name1,city1,state1,nickname1,country1
#  privateID2,call2,name2,,,nickname2,
#  privateID3,call3,name3,,,,
#
# The static file can be pulled in on-the-fly from an FTP server
# defined in static.host file
#
. ~/md380tools-vm/strings

FILESTATIC=~/static.users
FILENICK=user.nicknames
FILENICKZIP=~/md380tools-vm/nicknames.zip
STATICFILEHOST=~/static.host

#radioid : full data, missing 2nnnnnn
#RIDATAURL="https://www.radioid.net/api/dmr/user/?country=%&format=csv"
RIDATAURL="https://database.radioid.net/static/user.csv"
RIDATARAW=~/ri.raw
RIERROR=~/radioid.errors
FINALRI=~/ri.final
TOOLSDB=~/md380tools/db/stripped.csv

#ham-digital :  minimal info, all ids
HDSTATICUSERSURL=https://ham-digital.org/status/users.csv
HDSTATICFILE=~/hd.raw
HDERROR=~/hamdigital.errors
FINALHD=~/hd.final

USERDB=~/user.bin
# With expressed permission from Wilfred Pontfoort
TMPALTERNATE=~/alt-database.raw
FINALALTERNATE=~/alt-database.final
ALTERNATESEMADISABLE=~/extra.data.disable
ALTERNATEURL=https://github.com/DMR-Database/database/raw/master/user.bin
MERGEERRORS=~/merge.errors
ALTFILEDLERROR=~/alternate.download.error
EXCLUDEIDS=~/exclude.ids
EXCLUDECOUNTRIES=~/exclude.countries
INCLUDEIDS=~/include.ids
INCLUDECOUNTRIES=~/include.countries
MERGECONFIG=~/md380tools-vm/mergeconfig
FILTERDISABLESEMA=~/filter.disable

if [ -f $FILENICKZIP ]; then
    if [ -f $FILENICK ]; then rm $FILENICK ; fi
    unzip $FILENICKZIP $FILENICK
	if [ "$?" -ne 0 ]; then
	  echo -e "${RED}An Error was returned from the unzip application. ($?)${NC}"
	  sleep 1
	fi
	if [ ! -f $FILENICK ]; then
	  echo -e "${RED}Unable to extract compressed file from $FILENICKZIP.${NC}"
	  sleep 2
	  echo -e "${GREEN}Continuing without nicknames.${NC}"
	else
		sed -i '/^[[:space:]]*$/d' $FILENICK
		if [ -f ~/keep.seed.file ]; then
			echo -e "${GREEN}Keeping seed file for next ${YELLOW}glv."
			echo -e "${GREEN}To force an update next glv: rm ~/keep.seed.file${NC}"
			cp $FILENICKZIP ~
			sleep 2
		fi
	fi
fi


echo -e -n "${WHITE}Downloading data from RadioId.net"
if [ -f ~/keep.tmp ]; then
		timeout 120 wget --no-check-certificate --wait=3 $RIDATAURL -O $RIDATARAW
else
		timeout 120 wget --no-check-certificate --wait=3 $RIDATAURL -O $RIDATARAW  2>$RIERROR
fi
sed -i '/Callsign/d' $RIDATARAW
sizeRI=$(ls -l $RIDATARAW | cut -d" " -f5)
countRI=$(wc --lines $RIDATARAW | cut -d" " -f1)
if (( countRI < 10 )); then
	echo -e "${RED}"
	cat $RIERROR
	echo -e "${WHITE}"
    if [ -f $RIDATARAW ]; then rm $RIDATARAW ; fi
fi
if [ -f $RIDATARAW ]; then
   echo -e -n "${WHITE}...Normalizing"
   if [ -f ~/keep.tmp ]; then
      cat $RIDATARAW >ridata_raw.csv
   fi
   sed -i '1,2d' $RIDATARAW
   #sed -i -e 's/\",\"/\^/g' $RIDATARAW
   #sed -i -e 's/,//g' $RIDATARAW
   #sed -i -e 's/\"//g' $RIDATARAW
   # format from RadioId.net changes like the weather.  Come on guys, quit changing the column format
   # 27 April 2019 = RADIO_ID,CALLSIGN,FIRST_NAME,LAST_NAME,CITY,STATE,COUNTRY,REMARKS
   #awk -F^ '{print $5","$1","$4" "$8","$2","$7",,"$3}' $RIDATARAW >$FINALRI
   awk -F, 'length($1) {print $1","$2","$3" "$4","$5","$6",,"$7}' $RIDATARAW >$FINALRI
   sed -i -e 's/\@//g' $FINALRI   
   cat $FINALRI >$TOOLSDB
   echo -e "...done${NC}"
fi
echo -e -n "${WHITE}Downloading data from HamDigital.org"
if [ -f ~/keep.tmp ]; then
		timeout 120 wget --no-check-certificate --wait=3 $HDSTATICUSERSURL -O $HDSTATICFILE
else
		timeout 120 wget --no-check-certificate --wait=3 $HDSTATICUSERSURL -O $HDSTATICFILE  2>$HDERROR
fi 

sizeHD=$(ls -l $HDSTATICFILE | cut -d" " -f5)
countHD=$(wc --lines $HDSTATICFILE | cut -d" " -f1)
if (( countHD < 10 )); then
	echo -e "${RED}"
	cat $HDERROR
	echo -e "${WHITE}"
    if [ -f $HDSTATICFILE ]; then rm $HDSTATICFILE ; fi
fi
if [ -f $HDSTATICFILE ]; then
	echo -e -n "${WHITE}...Normalizing"
   awk -F, '{print $1","$2","$3","$4","$5",,"$6}' $HDSTATICFILE >$FINALHD
   echo -e "...done${NC}"
fi


if [ -f $TMPALTERNATE ]; then rm $TMPALTERNATE ; fi
if [ ! -f $ALTERNATESEMADISABLE ]; then
	OPTS="-quiet --no-verbose"
	if [ -f $TMPALTERNATE ] ; then rm $TMPALTERNATE ; fi
	echo -e -n "${WHITE}Downloading alternate data."
	if [ -f ~/keep.tmp ]; then
		timeout 120 wget --no-check-certificate --wait=3 $ALTERNATEURL -O $TMPALTERNATE
	else
		timeout 120 wget --no-check-certificate --wait=3 $ALTERNATEURL -O $TMPALTERNATE  2>$ALTFILEDLERROR
	fi 
	 
	if [ ! -f $TMPALTERNATE ]; then 
			echo -e " ${RED}Alternate data not downloaded.  See ${ALTFILEDLERROR} ${NC}"
	else
			echo -e "...done${NC}"
	fi
	
	if [ -f $TMPALTERNATE ]; then
		sed -i '1,1d' $TMPALTERNATE
		sed -i -e 's/\r//' $TMPALTERNATE
#		if (( counthd > 100 )); then
#			python ~/md380tools-vm/merge_users.py $TMPALTERNATE --abbrevCountries | awk -F, '!($7=="CA" || $7=="US") {print}' - >$FINALALTERNATE
#		else
			python ~/md380tools-vm/merge_users.py $TMPALTERNATE --abbrevCountries >$FINALALTERNATE
#		fi
	fi
fi

if [ -f $STATICFILEHOST ]; then
	if [ -f $FILESTATIC ]; then rm $FILESTATIC ; fi
    echo -e "${WHITE}Downloading ${FILESTATIC} from ${STATICFILEHOST}${NC}"
    timeout 120 wget --no-check-certificate --wait=3 -i $STATICFILEHOST -O $FILESTATIC
fi

if [ -f $FILESTATIC ]; then
  echo -e "${WHITE} Using ${FILESTATIC}${NC}"
  sed -i -e 's/\r//' $FILESTATIC  
fi

echo -e "${WHITE}Merging data...${NC}"

if [ -f $MERGECONFIG ]; then rm $MERGECONFIG; fi
if [ ! -f $FILTERDISABLESEMA ]; then
	if [ -f $EXCLUDECOUNTRIES ]; then
	  awk -F, 'length($1) {print "excludecountry * "$1}' $EXCLUDECOUNTRIES >>$MERGECONFIG
	fi
	if [ -f $EXCLUDEIDS ]; then
	  awk -F, 'length($1) {print "excludeID * "$1}' $EXCLUDEIDS >>$MERGECONFIG
	fi
	if [ -f $INCLUDECOUNTRIES ]; then
	  awk -F, 'length($1) {print "includecountry * "$1}' $INCLUDECOUNTRIES >>$MERGECONFIG
	fi
	if [ -f $INCLUDEIDS ]; then
	  awk -F, 'length($1) {print "includeID * "$1}' $INCLUDEIDS >>$MERGECONFIG
	fi
fi

FILELIST=""
if [ -f $FINALHD ]; then
	FILELIST="$FINALHD"
fi
if [ -f $FINALRI ]; then
	FILELIST="$FILELIST $FINALRI"
fi
if [ -f $FINALALTERNATE ]; then
	FILELIST="$FILELIST $FINALALTERNATE"
fi
if [ -f $FINALRI ]; then
	FILELIST="$FILELIST $FINALRI"
fi
if [ -f $FILESTATIC ]; then
	FILELIST="$FILELIST $FILESTATIC"
fi
if [ -f $FILENICK ]; then
	FILELIST="$FILELIST $FILENICK"
fi
OPTS="--abbrevCountries --abbrevStates --noHeader --abbrevDirections"
if [ -f $MERGECONFIG ]; then
  OPTS="$OPTS --config $MERGECONFIG"
  echo -e "${WHITE}Applying contact filters... ${YELLOW}"
  cat $MERGECONFIG
fi
if [ -f ~/keep.tmp ]; then
 echo -e "Filelist=$FILELIST Options=$OPTS" 
fi
if [ -f $MERGEERRORS ]; then rm $MERGEERRORS ; fi
echo -e -n "${RED}"

# Thanks to Dale Farnsworth dale@farnsworth.org for providing this complex merge routine in python.
python ~/md380tools-vm/merge_users.py $FILELIST $OPTS >$USERDB 2>$MERGEERRORS

echo -e "${WHITE}Contact Merge completed.${NC}"

countstatic=0
countalter=0
countseed=0
if [ -f $FILESTATIC ] ; then
  countstatic=$(wc --lines $FILESTATIC | cut -d" " -f1)
fi
if [ -f $FILENICK ] ; then
  countnick=$(wc --lines $FILENICK | cut -d" " -f1)
fi
if [ -f $FINALALTERNATE ] ; then
  countalter=$(wc --lines $FINALALTERNATE | cut -d" " -f1)
  sizealter=$(ls -l $FINALALTERNATE | cut -d" " -f5)
fi


countall=$(wc --lines $USERDB | cut -d" " -f1)
sizeall=$(ls -l $USERDB | cut -d" " -f5)
sizemergeerrors=$(ls -l $MERGEERRORS | cut -d" " -f5)
countmergeerrors=$(wc --lines $MERGEERRORS | cut -d" " -f1)
if [ -f ~/keep.tmp ]; then
  if (( sizemergeerrors > 10 )); then
	echo -e "${RED} ${countmergeerrors} records were not merged due to line format issues."
	echo -e " See the file ${MERGEERRORS} for details.${NC}"
	sleep 6
  fi
fi
if [ -f $FINALRI ]; then
echo -e "${GREEN}  RadioId count       :  ${YELLOW}${countRI}  records  ${GREEN}(${sizeRI}  bytes)"
fi
if [ -f $FINALHD ]; then
echo -e "${GREEN}  HamDigital count    :  ${YELLOW}${countHD}  records  ${GREEN}(${sizeHD}  bytes)"
fi
if [ -f $FINALALTERNATE ] ; then
echo -e "${GREEN}  Alternate count     :  ${YELLOW}${countalter}  records  ${GREEN}(${sizealter}  bytes)"
fi
if [ -f $FILESTATIC ] ; then
echo -e "${GREEN}  Static ID count     :  ${YELLOW}${countstatic}  records"
fi
if [ -f $FILENICK ] ; then
echo -e "${GREEN}  Nickname count      :  ${YELLOW}${countnick}  records"
fi
if (( sizemergeerrors > 10 )); then
echo -e "${GREEN}  Records skipped     :  ${RED}${countmergeerrors}     ${YELLOW}records  ${GREEN}See ${MERGEERRORS}"
fi
if (( countall < 1000 )); then
  echo -e "${RED} ** The Downloads for Contact information may have failed - run ${YELLOW}glvusers${RED} again"
  echo -e "${RED} ** before running ${YELLOW}flashdb${RED}, as the user.bin is not normal size.${NC}"
  echo -e "\a"
  sleep 8
fi
echo -e "${GREEN}  Merged total        :  ${YELLOW}${countall}  records  ${GREEN}(${sizeall}  bytes)${NC}"

if [ ! -f ~/keep.tmp ]; then
    if [ -f $TMPALTERNATE ]; then rm $TMPALTERNATE ; fi	
	if [ -f $FILENICK ]; then rm $FILENICK ; fi
fi