couchdb-dump.sh

#!/bin/bash
##
#    AUTHOR: DANIELE BAILO
#    https://github.com/danielebailo
#    www.danielebailo.it
#
#    Contributors:
#     * dalgibbard      - http://github.com/dalgibbard
#     * epos-eu         - http://github.com/epos-eu
#     * maximilianhuber - http://github.com/maximilianhuber
#     * ahodgkinson     - http://github.com/ahodgkinson (quiet-mode, timestamp, compress)
##

## This script allow for the Backup and Restore of a CouchDB Database.
## Backups are produced in a format that can be later uploaded with the bulk docs directive (as used by this script)

## USAGE
## * To Backup:
## ** example: ./couchdb-dump.sh -b -H 127.0.0.1 -d mydb -u admin -p password -f mydb.json
## * To Restore:
## ** example: ./couchdb-dump.sh -r -H 127.0.0.1 -d mydb -u admin -p password -f mydb.json


###################### CODE STARTS HERE ###################
scriptversionnumber="1.1.10"

##START: FUNCTIONS
usage(){
    echo
    echo "Usage: $0 [-b|-r] -H <COUCHDB_HOST> -d <DB_NAME> -f <BACKUP_FILE> [-u <username>] [-p <password>] [-P <port>] [-l <lines>] [-t <threads>] [-a <import_attempts>]"
    echo -e "\t-b   Run script in BACKUP mode."
    echo -e "\t-r   Run script in RESTORE mode."
    echo -e "\t-H   CouchDB Hostname or IP. Can be provided with or without 'http(s)://'"
    echo -e "\t-d   CouchDB Database name to backup/restore."
    echo -e "\t-f   File to Backup-to/Restore-from."
    echo -e "\t-P   Provide a port number for CouchDB [Default: 5984]"
    echo -e "\t-u   Provide a username for auth against CouchDB [Default: blank]"
    echo -e "\t       -- can also set with 'COUCHDB_USER' environment var"
    echo -e "\t-p   Provide a password for auth against CouchDB [Default: blank]"
    echo -e "\t       -- can also set with 'COUCHDB_PASS' environment var"
    echo -e "\t-l   Number of lines (documents) to Restore at a time. [Default: 5000] (Restore Only)"
    echo -e "\t-t   Number of CPU threads to use when parsing data [Default: nProcs-1] (Backup Only)"
    echo -e "\t-a   Number of times to Attempt import before failing [Default: 3] (Restore Only)"
    echo -e "\t-c   Create DB on demand, if they are not listed."
    echo -e "\t-q   Run in quiet mode. Suppress output, except for errors and warnings."
    echo -e "\t-z   Compress output file (Backup Only)"
    echo -e "\t-T   Add datetime stamp to output file name (Backup Only)"
    echo -e "\t-V   Display version information."
    echo -e "\t-h   Display usage information."
    echo
    echo "Example: $0 -b -H 127.0.0.1 -d mydb -f dumpedDB.json -u admin -p password"
    echo
    exit 1
}

scriptversion(){
    echo
    echo -e "\t** couchdb-dump version: $scriptversionnumber **"
    echo
    echo -e "\t URL:\thttps://github.com/danielebailo/couchdb-dump"
    echo
    echo -e "\t Authors:"
    echo -e "\t Daniele Bailo    (bailo.daniele@gmail.com)"
    echo -e "\t Darren Gibbard   (dalgibbard@gmail.com)"
    echo -e "\t Maximilian Huber (maximilian.huber@tngtech.com)"
    echo
    exit 1
}

checkdiskspace(){
## This function checks available diskspace for a required path, vs space required
## Example call:   checkdiskspace /path/to/file/to/create 1024
    location=$1
    KBrequired=$2
    if [ "x$location" = "x" ]||[ "x$KBrequired" = "x" ]; then
        echo "... ERROR: checkdiskspace() was not passed the correct arguments."
        exit 1
    fi

    stripdir=${location%/*}
    KBavail=$(df -P -k ${stripdir} | tail -n 1 | awk '{print$4}' | $sed_cmd -e 's/K$//')

    if [ $KBavail -ge $KBrequired ]; then
        return 0
    else
        echo
        echo "... ERROR: Insufficient Disk Space Available:"
        echo "        * Full Path:            ${location}"
        echo "        * Affected Directory:   ${stripdir}"
        echo "        * Space Available:      ${KBavail} KB"
        echo "        * Total Space Required: ${KBrequired} KB"
        echo "        * Additional Space Req: $(expr $KBrequired - $KBavail) KB"
        echo
        exit 1
    fi
}
## END FUNCTIONS

# Catch no args:
if [ "x$1" = "x" ]; then
    usage
fi

# Default Args
username=""
password=""
backup=false
restore=false
port=5984
OPTIND=1
lines=5000
attempts=3
createDBsOnDemand=false
verboseMode=true
compress=false
timestamp=false

while getopts ":h?H:d:f:u:p:P:l:t:a:c?q?z?T?V?b?B?r?R?" opt; do
    case "$opt" in
        h) usage;;
        b|B) backup=true ;;
        r|R) restore=true ;;
        H) url="$OPTARG" ;;
        d) db_name="$OPTARG" ;;
        f) file_name="$OPTARG" ;;
        u) username="${OPTARG}";;
        p) password="${OPTARG}";;
        P) port="${OPTARG}";;
        l) lines="${OPTARG}" ;;
        t) threads="${OPTARG}" ;;
        a) attempts="${OPTARG}";;
        c) createDBsOnDemand=true;;
        q) verboseMode=false;;
        z) compress=true;;
        T) timestamp=true;;
        V) scriptversion;;
        :) echo "... ERROR: Option \"-${OPTARG}\" requires an argument"; usage ;;
        *|\?) echo "... ERROR: Unknown Option \"-${OPTARG}\""; usage;;
    esac
done

# If quiet option: Setup echo mode and curl '--silent' opt
if [ "$verboseMode" = true ]; then
  curlSilentOpt=""
  echoVerbose=true
else
  curlSilentOpt="--silent"
  echoVerbose=false
fi

# Trap unexpected extra args
shift $((OPTIND-1))
[ "$1" = "--" ] && shift
if [ ! "x$@" = "x" ]; then
    echo "... ERROR: Unknown Option \"$@\""
    usage
fi

# Handle invalid backup/restore states:
if [ $backup = true ]&&[ $restore = true ]; then
    echo "... ERROR: Cannot pass both '-b' and '-r'"
    usage
elif [ $backup = false ]&&[ $restore = false ]; then
    echo "... ERROR: Missing argument '-b' (Backup), or '-r' (Restore)"
    usage
fi
# Handle empty args
# url
if [ "x$url" = "x" ]; then
    echo "... ERROR: Missing argument '-H <COUCHDB_HOST>'"
    usage
fi
# db_name
if [ "x$db_name" = "x" ]; then
    echo "... ERROR: Missing argument '-d <DB_NAME>'"
    usage
fi
# file_name
if [ "x$file_name" = "x" ]; then
    echo "... ERROR: Missing argument '-f <FILENAME>'"
    usage
fi
file_name_orig=$file_name

# Get OS TYPE (Linux for Linux, Darwin for MacOSX)
os_type=`uname -s`

# Pick sed or gsed
if [ "$os_type" = "FreeBSD" ]||[ "$os_type" = "Darwin" ]; then
    sed_cmd="gsed";
else
    sed_cmd="sed";
fi
## Make sure it's installed
echo | $sed_cmd 's/a//' >/dev/null 2>&1 
if [ ! $? = 0 ]; then
    echo "... ERROR: please install $sed_cmd (gnu-sed) and ensure it is in your path"
    exit 1
fi

# Validate thread count
## If we're on a Mac, use sysctl
if [ "$os_type" = "Darwin" ]; then
    cores=`sysctl -n hw.ncpu`
## If we're on FreeBSD, use sysctl
elif [ "$os_type" = "FreeBSD" ]; then
    cores=`sysctl kern.smp.cpus | awk -F ": " '{print $2}'`;
## Check if nproc available- set cores=1 if not
elif ! type nproc >/dev/null; then
    cores=1
## Otherwise use nproc
else
    cores=`nproc`
fi
if [ ! "x$threads" = "x" ]; then
    if [ $threads -gt $cores ]; then
        echo "... WARN: Thread setting of $threads is more than CPU count. Setting to $cores"
        threads=$cores
    else
        $echoVerbose && echo "... INFO: Setting parser threads to $threads"
    fi
else
    threads=`expr $cores - 1`
fi

# Validate Attempts, set to no-retry if zero/invalid.
case $attempts in
    ''|0|*[!0-9]*) echo "... WARN: Retry Attempt value of \"$attempts\" is invalid. Disabling Retry-on-Error."; attempts=1 ;;
    *) true ;;
esac

## Manage the passing of http/https for $url:
# Note; if the user wants to use 'https://' on a non-443 port they must specify it exclusively in the '-H <HOSTNAME>' arg.
if [ ! "`echo $url | grep -c http`" = 1 ]; then
    if [ "$port" == "443" ]; then
        url="https://$url";
    else
        url="http://$url";
    fi
fi

# Manage the addition of port
# If a port isn't already on our URL...
if [ ! "`echo $url | egrep -c ":[0-9]*$"`" = "1" ]; then
    # add it.
    url="$url:$port"
fi	

# Check for empty user/pass and try reading in from Envvars
if [ "x$username" = "x" ]; then
    username="$COUCHDB_USER"
fi
if [ "x$password" = "x" ]; then
    password="$COUCHDB_PASS"
fi

## Manage the addition of user+pass if needed:
# Ensure, if one is set, both are set.
if [ ! "x${username}" = "x" ]; then
    if [ "x${password}" = "x" ]; then
        echo "... ERROR: Password cannot be blank, if username is specified."
        usage
    fi
elif [ ! "x${password}" = "x" ]; then
    if [ "x${username}" = "x" ]; then
        echo "... ERROR: Username cannot be blank, if password is specified."
        usage
    fi
fi

# Check for sed option
sed_edit_in_place='-i.sedtmp'
if [ "$os_type" = "Darwin" ]; then
    sed_regexp_option='E'
else
    sed_regexp_option='r'
fi
# Allow for self-signed/invalid certs if method is HTTPS:
if [ "`echo $url | grep -ic "^https://"`" = "1" ]; then
	curlopt="-k"
fi

if [ ! "x${username}" = "x" ]&&[ ! "x${password}" = "x" ]; then
    curlopt="${curlopt} -u ${username}:${password}"
fi

## Check for curl
curl --version >/dev/null 2>&1 || ( echo "... ERROR: This script requires 'curl' to be present."; exit 1 )

# Check for tr
echo | tr -d "" >/dev/null 2>&1 || ( echo "... ERROR: This script requires 'tr' to be present."; exit 1 )

##### SETUP OUR LARGE VARS FOR SPLIT PROCESSING (due to limitations in split on Darwin/BSD)
AZ2="`echo {a..z}{a..z}`"
AZ3="`echo {a..z}{a..z}{a..z}`"

### If user selected BACKUP, run the following code:
if [ $backup = true ]&&[ $restore = false ]; then
    #################################################################
    ##################### BACKUP START ##############################
    #################################################################

    # If -T (timestamp) option, append datetime stamp ("-YYYYMMDD-hhmmss") before file extension
    if [ "$timestamp" = true ]; then
      datetime=`date "+%Y%m%d-%H%M%S"`						# Format: YYYYMMDD-hhmmss
      # Check for file_name extension, if so add the timestamp before it
      if [[ $file_name =~ \.[a-zA-Z0-9][a-zA-Z0-9_]* ]]; then
        file_name_ext=` echo "$file_name" | $sed_cmd 's/.*\.//'`		# Get text after last '.'
        file_name_base=`echo "$file_name" | $sed_cmd "s/\.${file_name_ext}$//"`	# file_name without '.' & extension
        file_name="$file_name_base-$datetime.$file_name_ext"
      else # Otherwise add timestamp to the end of file_name
        file_name="$file_name-$datetime"
      fi
    fi
    $echoVerbose && echo "... INFO: Output file ${file_name}"

    # Check if output already exists:
    if [ -f ${file_name} ]; then
        echo "... ERROR: Output file ${file_name} already exists."
        exit 1
    fi

    # Grab our data from couchdb
    curl ${curlSilentOpt} ${curlopt} -X GET "$url/$db_name/_all_docs?include_docs=true&attachments=true" -o ${file_name}
    # Check for curl errors
    if [ ! $? = 0 ]; then
        echo "... ERROR: Curl encountered an issue whilst dumping the database."
        rm -f ${file_name} 2>/dev/null
        exit 1
    fi
    # Check for export errors
    ERR_CHECK="`head -n 1 ${file_name} | grep '^{"error'`"
    if [ ! "x${ERR_CHECK}" = "x" ]; then
        echo "... ERROR: CouchDB reported: $ERR_CHECK"
        exit 1
    fi

    # CouchDB has a tendancy to output Windows carriage returns in it's output -
    # This messes up us trying to sed things at the end of lines!
    if grep -qU $'\x0d' $file_name; then
        $echoVerbose && echo "... INFO: File may contain Windows carriage returns- converting..."
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        checkdiskspace "${file_name}" $filesize
        tr -d '\r' < ${file_name} > ${file_name}.tmp
        if [ $? = 0 ]; then
            mv ${file_name}.tmp ${file_name}
            if [ $? = 0 ]; then
                $echoVerbose && echo "... INFO: Completed successfully."
            else
                echo "... ERROR: Failed to overwrite ${file_name} with ${file_name}.tmp"
                exit 1
            fi
        else
            echo ".. ERROR: Failed to convert file."
            exit 1
        fi
    fi

    ## Now we parse the output file to make it suitable for re-import.
    $echoVerbose && echo "... INFO: Amending file to make it suitable for Import."
    $echoVerbose && echo "... INFO: Stage 1 - Document filtering"

    # If the input file is larger than 250MB, multi-thread the parsing:
    if [ $(du -P -k ${file_name} | awk '{print$1}') -ge 256000 ]&&[ ! $threads -le 1 ]; then
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        KBreduction=$(($((`wc -l ${file_name} | awk '{print$1}'` * 80)) / 1024))
        filesize=`expr $filesize + $(expr $filesize - $KBreduction)`
        checkdiskspace "${file_name}" $filesize
        $echoVerbose && echo "... INFO: Multi-Threaded Parsing Enabled."
        if [ -f ${file_name}.thread000000 ]; then
            echo "... ERROR: Split files \"${file_name}.thread*\" already present. Please remove before continuing."
            exit 1
        elif [ -f ${file_name}.tmp ]; then
            echo "... ERROR: Tempfile ${file_name}.tmp already present. Please remove before continuing."
            exit 1
        fi

        ### SPLIT INTO THREADS
        split_cal=$(( $((`wc -l ${file_name} | awk '{print$1}'` / $threads)) + $threads ))
        #split --numeric-suffixes --suffix-length=6 -l ${split_cal} ${file_name} ${file_name}.thread
        split -a 2 -l ${split_cal} ${file_name} ${file_name}.thread
        if [ ! "$?" = "0" ]; then
            echo "... ERROR: Unable to create split files."
            exit 1
        fi

        # Capture if someone happens to breach the defined limits of AZ2 var. If this happens, we'll need to switch it out for AZ3 ...
        if [[ $threads -gt 650 ]]; then
            echo "Whoops- we hit a maximum limit here... \$AZ2 only allows for a maximum of 650 cores..."
            exit 1
        fi

        count=0
        for suffix in ${AZ2}; do
            (( count++ ))
            if [[ $count -gt $threads ]]; then
                break
            fi
            PADNAME="${file_name}.thread${suffix}"
            $sed_cmd ${sed_edit_in_place} 's/{"id".*,"doc"://g' ${PADNAME} &
        done
        wait
        count=0
        for suffix in ${AZ2}; do
            (( count++ ))
            if [[ $count -gt $threads ]]; then
                break
            fi
            PADNAME="${file_name}.thread${suffix}"
            cat ${PADNAME} >> ${file_name}.tmp
            rm -f ${PADNAME} ${PADNAME}.sedtmp
            (( NUM++ ))
        done
        if [ `wc -l ${file_name} | awk '{print$1}'` = `wc -l ${file_name}.tmp | awk '{print$1}'` ]; then
            mv ${file_name}{.tmp,}
            if [ ! $? = 0 ]; then
                echo "... ERROR: Failed to overwrite ${file_name}"
                exit 1
            fi
        else
            echo "... ERROR: Multi-threaded data parsing encountered an error."
            exit 1
        fi

    else
        # Estimating 80byte saving per line... probably a little conservative depending on keysize.
        KBreduction=$(($((`wc -l ${file_name} | awk '{print$1}'` * 80)) / 1024))
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        filesize=`expr $filesize - $KBreduction`
        checkdiskspace "${file_name}" $filesize
        $sed_cmd ${sed_edit_in_place} 's/{"id".*,"doc"://g' $file_name && rm -f ${file_name}.sedtmp
        if [ ! $? = 0 ];then
            echo "Stage failed."
            exit 1
        fi
    fi

    $echoVerbose && echo "... INFO: Stage 2 - Duplicate curly brace removal"
    # Approx 1Byte per line removed
    KBreduction=$((`wc -l ${file_name} | awk '{print$1}'` / 1024))
    filesize=$(du -P -k ${file_name} | awk '{print$1}')
    filesize=`expr $filesize - $KBreduction`
    checkdiskspace "${file_name}" $filesize
    $sed_cmd ${sed_edit_in_place} 's/}},$/},/g' ${file_name} && rm -f ${file_name}.sedtmp
    if [ ! $? = 0 ];then
        echo "Stage failed."
        exit 1
    fi
    $echoVerbose && echo "... INFO: Stage 3 - Header Correction"
    filesize=$(du -P -k ${file_name} | awk '{print$1}')
    checkdiskspace "${file_name}" $filesize
    $sed_cmd ${sed_edit_in_place} '1s/^.*/{"new_edits":false,"docs":[/' ${file_name} && rm -f ${file_name}.sedtmp
    if [ ! $? = 0 ];then
        echo "Stage failed."
        exit 1
    fi
    $echoVerbose && echo "... INFO: Stage 4 - Final document line correction"
    filesize=$(du -P -k ${file_name} | awk '{print$1}')
    checkdiskspace "${file_name}" $filesize
    $sed_cmd ${sed_edit_in_place} 's/}}$/}/g' ${file_name} && rm -f ${file_name}.sedtmp
    if [ ! $? = 0 ];then
        echo "Stage failed."
        exit 1
    fi

    # If -z (compress) option then compress output file
    if [ "$compress" = true ]; then
      $echoVerbose && echo "... INFO: Stage 5 - File compression"
      gzip $file_name
      file_name="$file_name.gz"
    fi

    $echoVerbose && echo "... INFO: Export completed successfully. File available at: ${file_name}"
    exit 0

### Else if user selected Restore:
elif [ $restore = true ]&&[ $backup = false ]; then
    #################################################################
    ##################### RESTORE START #############################
    #################################################################
    # Check if input exists:
    if [ ! -f ${file_name} ]; then
        echo "... ERROR: Input file ${file_name} not found."
        exit 1
    fi

    #### VALIDATION END

    $echoVerbose && echo "... INFO: Checking for database"
    attemptcount=0
    A=0
    until [ $A = 1 ]; do
        (( attemptcount++ ))
        existing_dbs=$(curl $curlSilentOpt $curlopt -X GET "${url}/_all_dbs")
        if [ ! $? = 0 ]; then
            if [ $attemptcount = $attempts ]; then
                echo "... ERROR: Curl failed to get the list of databases - Stopping"
                exit 1
            else
                echo "... WARN: Curl failed to get the list of databases - Attempt ${attemptcount}/${attempts}. Retrying..."
                sleep 1
            fi
        else
            A=1
        fi
    done
    if [[ ! "$existing_dbs" = "["*"]" ]]; then
        echo "... WARN: Curl failed to get the list of databases - Continuing"
        if [ "x$existing_dbs" = "x" ]; then
            echo "... WARN: Curl just returned: $existing_dbs"
        fi
    elif [[ ! "$existing_dbs" = *"\"${db_name}\""* ]]; then
        # database was not listed as existing databasa
        if [ $createDBsOnDemand = true ]; then
            attemptcount=0
            A=0
            until [ $A = 1 ]; do
                (( attemptcount++ ))
                curl $curlSilentOpt $curlopt -X PUT "${url}/${db_name}" -o tmp.out
                # If curl threw an error:
                if [ ! $? = 0 ]; then
                    if [ $attemptcount = $attempts ]; then
                        echo "... ERROR: Curl failed to create the database ${db_name} - Stopping"
                        if [ -f tmp.out ]; then
                            echo -n "... ERROR: Error message was:   "
                            cat tmp.out
                        else
                            echo ".. ERROR: See above for any errors"
                        fi
                        exit 1
                    else
                        echo "... WARN: Curl failed to create the database ${db_name} - Attempt ${attemptcount}/${attempts}. Retrying..."
                        sleep 1
                    fi
                # If curl was happy, but CouchDB returned an error in the return JSON:
                elif [ ! "`head -n 1 tmp.out | grep -c '^{"error":'`" = 0 ]; then
                    if [ $attemptcount = $attempts ]; then
                        echo "... ERROR: CouchDB Reported: `head -n 1 tmp.out`"
                        exit 1
                    else
                        echo "... WARN: CouchDB Reported an error during db creation - Attempt ${attemptcount}/${attempts} - Retrying..."
                        sleep 1
                    fi
                # Otherwise, if everything went well, delete our temp files.
                else
                    rm tmp.out
                    A=1
                fi
            done
        else
            echo "... ERROR: corresponding database ${db_name} not yet created - Stopping"
            $echoVerbose && echo "... HINT: you could add the -c flag to create the database automatically"
            exit 1
        fi
    fi

    ## Stop bash mangling wildcard...
    set -o noglob
    # Manage Design Documents as a priority, and remove them from the main import job
    $echoVerbose && echo "... INFO: Checking for Design documents"
    # Find all _design docs, put them into another file
    design_file_name=${file_name}-design
    grep '^{"_id":"_design' ${file_name} > ${design_file_name}

    # Count the design file (if it even exists)
    DESIGNS="`wc -l ${design_file_name} 2>/dev/null | awk '{print$1}'`"
    # If there's no design docs for import...
    if [ "x$DESIGNS" = "x" ]||[ "$DESIGNS" = "0" ]; then 
        # Cleanup any null files
        rm -f ${design_file_name} 2>/dev/null
        $echoVerbose && echo "... INFO: No Design Documents found for import."
    else
        $echoVerbose && echo "... INFO: Duplicating original file for alteration"
        # Duplicate the original DB file, so we don't mangle the user's input file:
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        checkdiskspace "${file_name}" $filesize
        cp -f ${file_name}{,-nodesign}
        # Re-set file_name to be our new file.
        file_name=${file_name}-nodesign
        # Remove these design docs from (our new) main file.
        $echoVerbose && echo "... INFO: Stripping _design elements from regular documents"
        checkdiskspace "${file_name}" $filesize
        $sed_cmd ${sed_edit_in_place} '/^{"_id":"_design/d' ${file_name} && rm -f ${file_name}.sedtmp
        # Remove the final document's trailing comma
        $echoVerbose && echo "... INFO: Fixing end document"
        line=$(expr `wc -l ${file_name} | awk '{print$1}'` - 1)
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        checkdiskspace "${file_name}" $filesize
        $sed_cmd ${sed_edit_in_place} "${line}s/,$//" ${file_name} && rm -f ${file_name}.sedtmp

        $echoVerbose && echo "... INFO: Inserting Design documents"
        designcount=0
        # For each design doc...
        while IFS="" read -r; do
            line="${REPLY}"
            # Split the ID out for use as the import URL path
            URLPATH=$(echo $line | awk -F'"' '{print$4}')
            # Scrap the ID and Rev from the main data, as well as any trailing ','
            echo "${line}" | $sed_cmd -${sed_regexp_option}e "s@^\{\"_id\":\"${URLPATH}\",\"_rev\":\"[0-9]*-[0-9a-zA-Z_\-]*\",@\{@" | $sed_cmd -e 's/,$//' > ${design_file_name}.${designcount}
            # Fix Windows CRLF
            if grep -qU $'\x0d' ${design_file_name}.${designcount}; then
                $echoVerbose && echo "... INFO: File contains Windows carriage returns- converting..."
                filesize=$(du -P -k ${design_file_name}.${designcount} | awk '{print$1}')
                checkdiskspace "${file_name}" $filesize
                tr -d '\r' < ${design_file_name}.${designcount} > ${design_file_name}.${designcount}.tmp
                if [ $? = 0 ]; then
                    mv ${design_file_name}.${designcount}.tmp ${design_file_name}.${designcount}
                    if [ $? = 0 ]; then
                        $echoVerbose && echo "... INFO: Completed successfully."
                    else
                        echo "... ERROR: Failed to overwrite ${design_file_name}.${designcount} with ${design_file_name}.${designcount}.tmp"
                        exit 1
                    fi
                else
                    echo ".. ERROR: Failed to convert file."
                    exit 1
                fi
            fi

            # Insert this file into the DB
            A=0
            attemptcount=0
            until [ $A = 1 ]; do
                (( attemptcount++ ))
                curl $curlSilentOpt ${curlopt} -T ${design_file_name}.${designcount} -X PUT "${url}/${db_name}/${URLPATH}" -H 'Content-Type: application/json' -o ${design_file_name}.out.${designcount}
                # If curl threw an error:
                if [ ! $? = 0 ]; then
                     if [ $attemptcount = $attempts ]; then
                         echo "... ERROR: Curl failed trying to restore ${design_file_name}.${designcount} - Stopping"
                         exit 1
                     else
                         echo "... WARN: Import of ${design_file_name}.${designcount} failed - Attempt ${attemptcount}/${attempts}. Retrying..."
                         sleep 1
                     fi
                # If curl was happy, but CouchDB returned an error in the return JSON:
                elif [ ! "`head -n 1 ${design_file_name}.out.${designcount} | grep -c '^{"error":'`" = 0 ]; then
                     if [ $attemptcount = $attempts ]; then
                         echo "... ERROR: CouchDB Reported: `head -n 1 ${design_file_name}.out.${designcount}`"
                         exit 1
                     else
                         echo "... WARN: CouchDB Reported an error during import - Attempt ${attemptcount}/${attempts} - Retrying..."
                         sleep 1
                     fi
                # Otherwise, if everything went well, delete our temp files.
                else
                     A=1
                     rm -f ${design_file_name}.out.${designcount}
                     rm -f ${design_file_name}.${designcount}
                fi
            done
            # Increase design count - mainly used for the INFO at the end.
            (( designcount++ ))
        # NOTE: This is where we insert the design lines exported from the main block
        done < <(cat ${design_file_name})
        $echoVerbose && echo "... INFO: Successfully imported ${designcount} Design Documents"
    fi
    set +o noglob

    # If the size of the file to import is less than our $lines size, don't worry about splitting
    if [ `wc -l $file_name | awk '{print$1}'` -lt $lines ]; then
        $echoVerbose && echo "... INFO: Small dataset. Importing as a single file."
        A=0
        attemptcount=0
        until [ $A = 1 ]; do
            (( attemptcount++ ))
            curl $curlSilentOpt $curlopt -T $file_name -X POST "$url/$db_name/_bulk_docs" -H 'Content-Type: application/json' -o tmp.out
            if [ "`head -n 1 tmp.out | grep -c '^{"error":'`" -eq 0 ]; then
                $echoVerbose && echo "... INFO: Imported ${file_name_orig} Successfully."
                rm -f tmp.out
                rm -f ${file_name_orig}-design
                rm -f ${file_name_orig}-nodesign
                exit 0
            else
                if [ $attemptcount = $attempts ]; then
                    echo "... ERROR: Import of ${file_name_orig} failed."
                    if [ -f tmp.out ]; then
                        echo -n "... ERROR: Error message was:   "
                        cat tmp.out
                    else
                        echo ".. ERROR: See above for any errors"
                    fi
                    rm -f tmp.out
                    exit 1
                else
                    echo "... WARN: Import of ${file_name_orig} failed - Attempt ${attemptcount}/${attempts} - Retrying..."
                    sleep 1
                fi
            fi
        done
    # Otherwise, it's a large import that requires bulk insertion.
    else
        $echoVerbose && echo "... INFO: Block import set to ${lines} lines."
        if [ -f ${file_name}.splitaaa ]; then
            echo "... ERROR: Split files \"${file_name}.split*\" already present. Please remove before continuing."
            exit 1
        fi
        importlines=`cat ${file_name} | grep -c .`

        # Due to the file limit imposed by the pre-calculated AZ3 variable, max split files is 15600 (alpha x 3positions)
        if [[ `expr ${importlines} / ${lines}` -gt 15600 ]]; then
            echo "... ERROR: Pre-processed split variable limit of 15600 files reached."
            echo "           Please increase the '-l' parameter (Currently: $lines) and try again."
            exit 1
        fi

        $echoVerbose && echo "... INFO: Generating files to import"
        filesize=$(du -P -k ${file_name} | awk '{print$1}')
        checkdiskspace "${file_name}" $filesize
        ### Split the file into many
        split -a 3 -l ${lines} ${file_name} ${file_name}.split
        if [ ! "$?" = "0" ]; then
            echo "... ERROR: Unable to create split files."
            exit 1
        fi
        HEADER="`head -n 1 $file_name`"
        FOOTER="`tail -n 1 $file_name`"

        count=0
        for PADNUM in $AZ3; do
            PADNAME="${file_name}.split${PADNUM}"
            if [ ! -f ${PADNAME} ]; then
                echo "... INFO: Import Cycle Completed."
                break
            fi

            if [ ! "`head -n 1 ${PADNAME}`" = "${HEADER}" ]; then
                $echoVerbose && echo "... INFO: Adding header to ${PADNAME}"
                filesize=$(du -P -k ${PADNAME} | awk '{print$1}')
                checkdiskspace "${PADNAME}" $filesize
                $sed_cmd ${sed_edit_in_place} "1i${HEADER}" ${PADNAME} && rm -f ${PADNAME}.sedtmp
            else
                $echoVerbose && echo "... INFO: Header already applied to ${PADNAME}"
            fi
            if [ ! "`tail -n 1 ${PADNAME}`" = "${FOOTER}" ]; then
                $echoVerbose && echo "... INFO: Adding footer to ${PADNAME}"
                filesize=$(du -P -k ${PADNAME} | awk '{print$1}')
                checkdiskspace "${PADNAME}" $filesize
                $sed_cmd ${sed_edit_in_place} '$s/,$//g' ${PADNAME} && rm -f ${PADNAME}.sedtmp
                echo "${FOOTER}" >> ${PADNAME}
            else
                $echoVerbose && echo "... INFO: Footer already applied to ${PADNAME}"
            fi

            $echoVerbose && echo "... INFO: Inserting ${PADNAME}"
            A=0
            attemptcount=0
            until [ $A = 1 ]; do
                (( attemptcount++ ))
                curl $curlSilentOpt $curlopt -T ${PADNAME} -X POST "$url/$db_name/_bulk_docs" -H 'Content-Type: application/json' -o tmp.out
                if [ ! $? = 0 ]; then
                    if [ $attemptcount = $attempts ]; then
                        echo "... ERROR: Curl failed trying to restore ${PADNAME} - Stopping"
                        exit 1
                    else
                        echo "... WARN: Failed to import ${PADNAME} - Attempt ${attemptcount}/${attempts} - Retrying..."
                        sleep 1
                    fi
                elif [ ! "`head -n 1 tmp.out | grep -c '^{"error":'`" = 0 ]; then
                    if [ $attemptcount = $attempts ]; then
                        echo "... ERROR: CouchDB Reported: `head -n 1 tmp.out`"
                        exit 1
                    else
                        echo "... WARN: CouchDB Reported and error during import - Attempt ${attemptcount}/${attempts} - Retrying..."
                        sleep 1
                    fi
                else
                    A=1
                    rm -f ${PADNAME}
                    rm -f tmp.out
                    (( count++ ))
                fi
            done

            $echoVerbose && echo "... INFO: Successfully Imported `expr ${count}` Files"
            A=1
            rm -f ${file_name_orig}-design
            rm -f ${file_name_orig}-nodesign
        done
    fi
fi