diff --git a/examples/receipts/2020-01-07t1205_lunch.pdf b/examples/receipts/2020-01-07t1205_lunch.pdf new file mode 100644 index 0000000..6255e40 Binary files /dev/null and b/examples/receipts/2020-01-07t1205_lunch.pdf differ diff --git a/scripts/csv2yaml/dkb-visa.js b/scripts/csv2yaml/dkb-visa.js index 45b0c2d..4d2cd72 100644 --- a/scripts/csv2yaml/dkb-visa.js +++ b/scripts/csv2yaml/dkb-visa.js @@ -7,6 +7,7 @@ const { rmEmptyString, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') @@ -74,11 +75,7 @@ function normalizeAndPrint (filePathTemp) { .localeCompare(String(transB.utc), 'en'), ) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2') - .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/csv2yaml/dkb.js b/scripts/csv2yaml/dkb.js new file mode 100644 index 0000000..f217a57 --- /dev/null +++ b/scripts/csv2yaml/dkb.js @@ -0,0 +1,91 @@ +const fse = require('fs-extra') +const yaml = require('js-yaml') +const csvnorm = require('csvnorm') +const converter = require('converter') + +const { + rmEmptyString, + keysToEnglish, + noteToAccount, + sanitizeYaml, +} = require('../helpers.js') + + +function normalizeAndPrint (filePathTemp) { + const csv2json = converter({ + from: 'csv', + to: 'json', + }) + + let jsonTemp = '' + csv2json.on('data', chunk => { + jsonTemp += chunk + }) + csv2json.on('end', () => { + const transactions = JSON + .parse(jsonTemp) + .map(keysToEnglish) + .map(transaction => { + const note = transaction.note + .replace(//g, '\n') + const amount = transaction.amount + ' €' + const sortedTransaction = { + utc: transaction['value-utc'] < transaction['entry-utc'] + ? transaction['value-utc'] + : transaction['entry-utc'], + } + + if (transaction['value-utc'] !== sortedTransaction.utc) { + sortedTransaction['value-utc'] = transaction['value-utc'] + } + if (transaction['entry-utc'] !== sortedTransaction.utc) { + sortedTransaction['entry-utc'] = transaction['entry-utc'] + } + + sortedTransaction.type = transaction.type + sortedTransaction.note = note + + const transfersObj = transaction.amount.startsWith('-') + ? { + transfers: [{ + from: 'dkb:visa', + to: noteToAccount(note), + amount: amount.slice(1), + 'original-amount': transaction['original-amount'], + }], + } + : { + transfers: [{ + from: noteToAccount(note), + to: 'dkb:visa', + // TODO: Remove when github.com/adius/csvnorm/issues/1 is solved + amount: transaction.amount === '0,00' ? '0 €' : amount, + 'original-amount': transaction['original-amount'], + }], + } + const newTransaction = Object.assign(sortedTransaction, transfersObj) + + delete newTransaction.amount + + return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) + }) + .sort((transA, transB) => + // Oldest first + String(transA.utc) + .localeCompare(String(transB.utc), 'en'), + ) + + const yamlString = sanitizeYaml(yaml.dump({transactions})) + + console.info(yamlString) + }) + + csvnorm.default({ + encoding: 'latin1', + readableStream: fse.createReadStream(filePathTemp), + skipLinesStart: 6, + writableStream: csv2json, + }) +} + +normalizeAndPrint(process.argv[2]) diff --git a/scripts/csv2yaml/fidor.js b/scripts/csv2yaml/fidor.js new file mode 100644 index 0000000..9616f67 --- /dev/null +++ b/scripts/csv2yaml/fidor.js @@ -0,0 +1,74 @@ +const fse = require('fs-extra') +const yaml = require('js-yaml') +const csvnorm = require('csvnorm') +const converter = require('converter') + +const { + rmEmptyString, + keysToEnglish, + noteToAccount, + sanitizeYaml, +} = require('../helpers.js') + + +function normalizeAndPrint (filePathTemp) { + const csv2json = converter({ + from: 'csv', + to: 'json', + }) + + let jsonTemp = '' + csv2json.on('data', chunk => { + jsonTemp += chunk + }) + csv2json.on('end', () => { + const transactions = JSON + .parse(jsonTemp) + .map(keysToEnglish) + .reverse() // Now sorted ascending by value date + .map(transaction => { + const currency = ' €' + const sortedTransaction = { + utc: transaction.date, + note: transaction.note + '\n' + transaction.note2, + } + const account = noteToAccount(transaction.note) || '_todo_' + const transfersObj = transaction.amount.startsWith('-') + ? { + transfers: [{ + from: 'fidor:giro', + to: account, + amount: transaction.amount.slice(1) + currency, + }], + } + : { + transfers: [{ + from: account, + to: 'fidor:giro', + // TODO: Remove when https://github.com/adius/csvnorm/issues/1 + // is solved + amount: transaction.amount === '0,00' + ? 0 + : transaction.amount + currency, + }], + } + const newTransaction = Object.assign(sortedTransaction, transfersObj) + + delete newTransaction.amount + + return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) + }) + + const yamlString = sanitizeYaml(yaml.dump({transactions})) + + console.info(yamlString) + }) + + csvnorm.default({ + encoding: 'utf-8', + readableStream: fse.createReadStream(filePathTemp), + writableStream: csv2json, + }) +} + +normalizeAndPrint(process.argv[2]) diff --git a/scripts/csv2yaml/finvesto.js b/scripts/csv2yaml/finvesto.js index 597182a..8cf61c6 100644 --- a/scripts/csv2yaml/finvesto.js +++ b/scripts/csv2yaml/finvesto.js @@ -7,6 +7,7 @@ const { rmEmptyString, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') @@ -69,11 +70,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n {2}-\n {4}') - .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2') - .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/csv2yaml/hypovereinsbank.js b/scripts/csv2yaml/hypovereinsbank.js index b14078b..deca67b 100644 --- a/scripts/csv2yaml/hypovereinsbank.js +++ b/scripts/csv2yaml/hypovereinsbank.js @@ -7,6 +7,7 @@ const { rmEmptyString, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') @@ -66,11 +67,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2') - .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/csv2yaml/mbs.js b/scripts/csv2yaml/mbs.js index bf08952..2dd02fa 100644 --- a/scripts/csv2yaml/mbs.js +++ b/scripts/csv2yaml/mbs.js @@ -7,6 +7,7 @@ const { rmEmptyString, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') @@ -41,7 +42,7 @@ function normalizeAndPrint (filePathTemp) { delete sortedTransaction['entry-utc'] } - const account = noteToAccount(transaction.to) + const account = noteToAccount(transaction.to) || '_todo_' const transfersObj = transaction.amount.startsWith('-') ? { transfers: [{ @@ -69,11 +70,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2') - .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/csv2yaml/paypal.js b/scripts/csv2yaml/paypal.js index 8b0330c..4ce1fdc 100644 --- a/scripts/csv2yaml/paypal.js +++ b/scripts/csv2yaml/paypal.js @@ -8,6 +8,8 @@ const chrono = require('chrono-node') const { rmEmptyString, keysToEnglish, + noteToAccount, + sanitizeYaml, } = require('../helpers.js') @@ -46,21 +48,18 @@ function normalizeAndPrint (filePathTemp) { }, transaction, ) + const account = noteToAccount(transaction.Name) const transfer = transaction.Gross.startsWith('-') ? { from: '_todo_:paypal:' + transaction.Currency .toLowerCase() .trim(), - to: transaction.Name - ? `${transaction.Name} <${transaction['To Email Address']}>` - : 'paypal', + to: account ? account : 'paypal', amount: transaction.Gross.slice(1) + ' ' + currency, } : { - from: transaction.Name - ? `${transaction.Name} <${transaction['From Email Address']}>` - : 'paypal', + from: account ? account : 'paypal', to: '_todo_:paypal:' + transaction.Currency .toLowerCase() @@ -101,11 +100,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^([\w- ]+): '(.+)'$/gm, '$1: $2') - .replace(/utc: 20(.+)$/gm, 'utc: \'20$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/helpers.js b/scripts/helpers.js index 0ac2e71..d6bd5b1 100644 --- a/scripts/helpers.js +++ b/scripts/helpers.js @@ -4,8 +4,20 @@ module.exports = { prettyFormat, prettyPrint, rmEmptyString, - toDdotMdotYYYY, + sanitizeYaml, toDDdotMMdotYYYY, + toDdotMdotYYYY, +} + + +function sanitizeYaml (yaml) { + if (typeof yaml !== 'string') { + throw new Error('YAML must be passed as a string') + } + return yaml + .replace(/^ {2}- /gm, '\n -\n ') + .replace(/^ {2}([\w- ]+): '(.+)'$/gm, ' $1: $2') + .replace(/utc: ([0-9TZ:.-]+)$/gm, 'utc: \'$1\'') } @@ -80,6 +92,7 @@ function keysToEnglish (object) { .replace('Verwendungszweck', 'note') .replace('Waehrung', 'currency') .replace('Wertstellung', 'value-utc') + .replace('Wert', 'amount') .replace('Zahlungsbetrag in ZW', 'amount') .replace('Zahlungswährung (ZW)', 'currency') @@ -89,6 +102,9 @@ function keysToEnglish (object) { } function noteToAccount (note) { + // Remove misleading terms + note = note.replace('Apple Pay', '') + // Sorted by ascending importance // I.e. later keywords overwrite selection /* eslint-disable quote-props */ @@ -98,11 +114,13 @@ function noteToAccount (note) { 'amazon': 'amazon', 'amazon prime': 'amazon:prime', 'patreon': 'patreon', + 'facebook': 'facebook', 'google ireland limited cloud platform': 'google:cloud', 'day night sports gmbh': 'day_night_sports', 'spotify': 'spotify', 'apple': 'apple', 'itunes': 'apple:itunes', + 'mozilla foundation': 'mozilla_foundation', 'mozilla': 'mozilla', 'namecheap': 'namecheap', 'name-cheap': 'namecheap', @@ -130,6 +148,14 @@ function noteToAccount (note) { 'jimmy joy': 'jimmy_joy', 'wecircberlin': 'circ', 'lime ride': 'lime', + 'lime electric': 'lime', + 'bird rides': 'bird', + 'google': 'google', + 'free software foundation': 'free_software_foundation', + 'landr audio': 'landr', + 'mzla technologies': 'mzla_technologies', + 'vodafone': 'vodafone', + 'mailgun': 'mailgun', // German 'ihk ': 'ihk', @@ -159,11 +185,15 @@ function noteToAccount (note) { 'qthority': 'qthority', 'nosh good taste': 'nosh', 'musikhaus thomann': 'thomann', - 'fratellis frankfurt': 'fratellis_frankfurt', + 'fratellis frankfurt': 'fratellis_ristorante', + 'fratellis rist': 'fratellis_ristorante', 'hansemerkur speziale kv': 'hansemerkur', 'zwanzigeins e.v.': 'zwanzigeins', 'tier de': 'tier', 'zalando': 'zalando', + 'hetzner online': 'hetzner', + 'golem media': 'golem', + 'pro rauchfrei': 'pro_rauchfrei', } /* eslint-enable quote-props */ let account = note @@ -180,5 +210,5 @@ function noteToAccount (note) { } }) - return account || '_todo_' + return account } diff --git a/scripts/transactions/dkb-visa.js b/scripts/transactions/dkb-visa.js index 5a18b43..fde06be 100644 --- a/scripts/transactions/dkb-visa.js +++ b/scripts/transactions/dkb-visa.js @@ -12,6 +12,7 @@ const { toDdotMdotYYYY, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') const prompt = inquirer.createPromptModule({ output: process.stderr }) @@ -62,18 +63,19 @@ function normalizeAndPrint (filePathTemp) { sortedTransaction.type = transaction.type sortedTransaction.note = note + const account = noteToAccount(note) const transfersObj = transaction.amount.startsWith('-') ? { transfers: [{ from: 'dkb:visa', - to: noteToAccount(note), + to: account, amount: amount.slice(1), 'original-amount': transaction['original-amount'], }], } : { transfers: [{ - from: noteToAccount(note), + from: account, to: 'dkb:visa', // TODO: Remove when github.com/adius/csvnorm/issues/1 is solved amount: transaction.amount === '0,00' ? '0 €' : amount, @@ -92,10 +94,7 @@ function normalizeAndPrint (filePathTemp) { .localeCompare(String(transB.utc), 'en'), ) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n-\n ') - .replace(/^ {2}([\w- ]+): '(.+)'$/gm, '$1: $2') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/transactions/dkb.js b/scripts/transactions/dkb.js index 8d2a489..b3065b1 100644 --- a/scripts/transactions/dkb.js +++ b/scripts/transactions/dkb.js @@ -12,6 +12,7 @@ const { toDDdotMMdotYYYY, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') const prompt = inquirer.createPromptModule({ output: process.stderr }) @@ -90,11 +91,7 @@ function normalizeAndPrint (filePathTemp) { .localeCompare(String(transB.utc), 'en'), ) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^ {2}([\w- ]+): '(.+)'$/gm, ' $1: $2') - .replace(/utc: ([0-9TZ:.-]+)$/gm, 'utc: \'$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/transactions/hypovereinsbank.js b/scripts/transactions/hypovereinsbank.js index 4225457..3bc3f4c 100644 --- a/scripts/transactions/hypovereinsbank.js +++ b/scripts/transactions/hypovereinsbank.js @@ -13,6 +13,7 @@ const { toDdotMdotYYYY, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') const prompt = inquirer.createPromptModule({ output: process.stderr }) @@ -54,17 +55,18 @@ function normalizeAndPrint (filePathTemp) { sortedTransaction.note = transaction.note + const account = noteToAccount(transaction.note) const transfersObj = transaction.amount.startsWith('-') ? { transfers: [{ from: 'hypo:giro', - to: noteToAccount(transaction.note), + to: account, amount: transaction.amount.slice(1) + currency, }], } : { transfers: [{ - from: noteToAccount(transaction.note), + from: account, to: 'hypo:giro', // TODO: Remove when github.com/adius/csvnorm/issues/1 is solved amount: transaction.amount === '0,00' @@ -83,11 +85,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n -\n ') - .replace(/^ {2}([\w- ]+): '(.+)'$/gm, ' $1: $2') - .replace(/utc: ([0-9TZ:.-]+)$/gm, 'utc: \'$1\'') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) }) diff --git a/scripts/transactions/mbs.js b/scripts/transactions/mbs.js index 31209b5..4b01936 100644 --- a/scripts/transactions/mbs.js +++ b/scripts/transactions/mbs.js @@ -15,6 +15,7 @@ const { toDdotMdotYYYY, keysToEnglish, noteToAccount, + sanitizeYaml, } = require('../helpers.js') nightmareDownloadManager(Nightmare) @@ -82,10 +83,7 @@ function normalizeAndPrint (filePathTemp) { return JSON.parse(JSON.stringify(newTransaction, rmEmptyString)) }) - const yamlString = yaml - .dump({transactions}) - .replace(/^ {2}- /gm, '\n-\n ') - .replace(/^ {2}([\w- ]+): '(.+)'$/gm, '$1: $2') + const yamlString = sanitizeYaml(yaml.dump({transactions})) console.info(yamlString) })