Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update Findependent PDF Extractor #4348

Closed
wants to merge 1 commit into from
Closed

update Findependent PDF Extractor #4348

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 74 additions & 39 deletions ...hen.portfolio/src/name/abuchen/portfolio/datatransfer/pdf/FindependentAGPDFExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ public String getLabel()

private void addBuySellTransaction()
{
DocumentType type = new DocumentType("ETF\\-(Kauf|Verkauf)");
DocumentType type = new DocumentType("ETF.(Kauf|Verkauf|purchase|sale)");
this.addDocumentTyp(type);

Transaction<BuySellEntry> pdfTransaction = new Transaction<>();

Block firstRelevantLine = new Block("^ETF\\-(Kauf|Verkauf)$", "^Erstellt am:.*$");
Block firstRelevantLine = new Block("^ETF.(Kauf|Verkauf|purchase|sale)$", "^(Created on|Erstellt am):.*$");
type.addBlock(firstRelevantLine);
firstRelevantLine.set(pdfTransaction);

Expand All @@ -57,12 +57,14 @@ private void addBuySellTransaction()
return portfolioTransaction;
})

// Is type --> "Verkauf" change from BUY to SELL
// Is type --> "Verkauf|sale" change from BUY to SELL
.section("type").optional() //
.match("^ETF\\-(?<type>(Kauf|Verkauf))$") //
.match("^ETF.(?<type>(Kauf|Verkauf|purchase|sale))$") //
.assign((t, v) -> {
if ("Verkauf".equals(v.get("type")))
t.setType(PortfolioTransaction.Type.SELL);
if ("sale".equals(v.get("type")))
t.setType(PortfolioTransaction.Type.SELL);
})

// @formatter:off
Expand All @@ -71,31 +73,31 @@ private void addBuySellTransaction()
// Preis pro Anteil CHF 100.040
// @formatter:on
.section("name", "isin", "currency") //
.match("^ETF-Name (?<name>.*)$") //
.match("^(ETF-Name|ETF name) (?<name>.*)$") //
.match("^ISIN (?<isin>[A-Z]{2}[A-Z0-9]{9}[0-9])$") //
.match("^Preis pro Anteil (?<currency>[\\w]{3}) [\\.'\\d]+$") //
.match("^(Preis pro Anteil|Price per share) (?<currency>[\\w]{3}) [\\.'\\d]+$") //
.assign((t, v) -> t.setSecurity(getOrCreateSecurity(v)))

// @formatter:off
// Anzahl Anteile 2
// @formatter:on
.section("shares") //
.match("^Anzahl Anteile (?<shares>[\\',\\d]+)$") //
.match("^(Anzahl Anteile|Number of shares) (?<shares>[\\',\\d]+)$") //
.assign((t, v) -> t.setShares(asShares(v.get("shares"))))

// @formatter:off
// Valuta 01.12.2023
// @formatter:on
.section("date") //
.match("^Valuta (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.match("^(Valuta|Value date) (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.assign((t, v) -> t.setDate(asDate(v.get("date"))))

// @formatter:off
// Verrechneter Betrag CHF 200.43
// Gutgeschriebener Betrag CHF 700.02
// @formatter:on
.section("currency", "amount") //
.match("^(Verrechneter|Gutgeschriebener) Betrag (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.match("^(Amount (charged|credited)|(Verrechneter|Gutgeschriebener) Betrag) (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
Expand All @@ -107,9 +109,9 @@ private void addBuySellTransaction()
// Kaufpreis CHF CHF 798.94
// @formatter:on
.section("fxGross", "baseCurrency", "termCurrency", "exchangeRate", "gross").optional() //
.match("^Kaufpreis total [\\w]{3} (?<fxGross>[\\.'\\d]+)$") //
.match("^Wechselkurs (?<termCurrency>[\\w]{3})\\/(?<baseCurrency>[\\w]{3}) (?<exchangeRate>[\\.'\\d]+)$") //
.match("^Kaufpreis [\\w]{3} [\\w]{3} (?<gross>[\\.'\\d]+)$") //
.match("^(Kaufpreis total|Verkaufspreis total|Total purchase price|Total sales price) [\\w]{3} (?<fxGross>[\\.'\\d]+)$") //
.match("^(Wechselkurs|Exchange rate) (?<termCurrency>[\\w]{3})\\/(?<baseCurrency>[\\w]{3}) (?<exchangeRate>[\\.'\\d]+)$") //
.match("^(Kaufpreis|Verkaufspreis|Purchase price|Selling price) [\\w]{3} [\\w]{3} (?<gross>[\\.'\\d]+)$") //
.assign((t, v) -> {
ExtrExchangeRate rate = asExchangeRate(v);
type.getCurrentContext().putType(rate);
Expand All @@ -128,12 +130,12 @@ private void addBuySellTransaction()

private void addDividendsTransaction()
{
DocumentType type = new DocumentType("Ertragsaussch.ttung");
DocumentType type = new DocumentType("(Ertragsaussch.ttung|Income distribution)");
this.addDocumentTyp(type);

Transaction<AccountTransaction> pdfTransaction = new Transaction<>();

Block firstRelevantLine = new Block("^Ertragsaussch.ttung$");
Block firstRelevantLine = new Block("^(Ertragsaussch.ttung|Income distribution)$");
type.addBlock(firstRelevantLine);
firstRelevantLine.set(pdfTransaction);

Expand All @@ -151,33 +153,52 @@ private void addDividendsTransaction()
// Bruttoertrag pro Anteil CHF 0.1200
// @formatter:on
.section("name", "isin", "currency") //
.match("^ETF-Name (?<name>.*)$") //
.match("^(ETF-Name|ETF name) (?<name>.*)$") //
.match("^ISIN (?<isin>[A-Z]{2}[A-Z0-9]{9}[0-9])$") //
.match("^(Bruttoertrag|Ertrag) pro Anteil (?<currency>[\\w]{3}) [\\.'\\d]+$") //
.match("^((Bruttoertrag|Ertrag) pro Anteil|(Gross income|Income) per (unit|share)) (?<currency>[\\w]{3}) [\\.'\\d]+$") //
.assign((t, v) -> t.setSecurity(getOrCreateSecurity(v)))

// @formatter:off
// Anzahl Anteile 58
// @formatter:on
.section("shares") //
.match("^Anzahl Anteile (?<shares>[\\.'\\d]+)$") //
.match("^(Anzahl Anteile|Number of shares) (?<shares>[\\.'\\d]+)$") //
.assign((t, v) -> t.setShares(asShares(v.get("shares"))))

// @formatter:off
// Valuta 13.09.2023
// @formatter:on
.section("date") //
.match("^Valuta (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.match("^(Valuta|Value date) (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.assign((t, v) -> t.setDateTime(asDate(v.get("date"))))

// @formatter:off
// Ertrag total CHF CHF 4.52
// @formatter:on
.section("currency", "amount") //
.match("^Ertrag total [\\w]{3} (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.match("^(Ertrag total|Total income|Total gross income) [\\w]{3} (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
})

// @formatter:off
// Ertrag total USD 912.50
// Wechselkurs CHF/USD 0.87555
// Ertrag total CHF CHF 798.94
// @formatter:on
.section("fxGross", "baseCurrency", "termCurrency", "exchangeRate", "gross").optional() //
.match("^(Ertrag total|Total income|Total gross income) [\\w]{3} (?<fxGross>[\\.'\\d]+)$") //
.match("^(Wechselkurs|Exchange rate) (?<termCurrency>[\\w]{3})\\/(?<baseCurrency>[\\w]{3}) (?<exchangeRate>[\\.'\\d]+)$") //
.match("^(Ertrag total|Total income|Total gross income) [\\w]{3} [\\w]{3} (?<gross>[\\.'\\d]+)$") //
.assign((t, v) -> {
ExtrExchangeRate rate = asExchangeRate(v);
type.getCurrentContext().putType(rate);

Money gross = Money.of(rate.getTermCurrency(), asAmount(v.get("gross")));
Money fxGross = Money.of(rate.getBaseCurrency(), asAmount(v.get("fxGross")));

checkAndSetGrossUnit(gross, fxGross, t, type.getCurrentContext());
})

.wrap(TransactionItem::new);
Expand All @@ -188,16 +209,17 @@ private void addDividendsTransaction()

private void addAccountStatementTransaction()
{
DocumentType type = new DocumentType("Deinem Konto wurde (gut(ge)?schrieben|belastet):");
DocumentType type = new DocumentType(
"(Deinem Konto wurde (gut(ge)?schrieben|belastet)|Your account has been (credited|debited)):");
this.addDocumentTyp(type);

Transaction<AccountTransaction> pdfTransaction = new Transaction<>();

Block firstRelevantLine = new Block("^(Einzahlung" //
+ "|Willkommensbonus von findependent" //
+ "|Depotgeb.hren an" //
+ "|Verwaltungsgeb.hren an" //
+ "|Geb.hrenerstattung von).*$"); //
Block firstRelevantLine = new Block("^((Einzahlung|Incoming payment)|(Auszahlung|Outgoing payment)" //
+ "|(Willkommensbonus von findependent|Findependent welcome bonus)" //
+ "|(Depotgeb.hren an|Custody fees to)" //
+ "|(Verwaltungsgeb.hren an|Management fees to)" //
+ "|(Geb.hrenerstattung von|Reimbursement of fees from)).*$"); //
type.addBlock(firstRelevantLine);
firstRelevantLine.set(pdfTransaction);

Expand All @@ -214,20 +236,33 @@ private void addAccountStatementTransaction()
// Is type --> "Gebührenerstattung" change from DEPOSIT to FEES_REFUND
// @formatter:on
.section("type").optional() //
.match("^(?<type>(Depotgeb.hren|Verwaltungsgeb.hren|Geb.hrenerstattung)) (an|von).*$") //
// .match("^(?<type>(Depotgeb.hren|Verwaltungsgeb.hren|Geb.hrenerstattung))
// (an|von).*$") //
// // |Auszahlung|Outgoing payment).*$)") //
.match("^(?<type>(Depotgeb.hren|Verwaltungsgeb.hren|Geb.hrenerstattung|Custody fees|Management fees|Reimbursement of fees)) (an|von|to|from).*$") //
.assign((t, v) -> {
if ("Depotgebühren".equals(v.get("type")) || "Verwaltungsgebühren".equals(v.get("type")))
if ("Depotgebühren".equals(v.get("type")) || "Verwaltungsgebühren".equals(v.get("type"))
|| "Custody fees".equals(v.get("type"))
|| "Management fees".equals(v.get("type")))
t.setType(AccountTransaction.Type.FEES);

if ("Gebührenerstattung".equals(v.get("type")))
if ("Gebührenerstattung".equals(v.get("type"))
|| "Reimbursement of fees".equals(v.get("type")))
t.setType(AccountTransaction.Type.FEES_REFUND);
})

.section("type2").optional() //
.match("^(?<type2>(Auszahlung|Outgoing payment)).*$") //
.assign((t, v) -> {
if ("Auszahlung".equals(v.get("type2")) || "Outgoing payment".equals(v.get("type2")))
t.setType(AccountTransaction.Type.REMOVAL);
})

// @formatter:off
// Valuta 06.11.2023
// @formatter:on
.section("date") //
.match("^Valuta (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.match("^(Valuta|Value date) (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$") //
.assign((t, v) -> t.setDateTime(asDate(v.get("date"))))

.oneOf( //
Expand All @@ -236,7 +271,7 @@ private void addAccountStatementTransaction()
// @formatter:on
section -> section //
.attributes("currency", "amount") //
.match("^Betrag (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.match("^(Betrag|Amount) (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
Expand All @@ -247,7 +282,7 @@ private void addAccountStatementTransaction()
// @formatter:on
section -> section //
.attributes("currency", "amount") //
.match("^(Depotgeb.hren|Verwaltungsgeb.hren) (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.match("^(Depotgeb.hren|Verwaltungsgeb.hren|Custody fees|Management fees) (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$") //
.assign((t, v) -> {
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
t.setAmount(asAmount(v.get("amount")));
Expand All @@ -258,7 +293,7 @@ private void addAccountStatementTransaction()
// Zahlungsgrund findependent Gutschrift Q3
// @formatter:on
.section("note").optional() //
.match("^Zahlungsgrund (?<note>.*)$") //
.match("^(Zahlungsgrund|Payment reason) (?<note>.*)$") //
.assign((t, v) -> t.setNote(trim(v.get("note")))) //

// @formatter:off
Expand All @@ -269,8 +304,8 @@ private void addAccountStatementTransaction()
// Periode 01.10.2023 - 31.12.2023
// @formatter:on
.section("note1", "note2").optional() //
.match("^(?<note1>(Depotgeb.hren|Verwaltungsgeb.hren)) [\\w]{3} [\\.'\\d]+$") //
.match("^Periode (?<note2>.*)$") //
.match("^(?<note1>(Depotgeb.hren|Verwaltungsgeb.hren|Custody fees|Management fees)) [\\w]{3} [\\.'\\d]+$") //
.match("^(Periode|Period) (?<note2>.*)$") //
.assign((t, v) -> {
t.setNote(concatenate(t.getNote(), trim(v.get("note1")), " "));
t.setNote(concatenate(t.getNote(), trim(v.get("note2")), " "));
Expand All @@ -287,15 +322,15 @@ private <T extends Transaction<?>> void addTaxesSectionsTransaction(T transactio
// Verrechnungssteuer (35%) CHF -2.44
// @formatter:on
.section("currency", "tax").optional() //
.match("^Verrechnungssteuer \\([\\d]+%\\) (?<currency>[\\w]{3}) (\\-)?(?<tax>[\\.'\\d]+)") //
.match("^(Verrechnungssteuer|Withholding tax) \\([\\d]+%\\) (?<currency>[\\w]{3}) (\\-)?(?<tax>[\\.'\\d]+)") //
.assign((t, v) -> processTaxEntries(t, v, type))

// @formatter:off
// Stempelabgaben CHF 0.30
// Stempelabgaben CHF -1.05
// @formatter:on
.section("currency", "tax").optional() //
.match("^Stempelabgaben (?<currency>[\\w]{3}) (\\-)?(?<tax>[\\.'\\d]+)") //
.match("^(Stempelabgaben|Stamp duties) (?<currency>[\\w]{3}) (\\-)?(?<tax>[\\.'\\d]+)") //
.assign((t, v) -> processTaxEntries(t, v, type));
}

Expand All @@ -308,7 +343,7 @@ private <T extends Transaction<?>> void addFeesSectionsTransaction(T transaction
// Börsenabgaben CHF -0.05
// @formatter:on
.section("currency", "fee").optional() //
.match("^B.rsenabgaben (?<currency>[\\w]{3}) (\\-)?(?<fee>[\\.'\\d]+)$") //
.match("^(B.rsenabgaben|Stock exchange fees) (?<currency>[\\w]{3}) (\\-)?(?<fee>[\\.'\\d]+)$") //
.assign((t, v) -> processFeeEntries(t, v, type));
}

Expand Down