Docu (#2)

* docu and Docker improvements

* update docu

* small docu changes

* update docu

* 📚

* Update README.md

* Update TEST.md

* Update TEST.md

* Update TEST.md

* Update README.md

* Update README.md

* cleanup and documentation

* cleanup, minor fixes

* bugfixes

* fixes, docu end-tests

* push ebics client

Co-authored-by: Walter Strametz <[email protected]>
Co-authored-by: [email protected] <[email protected]>

Dockerfile

@@ -1,14 +1,17 @@
 # Local build and debugging commands - you might want to remove sudo if not needed
+# gradle  -Dspring.profiles.active=prod -Debics.outputDir=./out  bootRun
 # sudo docker build . -t ebics; 
-# sudo docker run ebics -p 8093
-# sudo docker run ebics -cp "ebics-cli-1.2.jar:lib/*" org.kopi.ebics.client.EbicsClient --help
+# sudo docker run  -p 8093:8093 ebics
+# sudo docker run  -p 8093:8093 -v $HOME/ebics:/root/ebics ebics 
+# sudo docker run  -p 8093:8093 -v $HOME/ebics:/root/ebics --env spring.profiles.active=prod ebics
+# sudo docker run ebics -cp "ebics-cli.jar:lib/*" org.kopi.ebics.client.EbicsClient --help
 # sudo docker run -it --entrypoint sh ebics
-# sudo docker run -v $HOME/ebics:/root/ebics ebics -cp "ebics-cli-1.2.jar:lib/*" org.kopi.ebics.client.EbicsClient --sta -o /root/ebics/out sta.txt
+# sudo docker run -v $HOME/ebics:/root/ebics ebics -cp "ebics-cli.jar:lib/*" org.kopi.ebics.client.EbicsClient --sta -o /root/ebics/out sta.txt 
 
 FROM gradle:6-jdk8-hotspot as build
 
 # build ebics-client jar and server jars;  
-RUN mkdir /app
+RUN ls -la; mkdir /app;
 COPY build.gradle /app
 COPY gradle.properties /app
 COPY settings.gradle /app
@@ -26,9 +29,9 @@ COPY --from=build /app/*.jar /app/
 COPY --from=build /app/lib/*.jar /app/lib/
 COPY --from=build /app/build/libs/app-1.0.0.jar /app
 #remove version form jar files in container and note the used version
-RUN FN=`(ls app-*.jar | head -1)`;  echo $FN; mv $FN ebics-service.jar; touch $FN.version
-RUN FN=`(ls ebics-*.jar | head -1)`;  echo $FN;  mv $FN ebics-cli.jar;  touch $FN.version
-#see application.yml
+RUN FN=`(ls app-*.jar   | head -1)`;  echo $FN; mv $FN ebics-service.jar;  touch $FN.version
+RUN FN=`(ls ebics-*.jar | head -1)`;  echo $FN;  mv $FN ebics-cli.jar;     touch $FN.version
+#see application*.yml or spring boot spring.active.profiles for port and other configs
 EXPOSE 8093
 ENTRYPOINT ["java"]
 CMD ["-jar", "ebics-service.jar" ]

HOWTO.md

@@ -59,3 +59,18 @@ or docker container.
 
 Daily statements can only be retrieved once - then you get an error if you request again. 
 This Ebics "error" sticks, until you have a new transaction on your accounts. 
+
+
+### ERROR - Command not supported
+
+Not all banks support the full set of commands, or you need to configure a differnt command to retrieve your
+daily statement. For example your bank does not support Zipping the daily statements (Z53 vs C53) -
+please see 
+[`OrderType.java`](https://github.com/element36-io/ebics-java-client/blob/master/src/main/java/org/kopi/ebics/session/OrderType.java) for supported commands and compare that with the list of supported commands of 
+your bank. 
+
+### XML Validation Errors & Debugging ebics XML files
+
+Put the XSD files (Schema) in the same directory as your generated files. Now open the generated files with
+VS Code using the XML Plugin from RedHat - it will help you if your Ebics interface just quits with a
+validation error and without further details. 

README.md

@@ -1,9 +1,9 @@
 ## Overview
 
-This project wraps a [java-based command-line tool](https://github.com/element36-io/ebics-java-client)
-into a REST service. It can connect with your bank using 
+This project connects with your bank account using 
 [Ebics](https://en.wikipedia.org/wiki/Electronic_Banking_Internet_Communication_Standard) 
-standard.
+standard. The project wraps a [java-based command-line tool](https://github.com/element36-io/ebics-java-client)
+into a REST service and add features like cron and webhooks. 
 
 Setting it up successfully means, that you can 
 - Read daily statements including account balance and transactions of multiple accounts
@@ -14,12 +14,26 @@ Setting it up successfully means, that you can
 
     docker run -p 8093:8093 e36io/ebics-service 
 
-
-Open [Swagger on localhost:8093](http://localhost:8093/ebics/swagger-ui/?url=/ebics/v2/api-docs/) in your 
-browser and test the `simulate` service. For other APIs you need to set up and connect to your
+Open [Swagger](http://localhost:8093/ebics/swagger-ui/?url=/ebics/v2/api-docs/) in your 
+browser and test the `simulate` service.  For other APIs you need to set up and connect to your
 banks Ebics API. 
 
-## Setup
+## Start the service in development and production mode
+
+Development mode is default. Ebics-java-client does not connect to your bank but ebics documents
+and commands get logged. 
+
+    docker run -v $HOME/ebics./root/ebics -p 8093:8093 e36io/ebics-service 
+
+Production mode (Spring `prod` profile) requires Ebics to be set up. 
+:warning: This may trigger real payments from your bank account!
+
+    docker run -v $HOME/ebics:/root/ebics --env spring.profiles.active=prod -p 8093:8093 e36io/ebics-service 
+
+Again, open Open [Swagger](http://localhost:8093/ebics/swagger-ui/?url=/ebics/v2/api-docs/)
+to test the API. All Ebics documents which are exchanged with the bank are stored in `HOME/ebics/client`. 
+
+## Setup Ebics Configuration
 
 Ebics (Electronic Banking Internet Communication Standard) defines the protocol and a series of document to access and exchange banking data. 
 You need to talk to your bank first and request to provide access to the Ebics interface.
@@ -29,43 +43,41 @@ files, which should be well known among banks. Most banks offer the service for
 A crucial part to understand of Ebics is the key-exchange- roughly it works like this:  
 
 - Generate keypair and send public to bank with the ebics client
-- Bank sends their client specific key via a physically printed letter to you
-- Add this key to the client 
-- All configurations for the setup are done in one specific file: $HOME/ebics/ebics.txt
+- You send client specific keys via a physically printed letter to the bank
+- All configurations for the setup are done in one specific file: `$HOME/ebics/client/ebics.txt` see [`ebics-template.txt`](ebics-template.txt) 
 
  Check [ebics.org](https://www.ebics.org/en/home) for details. 
 
 ## Supported banks and national flavours
 
 Ebics standard was initiated by banks German/French partnership later adopted by Austria and Switzerland
-and also by Banks across Europe - you will find many bank supporting Ebics, a general list of all
+and by Banks across Europe. You will find many bank supporting Ebics, a general list of all
 banks supporting Ebics was not found. 
 
 There are national flavours regarding the data on bank transfers and bank statements, which are
 represented by national changes of the original Ebics XML documents. For example Switzerland: 
 
 - In addition to a reference field (e.g. invoice 1234) you may use a dedicated
-  "reference number" on payment slips  which you don't find in other countries
+  "reference number" on payment slips which you don't find on the payment slips of other countries
 - Account statements, status reports are ZIP files available through separate Ebics commands. 
   Generally it is easy to add new commands to the `ebics-java-client` library. 
 
 ### Switzerland
 
-A selection of test interfaces for Ebics and test environments in Switzerland: 
+A national [overview](https://www.six-group.com/dam/download/banking-services/interbank-clearing/en/standardization/ebics/ebics.pdf) 
+and selection of test interfaces for Ebics and test environments in Switzerland: 
+
 
 | Bank  | URL for testing/validating | more info |
 |---|---|---|
-| SIX Group | [validation](https://validation.iso-payments.ch/)  | [Url](http://www.six-interbank-clearing.com/de/home/standardization/iso-payments/customer-bank/implementation-guidelines.html) |
+| SIX Group | [validation](https://validation.iso-payments.ch/)  | [Url](http://www.six-interbank-clearing.com/de/home/standardization/iso-payments/customer-buank/implementation-guidelines.html) |
 | Credit Suisse  | [test-API](https://credit-suisse.com/iso20022test) | [help](https://iso20022test.credit-suisse.com/help) |
 | PostFinance | [register](https://isotest.postfinance.ch/corporates/) ||
 | Raiffeisen  | [test-API](http://raiffeisen.ch/testbank) ||
 | UBS  | [register](https://ubs-paymentstandards.ch/login) | [info](https://www.ubs.com/ch/de/swissbank/unternehmen/zahlungsverkehr/harmonisierung/testplattform-iso-20022.html) ||
 | ZKB  | [test-platform](https://testplattform.zkb.ch/) | [help](https://testplattform.zkb.ch/help) |
 
 ### Other countries
-
-Besides France, Germany, Austria and Switzerland many other countries are covered with an Ebics- Service. 
-
 Non-exhaustive examples of European banks providing Ebics information:  
 
 - [BNP Paribas](https://cashmanagement.bnpparibas.com/our-solutions/solution/global-ebics)
@@ -77,11 +89,9 @@ Non-exhaustive examples of European banks providing Ebics information:
 
 ## Adapt to national flavours 
 
-The setup process does not need to be adapted. With the homogenization of European 
-banking interfaces (SEPA, ISO20022) you may use a common and minimal 
-set information for working with bank interfaces which is reflected by the REST interface of this
-project. Anyway you might need to adapt to local flavour of your country or your bank, which is 
-can be done in the `io.element36.cash36.ebics.strategy.*` package. Examples: 
+The setup process does not need to be adapted.  Anyway you might need to adapt to local flavour of your 
+country or your bank if they support different Ebics commands or have national modifications of their 
+Ebics XML documents. This can be done in the `io.element36.cash36.ebics.strategy.*` package. Examples: 
 
 
 - Adapt conversion between Ebics-XML documents: E.g. `io.element36.cash36.ebics.strategy.implStatementCamt53_001` for 
@@ -90,19 +100,21 @@ mapping of daily statements to Json Response.
  which are also directly transferred to the Ebics server as commands. Look for national mapping tables if you face this issue. E.g. 
  [Switzerland](https://www.six-group.com/dam/download/banking-services/interbank-clearing/en/standardization/ebics/mapping-table.pdf)
  and [Austria](https://www.stuzza.at/de/download/ebics/418-btf-mappingtabelle-at-v20210506.html)
+- Genereate new ebics documents based on XSD specifications in `build.gradle` and `generateSourcesForXsd`. 
 
 You may check out www.ebics.ch, www.ebics.de. www.ebics.at, www.ebics.org. 
 
-## Kudos and references: 
+## Kudos and references
 
 The project is forked form [Ebics Java Client](https://github.com/uwemaurer/ebics-java-client/), 
 which was based on a [sourceforge project](https://sourceforge.net/p/ebics/). 
 
 Main differences with this fork from ebics-java-client form uwemaurrer: 
 
 - Fixed vulnerabilities of bouncycastle and log4j
-- Support for new commands used in Switzerland.
-- Jar file in maven central repository.
-- Docker image is automatically built from master branch on Dockerhub. 
+- Support for new commands used in Switzerland
+- Jar file in maven central repository
+- Docker image is automatically built from master branch on Dockerhub.
 - Changed documentation for usage with docker
 
+Run & check tests [TEST.md](TEST.md). 

TEST.md

@@ -1,24 +1,67 @@
 ## Run unit tests
 
-Run tests for the ebics-java-client on linux - it mounts sources into a docker container with java and the maven build tool:
+Run tests for the ebics-java-client on linux - it mounts sources into a docker container with java and the maven build tool - 
+so you do not need to install java or gradle on your local machine: 
 
+    git clone  https://github.com/element36-io/ebics-java-service.git
+    cd ebics-java-service; mkdir ./app;
+    docker run -it -v $PWD:/app -w /app  gradle:6-jdk8-hotspot gradle test
+
+See `./build/reports/tests/test/index.html` for test results. With minimum Java 8 and Maven run tests directly with `gradle test`
 
-    git clone [email protected]:element36-io/ebics-java-client.git
-    cd ebics-java-client
-    docker run -it -v $PWD:/app -w /app  maven:3-jdk-8 mvn test surefire-report:report
+See [here](https://github.com/element36-io/ebics-java-client/blob/master/README.md) how to run tests on ebics-java-client. 
 
-See `./target` for test results. `surefire-report:report` is optional but it creates test report here: `./target/site/surefire-report.html
+## Test API manually
 
+    docker run -p 8093:8093 e36io/ebics-service
 
-See [here](https://github.com/element36-io/ebics-java-client/blob/master/README.md) how to run tests on ebics-java-client. 
+or with existing Ebics configuration
+
+    docker run -p 8093:8093 -v $HOME/ebics:/root/ebics  e36io/ebics-service
+
+or with existing Ebics configuration and in production mode 
+
+    docker run -p 8093:8093 -v $HOME/ebics:/root/ebics --env spring.profiles.active=prod e36io/ebics-service
+
+
+Open [Swagger](http://localhost:8093/ebics/swagger-ui/?url=/ebics/v2/api-docs/) in your 
+browser and test the API. Ony if you set the image to production mode, it will 
+try co connect with your bank. In dev mode it will log its commands and generated 
+documents. You may check and download the payment document (ebics document) under `./app`
+which can be tested manually against your bank. To set up and connect to your banks 
+Ebics API you need to [switch to productive spring boot
+profile](https://www.baeldung.com/spring-profiles) by using `export spring_profiles_active=prod`. 
+
+
+### Test  `/ebics/api-v1/simulate' 
+
+Create a test transaction - send funds form your bank account to somebody else. 
+The result shows the Ebics file generated by the request. 
+You may test this file with your bank prior to activating Ebics: Many
+banks offer buld-upload function which read Camt.053 format via web-interface. 
+
+Same behaviour in dev and prod environment. 
+
+### Test  `/ebics/api-v1/bankstatements' 
+
+Retrieves bank statement and translates Camt.053 format to Json. 
+
+In dev mode you see the command which is issued to query the daily statement. 
+
+### Test  `/ebics/api-v1/unpeg' 
+
+Create a tarnsaction with your pegging account configured in `ebics.pegging.account`. 
+:warning: In prod mode it sends real money - in dev mode it shows the command and the Ebics document in the result. 
 
-## Start the API and test manually
+### Test  `/ebics/api-v1/paymentstatus'
 
-    docker run -it -v $PWD:/app
+Check status of a transaction with command Z01. In dev mode you see the command which is issued to query the status of the transaction.
 
-http://localhost:8093/ebics/swagger-ui/?url=/ebics/v2/api-docs/
+### Test  `/ebics/api-v1/create'
 
+At your bank you can have more than one bank account. The Ebics interface allows you to access all of them at once: 
+Daily statements will cover transactions of all accounts - and you can create new transactions from any bank account 
+from the Ebics interface. 
 
-## Github Matrix tests
 
 

build.gradle

@@ -49,31 +49,17 @@ dependencies {
 	implementation 'javax.xml.bind:jaxb-api:'+jaxbVersion
 	implementation 'javax.activation:activation:1.1.1'
 	implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.4'
-	execebics 'io.github.element36-io:ebics-cli:1.2'
+	execebics 'io.github.element36-io:ebics-cli:1.5'
 
-
 	implementation 'org.springframework.boot:spring-boot-starter-validation'
 	testImplementation 'junit:junit'
 	implementation 'org.springdoc:springdoc-openapi-ui:1.5.8'
 
-	//implementation 'org.kopi:ebics:1.1'
 	implementation 'io.springfox:springfox-boot-starter:3.0.0'
 	implementation 'org.springframework.boot:spring-boot-starter-parent:2.4.0'
     implementation 'io.springfox:springfox-swagger-ui:2.9.2'
-
-	// vulns 10.4.21
-	//implementation 'com.thoughtworks.xstream:xstream:1.4.16' // added due to vuln
-    //implementation 'org.bouncycastle:bcprov-jdk15on:1.68' // vuln
-    // for vulnerabilities 05.03.2020
-	//implementation 'org.springframework.security.oauth:spring-security-oauth2:2.5.1.RELEASE'
 
     implementation 'org.springframework.boot:spring-boot-starter-web'
-    //implementation ( 'org.springframework.cloud:spring-cloud-starter-netflix-eureka-client' ) {
-     // 	exclude module: 'org.springframework.cloud:spring-cloud-starter-ribbon'
-      //	exclude module: 'om.netflix.ribbon:ribbon-eureka'
-//	}
-
-  //  implementation 'org.springframework.cloud:spring-cloud-starter-oauth2'
     implementation 'org.apache.commons:commons-lang3:3.6'
     implementation 'org.apache.commons:commons-exec:1.3'
     implementation 'commons-io:commons-io:2.6'
@@ -99,10 +85,23 @@ task copyEbicsJar(type: Copy) {
     from ("$projectDir/lib") {
     	include "ebics-cli-*.jar"
     }
+
     into "$projectDir"
+
+}
+String stripVersion(String fileNameWithVersion) {
+		String ext = fileNameWithVersion.substring(fileNameWithVersion.lastIndexOf("."),fileNameWithVersion.length())
+		int end = fileNameWithVersion.lastIndexOf("-"); //assumes that: name-version.ext. Will not work with name-version-SNAPSHOT.ext
+		String fileNameWithoutVersion = fileNameWithVersion.substring(0, end) + ext
+		return fileNameWithoutVersion
 }
 build.finalizedBy(copyToLib,copyEbicsJar)
 
+// then gradle -Dkey=value works
+bootRun {
+    systemProperties System.properties
+}
+
 dependencies {
     jaxb 'javax.xml.bind:jaxb-api:'+jaxbVersion
     jaxb 'com.sun.xml.bind:jaxb-xjc:'+jaxbVersion