Skip to content

Commit

Permalink
feat!: Now works with v6 API
Browse files Browse the repository at this point in the history
  • Loading branch information
@flozf
flozf committed May 27, 2024
1 parent cb7ba5c commit 42703b5
Show file tree
Hide file tree
Showing 6 changed files with 46 additions and 67 deletions.
2 changes: 1 addition & 1 deletion .vscode/settings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@
// SPDX-License-Identifier: Apache-2.0
//################################################################################

"pylint.extraPaths": [ "../server" ]
"pylint.path": [ "../server" ]
}
15 changes: 11 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,25 @@ See [CSV File Format](documentation/CSV%20File%20Format.md)

## Installation
```bash
# Install pipenv, if not already present in local Python setup
# Install pipenv, if it is not already present in your local Python setup.
# Use one of the commands below, suitable for your OS.
pip install pipenv
pip3 install pipenv
brew install pipenv
apt install pipenv

# change into main directory
# Change into the main directory
cd bpdm-upload-tool

# optionally: create local .venv directory if you want the environment
# Optionally: create a local .venv directory if you want the environment
# to be stored below the source tree and not in your home directory.
mkdir .venv

# install dependencies (will automatically detect the .venv directory)
# Install dependencies (will automatically detect the .venv directory)
pipenv install

# create uploads folder
mkdir uploads
```

## Credentials
Expand Down
36 changes: 15 additions & 21 deletions server/convert_csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,6 @@ def __init__(self):
self.validTo: datetime.datetime = None
self.type: StatesType = None

class CSVClassifications:
"""Classifications of a CSV-Record"""
def __init__(self):
self.type: ClassificationsType = None
self.code: str|None = None
self.value: str|None = None

class CSVRecord:
"""A CSV-Record"""
def __init__(self):
Expand All @@ -77,13 +70,15 @@ def __init__(self):
self.identifiers: list[CSVIdentifiers] = []
self.states: list[CSVStates] = []
self.roles: list[Roles] = []
self.isOwnCompanyData: bool = False
self.legalEntity_legalEntityBpn: str|None = None
self.legalEntity_legalName: str|None = None
self.legalEntity_shortName: str|None = None
self.legalEntity_legalForm: str|None = None
self.legalEntity_classifications: list[CSVClassifications] = []
self.legalEntity_states: list[CSVStates] = []
self.site_siteBpn: str|None = None
self.site_name: str|None = None
self.site_states: list[CSVStates] = []
self.address_addressBpn: str|None = None
self.address_name: str|None = None
self.address_addressType: AddressType|None = None
Expand Down Expand Up @@ -121,9 +116,7 @@ def __init__(self):
self.address_alternative_deliveryServiceType: DeliveryServiceType = None
self.address_alternative_deliveryServiceQualifier: str|None = None
self.address_alternative_deliveryServiceNumber: str|None = None
self.createdAt: datetime.datetime|None = None
self.updatedAt: datetime.datetime|None = None
self.isOwnCompanyData: bool = False
self.address_states: list[CSVStates] = []

def to_dict(self) -> dict[str, Any]:
"""Converts the object to a dictionary suitable for submitting to the API"""
Expand All @@ -141,12 +134,15 @@ def to_dict(self) -> dict[str, Any]:
"states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.states ],
"roles": [ r.name for r in self.roles ],
"legalEntity": {
"classifications": [ { "type": c.type.name, "code": c.code, "value": c.value } for c in self.legalEntity_classifications ]
"states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.legalEntity_states ]
},
"site": {
"states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.site_states ]
},
"site": {},
"address": {
"physicalPostalAddress": {},
"alternativePostalAddress": {}
"alternativePostalAddress": {},
"states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.address_states ]
},
"isOwnCompanyData": self.isOwnCompanyData
}
Expand Down Expand Up @@ -241,25 +237,23 @@ def convert( reader: csv.reader ) -> list[CSVRecord]:

# If one of the columns exists, all columns have to exist in the input file
related_columns = [
_get_all_columns( header, re.compile( r"^identifiers." )),
_get_all_columns( header, re.compile( r"^states." )),
_get_all_columns( header, re.compile( r"^legalEntity.classifications." )),
_get_all_columns( header, re.compile( r"^address.physical.geographicCoordinates.l" )), # only longitude/latitude, not altitude
_get_all_columns( header, re.compile( r"^address.alternative.geographicCoordinates.l" )), # only longitude/latitude, not altitude
]

# These columns can be repeated again on new lines for adding additional elements to the array
array_groups = [
ArrayGroup( "identifiers", "identifiers.", CSVIdentifiers ),
ArrayGroup( "states", "states.", CSVStates ),
ArrayGroup( "legalEntity_classifications", "legalEntity.classifications.", CSVClassifications ),
ArrayGroup( "identifiers", "identifiers.", CSVIdentifiers ),
ArrayGroup( "roles" ),
ArrayGroup( "states", "states.", CSVStates ),
ArrayGroup( "legalEntity_states", "legalEntity.states.", CSVStates ),
ArrayGroup( "site_states", "site.states.", CSVStates ),
ArrayGroup( "address_states", "address.states.", CSVStates ),
]

data, _ = csv_tools.read_header_csv_with_reader( reader,
header=header,
make_objects=CSVRecord,
ignore_unknown_columns=True,
related_columns=related_columns,
identifier_column="externalId",
array_groups=array_groups )
Expand Down
40 changes: 7 additions & 33 deletions server/convert_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,11 @@ def convert_record( input_record: dict[str,Any], external_id: str, column_name_t
convert_identifier( identifier, additional_lines, external_id )
continue

if full_key == "states":
if full_key == "states" or full_key.endswith( ".states" ):
if len( value ) > 0:
convert_states( value[0], dest, external_id )
convert_states( value[0], dest, external_id, full_key )
for state in value[1:]:
convert_states( state, additional_lines, external_id )
convert_states( state, additional_lines, external_id, full_key )
continue

if full_key == "roles":
Expand All @@ -130,13 +130,6 @@ def convert_record( input_record: dict[str,Any], external_id: str, column_name_t
convert_roles( role, additional_lines, external_id )
continue

if full_key == "legalEntity.classifications":
if len( value ) > 0:
convert_legalentity_classifications( value[0], dest, external_id )
for classification in value[1:]:
convert_legalentity_classifications( classification, additional_lines, external_id )
continue

if isinstance( value, dict ):
convert_record( value, external_id, column_name_to_column, full_key + ".", dest, additional_lines, unknown_keys )
elif isinstance( value, list ):
Expand Down Expand Up @@ -169,7 +162,7 @@ def convert_identifier( identifier: dict[str,Any], dest: dict[str,str]|list, ext
d["externalId"] = external_id


def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id: str ):
def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id: str, full_key: str ):
"""Convert a state to a dictionary suitable for csv writing
dest can be a dictionary. In that case, the data is added to that dictionary.
Expand All @@ -182,9 +175,9 @@ def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id:
d = {}
dest.append( d )

d["states.validFrom"] = convert_value( state["validFrom"] )
d["states.validTo"] = convert_value( state["validTo"] )
d["states.type"] = convert_value( state["type"] )
d[full_key + ".validFrom"] = convert_value( state["validFrom"] )
d[full_key + ".validTo"] = convert_value( state["validTo"] )
d[full_key + ".type"] = convert_value( state["type"] )
d["externalId"] = external_id


Expand All @@ -205,25 +198,6 @@ def convert_roles( role: dict[str,Any], dest: dict[str,str]|list, external_id: s
d["externalId"] = external_id


def convert_legalentity_classifications( classification: dict[str,Any], dest: dict[str,str]|list, external_id: str ):
"""Convert a classification to a dictionary suitable for csv writing
dest can be a dictionary. In that case, the data is added to that dictionary.
Or it can be a list. In that case, the data is added to the list as a dictionary.
"""

if isinstance( dest, dict ):
d = dest
else:
d = {}
dest.append( d )

d["legalEntity.classifications.type"] = convert_value( classification["type"] )
d["legalEntity.classifications.code"] = convert_value( classification["code"] )
d["legalEntity.classifications.value"] = convert_value( classification["value"] )
d["externalId"] = external_id


def convert_value( value: Any ) -> str:
"""Convert a value to a string"""
if value is None:
Expand Down
16 changes: 10 additions & 6 deletions server/file_format.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,19 @@ def get_csv_header() -> list[Column]:
Column( "states.validTo", datetime.datetime, Empty.ToNone, optional=True ),
Column( "states.type", StatesType, Empty.ToNone, optional=True ),
Column( "roles", Roles, Empty.ToNone, optional=True ),
Column( "isOwnCompanyData", bool, Empty.NotOK ),
Column( "legalEntity.legalEntityBpn", str, Empty.ToNone, optional=True ),
Column( "legalEntity.legalName", str, Empty.ToNone, optional=True ),
Column( "legalEntity.shortName", str, Empty.ToNone, optional=True ),
Column( "legalEntity.legalForm", str, Empty.ToNone, optional=True ),
Column( "legalEntity.classifications.type", ClassificationsType,Empty.ToNone, optional=True ),
Column( "legalEntity.classifications.code", str, Empty.ToNone, optional=True ),
Column( "legalEntity.classifications.value", str, Empty.ToNone, optional=True ),
Column( "legalEntity.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ),
Column( "legalEntity.states.validTo", datetime.datetime, Empty.ToNone, optional=True ),
Column( "legalEntity.states.type", StatesType, Empty.ToNone, optional=True ),
Column( "site.siteBpn", str, Empty.ToNone, optional=True ),
Column( "site.name", str, Empty.ToNone, optional=True ),
Column( "site.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ),
Column( "site.states.validTo", datetime.datetime, Empty.ToNone, optional=True ),
Column( "site.states.type", StatesType, Empty.ToNone, optional=True ),
Column( "address.addressBpn", str, Empty.ToNone, optional=True ),
Column( "address.name", str, Empty.ToNone, optional=True ),
Column( "address.addressType", AddressType, Empty.ToNone, optional=True ),
Expand Down Expand Up @@ -161,9 +165,9 @@ def get_csv_header() -> list[Column]:
Column( "address.alternative.deliveryServiceType", DeliveryServiceType,Empty.ToNone, optional=True ),
Column( "address.alternative.deliveryServiceQualifier", str, Empty.ToNone, optional=True ),
Column( "address.alternative.deliveryServiceNumber", str, Empty.ToNone, optional=True ),
Column( "createdAt", datetime.datetime, Empty.ToNone, optional=True ),
Column( "updatedAt", datetime.datetime, Empty.ToNone, optional=True ),
Column( "isOwnCompanyData", bool, Empty.NotOK ),
Column( "address.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ),
Column( "address.states.validTo", datetime.datetime, Empty.ToNone, optional=True ),
Column( "address.states.type", StatesType, Empty.ToNone, optional=True ),
]

return header
4 changes: 2 additions & 2 deletions server/processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@
CHUNK_SIZE = 100

# Swagger: https://business-partners.int.demo.catena-x.net/companies/test-company/ui/swagger-ui/index.html#/business-partner-controller/upsertBusinessPartnersInput
AUTH_URL = "https://centralidp.int.demo.catena-x.net/auth/realms/CX-Central/protocol/openid-connect/token"
BASE_URL = "https://business-partners.int.demo.catena-x.net/companies/test-company/api/catena/"
AUTH_URL = "https://centralidp.dev.demo.catena-x.net/auth/realms/CX-Central/protocol/openid-connect/token"
BASE_URL = "https://business-partners.dev.demo.catena-x.net/companies/test-company/v6/"

CREDENTIALS = None # will be set in init()

Expand Down

0 comments on commit 42703b5

Please sign in to comment.