diff --git a/.vscode/settings.json b/.vscode/settings.json index 1f0500f..426b7c3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,5 +18,5 @@ // SPDX-License-Identifier: Apache-2.0 //################################################################################ - "pylint.extraPaths": [ "../server" ] + "pylint.path": [ "../server" ] } diff --git a/README.md b/README.md index a1763fc..a4a0ba9 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,25 @@ See [CSV File Format](documentation/CSV%20File%20Format.md) ## Installation ```bash -# Install pipenv, if not already present in local Python setup +# Install pipenv, if it is not already present in your local Python setup. +# Use one of the commands below, suitable for your OS. pip install pipenv +pip3 install pipenv +brew install pipenv +apt install pipenv -# change into main directory +# Change into the main directory cd bpdm-upload-tool -# optionally: create local .venv directory if you want the environment +# Optionally: create a local .venv directory if you want the environment # to be stored below the source tree and not in your home directory. mkdir .venv -# install dependencies (will automatically detect the .venv directory) +# Install dependencies (will automatically detect the .venv directory) pipenv install + +# create uploads folder +mkdir uploads ``` ## Credentials diff --git a/server/convert_csv.py b/server/convert_csv.py index cbeb59b..8051d7f 100644 --- a/server/convert_csv.py +++ b/server/convert_csv.py @@ -54,13 +54,6 @@ def __init__(self): self.validTo: datetime.datetime = None self.type: StatesType = None -class CSVClassifications: - """Classifications of a CSV-Record""" - def __init__(self): - self.type: ClassificationsType = None - self.code: str|None = None - self.value: str|None = None - class CSVRecord: """A CSV-Record""" def __init__(self): @@ -77,13 +70,15 @@ def __init__(self): self.identifiers: list[CSVIdentifiers] = [] self.states: list[CSVStates] = [] self.roles: list[Roles] = [] + self.isOwnCompanyData: bool = False self.legalEntity_legalEntityBpn: str|None = None self.legalEntity_legalName: str|None = None self.legalEntity_shortName: str|None = None self.legalEntity_legalForm: str|None = None - self.legalEntity_classifications: list[CSVClassifications] = [] + self.legalEntity_states: list[CSVStates] = [] self.site_siteBpn: str|None = None self.site_name: str|None = None + self.site_states: list[CSVStates] = [] self.address_addressBpn: str|None = None self.address_name: str|None = None self.address_addressType: AddressType|None = None @@ -121,9 +116,7 @@ def __init__(self): self.address_alternative_deliveryServiceType: DeliveryServiceType = None self.address_alternative_deliveryServiceQualifier: str|None = None self.address_alternative_deliveryServiceNumber: str|None = None - self.createdAt: datetime.datetime|None = None - self.updatedAt: datetime.datetime|None = None - self.isOwnCompanyData: bool = False + self.address_states: list[CSVStates] = [] def to_dict(self) -> dict[str, Any]: """Converts the object to a dictionary suitable for submitting to the API""" @@ -141,12 +134,15 @@ def to_dict(self) -> dict[str, Any]: "states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.states ], "roles": [ r.name for r in self.roles ], "legalEntity": { - "classifications": [ { "type": c.type.name, "code": c.code, "value": c.value } for c in self.legalEntity_classifications ] + "states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.legalEntity_states ] + }, + "site": { + "states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.site_states ] }, - "site": {}, "address": { "physicalPostalAddress": {}, - "alternativePostalAddress": {} + "alternativePostalAddress": {}, + "states": [ { "validFrom": s.validFrom.isoformat(), "validTo": s.validTo.isoformat(), "type": s.type.name } for s in self.address_states ] }, "isOwnCompanyData": self.isOwnCompanyData } @@ -241,25 +237,23 @@ def convert( reader: csv.reader ) -> list[CSVRecord]: # If one of the columns exists, all columns have to exist in the input file related_columns = [ - _get_all_columns( header, re.compile( r"^identifiers." )), - _get_all_columns( header, re.compile( r"^states." )), - _get_all_columns( header, re.compile( r"^legalEntity.classifications." )), _get_all_columns( header, re.compile( r"^address.physical.geographicCoordinates.l" )), # only longitude/latitude, not altitude _get_all_columns( header, re.compile( r"^address.alternative.geographicCoordinates.l" )), # only longitude/latitude, not altitude ] # These columns can be repeated again on new lines for adding additional elements to the array array_groups = [ - ArrayGroup( "identifiers", "identifiers.", CSVIdentifiers ), - ArrayGroup( "states", "states.", CSVStates ), - ArrayGroup( "legalEntity_classifications", "legalEntity.classifications.", CSVClassifications ), + ArrayGroup( "identifiers", "identifiers.", CSVIdentifiers ), ArrayGroup( "roles" ), + ArrayGroup( "states", "states.", CSVStates ), + ArrayGroup( "legalEntity_states", "legalEntity.states.", CSVStates ), + ArrayGroup( "site_states", "site.states.", CSVStates ), + ArrayGroup( "address_states", "address.states.", CSVStates ), ] data, _ = csv_tools.read_header_csv_with_reader( reader, header=header, make_objects=CSVRecord, - ignore_unknown_columns=True, related_columns=related_columns, identifier_column="externalId", array_groups=array_groups ) diff --git a/server/convert_json.py b/server/convert_json.py index cb29dea..d3a87bb 100644 --- a/server/convert_json.py +++ b/server/convert_json.py @@ -116,11 +116,11 @@ def convert_record( input_record: dict[str,Any], external_id: str, column_name_t convert_identifier( identifier, additional_lines, external_id ) continue - if full_key == "states": + if full_key == "states" or full_key.endswith( ".states" ): if len( value ) > 0: - convert_states( value[0], dest, external_id ) + convert_states( value[0], dest, external_id, full_key ) for state in value[1:]: - convert_states( state, additional_lines, external_id ) + convert_states( state, additional_lines, external_id, full_key ) continue if full_key == "roles": @@ -130,13 +130,6 @@ def convert_record( input_record: dict[str,Any], external_id: str, column_name_t convert_roles( role, additional_lines, external_id ) continue - if full_key == "legalEntity.classifications": - if len( value ) > 0: - convert_legalentity_classifications( value[0], dest, external_id ) - for classification in value[1:]: - convert_legalentity_classifications( classification, additional_lines, external_id ) - continue - if isinstance( value, dict ): convert_record( value, external_id, column_name_to_column, full_key + ".", dest, additional_lines, unknown_keys ) elif isinstance( value, list ): @@ -169,7 +162,7 @@ def convert_identifier( identifier: dict[str,Any], dest: dict[str,str]|list, ext d["externalId"] = external_id -def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id: str ): +def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id: str, full_key: str ): """Convert a state to a dictionary suitable for csv writing dest can be a dictionary. In that case, the data is added to that dictionary. @@ -182,9 +175,9 @@ def convert_states( state: dict[str,Any], dest: dict[str,str]|list, external_id: d = {} dest.append( d ) - d["states.validFrom"] = convert_value( state["validFrom"] ) - d["states.validTo"] = convert_value( state["validTo"] ) - d["states.type"] = convert_value( state["type"] ) + d[full_key + ".validFrom"] = convert_value( state["validFrom"] ) + d[full_key + ".validTo"] = convert_value( state["validTo"] ) + d[full_key + ".type"] = convert_value( state["type"] ) d["externalId"] = external_id @@ -205,25 +198,6 @@ def convert_roles( role: dict[str,Any], dest: dict[str,str]|list, external_id: s d["externalId"] = external_id -def convert_legalentity_classifications( classification: dict[str,Any], dest: dict[str,str]|list, external_id: str ): - """Convert a classification to a dictionary suitable for csv writing - - dest can be a dictionary. In that case, the data is added to that dictionary. - Or it can be a list. In that case, the data is added to the list as a dictionary. - """ - - if isinstance( dest, dict ): - d = dest - else: - d = {} - dest.append( d ) - - d["legalEntity.classifications.type"] = convert_value( classification["type"] ) - d["legalEntity.classifications.code"] = convert_value( classification["code"] ) - d["legalEntity.classifications.value"] = convert_value( classification["value"] ) - d["externalId"] = external_id - - def convert_value( value: Any ) -> str: """Convert a value to a string""" if value is None: diff --git a/server/file_format.py b/server/file_format.py index 0f47ade..320e3f3 100644 --- a/server/file_format.py +++ b/server/file_format.py @@ -115,15 +115,19 @@ def get_csv_header() -> list[Column]: Column( "states.validTo", datetime.datetime, Empty.ToNone, optional=True ), Column( "states.type", StatesType, Empty.ToNone, optional=True ), Column( "roles", Roles, Empty.ToNone, optional=True ), + Column( "isOwnCompanyData", bool, Empty.NotOK ), Column( "legalEntity.legalEntityBpn", str, Empty.ToNone, optional=True ), Column( "legalEntity.legalName", str, Empty.ToNone, optional=True ), Column( "legalEntity.shortName", str, Empty.ToNone, optional=True ), Column( "legalEntity.legalForm", str, Empty.ToNone, optional=True ), - Column( "legalEntity.classifications.type", ClassificationsType,Empty.ToNone, optional=True ), - Column( "legalEntity.classifications.code", str, Empty.ToNone, optional=True ), - Column( "legalEntity.classifications.value", str, Empty.ToNone, optional=True ), + Column( "legalEntity.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ), + Column( "legalEntity.states.validTo", datetime.datetime, Empty.ToNone, optional=True ), + Column( "legalEntity.states.type", StatesType, Empty.ToNone, optional=True ), Column( "site.siteBpn", str, Empty.ToNone, optional=True ), Column( "site.name", str, Empty.ToNone, optional=True ), + Column( "site.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ), + Column( "site.states.validTo", datetime.datetime, Empty.ToNone, optional=True ), + Column( "site.states.type", StatesType, Empty.ToNone, optional=True ), Column( "address.addressBpn", str, Empty.ToNone, optional=True ), Column( "address.name", str, Empty.ToNone, optional=True ), Column( "address.addressType", AddressType, Empty.ToNone, optional=True ), @@ -161,9 +165,9 @@ def get_csv_header() -> list[Column]: Column( "address.alternative.deliveryServiceType", DeliveryServiceType,Empty.ToNone, optional=True ), Column( "address.alternative.deliveryServiceQualifier", str, Empty.ToNone, optional=True ), Column( "address.alternative.deliveryServiceNumber", str, Empty.ToNone, optional=True ), - Column( "createdAt", datetime.datetime, Empty.ToNone, optional=True ), - Column( "updatedAt", datetime.datetime, Empty.ToNone, optional=True ), - Column( "isOwnCompanyData", bool, Empty.NotOK ), + Column( "address.states.validFrom", datetime.datetime, Empty.ToNone, optional=True ), + Column( "address.states.validTo", datetime.datetime, Empty.ToNone, optional=True ), + Column( "address.states.type", StatesType, Empty.ToNone, optional=True ), ] return header diff --git a/server/processing.py b/server/processing.py index 32b6b38..b06c280 100644 --- a/server/processing.py +++ b/server/processing.py @@ -40,8 +40,8 @@ CHUNK_SIZE = 100 # Swagger: https://business-partners.int.demo.catena-x.net/companies/test-company/ui/swagger-ui/index.html#/business-partner-controller/upsertBusinessPartnersInput -AUTH_URL = "https://centralidp.int.demo.catena-x.net/auth/realms/CX-Central/protocol/openid-connect/token" -BASE_URL = "https://business-partners.int.demo.catena-x.net/companies/test-company/api/catena/" +AUTH_URL = "https://centralidp.dev.demo.catena-x.net/auth/realms/CX-Central/protocol/openid-connect/token" +BASE_URL = "https://business-partners.dev.demo.catena-x.net/companies/test-company/v6/" CREDENTIALS = None # will be set in init()