Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

places, concepts: correct MEF creations/updates #175

Merged
merged 2 commits into from
Nov 18, 2024
Merged

places, concepts: correct MEF creations/updates #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad36b25
places, concepts: GND closeMatch corrections
rerowep Oct 31, 2024
f14bdac
places, concepts: correct MEF creations/updates
rerowep Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 11 additions & 12 deletions poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions rero_mef/alembic/d8536341fc5e_delete_identifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def upgrade():
ids.append(id_)
rec = agent_cls.get_record(id_)
rec.pop("identifier", None)
rec.update(data=rec, bcommit=False, reindex=True)
rec.update(data=rec, dbcommit=False, reindex=True)
if idx % 1000 == 0:
print(f" {idx} commit", end=" | ", flush=True)
db.session.commit()
Expand Down Expand Up @@ -102,7 +102,7 @@ def downgrade():
ids.append(id_)
rec = agent_cls.get_record(id_)
rec["identifier"] = f'"{url}{rec.pid}"'
rec.update(data=rec, bcommit=False, reindex=True)
rec.update(data=rec, dbcommit=False, reindex=True)
if idx % 1000 == 0:
print(f" {idx} commit", end=" | ", flush=True)
db.session.commit()
Expand Down
30 changes: 22 additions & 8 deletions rero_mef/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,11 +613,11 @@ def get_mef_record(mef_cls, name, pid):
)
}
if (
not mef_self_association_pid
and not mef_other_association_pid
and mef_other_pid
not bool(mef_self_association_pid)
and not bool(mef_other_association_pid)
and bool(mef_other_pid)
):
# Delete associated ref from MEF an create a new one
# Delete associated ref from MEF and create a new one
new_mef_record.pop(association_name)
if association_record := association_info[
"record_cls"
Expand All @@ -631,10 +631,10 @@ def get_mef_record(mef_cls, name, pid):
)
actions |= action
if (
mef_self_pid
and not mef_self_association_pid
and not mef_other_pid
and mef_other_association_pid
bool(mef_self_pid)
and not bool(mef_self_association_pid)
and not bool(mef_other_pid)
and bool(mef_other_association_pid)
):
# Delete entity from old MEF and add it to new MEF
ref = mef_associated_record.pop(association_name)
Expand All @@ -643,6 +643,20 @@ def get_mef_record(mef_cls, name, pid):
)
actions[associated_mef_record.pid] = Action.DELETE_ENTITY
new_mef_record[association_name] = ref
if (
bool(mef_self_pid)
and not bool(mef_self_association_pid)
and bool(mef_other_pid)
and bool(mef_other_association_pid)
):
# Delete entity from new MEF and add it to old MEF
ref = new_mef_record.pop(self.name)
new_mef_record.replace(
data=new_mef_record, dbcommit=dbcommit, reindex=reindex
)
actions[new_mef_record.pid] = Action.DELETE_ENTITY
mef_associated_record[self.name] = ref
new_mef_record = mef_associated_record

mef_record = new_mef_record.replace(
data=new_mef_record, dbcommit=dbcommit, reindex=reindex
Expand Down
33 changes: 20 additions & 13 deletions rero_mef/marctojson/do_gnd_concepts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,14 +215,8 @@ def trans_gnd_relation(self):
if value:
self.json_dict[relation] = value

def trans_gnd_classification(self):
"""Transformation classification from field 686."""
if self.logger and self.verbose:
self.logger.info("Call Function", "trans_gnd_classification")
# TODO: find classification

def trans_gnd_match(self):
"""Transformation closeMatch and exactfrom field 750."""
"""Transformation closeMatch and exactMatch from field 750."""
if self.logger and self.verbose:
self.logger.info("Call Function", "trans_gnd_match")
for field_750 in self.marc.get_fields("750"):
Expand Down Expand Up @@ -253,29 +247,42 @@ def trans_gnd_match(self):
if authorized_ap := build_string_from_field(
field=field_750, subfields=subfields, tag_grouping=tag_grouping
):
match = {
match_data = {
"authorized_access_point": authorized_ap,
"source": "GND",
}
identified_by = []
other_source = None
for subfield_0 in field_750.get_subfields("0"):
if subfield_0.startswith("http"):
match.setdefault("identifiedBy", []).append(
identified_by.insert(
0,
{
"type": "uri",
"value": subfield_0,
}
},
)
if other_source:
identified_by[0]["source"] = other_source
else:
source, id_ = get_source_and_id(subfield_0)
if source:
match.setdefault("identifiedBy", []).append(
insert_pos = -1
if source != "GND":
other_source = source
match_data["source"] = other_source
insert_pos = 0
identified_by.insert(
insert_pos,
{
"source": source,
"type": "bf:Nbn",
"value": id_,
}
},
)
self.json_dict.setdefault(match_type, []).append(match)
if identified_by:
match_data["identifiedBy"] = identified_by
self.json_dict.setdefault(match_type, []).append(match_data)

def trans_gnd_note(self):
"""Transformation notes from field.
Expand Down
25 changes: 19 additions & 6 deletions rero_mef/marctojson/do_gnd_places.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,29 +270,42 @@ def trans_gnd_match(self):
if authorized_ap := build_string_from_field(
field=field_751, subfields=subfields, tag_grouping=tag_grouping
):
match = {
match_data = {
"authorized_access_point": authorized_ap,
"source": "GND",
}
identified_by = []
other_source = None
for subfield_0 in field_751.get_subfields("0"):
if subfield_0.startswith("http"):
match.setdefault("identifiedBy", []).append(
identified_by.insert(
0,
{
"type": "uri",
"value": subfield_0,
}
},
)
if other_source:
identified_by[0]["source"] = other_source
else:
source, id_ = get_source_and_id(subfield_0)
if source:
match.setdefault("identifiedBy", []).append(
insert_pos = -1
if source != "GND":
other_source = source
match_data["source"] = other_source
insert_pos = 0
identified_by.insert(
insert_pos,
{
"source": source,
"type": "bf:Nbn",
"value": id_,
}
},
)
self.json_dict.setdefault(match_type, []).append(match)
if identified_by:
match_data["identifiedBy"] = identified_by
self.json_dict.setdefault(match_type, []).append(match_data)

def trans_gnd_note(self):
"""Transformation notes from field.
Expand Down
42 changes: 42 additions & 0 deletions tests/ui/places/test_places_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,45 @@ def test_create_place_record(app, place_idref_data, place_gnd_data, tmpdir):
"pid": "1",
"type": "bf:Place",
}

# test idref changes to other gnd
place_gnd_data["pid"] = "TEST2"
gnd_record_2 = PlaceGndRecord.create(
data=place_gnd_data, dbcommit=True, reindex=True, delete_pid=False
)
assert gnd_record_2.pid == "TEST2"
m_gnd_record_2, m_action = gnd_record_2.create_or_update_mef(
dbcommit=True, reindex=True
)
assert m_action == {"4": Action.CREATE}
assert m_gnd_record_2 == {
"$schema": "https://mef.rero.ch/schemas/places_mef/mef-place-v0.0.1.json",
"gnd": {"$ref": "https://mef.rero.ch/api/places/gnd/TEST2"},
"pid": "4",
"type": "bf:Place",
}

for identified_by in idref_record["identifiedBy"]:
if identified_by.get("source") == "GND":
identified_by["value"] = "(DE-101)TEST2"
idref_record = idref_record.update(data=idref_record, dbcommit=True, reindex=True)

PlaceMefRecord.flush_indexes()
m_idref_record, m_action = idref_record.create_or_update_mef(
dbcommit=True, reindex=True
)
assert m_action == {"1": Action.DELETE_ENTITY, "4": Action.UPDATE}
assert m_idref_record == {
"$schema": f"{SCHEMA_URL}/mef-place-v0.0.1.json",
"idref": {"$ref": "https://mef.rero.ch/api/places/idref/271330163"},
"gnd": {"$ref": "https://mef.rero.ch/api/places/gnd/TEST2"},
"pid": "4",
"type": "bf:Place",
}
m_gnd_record = PlaceMefRecord.get_record_by_pid(m_gnd_record.pid)
assert m_gnd_record == {
"$schema": f"{SCHEMA_URL}/mef-place-v0.0.1.json",
"gnd": {"$ref": "https://mef.rero.ch/api/places/gnd/TEST"},
"pid": "1",
"type": "bf:Place",
}
61 changes: 46 additions & 15 deletions tests/unit/concepts/examples/xml_minimal_record.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,55 @@

<record>
<leader>00589nx a2200193 45 </leader>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">027630501</subfield>
<subfield code="9">sudoc</subfield>
<datafield tag="670" ind1=" " ind2=" ">
<subfield code="a">
Grand Larousse universel (art. : Livre)
</subfield>
</datafield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">frBN001940328</subfield>
<datafield tag="675" ind1=" " ind2=" ">
<subfield code="a">
Laval RVM (en ligne), 2004-11-23
</subfield>
</datafield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">frBN000000089</subfield>
<datafield tag="680" ind1=" " ind2=" ">
<subfield code="a">
Mers profondément engagées dans la masse des continents
</subfield>
</datafield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">FRBNF118620892</subfield>
<subfield code="z">FRBNF11862089</subfield>
<datafield tag="667" ind1=" " ind2=" ">
<subfield code="a">Note interne</subfield>
</datafield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">http://viaf.org/viaf/124265140</subfield>
<subfield code="2">VIAF</subfield>
<subfield code="C">VIAF</subfield>
<subfield code="d">20200302</subfield>
<datafield tag="260" ind1=" " ind2="9">
<subfield code="a">
Voir le descripteur Opposition (science politique)
</subfield>
</datafield>
<datafield tag="260" ind1=" " ind2="9">
<subfield code="a">
Combiner un des descripteurs Mouvements contestataires
</subfield>
</datafield>
<datafield tag="260" ind1=" " ind2=" ">
<subfield code="a">
Voir les vedettes : Mouvements contestataires ; Opposition
</subfield>
</datafield>
<datafield tag="260" ind1=" " ind2=" ">
<subfield code="a">
Voir les vedettes du type : Antifascisme ; Mouvements
</subfield>
</datafield>
<datafield tag="260" ind1=" " ind2=" ">
<subfield code="a">
Voir aux mouvements d'opposition particuliers, par ex. : Combat
</subfield>
</datafield>
<datafield tag="360" ind1=" " ind2=" ">
<subfield code="a">
Voir aussi aux mers et océans particuliers
</subfield>
</datafield>
<datafield tag="016" ind1=" " ind2=" ">
<subfield code="9">VF3, NC3, NC30</subfield>
</datafield>
</record>
Loading
Loading