From 36d084e95a4c430342ac415cc2ba1c1f6e7a1103 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Wed, 6 Oct 2021 13:44:02 +0200 Subject: [PATCH 01/11] [BUGFIX] Fixed tests error. Now float and int are also valid values --- notionsci/connections/notion/structures/content.py | 3 +++ notionsci/connections/notion/structures/properties.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/notionsci/connections/notion/structures/content.py b/notionsci/connections/notion/structures/content.py index 1b2d164..2137f64 100644 --- a/notionsci/connections/notion/structures/content.py +++ b/notionsci/connections/notion/structures/content.py @@ -31,6 +31,9 @@ class HasPropertiesMixin(Generic[PT]): def get_property(self, name: str) -> PT: return self.properties[name] + def get_propery_raw_value(self, name: str, default=None) -> Optional[Any]: + return self.get_property(name).raw_value() if name in self.properties else default + def get_propery_value(self, name: str, default=None) -> Optional[Any]: return self.get_property(name).value() if name in self.properties else default diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index f1cf8b3..6294fe9 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -76,6 +76,8 @@ def object_to_text_value(raw_value: Any): return raw_value.text_value() elif isinstance(raw_value, Dict): return str(raw_value) + elif isinstance(raw_value, int) or isinstance(raw_value, float): + return raw_value return str(raw_value) From b71078b973c278c7e8d99f62a94879b434d66e31 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Wed, 6 Oct 2021 14:01:30 +0200 Subject: [PATCH 02/11] [FEATURE] Added references and collections block deduction --- notionsci/cli/sync/zotero.py | 38 ++++++++++++------- .../connections/notion/structures/blocks.py | 10 +++-- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/notionsci/cli/sync/zotero.py b/notionsci/cli/sync/zotero.py index c002ea9..6e10e2e 100644 --- a/notionsci/cli/sync/zotero.py +++ b/notionsci/cli/sync/zotero.py @@ -4,7 +4,7 @@ from notionsci.cli.notion import duplicate from notionsci.config import config -from notionsci.connections.notion import parse_uuid, parse_uuid_callback +from notionsci.connections.notion import parse_uuid, parse_uuid_callback, BlockType, block_type_filter from notionsci.connections.zotero import ID from notionsci.sync.zotero import RefsOneWaySync, CollectionsOneWaySync @@ -46,37 +46,49 @@ def template(ctx: click.Context, parent: ID): click.echo(f'Found references database ({parse_uuid(refs_db.id)})') +def child_database_filter(title: str): + type_filter = block_type_filter(BlockType.child_database) + return lambda b: type_filter(b) and b.child_database.title == title + + @zotero.command() -@click.argument('references', callback=parse_uuid_callback, required=True) +@click.argument('template', callback=parse_uuid_callback, required=True) @click.option('--force', is_flag=True, default=False, help='Ensures up to date items are also pushed to Zotero') -@click.option('-c', '--collections', callback=lambda c, p, x: parse_uuid_callback(c, p, x) if x else x, required=False, - help='Collections database page ID or url to (optionally) add references to') -def refs(references: ID, collections: ID, force: bool): +def refs(template: ID, force: bool): """ Starts a one way Zotero references sync to Notion - REFERENCES: References database page ID or url - - When collecitons option is specified Unofficial Notion Api access is required + TEMPLATE: Template page ID or url """ notion = config.connections.notion.client() zotero = config.connections.zotero.client() - RefsOneWaySync(notion, zotero, references, collections_id=collections, force=force).sync() + template_page = notion.page_get(template, with_children=True) + references = next(iter(template_page.get_children(child_database_filter('Zotero References'))), None) + collections = next(iter(template_page.get_children(child_database_filter('Zotero Collections'))), None) + + if not references or not collections: + raise Exception('Please check whether child database called "Zotero References" and "Zotero Collections" ' + 'exist in given template') + + RefsOneWaySync(notion, zotero, references.id, collections_id=collections.id, force=force).sync() @zotero.command() -@click.argument('collections', callback=parse_uuid_callback, required=True) +@click.argument('template', callback=parse_uuid_callback, required=True) @click.option('--force', is_flag=True, default=False, help='Ensures up to date items are also pushed to Zotero') -def collections(collections: ID, force: bool): +def collections(template: ID, force: bool): """ Starts a one way Zotero references sync to Notion - COLLECTIONS: Collections database page ID or url + TEMPLATE: Template page ID or url """ notion = config.connections.notion.client() zotero = config.connections.zotero.client() - CollectionsOneWaySync(notion, zotero, collections, force=force).sync() + template_page = notion.page_get(template, with_children=True) + collections = next(iter(template_page.get_children(child_database_filter('Zotero Collections'))), None) + + CollectionsOneWaySync(notion, zotero, collections.id, force=force).sync() diff --git a/notionsci/connections/notion/structures/blocks.py b/notionsci/connections/notion/structures/blocks.py index ae5f77e..3577121 100644 --- a/notionsci/connections/notion/structures/blocks.py +++ b/notionsci/connections/notion/structures/blocks.py @@ -2,7 +2,7 @@ import urllib.parse from dataclasses import dataclass, field from enum import Enum -from typing import Optional, List, Union +from typing import Optional, List, Union, Callable import pandas as pd from dataclass_dict_convert import dataclass_dict_convert @@ -39,6 +39,10 @@ class BlockType(Enum): BlockConvertor = ListConvertor(ForwardRefConvertor('Block')) +def block_type_filter(t: 'BlockType'): + return lambda b: b.type == t + + @dataclass class ChildrenMixin: children: Optional[List['Block']] = None @@ -46,8 +50,8 @@ class ChildrenMixin: def set_children(self, children: List['Block']): self.children = children - def get_children(self) -> List['Block']: - return self.children + def get_children(self, predicate: Callable[['Block'], bool] = None) -> List['Block']: + return self.children if not predicate else list(filter(predicate, self.children)) @dataclass_dict_convert( From ba2bdf70bf6622f8e0bd679289552fe38e3fd63d Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Wed, 6 Oct 2021 15:48:33 +0200 Subject: [PATCH 03/11] [FEATURE] WIP Added markdown sync (notion -> filesystem --- notionsci/cli/sync/__init__.py | 2 + notionsci/cli/sync/markdown.py | 30 ++++ notionsci/connections/notion/client.py | 8 +- .../connections/notion/structures/content.py | 3 + .../notion/structures/properties.py | 19 ++- .../connections/notion/structures/results.py | 2 + notionsci/sync/markdown/__init__.py | 1 + notionsci/sync/markdown/pages.py | 142 ++++++++++++++++++ notionsci/sync/structure.py | 2 +- 9 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 notionsci/cli/sync/markdown.py create mode 100644 notionsci/sync/markdown/__init__.py create mode 100644 notionsci/sync/markdown/pages.py diff --git a/notionsci/cli/sync/__init__.py b/notionsci/cli/sync/__init__.py index b866ddf..513ff7a 100644 --- a/notionsci/cli/sync/__init__.py +++ b/notionsci/cli/sync/__init__.py @@ -1,6 +1,7 @@ import click from .zotero import zotero +from .markdown import markdown @click.group() @@ -12,3 +13,4 @@ def sync(): sync.add_command(zotero) +sync.add_command(markdown) diff --git a/notionsci/cli/sync/markdown.py b/notionsci/cli/sync/markdown.py new file mode 100644 index 0000000..785ceae --- /dev/null +++ b/notionsci/cli/sync/markdown.py @@ -0,0 +1,30 @@ +import click + +from notionsci.config import config +from notionsci.connections.notion import parse_uuid_callback, ID +from notionsci.sync.markdown import MarkdownPagesSync + + +@click.group() +def markdown(): + """ + Collection of Sync commands for Markdown documents + """ + pass + + +@markdown.command() +@click.argument('collection', callback=parse_uuid_callback, required=True) +@click.argument('dir', required=True) +@click.option('--force', is_flag=True, default=False, + help='Ensures up to date items are also pushed to Zotero') +def pages(collection: ID, dir: str, force: bool): + """ + Starts a sync of a collection to a folder of markdown files + + COLLECTION: Database ID or url + DIR: Directory path to sync to + """ + notion = config.connections.notion.client() + + MarkdownPagesSync(notion, collection, dir).sync() \ No newline at end of file diff --git a/notionsci/connections/notion/client.py b/notionsci/connections/notion/client.py index 1ed4dc8..8f3551c 100644 --- a/notionsci/connections/notion/client.py +++ b/notionsci/connections/notion/client.py @@ -111,22 +111,26 @@ def database_query_all( def search( self, + query: str = None, filter: Optional[QueryFilter] = None, sorts: Optional[List[SortObject]] = None, start_cursor: str = None, page_size: int = None ) -> QueryResult: - args = format_query_args(filter=filter, sorts=sorts, start_cursor=start_cursor, page_size=page_size) + args = format_query_args( + query=query, filter=filter, sorts=sorts, start_cursor=start_cursor, page_size=page_size + ) result_raw = self.client.search(**args) return QueryResult.from_dict(result_raw) def search_all( self, + query: str = None, filter: Optional[QueryFilter] = None, sorts: Optional[List[SortObject]] = None ) -> Iterator[Union[Page, Database]]: return traverse_pagination( - args=dict(filter=filter if filter else None, sorts=sorts, page_size=100), + args=dict(query=query, filter=filter if filter else None, sorts=sorts, page_size=100), query_fn=lambda **args: self.search(**args) ) diff --git a/notionsci/connections/notion/structures/content.py b/notionsci/connections/notion/structures/content.py index 2137f64..51deb09 100644 --- a/notionsci/connections/notion/structures/content.py +++ b/notionsci/connections/notion/structures/content.py @@ -28,6 +28,9 @@ class ParentType(Enum): class HasPropertiesMixin(Generic[PT]): properties: Dict[str, PT] = field(default_factory=dict) + def has_property(self, name: str): + return name in self.properties + def get_property(self, name: str) -> PT: return self.properties[name] diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index 6294fe9..4aca548 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -42,10 +42,19 @@ class SelectValue: @dataclass_dict_convert(dict_letter_case=snakecase) @dataclass -class DateValue: +class DateValue(ToMarkdownMixin): start: str end: Optional[str] = None + @staticmethod + def from_date(value: dt.datetime): + return DateValue(value.isoformat()) + + def to_markdown(self, context: MarkdownContext) -> str: + return self.start if not self.end else f'{self.start} - {self.end}' + + + @dataclass_dict_convert(dict_letter_case=snakecase) @dataclass @@ -125,12 +134,18 @@ class Property(ToMarkdownMixin): def _value(self): return getattr(self, self.type.value) + def _set_value(self, value): + return setattr(self, self.type.value, value) + def raw_value(self): if self.type == PropertyType.date: return dt.datetime.fromisoformat(self.date.start) else: return self._value() + def set_raw_value(self, value): + self._set_value(value) + def value(self): return object_to_text_value(self.raw_value()) @@ -165,7 +180,7 @@ def as_number(number: int) -> 'Property': def as_date(date: dt.datetime) -> 'Property': return Property( type=PropertyType.date, - date=DateValue(date.isoformat()) + date=DateValue.from_date(date) ) @staticmethod diff --git a/notionsci/connections/notion/structures/results.py b/notionsci/connections/notion/structures/results.py index 1ac59fd..5c60af5 100644 --- a/notionsci/connections/notion/structures/results.py +++ b/notionsci/connections/notion/structures/results.py @@ -59,12 +59,14 @@ class SortObject: def format_query_args( + query: str = None, filter: Optional[QueryFilter] = None, sorts: Optional[List[SortObject]] = None, start_cursor: str = None, page_size: int = None ) -> dict: return filter_none_dict(dict( + query=query, filter=filter if filter else None, sorts=[sort.to_dict() for sort in sorts] if sorts else None, start_cursor=start_cursor, page_size=page_size diff --git a/notionsci/sync/markdown/__init__.py b/notionsci/sync/markdown/__init__.py new file mode 100644 index 0000000..3ea5990 --- /dev/null +++ b/notionsci/sync/markdown/__init__.py @@ -0,0 +1 @@ +from .pages import * \ No newline at end of file diff --git a/notionsci/sync/markdown/pages.py b/notionsci/sync/markdown/pages.py new file mode 100644 index 0000000..081bb4a --- /dev/null +++ b/notionsci/sync/markdown/pages.py @@ -0,0 +1,142 @@ +import datetime as dt +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Dict + +from notionsci.connections.notion import Page, ID, NotionClient, SortObject, SortDirection, DateValue, Property +from notionsci.sync import Action +from notionsci.sync.structure import Sync, ActionTarget, ActionType +from notionsci.utils import sanitize_filename, MarkdownContext + + +@dataclass +class MarkdownPage: + filename: str + path: Optional[str] = None + created_at: Optional[dt.datetime] = None + updated_at: Optional[dt.datetime] = None + synced_at: Optional[dt.datetime] = None + deleted: bool = False + + def __post_init__(self): + # On some filesystems, created at can be greater than modified at + if self.created_at is not None and self.updated_at is not None: + upperbound_time = min(self.created_at, self.synced_at) if self.synced_at else self.created_at + self.updated_at = self.updated_at if self.updated_at >= upperbound_time else upperbound_time + + def get_title(self): + return self.path.replace('.md', '') + + +PROPERTY_PATTERN = r'\|\s*{property_name}\s*\|(.*)\|' +SYNCED_AT_PATTERN = PROPERTY_PATTERN.format(property_name='Synced At') + + +@dataclass +class MarkdownPagesSync(Sync[MarkdownPage, Page]): + notion: NotionClient + database_id: ID + markdown_dir: str + + def fetch_items_a(self) -> Dict[str, MarkdownPage]: + print('Loading existing Markdown pages') + path = Path(self.markdown_dir) + pages = [] + for file in path.glob('**/*.md'): + if not file.is_file(): + continue + + content = file.read_text() + synced_at = re.search(SYNCED_AT_PATTERN, content) + + pages.append(MarkdownPage( + file.name, + str(file.absolute()), + created_at=dt.datetime.fromtimestamp(os.path.getctime(file)), + updated_at=dt.datetime.fromtimestamp(os.path.getmtime(file)), + synced_at=dt.datetime.fromisoformat(synced_at.group(1).strip()) if synced_at else None, + deleted='deleted' in file.parts + )) + + return {page.filename.replace('.md', ''): page for page in pages} + + def fetch_items_b(self) -> Dict[str, Page]: + print('Loading existing Notion pages') + return { + sanitize_filename(page.get_title()): page + for page in self.notion.database_query_all( + self.database_id, + sorts=[SortObject(property='Modified At', direction=SortDirection.descending)], + filter=None + ) + } + + def compare(self, a: Optional[MarkdownPage], b: Optional[Page]) -> Action[MarkdownPage, Page]: + if a is None: # Doesn't exist in markdown + return Action.push(ActionTarget.A, a, b) + if b is None: # Doesn't exist in notion + if a.deleted: + return Action.ignore() + else: + return Action.push(ActionTarget.B, a, b) + + if a.deleted: + return Action.delete(ActionTarget.B, a, b) + + locally_modified = a.synced_at is None or a.updated_at > a.synced_at + remote_modified = b.get_propery_value('Synced At') is None \ + or b.get_propery_value('Modified At') > b.get_propery_value('Synced At') + if locally_modified and remote_modified: + return Action.merge(a, b) + elif locally_modified: + return Action.push(ActionTarget.B, a, b) + elif remote_modified: + return Action.push(ActionTarget.A, a, b) + + return Action.ignore() + + def execute_a(self, action: Action[MarkdownPage, Page]): + if action.action_type == ActionType.PUSH: + # Load page property + page = action.b + self.notion.load_children(page, recursive=True, databases=True) + path = os.path.join(self.markdown_dir, f'{sanitize_filename(page.get_title())}.md') + synced_at = dt.datetime.now() + + # Update synced at property + if page.has_property('Synced At'): + page.get_property('Synced At').set_raw_value(DateValue.from_date(synced_at)) + else: + page.extend_properties({ + 'Synced At': Property.as_date(synced_at) + }) + + # TODO: Save to notion + + # Download page to markdown + content = page.to_markdown(MarkdownContext()) + with open(path, 'w') as f: + f.write(content) + + # Update file modified at such that it is before synced at + os.utime(path, (synced_at.timestamp() - 5, synced_at.timestamp() - 5)) + print(f'- [MARKDOWN] Updated: {action.b.get_title()}') + elif action.action_type == ActionType.PUSH: + page = action.b + path = os.path.join(self.markdown_dir, f'{sanitize_filename(page.get_title())}.md') + os.remove(path) + print(f'- [MARKDOWN] Deleted: {action.b.get_title()}') + else: + return super().execute_a(action) + + def execute_b(self, action: Action[MarkdownPage, Page]): + if action.action_type == ActionType.PUSH: + pass + print(f'- [NOTION] Updated: {action.a.get_title()}') + elif action.action_type == ActionType.PUSH: + pass + print(f'- [NOTION] Deleted: {action.b.get_title()}') + else: + return super().execute_a(action) diff --git a/notionsci/sync/structure.py b/notionsci/sync/structure.py index 6bedbf7..72d3dd6 100644 --- a/notionsci/sync/structure.py +++ b/notionsci/sync/structure.py @@ -81,7 +81,7 @@ def fetch_items_a(self) -> Dict[str, A]: def fetch_items_b(self) -> Dict[str, B]: pass - def compare(self, a: Optional[A], b: Optional[A]) -> Action[A, B]: + def compare(self, a: Optional[A], b: Optional[B]) -> Action[A, B]: if a is None or b is None: return Action.push(ActionTarget.A if not a else ActionTarget.B, a, b) return None From bcb3a9ae468237555ed7b66c3b375e139e1184c1 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 11:05:12 +0200 Subject: [PATCH 04/11] [FEATURE] Added support for table of contents and dividers --- .../connections/notion/structures/blocks.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/notionsci/connections/notion/structures/blocks.py b/notionsci/connections/notion/structures/blocks.py index 3577121..b361602 100644 --- a/notionsci/connections/notion/structures/blocks.py +++ b/notionsci/connections/notion/structures/blocks.py @@ -34,6 +34,8 @@ class BlockType(Enum): code = 'code' unsupported = 'unsupported' child_database = 'child_database' + table_of_contents = 'table_of_contents' + divider = 'divider' BlockConvertor = ListConvertor(ForwardRefConvertor('Block')) @@ -257,6 +259,20 @@ def to_markdown(self, context: MarkdownContext) -> str: return MarkdownBuilder.code(content, self.language) +@dataclass_dict_convert(dict_letter_case=snakecase) +@dataclass +class TableOfContentsBlock(ToMarkdownMixin): + def to_markdown(self, context: MarkdownContext) -> str: + return MarkdownBuilder.table_of_contents() + + +@dataclass_dict_convert(dict_letter_case=snakecase) +@dataclass +class DividerBlock(ToMarkdownMixin): + def to_markdown(self, context: MarkdownContext) -> str: + return MarkdownBuilder.divider() + + @dataclass_dict_convert( dict_letter_case=snakecase, **ignore_fields(['database', 'children']) @@ -315,6 +331,8 @@ class Block(ToMarkdownMixin): equation: Optional[EquationBlock] = None code: Optional[CodeBlock] = None child_database: Optional[ChildDatabaseBlock] = None + table_of_contents: Optional[TableOfContentsBlock] = None + divider: Optional[DividerBlock] = None unsupported: Optional[str] = None def to_markdown(self, context: MarkdownContext) -> str: From 17a81fc26779fa8a8e078e4f257b663a2dcd85a3 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 11:07:03 +0200 Subject: [PATCH 05/11] [FEATURE] Added support for table of contents and dividers --- notionsci/utils/markdown.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/notionsci/utils/markdown.py b/notionsci/utils/markdown.py index 77f1d75..552414a 100644 --- a/notionsci/utils/markdown.py +++ b/notionsci/utils/markdown.py @@ -165,6 +165,14 @@ def code(content, language): def equation(content): return f'$$\n{content}\n$$' + @staticmethod + def table_of_contents(): + return '[TOC]\n' + + @staticmethod + def divider(): + return '----\n' + def chain_to_markdown(items: List[ToMarkdownMixin], context: MarkdownContext, sep='', prefix=''): result = [] From c846839ea1b6eed98e1fb0692f4b50ef27c10478 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 12:52:05 +0200 Subject: [PATCH 06/11] [FEATURE] Fixed issues regarding tests and new database deduction from page --- notionsci/cli/sync/zotero.py | 25 +++++++++++++++++-------- test/TestZoteroSync.py | 13 +++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/notionsci/cli/sync/zotero.py b/notionsci/cli/sync/zotero.py index 6e10e2e..0f7dd36 100644 --- a/notionsci/cli/sync/zotero.py +++ b/notionsci/cli/sync/zotero.py @@ -7,6 +7,7 @@ from notionsci.connections.notion import parse_uuid, parse_uuid_callback, BlockType, block_type_filter from notionsci.connections.zotero import ID from notionsci.sync.zotero import RefsOneWaySync, CollectionsOneWaySync +from notionsci.utils import take_1 @click.group() @@ -33,15 +34,16 @@ def template(ctx: click.Context, parent: ID): ctx.invoke(duplicate, source=source, parent=parent, target_id=target_id) # Extract the children from the command - unotion = config.connections.notion_unofficial.client() - page = unotion.get_block(target_id) - children = list(page.children) + notion = config.connections.notion.client() + page = notion.page_get(target_id, with_children=True) + + click.echo(f'Created page ({parse_uuid(page.id)})') - collections_db = next(filter(lambda x: 'Collections' in x.title, children), None) + collections_db = take_1(page.get_children(child_database_filter('Zotero Collections'))) if collections_db: click.echo(f'Found collection database ({parse_uuid(collections_db.id)})') - refs_db = next(filter(lambda x: 'References' in x.title, children), None) + refs_db = take_1(page.get_children(child_database_filter('Zotero References'))) if refs_db: click.echo(f'Found references database ({parse_uuid(refs_db.id)})') @@ -63,16 +65,23 @@ def refs(template: ID, force: bool): """ notion = config.connections.notion.client() zotero = config.connections.zotero.client() + sync_config = config.sync.zotero.get('refs', {}) template_page = notion.page_get(template, with_children=True) - references = next(iter(template_page.get_children(child_database_filter('Zotero References'))), None) - collections = next(iter(template_page.get_children(child_database_filter('Zotero Collections'))), None) + references = take_1(template_page.get_children(child_database_filter('Zotero References'))) + collections = take_1(template_page.get_children(child_database_filter('Zotero Collections'))) if not references or not collections: raise Exception('Please check whether child database called "Zotero References" and "Zotero Collections" ' 'exist in given template') - RefsOneWaySync(notion, zotero, references.id, collections_id=collections.id, force=force).sync() + RefsOneWaySync( + notion, zotero, + references.id, + collections_id=collections.id, + force=force, + **sync_config + ).sync() @zotero.command() diff --git a/test/TestZoteroSync.py b/test/TestZoteroSync.py index 7dc9c36..604c2f2 100644 --- a/test/TestZoteroSync.py +++ b/test/TestZoteroSync.py @@ -41,17 +41,16 @@ def setupTemplate(self): ) self.assertEqual(code, 0) - self.collection_db = re.search( - r"Found collection database \((.*)\)", output + self.template = re.search( + r"Created page \((.*)\)", output ).group(1) - self.refs_db = re.search(r"Found references database \((.*)\)", output).group(1) def test_sync_refs(self): self.setupTemplate() code, output = capture_cmd( lambda: cli( - ["sync", "zotero", "refs", parse_uuid_or_url(self.refs_db), "--force",] + ["sync", "zotero", "refs", parse_uuid_or_url(self.template), "--force",] ) ) @@ -68,9 +67,7 @@ def test_sync_refs_with_collections(self): "sync", "zotero", "refs", - parse_uuid_or_url(self.refs_db), - "-c", - parse_uuid_or_url(self.collection_db), + parse_uuid_or_url(self.template), "--force", ] ) @@ -90,7 +87,7 @@ def test_sync_collections(self): "sync", "zotero", "collections", - parse_uuid_or_url(self.collection_db), + parse_uuid_or_url(self.template), "--force", ] ) From 4f6d678f1e5032f63ca7a96eecb4a29175bc26bb Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 12:53:39 +0200 Subject: [PATCH 07/11] [FEATURE] Added special tag filtering [FEATURE] Added database schema ensurance --- notionsci/cli/notion.py | 18 +++ notionsci/config/structure.py | 8 +- notionsci/connections/notion/client.py | 18 ++- .../notion/structures/properties.py | 5 + notionsci/connections/zotero/client.py | 125 ++++++++---------- notionsci/sync/zotero/refs.py | 34 ++++- notionsci/utils/__init__.py | 3 +- notionsci/utils/iterators.py | 7 + notionsci/utils/serialization.py | 6 +- 9 files changed, 149 insertions(+), 75 deletions(-) create mode 100644 notionsci/utils/iterators.py diff --git a/notionsci/cli/notion.py b/notionsci/cli/notion.py index 687fcad..9ddf10b 100644 --- a/notionsci/cli/notion.py +++ b/notionsci/cli/notion.py @@ -119,3 +119,21 @@ def download_md(page: ID, output: str): f.write(content) + +@notion.command() +@click.argument('parent', callback=parse_uuid_callback) +@click.argument('file') +def upload_md(parent: ID, file: str): + """ + Uploads given markdown FILE as a child page of the given PARENT + + :param parent: + :param file: + :return: + """ + notion = config.connections.notion.client() + + click.echo(f'Reading file {file}') + with open(file, 'r') as f: + content = f.read() + diff --git a/notionsci/config/structure.py b/notionsci/config/structure.py index 7675ca7..198b346 100644 --- a/notionsci/config/structure.py +++ b/notionsci/config/structure.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from notion_client import Client from simple_parsing import Serializable @@ -64,9 +64,15 @@ class Development(Serializable): test_page: str = DEV_TESTS_PAGE +@dataclass +class Sync(Serializable): + zotero: dict = field(default_factory=dict) + + @dataclass class Config(Serializable): version: int = CONFIG_VERSION connections: Connections = Connections() templates: Templates = Templates() development: Development = Development() + sync: Sync = Sync() diff --git a/notionsci/connections/notion/client.py b/notionsci/connections/notion/client.py index 8f3551c..c9f4caa 100644 --- a/notionsci/connections/notion/client.py +++ b/notionsci/connections/notion/client.py @@ -3,7 +3,7 @@ from notion_client import Client -from notionsci.connections.notion import BlockType +from notionsci.connections.notion import BlockType, PropertyDef from notionsci.connections.notion.structures import Database, SortObject, QueryFilter, format_query_args, ContentObject, \ PropertyType, Page, ID, QueryResult, Block from notionsci.utils import strip_none_field, filter_none_dict @@ -81,6 +81,11 @@ def database_get(self, id: ID) -> Database: result = self.client.databases.retrieve(id) return Database.from_dict(result) + def database_update(self, database: Database) -> Page: + args = strip_none_field(strip_readonly_props(database).to_dict()) + result = self.client.databases.update(database.id, **args) + return Database.from_dict(result) + def database_create(self, database: Database) -> Database: args = strip_none_field(strip_readonly_props(database).to_dict()) result = self.client.databases.create(**args) @@ -165,3 +170,14 @@ def load_children(self, item: Union[Page, Block], recursive=False, databases=Fal elif databases and child.type == BlockType.child_database: child.child_database.database = self.database_get(child.id) child.child_database.children = list(self.database_query_all(child.id)) # eager load (mayby dont?) + + def ensure_database_schema(self, schema: Dict[str, PropertyDef], db: Database): + new_props = { + prop_name: prop for prop_name, prop in schema.items() + if not db.has_property(prop_name) + } + if len(new_props) > 0: + db.extend_properties(new_props) + db = self.database_update(db) + + return db diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index 4aca548..af6871b 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -300,6 +300,11 @@ def as_last_edited_time() -> 'PropertyDef': def as_date() -> 'PropertyDef': return PropertyDef(type=PropertyType.date, date={}) + @staticmethod + def as_number() -> 'PropertyDef': + return PropertyDef(type=PropertyType.number, number={}) + + @staticmethod def as_relation(database: ID) -> 'PropertyDef': return PropertyDef(type=PropertyType.relation, relation=RelationDef(database)) diff --git a/notionsci/connections/zotero/client.py b/notionsci/connections/zotero/client.py index 681c726..50432f8 100644 --- a/notionsci/connections/zotero/client.py +++ b/notionsci/connections/zotero/client.py @@ -1,10 +1,12 @@ from dataclasses import dataclass -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Callable, Any, Iterator, TypeVar, Union from pyzotero.zotero import Zotero from notionsci.connections.notion import NotionNotAttachedException +from notionsci.connections.zotero import Entity from notionsci.connections.zotero.structures import SearchParameters, SearchPagination, Item, ID, Collection +from notionsci.utils import list_from_dict class ZoteroNotAttachedException(Exception): @@ -44,26 +46,16 @@ def collections( **(pagination.to_query() if pagination else {}), **kwargs ) - return [Collection.from_dict(item) for item in result_raw] + return list_from_dict(Collection, result_raw) def all_collections( self, params: Optional[SearchParameters] = None, pagination: Optional[SearchPagination] = None, ): - pagination = pagination if pagination else SearchPagination() - done = False - while not done: - result = self.collections( - params, - pagination - ) - - if len(result) < pagination.limit: - done = True - pagination.start = pagination.start + pagination.limit - - yield from result + yield from traverse_pagination( + pagination, lambda pagination: self.collections(params, pagination) + ) def items( self, @@ -76,71 +68,66 @@ def items( **(pagination.to_query() if pagination else {}), **kwargs ) - return [Item.from_dict(item) for item in result_raw] + return list_from_dict(Item, result_raw) def all_items( self, params: Optional[SearchParameters] = None, pagination: Optional[SearchPagination] = None, ): - pagination = pagination if pagination else SearchPagination() - done = False - while not done: - result = self.items( - params, - pagination - ) + yield from traverse_pagination( + pagination, lambda pagination: self.items(params, pagination) + ) - if len(result) < pagination.limit: - done = True - pagination.start = pagination.start + pagination.limit + def all_items_grouped(self, delete_children: bool = True) -> List[Item]: + return group_entities(self.all_items(), lambda x: x.data.parent_item, delete_children) - yield from result + def all_collections_grouped(self, delete_children: bool = True) -> List[Collection]: + return group_entities(self.all_collections(), lambda x: x.data.parent_collection, delete_children) - def all_items_grouped(self, delete_children: bool = True) -> List[Item]: - items: Dict[ID, Item] = { - x.key: x - for x in self.all_items() - } - # Assign items to the right parents - known_children = set() - for child_key, child in items.items(): - if not child.data.parent_item: - continue +T = TypeVar('T') - if child.data.parent_item in items: - parent = items[child.data.parent_item] - if not parent.children: parent.children = {} - parent.children[child_key] = child - known_children.add(child_key) - if delete_children: - for k in known_children: - del items[k] +def traverse_pagination( + args: Optional[SearchPagination], + query_fn: Callable[[SearchPagination], List[T]] +) -> Iterator[T]: + if not args: + args = SearchPagination() - return list(items.values()) + done = False + while not done: + result = query_fn(args) + if len(result) < args.limit: + done = True + args.start = args.start + args.limit - def all_collections_grouped(self, delete_children: bool = True) -> List[Collection]: - collections: Dict[ID, Collection] = { - x.key: x - for x in self.all_collections() - } - - # Assign collections to the right parents - known_children = set() - for child_key, child in collections.items(): - if not child.data.parent_collection: - continue - - if child.data.parent_collection in collections: - parent = collections[child.data.parent_collection] - if not parent.children: parent.children = {} - parent.children[child_key] = child - known_children.add(child_key) - - if delete_children: - for k in known_children: - del collections[k] - - return list(collections.values()) + yield from result + + +def group_entities( + items: Iterator[Union[Entity, T]], + parent_key: Callable[[T], ID], + delete_children: bool = True +): + results: Dict[ID, T] = { + x.key: x for x in items + } + + # Assign items to the right parents + known_children = set() + for child_key, child in results.items(): + if not parent_key(child) or parent_key(child) not in results: + continue + + parent = results[parent_key(child)] + if not parent.children: parent.children = {} + parent.children[child_key] = child + known_children.add(child_key) + + if delete_children: + for k in known_children: + del results[k] + + return list(results.values()) diff --git a/notionsci/sync/zotero/refs.py b/notionsci/sync/zotero/refs.py index aba88e5..fc9a0e6 100644 --- a/notionsci/sync/zotero/refs.py +++ b/notionsci/sync/zotero/refs.py @@ -1,14 +1,32 @@ import datetime as dt +import re from dataclasses import dataclass, field from typing import Optional, Dict, List, Set from notionsci.connections.notion import Page, ID, SortObject, SortDirection, Property, \ - RelationItem + RelationItem, PropertyDef from notionsci.connections.zotero import Item, generate_citekey, Collection, build_inherency_tree from notionsci.sync.structure import B, A from notionsci.sync.zotero.base import ZoteroNotionOneWaySync, PROP_SYNCED_AT, PROP_VERSION from notionsci.utils import key_by, flatten +SCHEMA = { + 'ID': PropertyDef.as_rich_text(), + 'Type': PropertyDef.as_select(), + 'Cite Key': PropertyDef.as_title(), + 'Title': PropertyDef.as_rich_text(), + 'Authors': PropertyDef.as_rich_text(), + 'Publication Date': PropertyDef.as_rich_text(), + 'Abstract': PropertyDef.as_rich_text(), + 'URL': PropertyDef.as_url(), + 'Publication': PropertyDef.as_rich_text(), + 'Tags': PropertyDef.as_multi_select(), + 'Special Tags': PropertyDef.as_multi_select(), + 'Collections': PropertyDef.as_multi_select(), + PROP_SYNCED_AT: PropertyDef.as_date(), + PROP_VERSION: PropertyDef.as_date(), +} + @dataclass class RefsOneWaySync(ZoteroNotionOneWaySync[Item]): @@ -16,6 +34,7 @@ class RefsOneWaySync(ZoteroNotionOneWaySync[Item]): notion_collections: Dict[ID, Page] = field(default_factory=dict) zotero_collections: Dict[ID, Collection] = field(default_factory=dict) collection_sets: Dict[ID, Set[ID]] = field(default_factory=dict) + special_tags_regex: str = None def fetch_items_a(self) -> Dict[str, A]: print('Loading existing Zotero items') @@ -25,6 +44,9 @@ def fetch_items_a(self) -> Dict[str, A]: } def preprocess(self, items_a: Dict[str, A], items_b: Dict[str, B], keys: List[str]): + db = self.notion.database_get(self.database_id) + self.notion.ensure_database_schema(SCHEMA, db) + if self.collections_id: print('Loading existing Notion collections') self.notion_collections = { @@ -42,6 +64,9 @@ def preprocess(self, items_a: Dict[str, A], items_b: Dict[str, B], keys: List[st return super().preprocess(items_a, items_b, keys) + def is_special_tag(self, tag: str) -> bool: + return self.special_tags_regex is not None and re.match(self.special_tags_regex, tag) is not None + def collect_props(self, a: A): item_collections = set(flatten( [self.collection_sets.get(col_key, []) for col_key in a.data.collections] @@ -58,7 +83,12 @@ def collect_props(self, a: A): 'Abstract': Property.as_rich_text(a.data.abstract), 'URL': Property.as_url(a.data.url), 'Publication': Property.as_rich_text(a.data.publication), - 'Tags': Property.as_multi_select([tag.tag for tag in a.data.tags]), + 'Tags': Property.as_multi_select( + [tag.tag for tag in a.data.tags if not self.is_special_tag(tag.tag)] + ), + 'Special Tags': Property.as_multi_select( + [tag.tag for tag in a.data.tags if self.is_special_tag(tag.tag)] + ), 'Collections': Property.as_multi_select([ self.zotero_collections[col_key].title for col_key in item_collections diff --git a/notionsci/utils/__init__.py b/notionsci/utils/__init__.py index 811d021..27f3cba 100644 --- a/notionsci/utils/__init__.py +++ b/notionsci/utils/__init__.py @@ -1,4 +1,5 @@ +from .iterators import * from .dicts import * from .serialization import * from .markdown import * -from .io import * \ No newline at end of file +from .io import * diff --git a/notionsci/utils/iterators.py b/notionsci/utils/iterators.py new file mode 100644 index 0000000..0925981 --- /dev/null +++ b/notionsci/utils/iterators.py @@ -0,0 +1,7 @@ +from typing import Iterable, List, TypeVar, Optional, Union + +T = TypeVar('T') + + +def take_1(x: Union[Iterable[T], List[T]]) -> Optional[T]: + return next(iter(x), None) diff --git a/notionsci/utils/serialization.py b/notionsci/utils/serialization.py index 06bb105..d091741 100644 --- a/notionsci/utils/serialization.py +++ b/notionsci/utils/serialization.py @@ -60,4 +60,8 @@ def ignore_fields(fields): field: lambda x: None for field in fields } - ) \ No newline at end of file + ) + + +def list_from_dict(type, data: List[dict]) -> List[Any]: + return [type.from_dict(x) for x in data] From 162ff39732dda7bedaef2f4af2acda34cf8a3874 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 19:08:46 +0200 Subject: [PATCH 08/11] [FEATURE] Undefineable field to hide "undefined" fields [FEATURE] Added a way to update zotero references --- notionsci/connections/zotero/client.py | 5 ++ .../connections/zotero/structures/item.py | 19 +++-- notionsci/utils/serialization.py | 72 ++++++++++++++++++- 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/notionsci/connections/zotero/client.py b/notionsci/connections/zotero/client.py index 50432f8..58b0de8 100644 --- a/notionsci/connections/zotero/client.py +++ b/notionsci/connections/zotero/client.py @@ -57,6 +57,11 @@ def all_collections( pagination, lambda pagination: self.collections(params, pagination) ) + def update_items(self, items: List[Item]) -> Item: + args = [Item.to_dict(item)['data'] for item in items] + res = self.client.update_items(args) + return res + def items( self, params: Optional[SearchParameters] = None, diff --git a/notionsci/connections/zotero/structures/item.py b/notionsci/connections/zotero/structures/item.py index c7e46f8..ddcc903 100644 --- a/notionsci/connections/zotero/structures/item.py +++ b/notionsci/connections/zotero/structures/item.py @@ -7,7 +7,7 @@ from stringcase import camelcase from notionsci.connections.zotero.structures import ID, Links, Library, Meta, Entity -from notionsci.utils import ignore_unknown +from notionsci.utils import serde, Undefinable class ItemType(Enum): @@ -49,24 +49,21 @@ class ItemType(Enum): webpage = 'webpage' -@dataclass_dict_convert(dict_letter_case=camelcase) +@serde(camel=True) @dataclass class Tag: tag: str - type: Optional[int] = None + type: Undefinable[int] = None -@dataclass_dict_convert( - dict_letter_case=camelcase, - on_unknown_field=ignore_unknown -) +@serde(camel=True, ignore_unknown=True) @dataclass class ItemData(Entity): item_type: ItemType date_added: dt.datetime date_modified: dt.datetime - parent_item: Optional[ID] = None + parent_item: Undefinable[ID] = None tags: Optional[List[Tag]] = None collections: Optional[List[ID]] = None relations: Optional[Dict] = None @@ -134,13 +131,15 @@ def on_unknown_field(field: str): def item_to_dict_converter(): def wrap(val: ItemData): result = ItemData.to_dict(val) + result.update(result['properties']) + del result['properties'] return result return wrap -@dataclass_dict_convert( - dict_letter_case=camelcase, +@serde( + camel=True, custom_from_dict_convertors={ 'data': item_from_dict_converter() }, diff --git a/notionsci/utils/serialization.py b/notionsci/utils/serialization.py index d091741..b50a2a7 100644 --- a/notionsci/utils/serialization.py +++ b/notionsci/utils/serialization.py @@ -1,6 +1,74 @@ -from typing import List, Dict, Union, Any, Callable +import types +from typing import List, Dict, Union, Any, Callable, Optional, get_type_hints -from dataclass_dict_convert.convert import SimpleTypeConvertor +from dataclass_dict_convert.convert import SimpleTypeConvertor, dataclass_dict_convert +from stringcase import camelcase, snakecase + + +class ExplicitNone(): + def __nonzero__(self): + return False + + +class UndefinableMeta(type): + def __getitem__(cls, key): + new_cls = types.new_class( + f"{cls.__name__}_{key.__name__}", + (cls,), + {}, + lambda ns: ns.__setitem__("type", Optional[key]) + ) + new_cls.__origin__ = Union + new_cls.__args__ = [key, type(None)] # , ExplicitNone] + return new_cls + + +class Undefinable(metaclass=UndefinableMeta): pass + + +def serde(*, camel=False, ignore_unknown=False, **kwargs): + case = camelcase if camel else snakecase + + def wrap(cls): + types = get_type_hints(cls) + undefinable_fields = [ + field_name + for field_name, field_type in types.items() + if isinstance(field_type, UndefinableMeta) + ] + + cls = dataclass_dict_convert( + dict_letter_case=case, + on_unknown_field=ignore_unknown if ignore_unknown else None, + **kwargs, + )(cls) + + original_to_dict = cls.to_dict + original_from_dict = cls.from_dict + + def to_dict(self, *args, **kwargs): + result: dict = original_to_dict(self, *args, **kwargs) + for field_name in undefinable_fields: + field_name = case(field_name) + if field_name in result and result[field_name] is None: + del result[field_name] + return result + + def from_dict(dict, *args, **kwargs): + result = original_from_dict(dict, *args, **kwargs) + for field_name in undefinable_fields: + field_name = case(field_name) + if field_name in dict and dict[field_name] is None: + setattr(result, field_name, ExplicitNone()) + + return result + + cls.to_dict = to_dict + cls.from_dict = from_dict + + return cls + + return wrap class ListConvertor(SimpleTypeConvertor): From 70ce205bca377d0c270c090657ffac1d9a39255a Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 21:50:50 +0200 Subject: [PATCH 09/11] [FEATURE] Updated models to use undefinable property [FEATURE] Added two-way zotero refs sync (only tags) --- notionsci/cli/sync/zotero.py | 6 +- notionsci/connections/notion/client.py | 10 +-- .../connections/notion/structures/common.py | 35 +++++----- .../connections/notion/structures/content.py | 21 +++--- .../notion/structures/properties.py | 67 +++++++++---------- notionsci/sync/structure.py | 11 ++- notionsci/sync/zotero/base.py | 47 +++++++++---- notionsci/sync/zotero/collections.py | 9 ++- notionsci/sync/zotero/refs.py | 42 ++++++++++-- notionsci/utils/dicts.py | 14 ---- notionsci/utils/serialization.py | 23 +++++-- 11 files changed, 175 insertions(+), 110 deletions(-) diff --git a/notionsci/cli/sync/zotero.py b/notionsci/cli/sync/zotero.py index 0f7dd36..c5b148c 100644 --- a/notionsci/cli/sync/zotero.py +++ b/notionsci/cli/sync/zotero.py @@ -6,7 +6,7 @@ from notionsci.config import config from notionsci.connections.notion import parse_uuid, parse_uuid_callback, BlockType, block_type_filter from notionsci.connections.zotero import ID -from notionsci.sync.zotero import RefsOneWaySync, CollectionsOneWaySync +from notionsci.sync.zotero import RefsSync, CollectionsSync from notionsci.utils import take_1 @@ -75,7 +75,7 @@ def refs(template: ID, force: bool): raise Exception('Please check whether child database called "Zotero References" and "Zotero Collections" ' 'exist in given template') - RefsOneWaySync( + RefsSync( notion, zotero, references.id, collections_id=collections.id, @@ -100,4 +100,4 @@ def collections(template: ID, force: bool): template_page = notion.page_get(template, with_children=True) collections = next(iter(template_page.get_children(child_database_filter('Zotero Collections'))), None) - CollectionsOneWaySync(notion, zotero, collections.id, force=force).sync() + CollectionsSync(notion, zotero, collections.id, force=force).sync() diff --git a/notionsci/connections/notion/client.py b/notionsci/connections/notion/client.py index c9f4caa..66306ec 100644 --- a/notionsci/connections/notion/client.py +++ b/notionsci/connections/notion/client.py @@ -6,7 +6,7 @@ from notionsci.connections.notion import BlockType, PropertyDef from notionsci.connections.notion.structures import Database, SortObject, QueryFilter, format_query_args, ContentObject, \ PropertyType, Page, ID, QueryResult, Block -from notionsci.utils import strip_none_field, filter_none_dict +from notionsci.utils import filter_none_dict class NotionNotAttachedException(Exception): @@ -65,12 +65,12 @@ def page_get(self, id: ID, with_children=False) -> Page: return page def page_update(self, page: Page) -> Page: - args = strip_none_field(strip_readonly_props(page).to_dict()) + args = strip_readonly_props(page).to_dict() result = self.client.pages.update(page.id, **args) return Page.from_dict(result) def page_create(self, page: Page) -> Page: - args = strip_none_field(strip_readonly_props(page).to_dict()) + args = strip_readonly_props(page).to_dict() result = self.client.pages.create(**args) return Page.from_dict(result) @@ -82,12 +82,12 @@ def database_get(self, id: ID) -> Database: return Database.from_dict(result) def database_update(self, database: Database) -> Page: - args = strip_none_field(strip_readonly_props(database).to_dict()) + args = strip_readonly_props(database).to_dict() result = self.client.databases.update(database.id, **args) return Database.from_dict(result) def database_create(self, database: Database) -> Database: - args = strip_none_field(strip_readonly_props(database).to_dict()) + args = strip_readonly_props(database).to_dict() result = self.client.databases.create(**args) return Database.from_dict(result) diff --git a/notionsci/connections/notion/structures/common.py b/notionsci/connections/notion/structures/common.py index d4832f9..31d3deb 100644 --- a/notionsci/connections/notion/structures/common.py +++ b/notionsci/connections/notion/structures/common.py @@ -6,7 +6,8 @@ from dataclass_dict_convert import dataclass_dict_convert from stringcase import snakecase -from notionsci.utils import UnionConvertor, ToMarkdownMixin, MarkdownContext, MarkdownBuilder, chain_to_markdown +from notionsci.utils import UnionConvertor, ToMarkdownMixin, MarkdownContext, MarkdownBuilder, chain_to_markdown, \ + Undefinable, serde Color = str ID = str @@ -83,28 +84,28 @@ class PageMention: id: ID -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class MentionObject: type: str - user: Optional[UserObject] = None - page: Optional[PageMention] = None + user: Undefinable[UserObject] = None + page: Undefinable[PageMention] = None def get_text(self) -> str: return self.type -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class RichText(ToMarkdownMixin): type: RichTextType - plain_text: Optional[str] = None - annotations: Optional[Annotation] = None - href: Optional[str] = None - text: Optional[TextObject] = None - equation: Optional[EquationObject] = None - mention: Optional[MentionObject] = None + plain_text: Undefinable[str] = None + annotations: Undefinable[Annotation] = None + href: Undefinable[str] = None + text: Undefinable[TextObject] = None + equation: Undefinable[EquationObject] = None + mention: Undefinable[MentionObject] = None def raw_value(self): if self.type == RichTextType.text: @@ -149,21 +150,21 @@ class FileType(Enum): external = 'external' -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class FileTypeObject: url: str expiry_time: Optional[dt.datetime] = None -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class FileObject(ToMarkdownMixin): type: FileType - caption: Optional[List[RichText]] = None - file: Optional[FileTypeObject] = None - external: Optional[FileTypeObject] = None - name: Optional[str] = None # Only filled for when used as property value + caption: Undefinable[List[RichText]] = None + file: Undefinable[FileTypeObject] = None + external: Undefinable[FileTypeObject] = None + name: Undefinable[str] = None # Only filled for when used as property value def get_url(self) -> str: return self.file.url if self.type == FileType.file else self.external.url diff --git a/notionsci/connections/notion/structures/content.py b/notionsci/connections/notion/structures/content.py index 51deb09..7a24b3b 100644 --- a/notionsci/connections/notion/structures/content.py +++ b/notionsci/connections/notion/structures/content.py @@ -11,7 +11,8 @@ from notionsci.connections.notion.structures.common import FileObject, ID, \ UnionEmojiFileConvertor, EmojiFileType from notionsci.connections.notion.structures.properties import PropertyDef, TitleValue, PropertyType, Property -from notionsci.utils import ToMarkdownMixin, MarkdownContext, chain_to_markdown, MarkdownBuilder, filter_not_none +from notionsci.utils import ToMarkdownMixin, MarkdownContext, chain_to_markdown, MarkdownBuilder, filter_not_none, \ + Undefinable, serde class ParentType(Enum): @@ -54,9 +55,9 @@ def extend_properties(self, properties: Dict[str, PT]): @dataclass class Parent: type: ParentType - database_id: Optional[ID] = None - page_id: Optional[ID] = None - workspace: Optional[bool] = None + database_id: Undefinable[ID] = None + page_id: Undefinable[ID] = None + workspace: Undefinable[bool] = None @staticmethod def page(id: ID) -> 'Parent': @@ -85,9 +86,9 @@ class ContentObject: last_edited_time: Optional[dt.datetime] = None -@dataclass_dict_convert( - dict_letter_case=snakecase, - custom_type_convertors=[UnionEmojiFileConvertor, BlockConvertor] +@serde( + custom_type_convertors=[UnionEmojiFileConvertor, BlockConvertor], + exclude=['children'] ) @dataclass class Page(ContentObject, ToMarkdownMixin, ChildrenMixin, HasPropertiesMixin[Property]): @@ -114,9 +115,9 @@ def get_title(self): return next(map(lambda x: x.value(), self.get_property_by_type(PropertyType.title)), '') -@dataclass_dict_convert( - dict_letter_case=snakecase, - custom_type_convertors=[UnionEmojiFileConvertor] +@serde( + custom_type_convertors=[UnionEmojiFileConvertor], + exclude=['children'] ) @dataclass class Database(ContentObject, HasPropertiesMixin[PropertyDef]): diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index af6871b..e3490b9 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -3,11 +3,8 @@ from enum import Enum from typing import Optional, List, Dict, Any -from dataclass_dict_convert import dataclass_dict_convert -from stringcase import snakecase - from notionsci.connections.notion.structures.common import RichText, Color, ID, FileObject -from notionsci.utils import ExplicitNone, ToMarkdownMixin, MarkdownContext +from notionsci.utils import ExplicitNone, ToMarkdownMixin, MarkdownContext, serde, Undefinable class PropertyType(Enum): @@ -32,19 +29,19 @@ class PropertyType(Enum): last_edited_by = 'last_edited_by' -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class SelectValue: name: str - id: Optional[str] = None - color: Optional[Color] = None + id: Undefinable[str] = None + color: Undefinable[Color] = None -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class DateValue(ToMarkdownMixin): start: str - end: Optional[str] = None + end: Undefinable[str] = None @staticmethod def from_date(value: dt.datetime): @@ -56,7 +53,7 @@ def to_markdown(self, context: MarkdownContext) -> str: -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class RelationItem: id: ID @@ -104,32 +101,32 @@ def object_to_markdown(raw_value: Any, context: MarkdownContext, sep=' '): ## Property Definition Types -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class Property(ToMarkdownMixin): type: PropertyType - id: Optional[str] = None - title: Optional[TitleValue] = None - - rich_text: Optional[RichTextValue] = None - number: Optional[NumberValue] = None - select: Optional[SelectValue] = None - multi_select: Optional[MultiSelectValue] = None - date: Optional[DateValue] = None - people: Optional[PeopleValue] = None - files: Optional[FilesValue] = None - checkbox: Optional[CheckboxValue] = None - url: Optional[UrlValue] = None - email: Optional[EmailValue] = None - phone_number: Optional[Dict] = None - formula: Optional[Dict] = None - relation: Optional[RelationValue] = None - rollup: Optional[Dict] = None - created_time: Optional[CreatedTimeValue] = None - created_by: Optional[CreatedByValue] = None - last_edited_time: Optional[LastEditedTimeValue] = None - last_edited_by: Optional[LastEditedByValue] = None + id: Undefinable[str] = None + + title: Undefinable[TitleValue] = None + rich_text: Undefinable[RichTextValue] = None + number: Undefinable[NumberValue] = None + select: Undefinable[SelectValue] = None + multi_select: Undefinable[MultiSelectValue] = None + date: Undefinable[DateValue] = None + people: Undefinable[PeopleValue] = None + files: Undefinable[FilesValue] = None + checkbox: Undefinable[CheckboxValue] = None + url: Undefinable[UrlValue] = None + email: Undefinable[EmailValue] = None + phone_number: Undefinable[Dict] = None + formula: Undefinable[Dict] = None + relation: Undefinable[RelationValue] = None + rollup: Undefinable[Dict] = None + created_time: Undefinable[CreatedTimeValue] = None + created_by: Undefinable[CreatedByValue] = None + last_edited_time: Undefinable[LastEditedTimeValue] = None + last_edited_by: Undefinable[LastEditedByValue] = None def _value(self): return getattr(self, self.type.value) @@ -216,13 +213,13 @@ def as_relation(relations: RelationValue) -> 'Property': ) -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class SelectDef: options: List[SelectValue] -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class RelationDef: database_id: ID @@ -245,7 +242,7 @@ class RelationDef: MultiSelectDef = SelectDef -@dataclass_dict_convert(dict_letter_case=snakecase) +@serde() @dataclass class PropertyDef: type: PropertyType diff --git a/notionsci/sync/structure.py b/notionsci/sync/structure.py index 72d3dd6..8030bd9 100644 --- a/notionsci/sync/structure.py +++ b/notionsci/sync/structure.py @@ -63,9 +63,18 @@ def sync(self): ] # TODO: apply topo sort on resulting graph + print('Checking for conflicts') + for a in actions: + if a.action_type == ActionType.MERGE: + v = input(f'Merge conflict occurred for {a.b.get_title()}. Do you want to continue y/n').lower() + if v == 'n': + exit(1) + print('Executing actions') for a in actions: - if a.target == ActionTarget.A: + if a.action_type == ActionType.MERGE: + pass + elif a.target == ActionTarget.A: self.execute_a(a) elif a.target == ActionTarget.B: self.execute_b(a) diff --git a/notionsci/sync/zotero/base.py b/notionsci/sync/zotero/base.py index 49a9777..07797ff 100644 --- a/notionsci/sync/zotero/base.py +++ b/notionsci/sync/zotero/base.py @@ -12,16 +12,44 @@ PROP_VERSION = 'Version' -def compare_entity(a: Entity, b: Page, force: bool): - # modified_remote = b.get_propery_value(PROP_MODIFIED_AT) > b.get_propery_value(PROP_SYNCED_AT) - if a.version > b.get_propery_value(PROP_VERSION, 0) or force: +def twoway_compare_entity(a: Optional[Entity], b: Optional[Page], force: bool) -> Action[Entity, Page]: + if a is None: + return Action.delete(ActionTarget.B, a, b) + if b is None: + return Action.push(ActionTarget.B, a, b) + + a_changed = a.version > b.get_propery_value(PROP_VERSION, 0) + b_changed = b.get_propery_value('Synced At') is None \ + or b.get_propery_value('Modified At') > b.get_propery_value('Synced At') + + if force: + return Action.push(ActionTarget.B, a, b) + + if a_changed and b_changed: + return Action.merge(a, b) + elif a_changed: + return Action.push(ActionTarget.B, a, b) + elif b_changed: + return Action.push(ActionTarget.A, a, b) + else: + return Action.ignore() + + +def oneway_compare_entity(a: Optional[Entity], b: Optional[Page], force: bool) -> Action[Entity, Page]: + if a is None: + return Action.delete(ActionTarget.B, a, b) + if b is None: + return Action.push(ActionTarget.B, a, b) + + a_changed = a.version > b.get_propery_value(PROP_VERSION, 0) + if a_changed or force: return Action.push(ActionTarget.B, a, b) else: return Action.ignore() @dataclass -class ZoteroNotionOneWaySync(Sync[A, Page], ABC): +class ZoteroNotionSync(Sync[A, Page], ABC): notion: NotionClient zotero: ZoteroClient database_id: ID @@ -39,13 +67,6 @@ def fetch_items_b(self) -> Dict[str, B]: ) } - def compare(self, a: Optional[A], b: Optional[Page]) -> Action[A, B]: - if a is None: - return Action.delete(ActionTarget.B, a, b) - if b is None: - return Action.push(ActionTarget.B, a, b) - return compare_entity(a, b, self.force) - @abstractmethod def collect_props(self, a: A): pass @@ -59,8 +80,8 @@ def execute_b(self, action: Action[A, B]): action.b.extend_properties(self.collect_props(action.a)) action.b = self.notion.page_upsert(action.b) - print(f'- Updated: {action.b.get_title()}') + print(f'-[Notion] Updated: {action.b.get_title()}') elif action.action_type.DELETE: action.b.archived = True action.b = self.notion.page_update(action.b) - print(f'- Deleted: {action.b.get_title()}') + print(f'-[Notion] Deleted: {action.b.get_title()}') diff --git a/notionsci/sync/zotero/collections.py b/notionsci/sync/zotero/collections.py index 8c3d482..0ff2263 100644 --- a/notionsci/sync/zotero/collections.py +++ b/notionsci/sync/zotero/collections.py @@ -1,16 +1,16 @@ import datetime as dt from dataclasses import dataclass, field -from typing import Dict, List +from typing import Dict, List, Optional from notionsci.connections.notion import Property, \ RelationItem from notionsci.connections.zotero import Collection from notionsci.sync.structure import Action, B, ActionType, topo_sort, A -from notionsci.sync.zotero.base import ZoteroNotionOneWaySync, PROP_VERSION, PROP_SYNCED_AT +from notionsci.sync.zotero.base import ZoteroNotionSync, PROP_VERSION, PROP_SYNCED_AT, oneway_compare_entity @dataclass -class CollectionsOneWaySync(ZoteroNotionOneWaySync[Collection]): +class CollectionsSync(ZoteroNotionSync[Collection]): zotero_notion_ids: Dict[str, str] = field(default_factory=dict) def fetch_items_a(self) -> Dict[str, A]: @@ -35,6 +35,9 @@ def preprocess(self, items_a: Dict[str, A], items_b: Dict[str, B], keys: List[st return super().preprocess(items_a, items_b, keys) + def compare(self, a: Optional[A], b: Optional[B]) -> Action[A, B]: + return oneway_compare_entity(a, b, force=self.force) + def collect_props(self, a: A): return { 'ID': Property.as_rich_text(a.key), diff --git a/notionsci/sync/zotero/refs.py b/notionsci/sync/zotero/refs.py index fc9a0e6..891ce8a 100644 --- a/notionsci/sync/zotero/refs.py +++ b/notionsci/sync/zotero/refs.py @@ -3,11 +3,15 @@ from dataclasses import dataclass, field from typing import Optional, Dict, List, Set +import pytz + from notionsci.connections.notion import Page, ID, SortObject, SortDirection, Property, \ RelationItem, PropertyDef -from notionsci.connections.zotero import Item, generate_citekey, Collection, build_inherency_tree -from notionsci.sync.structure import B, A -from notionsci.sync.zotero.base import ZoteroNotionOneWaySync, PROP_SYNCED_AT, PROP_VERSION +from notionsci.connections.zotero import Item, generate_citekey, Collection, build_inherency_tree, Tag +from notionsci.sync import Action +from notionsci.sync.structure import B, A, ActionType +from notionsci.sync.zotero.base import ZoteroNotionSync, PROP_SYNCED_AT, PROP_VERSION, twoway_compare_entity, \ + oneway_compare_entity from notionsci.utils import key_by, flatten SCHEMA = { @@ -29,12 +33,13 @@ @dataclass -class RefsOneWaySync(ZoteroNotionOneWaySync[Item]): +class RefsSync(ZoteroNotionSync[Item]): collections_id: Optional[ID] = None notion_collections: Dict[ID, Page] = field(default_factory=dict) zotero_collections: Dict[ID, Collection] = field(default_factory=dict) collection_sets: Dict[ID, Set[ID]] = field(default_factory=dict) special_tags_regex: str = None + twoway: bool = True def fetch_items_a(self) -> Dict[str, A]: print('Loading existing Zotero items') @@ -64,6 +69,12 @@ def preprocess(self, items_a: Dict[str, A], items_b: Dict[str, B], keys: List[st return super().preprocess(items_a, items_b, keys) + def compare(self, a: Optional[A], b: Optional[B]) -> Action[A, B]: + if self.twoway: + return twoway_compare_entity(a, b, force=self.force) + else: + return oneway_compare_entity(a, b, force=self.force) + def is_special_tag(self, tag: str) -> bool: return self.special_tags_regex is not None and re.match(self.special_tags_regex, tag) is not None @@ -99,6 +110,27 @@ def collect_props(self, a: A): for k in (a.data.collections or []) if k in self.notion_collections ]), - PROP_SYNCED_AT: Property.as_date(dt.datetime.now()), + PROP_SYNCED_AT: Property.as_date(dt.datetime.now(pytz.utc)), PROP_VERSION: Property.as_number(a.version) } + + def execute_a(self, action: Action[Item, Page]): + if action.action_type == ActionType.PUSH: + if not action.a: + raise Exception('Creating new zotero items is not supported') + + tags = [Tag(tag=select.name, type=None) for select in action.b.get_propery_raw_value('Tags', [])] \ + + [Tag(tag=select.name, type=1) for select in action.b.get_propery_raw_value('Special Tags', [])] + + # Update zotero + action.a.data.tags = tags + self.zotero.update_items([action.a]) + + # Update synced at date in notion + action.b.extend_properties({ + PROP_SYNCED_AT: Property.as_date(dt.datetime.now(pytz.utc)), + }) + action.b = self.notion.page_upsert(action.b) + print(f'-[Zotero] Updated: {action.a.title}') + elif action.action_type == action.action_type.DELETE: + raise Exception('Deleting from zotero is not supported') diff --git a/notionsci/utils/dicts.py b/notionsci/utils/dicts.py index c49b854..fdf346e 100644 --- a/notionsci/utils/dicts.py +++ b/notionsci/utils/dicts.py @@ -19,20 +19,6 @@ def key_by( return {key(item): item for item in items} -class ExplicitNone: - pass - - -def strip_none_field(value: Any): - if isinstance(value, list): - return [strip_none_field(v) for v in value if v is not None] - if isinstance(value, dict): - return {k: strip_none_field(v) for k, v in value.items() if v is not None} - if isinstance(value, ExplicitNone): - return None - return value - - def flatten(t): return [item for sublist in t for item in sublist] diff --git a/notionsci/utils/serialization.py b/notionsci/utils/serialization.py index b50a2a7..c9b1618 100644 --- a/notionsci/utils/serialization.py +++ b/notionsci/utils/serialization.py @@ -3,6 +3,7 @@ from dataclass_dict_convert.convert import SimpleTypeConvertor, dataclass_dict_convert from stringcase import camelcase, snakecase +import itertools as it class ExplicitNone(): @@ -13,7 +14,7 @@ def __nonzero__(self): class UndefinableMeta(type): def __getitem__(cls, key): new_cls = types.new_class( - f"{cls.__name__}_{key.__name__}", + f"{cls.__name__}_{key.__name__ if hasattr(key, '__name__') else repr(key)}", (cls,), {}, lambda ns: ns.__setitem__("type", Optional[key]) @@ -26,8 +27,14 @@ def __getitem__(cls, key): class Undefinable(metaclass=UndefinableMeta): pass -def serde(*, camel=False, ignore_unknown=False, **kwargs): +def serde( + _cls=None, *, + camel=False, ignore_unknown=False, + exclude=None, + **kwargs +): case = camelcase if camel else snakecase + exclude = exclude if exclude else [] def wrap(cls): types = get_type_hints(cls) @@ -48,10 +55,15 @@ def wrap(cls): def to_dict(self, *args, **kwargs): result: dict = original_to_dict(self, *args, **kwargs) - for field_name in undefinable_fields: + for field_name in it.chain(undefinable_fields, exclude): field_name = case(field_name) if field_name in result and result[field_name] is None: del result[field_name] + + for k, v in result.items(): + if isinstance(v, ExplicitNone): + result[k] = None + return result def from_dict(dict, *args, **kwargs): @@ -68,7 +80,10 @@ def from_dict(dict, *args, **kwargs): return cls - return wrap + if _cls: + return wrap(_cls) + else: + return wrap class ListConvertor(SimpleTypeConvertor): From ff36f848ac83413e567c48985caa519897499cfe Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 22:00:03 +0200 Subject: [PATCH 10/11] [FEATURE] Updated models to use undefinable property --- .../notion/structures/properties.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index e3490b9..c100d16 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -246,28 +246,28 @@ class RelationDef: @dataclass class PropertyDef: type: PropertyType - id: Optional[str] = None - name: Optional[str] = None - - title: Optional[TitleDef] = None - rich_text: Optional[RichTextDef] = None - number: Optional[NumberDef] = None - select: Optional[SelectDef] = None - multi_select: Optional[MultiSelectDef] = None - date: Optional[DateDef] = None - people: Optional[PeopleDef] = None - files: Optional[Dict] = None - checkbox: Optional[CheckboxDef] = None - url: Optional[UrlDef] = None - email: Optional[EmailDef] = None - phone_number: Optional[Dict] = None - formula: Optional[Dict] = None - relation: Optional[RelationDef] = None - rollup: Optional[Dict] = None - created_time: Optional[CreatedTimeDef] = None - created_by: Optional[CreatedByDef] = None - last_edited_time: Optional[LastEditedTimeDef] = None - last_edited_by: Optional[LastEditedByDef] = None + id: Undefinable[str] = None + name: Undefinable[str] = None + + title: Undefinable[TitleDef] = None + rich_text: Undefinable[RichTextDef] = None + number: Undefinable[NumberDef] = None + select: Undefinable[SelectDef] = None + multi_select: Undefinable[MultiSelectDef] = None + date: Undefinable[DateDef] = None + people: Undefinable[PeopleDef] = None + files: Undefinable[Dict] = None + checkbox: Undefinable[CheckboxDef] = None + url: Undefinable[UrlDef] = None + email: Undefinable[EmailDef] = None + phone_number: Undefinable[Dict] = None + formula: Undefinable[Dict] = None + relation: Undefinable[RelationDef] = None + rollup: Undefinable[Dict] = None + created_time: Undefinable[CreatedTimeDef] = None + created_by: Undefinable[CreatedByDef] = None + last_edited_time: Undefinable[LastEditedTimeDef] = None + last_edited_by: Undefinable[LastEditedByDef] = None @staticmethod def as_title() -> 'PropertyDef': From 78e9ae7b2e06056de2ea37aa73ef42211fbebcdf Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Fri, 22 Oct 2021 22:03:53 +0200 Subject: [PATCH 11/11] [CI] Code style fixes --- test/TestZoteroSync.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test/TestZoteroSync.py b/test/TestZoteroSync.py index 604c2f2..f620482 100644 --- a/test/TestZoteroSync.py +++ b/test/TestZoteroSync.py @@ -41,9 +41,7 @@ def setupTemplate(self): ) self.assertEqual(code, 0) - self.template = re.search( - r"Created page \((.*)\)", output - ).group(1) + self.template = re.search(r"Created page \((.*)\)", output).group(1) def test_sync_refs(self): self.setupTemplate() @@ -63,13 +61,7 @@ def test_sync_refs_with_collections(self): code, output = capture_cmd( lambda: cli( - [ - "sync", - "zotero", - "refs", - parse_uuid_or_url(self.template), - "--force", - ] + ["sync", "zotero", "refs", parse_uuid_or_url(self.template), "--force",] ) )