Merge pull request #8 from EgorDm/develop

Develop

notionsci/cli/notion.py

@@ -119,3 +119,21 @@ def download_md(page: ID, output: str):
         f.write(content)
 
 
+
+@notion.command()
+@click.argument('parent', callback=parse_uuid_callback)
+@click.argument('file')
+def upload_md(parent: ID, file: str):
+    """
+    Uploads given markdown FILE as a child page of the given PARENT
+
+    :param parent:
+    :param file:
+    :return:
+    """
+    notion = config.connections.notion.client()
+
+    click.echo(f'Reading file {file}')
+    with open(file, 'r') as f:
+        content = f.read()
+

notionsci/cli/sync/__init__.py

@@ -1,6 +1,7 @@
 import click
 
 from .zotero import zotero
+from .markdown import markdown
 
 
 @click.group()
@@ -12,3 +13,4 @@ def sync():
 
 
 sync.add_command(zotero)
+sync.add_command(markdown)

notionsci/cli/sync/markdown.py

@@ -0,0 +1,30 @@
+import click
+
+from notionsci.config import config
+from notionsci.connections.notion import parse_uuid_callback, ID
+from notionsci.sync.markdown import MarkdownPagesSync
+
+
+@click.group()
+def markdown():
+    """
+    Collection of Sync commands for Markdown documents
+    """
+    pass
+
+
+@markdown.command()
+@click.argument('collection', callback=parse_uuid_callback, required=True)
+@click.argument('dir', required=True)
+@click.option('--force', is_flag=True, default=False,
+              help='Ensures up to date items are also pushed to Zotero')
+def pages(collection: ID, dir: str, force: bool):
+    """
+    Starts a sync of a collection to a folder of markdown files
+
+    COLLECTION: Database ID or url
+    DIR: Directory path to sync to
+    """
+    notion = config.connections.notion.client()
+
+    MarkdownPagesSync(notion, collection, dir).sync()

notionsci/cli/sync/zotero.py

@@ -4,9 +4,10 @@
 
 from notionsci.cli.notion import duplicate
 from notionsci.config import config
-from notionsci.connections.notion import parse_uuid, parse_uuid_callback
+from notionsci.connections.notion import parse_uuid, parse_uuid_callback, BlockType, block_type_filter
 from notionsci.connections.zotero import ID
-from notionsci.sync.zotero import RefsOneWaySync, CollectionsOneWaySync
+from notionsci.sync.zotero import RefsSync, CollectionsSync
+from notionsci.utils import take_1
 
 
 @click.group()
@@ -33,50 +34,70 @@ def template(ctx: click.Context, parent: ID):
     ctx.invoke(duplicate, source=source, parent=parent, target_id=target_id)
 
     # Extract the children from the command
-    unotion = config.connections.notion_unofficial.client()
-    page = unotion.get_block(target_id)
-    children = list(page.children)
+    notion = config.connections.notion.client()
+    page = notion.page_get(target_id, with_children=True)
+
+    click.echo(f'Created page ({parse_uuid(page.id)})')
 
-    collections_db = next(filter(lambda x: 'Collections' in x.title, children), None)
+    collections_db = take_1(page.get_children(child_database_filter('Zotero Collections')))
     if collections_db:
         click.echo(f'Found collection database ({parse_uuid(collections_db.id)})')
 
-    refs_db = next(filter(lambda x: 'References' in x.title, children), None)
+    refs_db = take_1(page.get_children(child_database_filter('Zotero References')))
     if refs_db:
         click.echo(f'Found references database ({parse_uuid(refs_db.id)})')
 
 
+def child_database_filter(title: str):
+    type_filter = block_type_filter(BlockType.child_database)
+    return lambda b: type_filter(b) and b.child_database.title == title
+
+
 @zotero.command()
-@click.argument('references', callback=parse_uuid_callback, required=True)
+@click.argument('template', callback=parse_uuid_callback, required=True)
 @click.option('--force', is_flag=True, default=False,
               help='Ensures up to date items are also pushed to Zotero')
-@click.option('-c', '--collections', callback=lambda c, p, x: parse_uuid_callback(c, p, x) if x else x, required=False,
-              help='Collections database page ID or url to (optionally) add references to')
-def refs(references: ID, collections: ID, force: bool):
+def refs(template: ID, force: bool):
     """
     Starts a one way Zotero references sync to Notion
 
-    REFERENCES: References database page ID or url
-
-    When collecitons option is specified Unofficial Notion Api access is required
+    TEMPLATE: Template page ID or url
     """
     notion = config.connections.notion.client()
     zotero = config.connections.zotero.client()
+    sync_config = config.sync.zotero.get('refs', {})
+
+    template_page = notion.page_get(template, with_children=True)
+    references = take_1(template_page.get_children(child_database_filter('Zotero References')))
+    collections = take_1(template_page.get_children(child_database_filter('Zotero Collections')))
 
-    RefsOneWaySync(notion, zotero, references, collections_id=collections, force=force).sync()
+    if not references or not collections:
+        raise Exception('Please check whether child database called "Zotero References" and "Zotero Collections" '
+                        'exist in given template')
+
+    RefsSync(
+        notion, zotero,
+        references.id,
+        collections_id=collections.id,
+        force=force,
+        **sync_config
+    ).sync()
 
 
 @zotero.command()
-@click.argument('collections', callback=parse_uuid_callback, required=True)
+@click.argument('template', callback=parse_uuid_callback, required=True)
 @click.option('--force', is_flag=True, default=False,
               help='Ensures up to date items are also pushed to Zotero')
-def collections(collections: ID, force: bool):
+def collections(template: ID, force: bool):
     """
     Starts a one way Zotero references sync to Notion
 
-    COLLECTIONS: Collections database page ID or url
+    TEMPLATE: Template page ID or url
     """
     notion = config.connections.notion.client()
     zotero = config.connections.zotero.client()
 
-    CollectionsOneWaySync(notion, zotero, collections, force=force).sync()
+    template_page = notion.page_get(template, with_children=True)
+    collections = next(iter(template_page.get_children(child_database_filter('Zotero Collections'))), None)
+
+    CollectionsSync(notion, zotero, collections.id, force=force).sync()

notionsci/config/structure.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from notion_client import Client
 from simple_parsing import Serializable
@@ -64,9 +64,15 @@ class Development(Serializable):
     test_page: str = DEV_TESTS_PAGE
 
 
+@dataclass
+class Sync(Serializable):
+    zotero: dict = field(default_factory=dict)
+
+
 @dataclass
 class Config(Serializable):
     version: int = CONFIG_VERSION
     connections: Connections = Connections()
     templates: Templates = Templates()
     development: Development = Development()
+    sync: Sync = Sync()

notionsci/connections/notion/client.py

@@ -3,10 +3,10 @@
 
 from notion_client import Client
 
-from notionsci.connections.notion import BlockType
+from notionsci.connections.notion import BlockType, PropertyDef
 from notionsci.connections.notion.structures import Database, SortObject, QueryFilter, format_query_args, ContentObject, \
     PropertyType, Page, ID, QueryResult, Block
-from notionsci.utils import strip_none_field, filter_none_dict
+from notionsci.utils import filter_none_dict
 
 
 class NotionNotAttachedException(Exception):
@@ -65,12 +65,12 @@ def page_get(self, id: ID, with_children=False) -> Page:
         return page
 
     def page_update(self, page: Page) -> Page:
-        args = strip_none_field(strip_readonly_props(page).to_dict())
+        args = strip_readonly_props(page).to_dict()
         result = self.client.pages.update(page.id, **args)
         return Page.from_dict(result)
 
     def page_create(self, page: Page) -> Page:
-        args = strip_none_field(strip_readonly_props(page).to_dict())
+        args = strip_readonly_props(page).to_dict()
         result = self.client.pages.create(**args)
         return Page.from_dict(result)
 
@@ -81,8 +81,13 @@ def database_get(self, id: ID) -> Database:
         result = self.client.databases.retrieve(id)
         return Database.from_dict(result)
 
+    def database_update(self, database: Database) -> Page:
+        args = strip_readonly_props(database).to_dict()
+        result = self.client.databases.update(database.id, **args)
+        return Database.from_dict(result)
+
     def database_create(self, database: Database) -> Database:
-        args = strip_none_field(strip_readonly_props(database).to_dict())
+        args = strip_readonly_props(database).to_dict()
         result = self.client.databases.create(**args)
         return Database.from_dict(result)
 
@@ -111,22 +116,26 @@ def database_query_all(
 
     def search(
             self,
+            query: str = None,
             filter: Optional[QueryFilter] = None,
             sorts: Optional[List[SortObject]] = None,
             start_cursor: str = None,
             page_size: int = None
     ) -> QueryResult:
-        args = format_query_args(filter=filter, sorts=sorts, start_cursor=start_cursor, page_size=page_size)
+        args = format_query_args(
+            query=query, filter=filter, sorts=sorts, start_cursor=start_cursor, page_size=page_size
+        )
         result_raw = self.client.search(**args)
         return QueryResult.from_dict(result_raw)
 
     def search_all(
             self,
+            query: str = None,
             filter: Optional[QueryFilter] = None,
             sorts: Optional[List[SortObject]] = None
     ) -> Iterator[Union[Page, Database]]:
         return traverse_pagination(
-            args=dict(filter=filter if filter else None, sorts=sorts, page_size=100),
+            args=dict(query=query, filter=filter if filter else None, sorts=sorts, page_size=100),
             query_fn=lambda **args: self.search(**args)
         )
 
@@ -161,3 +170,14 @@ def load_children(self, item: Union[Page, Block], recursive=False, databases=Fal
                 elif databases and child.type == BlockType.child_database:
                     child.child_database.database = self.database_get(child.id)
                     child.child_database.children = list(self.database_query_all(child.id))  # eager load (mayby dont?)
+
+    def ensure_database_schema(self, schema: Dict[str, PropertyDef], db: Database):
+        new_props = {
+            prop_name: prop for prop_name, prop in schema.items()
+            if not db.has_property(prop_name)
+        }
+        if len(new_props) > 0:
+            db.extend_properties(new_props)
+            db = self.database_update(db)
+
+        return db

notionsci/connections/notion/structures/blocks.py

@@ -2,7 +2,7 @@
 import urllib.parse
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Optional, List, Union
+from typing import Optional, List, Union, Callable
 
 import pandas as pd
 from dataclass_dict_convert import dataclass_dict_convert
@@ -34,20 +34,26 @@ class BlockType(Enum):
     code = 'code'
     unsupported = 'unsupported'
     child_database = 'child_database'
+    table_of_contents = 'table_of_contents'
+    divider = 'divider'
 
 
 BlockConvertor = ListConvertor(ForwardRefConvertor('Block'))
 
 
+def block_type_filter(t: 'BlockType'):
+    return lambda b: b.type == t
+
+
 @dataclass
 class ChildrenMixin:
     children: Optional[List['Block']] = None
 
     def set_children(self, children: List['Block']):
         self.children = children
 
-    def get_children(self) -> List['Block']:
-        return self.children
+    def get_children(self, predicate: Callable[['Block'], bool] = None) -> List['Block']:
+        return self.children if not predicate else list(filter(predicate, self.children))
 
 
 @dataclass_dict_convert(
@@ -253,6 +259,20 @@ def to_markdown(self, context: MarkdownContext) -> str:
         return MarkdownBuilder.code(content, self.language)
 
 
+@dataclass_dict_convert(dict_letter_case=snakecase)
+@dataclass
+class TableOfContentsBlock(ToMarkdownMixin):
+    def to_markdown(self, context: MarkdownContext) -> str:
+        return MarkdownBuilder.table_of_contents()
+
+
+@dataclass_dict_convert(dict_letter_case=snakecase)
+@dataclass
+class DividerBlock(ToMarkdownMixin):
+    def to_markdown(self, context: MarkdownContext) -> str:
+        return MarkdownBuilder.divider()
+
+
 @dataclass_dict_convert(
     dict_letter_case=snakecase,
     **ignore_fields(['database', 'children'])
@@ -311,6 +331,8 @@ class Block(ToMarkdownMixin):
     equation: Optional[EquationBlock] = None
     code: Optional[CodeBlock] = None
     child_database: Optional[ChildDatabaseBlock] = None
+    table_of_contents: Optional[TableOfContentsBlock] = None
+    divider: Optional[DividerBlock] = None
     unsupported: Optional[str] = None
 
     def to_markdown(self, context: MarkdownContext) -> str: