Add support for C# and F#

README.md

@@ -16,7 +16,7 @@ It can also convert these documents **into** Jupyter
 Notebooks, allowing you to synchronize content in both
 directions.
 
-The languages that are currently supported by Jupytext are: Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+, IDL, TypeScript, Javascript, Scala, Rust/Evxcr, PowerShell and Robot Framework. Extending Jupytext to more languages should be easy - read more at [CONTRIBUTING.md](https://github.com/mwouts/jupytext/blob/master/CONTRIBUTING.md). In addition, jupytext users can choose between two formats for notebooks as scripts:
+The languages that are currently supported by Jupytext are: Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+, IDL, TypeScript, Javascript, Scala, Rust/Evxcr, PowerShell, C#, F#, and Robot Framework. Extending Jupytext to more languages should be easy - read more at [CONTRIBUTING.md](https://github.com/mwouts/jupytext/blob/master/CONTRIBUTING.md). In addition, jupytext users can choose between two formats for notebooks as scripts:
 - The `percent` format, compatible with several IDEs, including Spyder, Hydrogen, VScode and PyCharm. In that format, cells are delimited with a commented `%%`.
 - The `light` format, designed for this project. Use that format to open standard scripts as notebooks, or to save notebooks as scripts with few cell markers - none when possible.
 

docs/introduction.md

@@ -10,7 +10,7 @@ Jupytext can save Jupyter notebooks as
 - Markdown and R Markdown documents,
 - Scripts in many languages.
 
-The languages that are currently supported by Jupytext are: Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+, IDL, TypeScript, Javascript, Scala, Rust/Evxcr, PowerShell and Robot Framework. Extending Jupytext to more languages should be easy - read more at [CONTRIBUTING.md](https://github.com/mwouts/jupytext/blob/master/CONTRIBUTING.md#). In addition, jupytext users can choose between two formats for notebooks as scripts:
+The languages that are currently supported by Jupytext are: Julia, Python, R, Bash, Scheme, Clojure, Matlab, Octave, C++, q/kdb+, IDL, TypeScript, Javascript, Scala, Rust/Evxcr, PowerShell, C#, F#,P and Robot Framework. Extending Jupytext to more languages should be easy - read more at [CONTRIBUTING.md](https://github.com/mwouts/jupytext/blob/master/CONTRIBUTING.md#). In addition, jupytext users can choose between two formats for notebooks as scripts:
 - The `percent` format, compatible with several IDEs, including Spyder, Hydrogen, VScode and PyCharm. In that format, cells are delimited with a commented `%%`.
 - The `light` format, designed for this project. Use that format to open standard scripts as notebooks, or to save notebooks as scripts with few cell markers - none when possible.
 

jupytext/cell_metadata.py

@@ -49,7 +49,7 @@
     'autoscroll', 'collapsed', 'scrolled', 'trusted', 'ExecuteTime'] +
                                  _JUPYTEXT_CELL_METADATA)
 
-_IDENTIFIER_RE = re.compile(r'^[a-zA-Z_\\.]+[a-zA-Z0-9_\\.]*$')
+_IDENTIFIER_RE = re.compile(r'^[a-zA-Z_\.]+[a-zA-Z0-9_\.]*$')
 
 
 def _r_logical_values(pybool):
@@ -345,7 +345,7 @@ def parse_key_equal_value(text):
     last_space_pos = text.rfind(' ')
 
     # Just an identifier?
-    if isidentifier(text[last_space_pos + 1:]):
+    if not text.startswith('--') and isidentifier(text[last_space_pos + 1:]):
         key = text[last_space_pos + 1:]
         value = None
         result = {key: value}

jupytext/cell_reader.py

@@ -263,7 +263,7 @@ def extract_content(self, lines):
 class MarkdownCellReader(BaseCellReader):
     """Read notebook cells from Markdown documents"""
     comment = ''
-    start_code_re = re.compile(r"^```({})($|\s(.*)$)".format(
+    start_code_re = re.compile(r"^```(\s*)({})($|\s.*$)".format(
         '|'.join(_JUPYTER_LANGUAGES_LOWER_AND_UPPER).replace('+', '\\+')))
     non_jupyter_code_re = re.compile(r"^```")
     end_code_re = re.compile(r"^```\s*$")

jupytext/cell_to_text.py

@@ -3,7 +3,7 @@
 import re
 import warnings
 from copy import copy
-from .languages import cell_language, comment_lines
+from .languages import cell_language, comment_lines, same_language
 from .cell_metadata import is_active, _IGNORE_CELL_METADATA
 from .cell_metadata import metadata_to_text, metadata_to_rmd_options, metadata_to_double_percent_options
 from .metadata_filter import filter_metadata
@@ -37,16 +37,18 @@ def __init__(self, cell, default_language, fmt=None):
         self.metadata = filter_metadata(cell.metadata,
                                         self.fmt.get('cell_metadata_filter'),
                                         _IGNORE_CELL_METADATA)
-        self.language, magic_args = cell_language(self.source) if self.parse_cell_language else (None, None)
+        if self.parse_cell_language:
+            self.language, magic_args = cell_language(self.source, default_language)
 
-        if self.language:
             if magic_args:
                 self.metadata['magic_args'] = magic_args
+        else:
+            self.language = None
 
-            if not self.ext.endswith('.Rmd'):
-                self.metadata['language'] = self.language
+        if self.language and not self.ext.endswith('.Rmd'):
+            self.metadata['language'] = self.language
 
-        self.language = self.language or default_language
+        self.language = self.language or cell.metadata.get('language', default_language)
         self.default_language = default_language
         self.comment = _SCRIPT_EXTENSIONS.get(self.ext, {}).get('comment', '#')
         self.comment_magics = self.fmt.get('comment_magics', self.default_comment_magics)
@@ -252,7 +254,7 @@ def is_code(self):
 
     def code_to_text(self):
         """Return the text representation of a code cell"""
-        active = is_active(self.ext, self.metadata, self.language == self.default_language)
+        active = is_active(self.ext, self.metadata, same_language(self.language, self.default_language))
         source = copy(self.source)
         escape_code_start(source, self.ext, self.language)
         comment_questions = self.metadata.pop('comment_questions', True)
@@ -378,7 +380,7 @@ def cell_to_text(self):
         if self.cell_type != 'code':
             self.metadata['cell_type'] = self.cell_type
 
-        active = is_active(self.ext, self.metadata, self.language == self.default_language)
+        active = is_active(self.ext, self.metadata, same_language(self.language, self.default_language))
         if self.cell_type == 'raw' and 'active' in self.metadata and self.metadata['active'] == '':
             del self.metadata['active']
 

jupytext/formats.py

@@ -583,6 +583,9 @@ def auto_ext_from_metadata(metadata):
     if auto_ext == '.r':
         return '.R'
 
+    if auto_ext == '.fs':
+        return '.fsx'
+
     return auto_ext
 
 

jupytext/jupytext.py

@@ -71,21 +71,22 @@ def reads(self, s, **_):
         if self.implementation.format_name and self.implementation.format_name.startswith('sphinx'):
             cells.append(new_code_cell(source='%matplotlib inline'))
 
-        cell_metadata = set()
         cell_metadata_json = False
 
         while lines:
             reader = self.implementation.cell_reader_class(self.fmt, default_language)
             cell, pos = reader.read(lines)
             cells.append(cell)
-            cell_metadata.update(cell.metadata.keys())
             cell_metadata_json = cell_metadata_json or reader.cell_metadata_json
             if pos <= 0:
                 raise Exception('Blocked at lines ' + '\n'.join(lines[:6]))  # pragma: no cover
             lines = lines[pos:]
 
-        update_metadata_filters(metadata, jupyter_md, cell_metadata)
         set_main_and_cell_language(metadata, cells, self.implementation.extension)
+        cell_metadata = set()
+        for cell in cells:
+            cell_metadata.update(cell.metadata.keys())
+        update_metadata_filters(metadata, jupyter_md, cell_metadata)
 
         if cell_metadata_json:
             metadata.setdefault('jupytext', {}).setdefault('cell_metadata_json', True)
@@ -130,17 +131,16 @@ def writes(self, nb, metadata=None, **kwargs):
             cells=nb.cells)
 
         metadata = nb.metadata
-        default_language = default_language_from_metadata_and_ext(metadata, self.implementation.extension) or 'python'
+        default_language = default_language_from_metadata_and_ext(metadata,
+                                                                  self.implementation.extension,
+                                                                  True) or 'python'
         self.update_fmt_with_notebook_options(nb.metadata)
         if 'use_runtools' not in self.fmt:
             for cell in nb.cells:
                 if cell.metadata.get('hide_input', False) or cell.metadata.get('hide_output', False):
                     self.fmt['use_runtools'] = True
                     break
 
-        if 'main_language' in metadata.get('jupytext', {}):
-            del metadata['jupytext']['main_language']
-
         header = encoding_and_executable(nb, metadata, self.ext)
         header_content, header_lines_to_next_cell = metadata_and_cell_to_header(nb, metadata,
                                                                                 self.implementation, self.ext)

jupytext/languages.py

@@ -24,44 +24,58 @@
                       '.ts': {'language': 'typescript', 'comment': '//'},
                       '.scala': {'language': 'scala', 'comment': '//'},
                       '.rs': {'language': 'rust', 'comment': '//'},
-                      '.robot': {'language': 'robotframework', 'comment': '#'}}
+                      '.robot': {'language': 'robotframework', 'comment': '#'},
+                      '.cs': {'language': 'csharp', 'comment': '//'},
+                      '.fsx': {'language': 'fsharp', 'comment': '//'},
+                      '.fs': {'language': 'fsharp', 'comment': '//'}}
 
 _COMMENT_CHARS = [_SCRIPT_EXTENSIONS[ext]['comment'] for ext in _SCRIPT_EXTENSIONS if
                   _SCRIPT_EXTENSIONS[ext]['comment'] != '#']
 
 _COMMENT = {_SCRIPT_EXTENSIONS[ext]['language']: _SCRIPT_EXTENSIONS[ext]['comment'] for ext in _SCRIPT_EXTENSIONS}
-_JUPYTER_LANGUAGES = set(_JUPYTER_LANGUAGES).union(_COMMENT.keys())
+_JUPYTER_LANGUAGES = set(_JUPYTER_LANGUAGES).union(_COMMENT.keys()).union(['c#', 'f#', 'cs', 'fs'])
 _JUPYTER_LANGUAGES_LOWER_AND_UPPER = _JUPYTER_LANGUAGES.union({str.upper(lang) for lang in _JUPYTER_LANGUAGES})
 
 
-def default_language_from_metadata_and_ext(metadata, ext):
+def default_language_from_metadata_and_ext(metadata, ext, pop_main_language=False):
     """Return the default language given the notebook metadata, and a file extension"""
     default_from_ext = _SCRIPT_EXTENSIONS.get(ext, {}).get('language')
 
-    language = (metadata.get('jupytext', {}).get('main_language')
-                or metadata.get('kernelspec', {}).get('language')
-                or default_from_ext)
+    main_language = metadata.get('jupytext', {}).get('main_language')
+    default_language = metadata.get('kernelspec', {}).get('language') or default_from_ext
+    language = main_language or default_language
+
+    if main_language is not None and main_language == default_language and pop_main_language:
+        metadata['jupytext'].pop('main_language')
 
     if language is None or language == 'R':
         return language
 
     if language.startswith('C++'):
         return 'c++'
 
-    return language.lower()
+    return language.lower().replace('#', 'sharp')
+
+
+def usual_language_name(language):
+    """Return the usual language name (one that may be found in _SCRIPT_EXTENSIONS above)"""
+    language = language.lower()
+    if language == 'r':
+        return 'R'
+    if language.startswith('c++'):
+        return 'c++'
+    if language == 'octave':
+        return 'matlab'
+    if language in ['cs', 'c#']:
+        return 'csharp'
+    if language in ['fs', 'f#']:
+        return 'fsharp'
+    return language
 
 
 def same_language(kernel_language, language):
     """Are those the same language?"""
-    if kernel_language == language:
-        return True
-    if kernel_language.lower() == language:
-        return True
-    if kernel_language.startswith('C++') and language == 'c++':
-        return True
-    if kernel_language == 'octave' and language == 'matlab':
-        return True
-    return False
+    return usual_language_name(kernel_language) == usual_language_name(language)
 
 
 def set_main_and_cell_language(metadata, cells, ext):
@@ -73,7 +87,7 @@ def set_main_and_cell_language(metadata, cells, ext):
         languages = {'python': 0.5}
         for cell in cells:
             if 'language' in cell['metadata']:
-                language = cell['metadata']['language']
+                language = usual_language_name(cell['metadata']['language'])
                 languages[language] = languages.get(language, 0.0) + 1
 
         main_language = max(languages, key=languages.get)
@@ -85,20 +99,35 @@ def set_main_and_cell_language(metadata, cells, ext):
     # Remove 'language' meta data and add a magic if not main language
     for cell in cells:
         if 'language' in cell['metadata']:
-            language = cell['metadata'].pop('language')
-            if language != main_language and language in _JUPYTER_LANGUAGES:
+            language = cell['metadata']['language']
+            if language == main_language:
+                cell['metadata'].pop('language')
+                continue
+
+            if usual_language_name(language) == main_language:
+                continue
+
+            if language in _JUPYTER_LANGUAGES:
+                cell['metadata'].pop('language')
+                magic = '%%' if main_language != 'csharp' else '#!'
                 if 'magic_args' in cell['metadata']:
                     magic_args = cell['metadata'].pop('magic_args')
-                    cell['source'] = u'%%{} {}\n'.format(language, magic_args) + cell['source']
+                    cell['source'] = u'{}{} {}\n'.format(magic, language, magic_args) + cell['source']
                 else:
-                    cell['source'] = u'%%{}\n'.format(language) + cell['source']
+                    cell['source'] = u'{}{}\n'.format(magic, language) + cell['source']
 
 
-def cell_language(source):
+def cell_language(source, default_language):
     """Return cell language and language options, if any"""
     if source:
         line = source[0]
-        if line.startswith('%%'):
+        if default_language == 'csharp':
+            if line.startswith('#!'):
+                lang = line[2:].strip()
+                if lang in _JUPYTER_LANGUAGES:
+                    source.pop(0)
+                    return lang, ''
+        elif line.startswith('%%'):
             magic = line[2:]
             if ' ' in magic:
                 lang, magic_args = magic.split(' ', 1)

jupytext/magics.py

@@ -2,7 +2,7 @@
 
 import re
 from .stringparser import StringParser
-from .languages import _SCRIPT_EXTENSIONS, _COMMENT
+from .languages import _SCRIPT_EXTENSIONS, _COMMENT, usual_language_name
 
 # A magic expression is a line or cell or metakernel magic (#94, #61) escaped zero, or multiple times
 _MAGIC_RE = {_SCRIPT_EXTENSIONS[ext]['language']: re.compile(
@@ -20,6 +20,11 @@
 _MAGIC_FORCE_ESC_RE['rust'] = re.compile(r"^(// |//)*:[a-zA-Z](.*)//\s*escape")
 _MAGIC_FORCE_ESC_RE['rust'] = re.compile(r"^(// |//)*:[a-zA-Z](.*)//\s*noescape")
 
+# C# magics start with '#!'
+_MAGIC_RE['csharp'] = re.compile(r"^(// |//)*#![a-zA-Z]")
+_MAGIC_FORCE_ESC_RE['csharp'] = re.compile(r"^(// |//)*#![a-zA-Z](.*)//\s*escape")
+_MAGIC_FORCE_ESC_RE['csharp'] = re.compile(r"^(// |//)*#![a-zA-Z](.*)//\s*noescape")
+
 # Commands starting with a question or exclamation mark have to be escaped
 _PYTHON_HELP_OR_BASH_CMD = re.compile(r"^(# |#)*(\?|!)\s*[A-Za-z]")
 
@@ -37,6 +42,7 @@
 
 def is_magic(line, language, global_escape_flag=True, explicitly_code=False):
     """Is the current line a (possibly escaped) Jupyter magic, and should it be commented?"""
+    language = usual_language_name(language)
     if language in ['octave', 'matlab'] or language not in _SCRIPT_LANGUAGES:
         return False
     if _MAGIC_FORCE_ESC_RE[language].match(line):

jupytext/metadata_filter.py

@@ -75,8 +75,6 @@ def metadata_filter_as_string(metadata_filter):
 def update_metadata_filters(metadata, jupyter_md, cell_metadata):
     """Update or set the notebook and cell metadata filters"""
 
-    cell_metadata = [m for m in cell_metadata if m not in ['language', 'magic_args']]
-
     if 'cell_metadata_filter' in metadata.get('jupytext', {}):
         metadata_filter = metadata_filter_as_dict(metadata.get('jupytext', {})['cell_metadata_filter'])
         if isinstance(metadata_filter.get('excluded'), list):

jupytext/version.py

@@ -1,3 +1,3 @@
 """Jupytext's version number"""
 
-__version__ = '1.3.3'
+__version__ = '1.3.3+dev'