func: convert ipynb to markdown (#1)

xiexianbin · web-flow · commit eb28308243a3 · 2023-06-24T16:41:48.000+08:00
Signed-off-by: xiexianbin &lt;me@xiexianbin.cn&gt;
diff --git a/.gitignore b/.gitignore
@@ -162,3 +162,4 @@ cython_debug/
 # self define
 .DS_Store
 .history
+.vscode
diff --git a/ChangeLog b/ChangeLog
@@ -1,8 +1,5 @@
 CHANGES
 =======
 
-v0.1.0
-------
-
-* feautre: base func, convert ipynb to hugo markdown files
-* init commit
+* init
+* Initial commit
diff --git a/Dockerfile b/Dockerfile
@@ -3,15 +3,15 @@ FROM ubuntu:22.04
 # Dockerfile build cache 
 ENV REFRESHED_AT 2023-05-07
 
-LABEL "com.github.actions.name"="nbcli"
-LABEL "com.github.actions.description"="nbcli"
+LABEL "com.github.actions.name"="nbctl"
+LABEL "com.github.actions.description"="nbctl"
 LABEL "com.github.actions.icon"="chevrons-right"
 LABEL "com.github.actions.color"="blue"
-LABEL "repository"="http://github.com/x-actions/python3-nbcli"
-LABEL "homepage"="http://github.com/x-actions/python3-nbcli"
+LABEL "repository"="http://github.com/x-actions/python3-nbctl"
+LABEL "homepage"="http://github.com/x-actions/python3-nbctl"
 LABEL "maintainer"="xiexianbin<me@xiexianbin.cn>"
 
-LABEL "Name"="nbcli"
+LABEL "Name"="nbctl"
 LABEL "Version"="1.0.0"
 
 ENV LC_ALL C.UTF-8
@@ -20,10 +20,10 @@ ENV LANGUAGE en_US.UTF-8
 
 # apt install -y docker-ce docker-ce-cli containerd.io; systemctl start docker
 RUN apt update && \
-    apt install -y git python3 python3-pip skopeo jq && \
+    apt install -y git python3 python3-pip && \
     cd ~ && \
-    git clone https://github.com/x-actions/python3-nbcli.git && \
-    cd python3-nbcli && \
+    git clone https://github.com/x-actions/python3-nbctl.git && \
+    cd python3-nbctl && \
     git checkout v1 && \
     pip3 install -r requirements.txt && \
     python3 setup.py --version && \
diff --git a/README.md b/README.md
@@ -1,16 +1,19 @@
-# python3-nbcli
+# python3-nbctl
 
-将ipynb文件转化为hugo markdown文件/Convert ipynb to hugo markdown files
+将ipynb文件转化为markdown文件/convert jupyter *.ipynb file to markdown files
 
-[![PyPI-python3-nbcli](https://img.shields.io/pypi/v/python3-nbcli.svg?maxAge=3600)](https://pypi.org/project/python3-nbcli/)
+[![PyPI-python3-nbctl](https://img.shields.io/pypi/v/python3-nbctl.svg?maxAge=3600)](https://pypi.org/project/python3-nbctl/)
 
-Github Actions for [Container Images Sync](https://github.com/marketplace/actions/nbcli)
+Github Actions for [Container ipynb to markdown](https://github.com/marketplace/actions/nbctl)
 
 ## How to Use by Github Actions
 
 ```
-    - name: convert ipynb to hugo markdown
-      uses: x-actions/python3-nbcli@v1
+      - name: convert jupyter *.ipynb file to markdown
+        uses: x-actions/python3-nbctl@v1
+        with:
+          script: >
+            find <path> -type f -name "*.ipynb" | grep -v ".ipynb_checkpoints" | xargs -I{} nbctl --input {} --force --debug
 ```
 
 ## Dev and Test
@@ -23,8 +26,11 @@ pip3 install -r requirements.txt
 python3 setup.py install
 
 # or
-pip3 install python3-nbcli
+pip3 install python3-nbctl
+```
+
+- help
 
-# run
-cisctl
+```
+nbctl --help
 ```
diff --git a/action.yml b/action.yml
@@ -1,10 +1,14 @@
-name: 'convert ipynb to hugo markdown'
-description: 'nbcli'
+name: 'nbctl'
+description: 'convert jupyter *.ipynb file to markdown'
 author: 'xiexianbin <me@xiexianbin>'
 inputs:
-  LOG_LEVEL:
+  log:
     description: 'logger level, value [DEBUG|INFO|WARN|ERROR|FATAL]'
     default: 'DEBUG'
+  script:
+    description: 'script to run'
+    default: 'find . -type f -name "*.ipynb" | xargs -I{} nbctl --input {} -f'
+
 runs:
   using: 'docker'
   image: 'Dockerfile'
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -1,10 +1,17 @@
 #!/bin/bash
 set -e
 
-export LOG_LEVEL=${LOG_LEVEL:-"DEBUG"}
+export LOG_LEVEL=${INPUT_LOG:-"DEBUG"}
+export INPUT_SCRIPT=${INPUT_SCRIPT:-"echo `date`"}
 
+echo $(pwd)
 echo "## Check Package Version ##################"
 bash --version
 git version
 
-nbcli
+echo "env is:"
+env
+
+echo "## Run nbctl ##################"
+
+eval ${INPUT_SCRIPT}
diff --git a/nbcli/shell.py b/nbcli/shell.py
diff --git a/nbctl/__init__.py b/nbctl/__init__.py
@@ -17,7 +17,7 @@
 __all__ = ['__version__']
 
 
-version_info = pbr.version.VersionInfo('python3-nbcli')
+version_info = pbr.version.VersionInfo('python3-nbctl')
 # We have a circular import problem when we first run python setup.py sdist
 # It's harmless, so deflect it.
 try:
diff --git a/nbctl/constants.py b/nbctl/constants.py
@@ -16,3 +16,4 @@
 
 CURRENT_PATH = os.getcwd()
 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG')
+DEFAULT_LOG_LEVEL = "INFO"
diff --git a/nbctl/logger.py b/nbctl/logger.py
@@ -19,31 +19,31 @@
 from nbctl import constants
 
 
-def gen_logger():
+def gen_logger(level=constants.LOG_LEVEL):
     # create logger
-    if constants.LOG_LEVEL == 'DEBUG':
+    if level == 'DEBUG':
         log_level = logging.DEBUG
-    elif constants.LOG_LEVEL == 'INFO':
+    elif level == 'INFO':
         log_level = logging.INFO
-    elif constants.LOG_LEVEL == 'WARN':
+    elif level == 'WARN':
         log_level = logging.WARN
-    elif constants.LOG_LEVEL == 'ERROR':
+    elif level == 'ERROR':
         log_level = logging.ERROR
-    elif constants.LOG_LEVEL == 'FATAL':
+    elif level == 'FATAL':
         log_level = logging.FATAL
     else:
         log_level = logging.DEBUG
     formatter = logging.Formatter(
-        fmt="%(asctime)-15s %(process)d %(levelname)s %(message)s - %(filename)s %(lineno)d",
+        fmt="%(asctime)-15s %(process)d - %(filename)s:%(lineno)d %(levelname)s %(message)s",
         datefmt="%a %d %b %Y %H:%M:%S")
 
-    _logger = logging.getLogger(name="cis")
+    _logger = logging.getLogger(name="nbctl")
     _logger.setLevel(log_level)
 
-    fh = logging.FileHandler(filename="cis.log")
-    fh.setLevel(log_level)
-    fh.setFormatter(formatter)
-    _logger.addHandler(fh)
+    # fh = logging.FileHandler(filename="nbctl.log")
+    # fh.setLevel(log_level)
+    # fh.setFormatter(formatter)
+    # _logger.addHandler(fh)
 
     oh = logging.StreamHandler(sys.stdout)
     oh.setLevel(log_level)
@@ -52,4 +52,4 @@ def gen_logger():
     return _logger
 
 
-logger = gen_logger()
+logger = gen_logger(constants.DEFAULT_LOG_LEVEL)
diff --git a/nbctl/shell.py b/nbctl/shell.py
@@ -0,0 +1,142 @@
+#  Copyright 2023 xiexianbin.cn
+#  All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#         http://www.apache.org/licenses/LICENSE-2.0
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""convert jupyter *.ipynb file to markdown"""
+
+import argparse
+import os
+import sys
+from typing import Dict
+
+import nbformat
+from nbconvert import MarkdownExporter
+from traitlets.config import Config
+
+from nbctl import constants
+from nbctl import logger
+
+
+parser = argparse.ArgumentParser(description='convert jupyter *.ipynb file to markdown')
+parser.add_argument('--debug', '-d', help='show debug info, default is INFO',
+                    action="store_true")
+parser.add_argument('--input', '-i', help='input *.ipynb file or dir', 
+                    required=True, default='', type=str)
+parser.add_argument('--output', '-o', help='output file or dir', 
+                    required=False, default='', type=str)
+parser.add_argument('--force', '-f', help='force overwrite output file', 
+                    action="store_true")
+args = parser.parse_args()
+
+
+class nbctl(object):
+
+    def __init__(self, input: str, output: str, debug: bool=False, force: bool=False):
+        self.input = input
+        self.output = output
+        self.logger = logger.logger
+        self.debug = debug
+        if self.debug:
+            self.logger = logger.gen_logger(constants.LOG_LEVEL)
+            self.logger.debug(f'run in debug model...')
+        self.force = force
+
+        # Setup config
+        self.config = Config()
+
+    def _parse_metadata(self, info: str) -> Dict:
+        if '\n' in info:
+            info = info.split('\n')
+        metadate = dict()
+        for i in info:
+            if i == '---':
+                continue
+            if ': ' in i:
+                key, value = i.split(': ')
+                metadate[key] = value
+        return metadate
+
+    def render_one(self, ipynb_path: str, markdown_dir: str) -> bool:
+        # ipynb convert
+        content = ''
+        with open(ipynb_path, 'r') as f:
+            content = ''.join(f.readlines())
+        jake_notebook = nbformat.reads(content, as_version=4)
+
+        for i in jake_notebook.cells:
+            self.logger.debug(f'cell_type: {i.cell_type}, i: {i}')
+
+        md_exporter = MarkdownExporter()  # config=self.config
+        (body, resources) = md_exporter.from_notebook_node(jake_notebook)
+
+        self.logger.debug(f'body:\n{body}')
+        self.logger.debug(f'resources:\n{resources}')
+
+        # output dir
+        if not markdown_dir:
+            self.logger.debug(f'[{ipynb_path}] not -o/--output set, try to parse output from .ipynb file ...')
+            metadata = self._parse_metadata(jake_notebook.cells[0]['source'])
+            _output = metadata.get('output', None)
+            if _output:
+                markdown_dir = _output
+            if not markdown_dir:
+                self.logger.warning(f'parse output from [{ipynb_path}] file failed.')
+                return False
+        if not os.path.exists(markdown_dir):
+            os.system(f'mkdir -p {markdown_dir}')
+
+        # output path
+        markdown_name = f"{'.'.join(os.path.basename(ipynb_path).split('.')[:-1])}.md"
+        markdown_path = os.path.join(markdown_dir, markdown_name)
+        if os.path.isfile(markdown_path):
+            self.logger.warning(f'file {markdown_path} is already exists, use -f/--force overwrite.')
+            if not self.force:
+                return False
+
+        # write to file
+        with open(markdown_path, "w") as f:
+            f.writelines(body)
+
+        return True
+
+
+    def do(self):
+        # check param
+        if os.path.exists(self.input):
+            if os.path.isdir(self.input):
+                for (dirpath, dirnames, filenames) in os.walk(self.input):
+                    self.logger.debug(f'dirpath: {dirpath}, dirnames: {dirnames}, filenames: {filenames}')
+            else:
+                self.render_one(ipynb_path=self.input, markdown_dir=self.output)
+        else:
+            print(f'dir {self.input} not exists, exit 1.')
+            sys.exit(1)
+
+        return True
+
+
+def main():
+    input = args.input
+    if not input:
+        print(f'not -i/--input set')
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        client = nbctl(input, args.output, debug=args.debug, force=args.force)
+        client.do()
+    except Exception as e:
+        print(e)
+        raise
+
+if __name__ == "__main__":
+    main()
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,5 @@
+# argparse==1.4.0 # Python Software Foundation
 pbr>=5.5.0 # Apache-2.0
 setuptools==65.1.1 # MIT
-nbconvert==7.3.1 # BSD
+nbconvert==7.6.0 # BSD
+nbformat==5.9.0 # BSD
diff --git a/setup.cfg b/setup.cfg

Original file line number	Diff line number	Diff line change
`@@ -16,3 +16,4 @@`
`16`	`16`
`17`	`17`	`CURRENT_PATH = os.getcwd()`
`18`	`18`	`LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG')`
	`19`	`+DEFAULT_LOG_LEVEL = "INFO"`