CommonServerPython reference docs (#476)

* csp doc initial commit

* add docspec package

* use pipenv

* run

* escape gt and lt chars

* html escape

* run html escape on line

* use pydocmd escaping, rm class def and add csp overview

* add api ref category

* rm escaping and trailing whitespaces

* mv demistomock rename to the py script

* rm trailing whitespace

* extend sidebar

* add filter processor and test for csp

* revert gitignore

* rm trailing whitespace

* exclude csp test data

* add cwd class test helper

* fix md csp link

* rm readline from test

* push api ref before content rn

* use get instead of direct key access

* skip none return type

* rm init rendering

* improve enum rendering

* add signature class prefix

* sort alphabetically

* add py formatting to examples

* add comment about removing first lines from csp

content-repo/.gitignore

@@ -1,6 +1,7 @@
 CommonServerPython.py
 demistomock.py
 !test_data/demistomock.py
+!test_data/CommonServerPython.py
 approved_tags.json
 approved_usecases.json
 .content-bucket

content-repo/create-docs.sh

@@ -175,28 +175,21 @@ cp ${CONTENT_GIT_DIR}/Tests/demistomock/demistomock.py .
 cp ${CONTENT_GIT_DIR}/Tests/Marketplace/approved_tags.json .
 cp ${CONTENT_GIT_DIR}/Tests/Marketplace/approved_usecases.json .
 
-ARTICLES_DIR=${SCRIPT_DIR}/extra-docs/articles
-DEMISTO_CLASS_OVERVIEW="All Python integrations and scripts have available as part of the runtime the \`demisto\` class object. The object exposes a series of API methods which are used to retrieve and send data to the Cortex XSOAR Server.
-
-:::note
-The \`demisto\` class is a low level API. For many operations we provide a simpler and more robust API as part of the  [Common Server Functions](https://xsoar.pan.dev/docs/integrations/code-conventions#common-server-functions).
-:::"
-DEMISTO_CLASS_DOCS_CMD=("./gen_pydocs.py" "-d" "${ARTICLES_DIR}" "-i" "demisto-class" "-t" "'Demisto Class'" "-m" "demisto" "-p" "demisto." "-o" "'${DEMISTO_CLASS_OVERVIEW}'")
-mv demistomock.py demisto.py
+# Removing the first lines from CommonServerPython.py which are a description of the script we don't need here
+echo "$(tail -n +6 CommonServerPython.py)" > CommonServerPython.py
 
 if [ -z "${NETLIFY}" ]; then
     echo "Not running in netlify. Using pipenv"
     echo "Installing pipenv..."
     pipenv install
-    echo "Generating Demisto class docs..."
-    pipenv run "${DEMISTO_CLASS_DOCS_CMD[@]}"
-    mv demisto.py demistomock.py
     echo "Generating docs..."
     pipenv run ./gendocs.py -t "${TARGET_DIR}" -d "${CONTENT_GIT_DIR}"
+    echo "Generating Demisto class and CommonServerPython docs..."
+    pipenv run ./gen_pydocs.py -t "${TARGET_DIR}"
+
 else
-    echo "Generating Demisto class docs..."
-    eval "${DEMISTO_CLASS_DOCS_CMD[@]}"
-    mv demisto.py demistomock.py
     echo "Generating docs..."
     ./gendocs.py -t "${TARGET_DIR}" -d "${CONTENT_GIT_DIR}"
+    echo "Generating Demisto class and CommonServerPython docs..."
+    ./gen_pydocs.py -t "${TARGET_DIR}"
 fi

content-repo/gen_pydocs.py

@@ -1,17 +1,21 @@
 #!/usr/bin/env python3
 
 import argparse
-import sys
-from io import StringIO
-from typing import Optional
+import html
+import json
+import os
+import re
+from typing import Dict, List, Optional
 
 import docspec
 from nr.databind.core import Field
-from pydoc_markdown import MarkdownRenderer, PydocMarkdown, PythonLoader
+from pydoc_markdown import (FilterProcessor, MarkdownRenderer, PydocMarkdown,
+                            PythonLoader, SmartProcessor)
+from pydoc_markdown.contrib.processors.sphinx import (
+    SphinxProcessor, generate_sections_markdown)
 
 
 class DemistoMarkdownRenderer(MarkdownRenderer):
-
     func_prefix = Field(str, default=None)
 
     module_overview = Field(str, default=None)
@@ -32,21 +36,146 @@ def _format_function_signature(self, func: docspec.Function, override_name: str
             function_signature = f'{self.func_prefix}{function_signature}'
         return function_signature
 
+    def _render_signature_block(self, fp, obj):
+        if '__init__' in obj.name:
+            return
+        super()._render_signature_block(fp, obj)
+
     def _render_header(self, fp, level, obj):
-        if isinstance(obj, docspec.Module) and self.module_overview:
-            fp.write(self.module_overview)
+        if '__init__' in obj.name:
+            return
+        if type(obj).__name__ == 'Data' and 'Enum' in self._get_parent(obj).docstring:
+            fp.write(f'- {obj.name}')
+            fp.write('\n\n')
+            return
+        super()._render_header(fp, level, obj)
+
+    def _render_object(self, fp, level, obj):
+        if not isinstance(obj, docspec.Module) or self.render_module_header:
+            self._render_header(fp, level, obj)
+        url = self.source_linker.get_source_url(obj) if self.source_linker else None
+        source_string = self.source_format.replace('{url}', str(url)) if url else None
+        if source_string and self.source_position == 'before signature':
+            fp.write(source_string + '\n\n')
+        self._render_signature_block(fp, obj)
+        if source_string and self.source_position == 'after signature':
+            fp.write(source_string + '\n\n')
+        if obj.docstring:
+            should_escape = self.escape_html_in_docstring and '>>>' not in obj.docstring
+            docstring = html.escape(obj.docstring) if should_escape else obj.docstring
+            lines = docstring.split('\n')
+            if self.docstrings_as_blockquote:
+                lines = ['> ' + x for x in lines]
+            fp.write('\n'.join(lines))
             fp.write('\n\n')
-        else:
-            super()._render_header(fp, level, obj)
+
+
+class CommonServerPythonProcessor(SphinxProcessor):
+    def _process(self, node):
+        if not node.docstring:
+            return
+
+        lines = []
+        in_codeblock = False
+        keyword = None
+        components: Dict[str, List[str]] = {}
+        param_type = ''
+        return_desc = ''
+
+        for line in node.docstring.split('\n'):
+            line = line.strip()
+
+            if line.startswith("```"):
+                in_codeblock = not in_codeblock
+
+            line_codeblock = line.startswith('    ')
+
+            if not in_codeblock and not line_codeblock:
+                match = re.match(r'\s*:(?:type)\s+(\w+)\s*:(.*)?$', line)
+                if match:
+                    param_type = match.group(2).strip().replace('`', '')
+                    continue
+
+                match = re.match(r'\s*:(?:arg|argument|param|parameter)\s+(\w+)\s*:(.*)?$', line)
+                if match:
+                    keyword = 'Arguments'
+                    param = match.group(1)
+                    text = match.group(2)
+                    text = text.strip()
+
+                    component = components.setdefault(keyword, [])
+                    component.append('- `{}` _{}_: {}'.format(param, param_type, text))
+                    continue
+
+                match = re.match(r'\s*:(?:return|returns)\s*:(.*)?$', line)
+                if match:
+                    return_desc = match.group(1).strip()
+                    continue
+
+                match = re.match(r'\s*:(?:rtype)\s*:(.*)?$', line)
+                if match:
+                    keyword = 'Returns'
+                    return_type = match.group(1).strip().replace('`', '')
+                    if 'None' in return_type:
+                        continue
+
+                    component = components.setdefault(keyword, [])
+                    component.append('- `{}` - {}'.format(return_type, return_desc))
+                    continue
+
+                match = re.match('\\s*:(?:raises|raise)\\s+(\\w+)\\s*:(.*)?$', line)
+                if match:
+                    keyword = 'Raises'
+                    exception = match.group(1)
+                    text = match.group(2)
+                    text = text.strip()
+
+                    component = components.setdefault(keyword, [])
+                    component.append('- `{}`: {}'.format(exception, text))
+                    continue
+
+            stripped_line = line.strip()
+            if stripped_line.endswith('Examples:'):
+                continue
+
+            if stripped_line.startswith('>>>'):
+                keyword = 'Examples'
+
+                component = components.setdefault(keyword, ['```python'])
+                component.append(stripped_line)
+                continue
+
+            if keyword is not None:
+                components[keyword].append(line)
+            else:
+                lines.append(line)
+
+        if 'Examples' in components:
+            components['Examples'].append('```')
+        generate_sections_markdown(lines, components)
+        node.docstring = '\n'.join(lines)
+
+
+class IgnoreDocstringProcessor(FilterProcessor):
+    def process(self, modules, _resolver):
+        filtered_modules = []
+        for member in getattr(modules[0], 'members', []):
+            if member.docstring and 'ignore docstring' not in member.docstring:
+                filtered_modules.append(member)
+            else:
+                print(f'Skipping {member}')
+        modules.clear()
+        modules.extend(filtered_modules)
+        modules.sort(key=lambda obj: obj.name)
 
 
 def generate_pydoc(
         module: str,
         article_id: str,
         article_title: str,
         target_dir: str,
-        func_prefix: Optional[str],
-        module_overview: Optional[str]
+        module_overview: str,
+        func_prefix: Optional[str] = None,
 ) -> None:
     """
     Args:
@@ -61,39 +190,87 @@ def generate_pydoc(
         None: No data returned.
     """
     pydocmd = PydocMarkdown()
+    pydocmd.processors[0] = IgnoreDocstringProcessor()
+    pydocmd.processors[1] = SmartProcessor(sphinx=CommonServerPythonProcessor())
     pydocmd.renderer = DemistoMarkdownRenderer(
         insert_header_anchors=False,
         func_prefix=func_prefix,
-        module_overview=module_overview
+        module_overview=module_overview,
+        escape_html_in_docstring=True,
+        classdef_code_block=False,
+        descriptive_class_title=False,
+        signature_with_decorators=False,
+        signature_class_prefix=True,
     )
     loader: PythonLoader = next((ldr for ldr in pydocmd.loaders if isinstance(ldr, PythonLoader)), None)
     loader.modules = [module]
     modules = pydocmd.load_modules()
     pydocmd.process(modules)
 
-    stdout = sys.stdout
-    sys.stdout = tmp_stdout = StringIO()
-    pydocmd.render(modules)
-    sys.stdout = stdout
-    pydoc = tmp_stdout.getvalue()
+    pydoc = pydocmd.renderer.render_to_string(modules)
 
     article_description = f'API reference documentation for {article_title}.'
-    content = f'---\nid: {article_id}\ntitle: {article_title}\ndescription: {article_description}\n---\n\n{pydoc}'
+    content = f'---\nid: {article_id}\ntitle: {article_title}\ndescription: {article_description}\n---\n\n{module_overview}\n\n{pydoc}'
     with open(f'{target_dir}/{article_id}.md', mode='w', encoding='utf-8') as f:
         f.write(content)
 
 
+def generate_demisto_class_docs(target_dir: str):
+    overview = """All Python integrations and scripts have available as part of the runtime the `demisto` class
+object. The object exposes a series of API methods which are used to retrieve and send data to the Cortex XSOAR Server.
+
+:::note
+The `demisto` class is a low level API. For many operations we provide a simpler and more robust API as
+part of the [Common Server Functions](https://xsoar.pan.dev/docs/integrations/code-conventions#common-server-functions).
+:::"""
+    generate_pydoc(
+        module='demisto',
+        article_id='demisto-class',
+        article_title='Demisto Class',
+        target_dir=target_dir,
+        func_prefix='demisto.',
+        module_overview=overview,
+    )
+
+
+def generate_common_server_python_docs(target_dir: str):
+    overview = 'Common functions that will be appended to the code of each integration/script before being executed.'
+    generate_pydoc(
+        module='CommonServerPython',
+        article_id='common-server-python',
+        article_title='Common Server Python',
+        target_dir=target_dir,
+        module_overview=overview,
+    )
+
+
 def main():
     parser = argparse.ArgumentParser(description='Generate Content Python Docs',
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('-d', '--dir', help='Target directory to generate docs at.', required=True)
-    parser.add_argument('-i', '--article_id', help='Article ID.', required=True)
-    parser.add_argument('-t', '--article_title', help='Article title.', required=True)
-    parser.add_argument('-m', '--module', help='Module to generate docs for.', required=True)
-    parser.add_argument('-p', '--func_prefix', help='Prefix to add to function signature.', required=False)
-    parser.add_argument('-o', '--module_overview', help='Module overview to add to the doc header.', required=False)
+    parser.add_argument('-t', '--target_dir', help='Target directory to generate docs at.', required=True)
     args = parser.parse_args()
-    generate_pydoc(args.module, args.article_id, args.article_title, args.dir, args.func_prefix, args.module_overview)
+    target_sub_dir = f'{args.target_dir}/api'
+    if not os.path.exists(target_sub_dir):
+        os.makedirs(target_sub_dir)
+    os.rename('demistomock.py', 'demisto.py')
+    generate_demisto_class_docs(target_sub_dir)
+    os.rename('demisto.py', 'demistomock.py')
+    generate_common_server_python_docs(target_sub_dir)
+    api_ref_path = f'{os.path.basename(args.target_dir)}/api'
+    sidebar = {
+        'type': 'category',
+        'label': 'API Reference',
+        'items': [
+            f'{api_ref_path}/demisto-class',
+            f'{api_ref_path}/common-server-python',
+        ]
+    }
+    with open(f'{args.target_dir}/sidebar.json', 'r+') as f:
+        data = json.load(f)
+        rn_item_index = next(data.index(item) for item in data if item.get('label') == 'Content Release Notes')
+        data.insert(rn_item_index, sidebar)
+        f.seek(0)
+        json.dump(data, f, indent=4)
 
 
 if __name__ == '__main__':