Merge pull request #174 from tbotnz/refactor

Refactor

README.md

@@ -273,7 +273,7 @@ Edit the `config/config.json` file to change any parameters
     "txtfsm_template_server":"http://textfsm.nornir.tech",
     "custom_scripts":"backend/plugins/custom_scripts/",
     "jinja2_config_templates":"backend/plugins/jinja2_templates/",
-    "jinja2_service_templates":"backend/plugins/service_templates/",
+    "python_service_templates":"backend/plugins/services/",
     "self_api_call_timeout":15
 }
 ```

config/defaults.json

@@ -36,7 +36,7 @@
   "txtfsm_template_server": "http://textfsm.nornir.tech",
   "custom_scripts": "netpalm/backend/plugins/extensibles/custom_scripts/",
   "jinja2_config_templates": "netpalm/backend/plugins/extensibles/j2_config_templates/",
-  "jinja2_service_templates": "netpalm/backend/plugins/extensibles/j2_service_templates/",
+  "python_service_templates": "netpalm/backend/plugins/extensibles/services/",
   "ttp_templates": "netpalm/backend/plugins/extensibles/ttp_templates/",
   "self_api_call_timeout": 15,
   "default_webhook_url": "https://9d4f355779c960d7509368ad5a7e3503.m.pipedream.net",

gen_service_package.py

@@ -0,0 +1,25 @@
+import argparse
+import os
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='netpalm service package generate')
+    required_files = parser.add_argument_group('required arguments')
+    required_files.add_argument('-n', '--name', help='service package name', required=True)
+    required_files.add_argument('-o', '--output', help='python | base64', default="python", required=True)
+    args = parser.parse_args()
+
+    package_name = args.name.replace(" ", "_")
+
+    if os.path.isdir(package_name) or os.path.isfile(package_name):
+        print("Please use a different package name, this is in use already")
+        exit()
+
+    os.mkdir(package_name)
+
+
+
+example_service = """
+"""
+
+    with open(f'{package_name}.py', 'w') as fp:
+        pass

netpalm/backend/core/confload/confload.py

@@ -78,7 +78,7 @@ def __init__(self, config_filename=None, search_tfsm=True):
         self.txtfsm_template_server = data["txtfsm_template_server"]
         self.custom_scripts = data["custom_scripts"]
         self.jinja2_config_templates = data["jinja2_config_templates"]
-        self.jinja2_service_templates = data["jinja2_service_templates"]
+        self.python_service_templates = data["python_service_templates"]
         self.self_api_call_timeout = data["self_api_call_timeout"]
         self.default_webhook_url = data["default_webhook_url"]
         self.default_webhook_ssl_verify = data["default_webhook_ssl_verify"]

netpalm/backend/core/manager/netpalm_manager.py

@@ -1,4 +1,9 @@
 import time
+import json
+
+import logging
+
+from typing import Any
 
 from netpalm.backend.core.redis.rediz import Rediz
 from fastapi.encoders import jsonable_encoder
@@ -16,15 +21,19 @@
 from netpalm.backend.core.models.ncclient import NcclientSetConfig
 from netpalm.backend.core.models.netmiko import NetmikoSetConfig
 from netpalm.backend.core.models.restconf import Restconf
-from netpalm.backend.core.models.task import Response
+from netpalm.backend.core.models.task import Response, ResponseBasic
 
-from netpalm.backend.core.models.service import ServiceModel, ServiceInventoryResponse
+from netpalm.backend.core.models.service import ServiceInstanceData, ServiceInstanceState
 from netpalm.backend.core.models.task import ServiceResponse, Response
 
 from netpalm.backend.core.models.models import Script
 
 from netpalm.backend.core.models.task import Response
 
+from netpalm.backend.plugins.utilities.webhook.webhook import exec_webhook_func
+
+log = logging.getLogger(__name__)
+
 
 class NetpalmManager(Rediz):
 
@@ -121,41 +130,66 @@ def execute_script(self, script: Script):
             req_data = script
         else:
             req_data = script.dict(exclude_none=True)
+
+        # check if pinned required
+        if req_data.get("queue_strategy") == "pinned":
+            if isinstance(req_data.get("connection_args"), dict):
+                req_data["connection_args"]["host"] = req_data["script"]
+            else:
+                req_data["connection_args"] = {}
+                req_data["connection_args"]["host"] = req_data["script"]
+
         r = self.execute_task(method="script", kwargs=req_data)
         resp = jsonable_encoder(r)
         return resp
 
-    def create_new_service_instance(self, service_model: str, service: ServiceModel):
+    def create_new_service_instance(self, service_model: str, service: Any):
         """ creates a netpalm service and adds it to the service inventory """
         if isinstance(service, dict):
             req_data = service
         else:
             req_data = service.dict(exclude_none=True)
-        req_data["service_model"] = service_model
-        r = self.execute_service_task(metho="render_service", kwargs=req_data)
+        r = self.execute_create_service_task(metho="service_create", model=service_model, kwargs=req_data)
         resp = jsonable_encoder(r)
         return resp
 
     def list_service_instances(self):
         """ lists services in the netpalm service inventory """
         r = self.get_service_instances()
-        resp = jsonable_encoder(r)
+        if r:
+            formatted_result = ResponseBasic(status="success", data={"task_result": r}).dict()
+        else:
+            formatted_result = ResponseBasic(status="success", data={"task_result": None}).dict()
+        resp = jsonable_encoder(formatted_result)
         return resp
 
     def get_service_instance(self, service_id: str):
         """ gets a from the service inventory """
         r = self.fetch_service_instance_args(sid=service_id)
         if r:
-            resp = jsonable_encoder(r)
+            formatted_result = ResponseBasic(status="success", data={"task_result": r}).dict()
+            resp = jsonable_encoder(formatted_result)
             return resp
         else:
             return False
 
     def validate_service_instance_state(self, service_id: str):
         """ runs the validate method on the service template """
-        r = self.validate_service_instance(sid=service_id)
-        resp = jsonable_encoder(r)
-        return resp
+        try:
+            r = self.validate_service_instance(sid=service_id)
+            resp = jsonable_encoder(r)
+            return resp
+        except Exception:
+            return False
+
+    def health_check_service_instance_state(self, service_id: str):
+        """ runs the validate method on the service template """
+        try:
+            r = self.health_check_service_instance(sid=service_id)
+            resp = jsonable_encoder(r)
+            return resp
+        except Exception:
+            return False
 
     def retrieve_service_instance_state(self, service_id: str):
         """ retrieves the service current state """
@@ -165,16 +199,40 @@ def retrieve_service_instance_state(self, service_id: str):
 
     def redeploy_service_instance_state(self, service_id: str):
         """ redeploys the service instance """
-        r = self.redeploy_service_instance(sid=service_id)
-        resp = jsonable_encoder(r)
-        return resp
+        try:
+            self.set_service_instance_status(self.service_id, state="deploying")
+            r = self.redeploy_service_instance(sid=service_id)
+            resp = jsonable_encoder(r)
+            return resp
+        except Exception:
+            return False
 
     def delete_service_instance_state(self, service_id: str):
         """ deletes the service instance """
         r = self.delete_service_instance(sid=service_id)
         resp = jsonable_encoder(r)
         return resp
 
+    def update_service_instance(self, service_id: str, service_data: Any):
+        """ deletes the service instance """
+
+        if isinstance(service_data, dict):
+            req_data = service_data
+        else:
+            req_data = service_data.dict(exclude_none=True)
+
+        data = self.fetch_service_instance(service_id)
+        if data:
+            service_json = json.loads(data)
+            service_json["service_data"] = req_data
+            service_json["service_meta"]["service_state"] = "deploying"
+            self.update_service_instance_data(service_id, service_json)
+            r = self.execute_task(method="service_update", kwargs=service_json)
+            resp = jsonable_encoder(r)
+            return resp
+        else:
+            return False
+
     def retrieve_task_result(self, netpalm_response: Response):
         """ waits for the task to complete the returns the result """
         if isinstance(netpalm_response, dict):
@@ -192,3 +250,45 @@ def retrieve_task_result(self, netpalm_response: Response):
                 time.sleep(0.3)
         else:
             return req_data
+
+    def retrieve_task_result_multiple(self, netpalm_response_list: list):
+        """
+            retrieves multiple task results in a sync fashion
+
+            Args:
+                netpalm_response_list: list of netpalm response objects
+
+
+            Returns:
+                list of netpalm responses objects with result
+        """
+
+        result = []
+        for netpalm_response in netpalm_response_list:
+            one_result = self.retrieve_task_result(netpalm_response)
+            result.append(one_result)
+
+        return result
+
+    def trigger_webhook(self, webhook_payload: dict, webhook_meta_data: dict):
+        """
+            executes a webhook call
+
+            can also run the job_data through a j2 template if the j2template name is specificed in the
+
+            Args:
+                webhook_payload: dictionary containing the result of the job to be passed into the webhook e.g a netpalm Response dict
+                webhook_meta_data: This is a dictionary describing the metadata of webhook itself e.g webhook name, user specified args to pass into the webhook itself
+                        {
+                            "name": "default_webhook", # webhook name
+                            "args": {
+                                "insert": "something useful" # args to pass into webhook
+                            },
+                            "j2template": "myj2template" # add this key if you want to run the job data through a j2template before passing it into the webhook
+                        }
+
+            Returns:
+                the result of executing the webhook
+        """
+        res = exec_webhook_func(jobdata=webhook_payload, webhook_payload=webhook_meta_data)
+        return res

netpalm/backend/core/models/service.py

@@ -1,11 +1,12 @@
 from enum import Enum
-from typing import Optional, List
+from typing import Optional, List, Any
 
 from pydantic import BaseModel
 
 from netpalm.backend.core.models.models import QueueStrategy
 
 
+# now redundant
 class ServiceLifecycle(str, Enum):
     create = "create"
     retrieve = "retrieve"
@@ -14,6 +15,26 @@ class ServiceLifecycle(str, Enum):
     script = "script"
 
 
+class ServiceInstanceState(str, Enum):
+    deployed = "deployed"
+    errored = "errored"
+    deploying = "deploying"
+
+
+class ServiceMeta(BaseModel):
+    service_model: str
+    created_at: str
+    updated_at: Optional[str] = None
+    service_id: str
+    service_state: Optional[ServiceInstanceState] = None
+
+
+class ServiceInstanceData(BaseModel):
+    service_meta: ServiceMeta
+    service_data: Any
+
+
+# now redundant
 class ServiceModel(BaseModel):
     operation: ServiceLifecycle
     args: dict
@@ -30,24 +51,23 @@ class Config:
             }
         }
 
-
+# now redundant
 class ServiceModelMethods(BaseModel):
     operation: ServiceLifecycle
     path: Optional[str] = None
     payload: dict
 
-
+# now redundant
 class ServiceModelSupportedMethods(BaseModel):
     supported_methods: List[ServiceModelMethods] = None
 
-
+# now redundant
 class ServiceModelTemplate(BaseModel):
     __root__: List[ServiceModelSupportedMethods]
 
 
 class ServiceInventorySchema(BaseModel):
-    service_model: str
-    service_id: str
+    service_meta: dict
 
 
 class ServiceInventoryResponse(BaseModel):