Merge branch 'master' into refactor

.dockerignore

@@ -1,2 +1,10 @@
 .git
+.github
+.gitignore
+.idea
+.vscode
+venv
+backend/plugins/extensibles/ntc-templates
 static
+docker-compose*.yml
+dockerfiles

.gitignore

@@ -1,5 +1,7 @@
-backend/plugins/extensibles/ntc-templates/
-*.pyc
-backend/plugins/extensibles/ntc-templates
 .vscode/
 .vscode/settings.json
+.idea
+*.pyc
+venv
+backend/plugins/extensibles/ntc-templates/
+backend/plugins/extensibles/ntc-templates

RUNNING_TESTS.md

@@ -89,6 +89,10 @@ docker-compose command to force this to background, but it's handy to see this o
 
 ## Running Cisgo tests
 
+```
+docker-compose -f docker-compose.ci.yml exec netpalm-controller pytest -m "not fulllab" -vv tests/integration
+```
+
 We'll actually run our tests from another terminal now.
 
 We're actually going to execute these from inside the controller container. 

docker-compose.ci.yml

@@ -29,6 +29,8 @@ services:
       - redis
     networks:
       - "netpalm-network"
+#    deploy:
+#      replicas: 2
 
   worker-fifo:
     image: netpalm_netpalm-controller
@@ -50,7 +52,9 @@ services:
       - "netpalm-network"
 
   cisgo:
-    image: apcela/cisshgo:v0.1.0
+    image: apcela/cisshgo:v0.1.1
+    ports:
+        - "10005:10005"  # one port just for convenience in case you need to ssh from outside for some reason
     networks:
       - "netpalm-network"
 

netpalm/backend/core/models/task.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, Any
+from typing import Optional, Any, List, Union, Dict
 
 from pydantic import BaseModel
 
@@ -18,11 +18,6 @@ class TaskStatusEnum(str, Enum):
     scheduled = "scheduled"
 
 
-class TaskMeta(BaseModel):
-    result: str
-    errors: list
-
-
 class TaskMetaData(BaseModel):
     enqueued_at: Optional[str]
     started_at: Optional[str]
@@ -32,14 +27,29 @@ class TaskMetaData(BaseModel):
     assigned_worker: Optional[str]
 
 
+class TaskError(BaseModel):
+    exception_class: str
+    exception_args: List[str]
+
+
+TaskErrorList = List[Union[str, TaskError]]
+
+class ServiceTaskHostError(BaseModel):
+    task_id: str
+    task_errors: TaskErrorList
+
+
+ServiceTaskErrors = List[Dict[str, ServiceTaskHostError]]
+
+
 class TaskResponse(BaseModel):
     task_id: str
     created_on: str
     task_queue: str
     task_meta: Optional[TaskMetaData] = None
     task_status: TaskStatusEnum
     task_result: Any
-    task_errors: list
+    task_errors: Union[TaskErrorList, ServiceTaskErrors]  # Needed to get service tasks to validate when they're polled from /task/:taskid
 
 
 class Response(BaseModel):

netpalm/backend/core/utilities/rediz_meta.py

@@ -1,17 +1,59 @@
+import inspect
+from logging import getLogger
+
 from rq import get_current_job
 
 from netpalm.backend.core.confload.confload import config
 from netpalm.backend.core.models.task import Response
+from netpalm.exceptions import NetpalmMetaProcessedException
+
+log = getLogger(__name__)
+
+
+def exception_full_name(exception: BaseException):
+    name = exception.__class__.__name__
+    if (module := inspect.getmodule(exception)) is None:
+        return name
+
+    name = f'{module.__name__}.{name}'
+    return name
+
+
+def yield_exception_chain(exc: BaseException):
+    yield exc
+    if exc.__context__ is None:
+        return
+    yield from yield_exception_chain(exc.__context__)
+
+
+def write_meta_error(exception: Exception):
+    """custom exception handler for within an rpc job"""
+    if isinstance(exception, NetpalmMetaProcessedException):
+        raise exception from None  # Don't process the same exception twice
+
+    log.exception('`write_meta_error` processing error')
+
+    job = get_current_job()
+    job.meta["result"] = "failed"
+
+    exception_chain = yield_exception_chain(exception)
+
+    for exception in reversed(list(exception_chain)):
+        task_error = {
+            'exception_class': exception_full_name(exception),
+            'exception_args': exception.args
+        }
+        job.meta["errors"].append(task_error)
+
+    job.save_meta()
+    raise NetpalmMetaProcessedException from exception
 
 
-def write_meta_error(data):
+def write_meta_error_string(data):
     """custom exception handler for within an rpc job"""
     job = get_current_job()
     job.meta["result"] = "failed"
-    if type(data) == list:
-        job.meta["errors"].append(data.split('\n'))
-    else:
-        job.meta["errors"].append(data)
+    job.meta["errors"].append(data)
     job.save_meta()
     raise Exception(f"failed: {data}")
 

netpalm/backend/core/utilities/rediz_worker_controller.py

@@ -5,6 +5,8 @@
 import logging
 import uuid
 
+from names_generator import generate_name
+
 from netpalm.backend.core.confload.confload import Config
 from netpalm.backend.core.models.models import PinnedStore
 
@@ -49,8 +51,9 @@ def __init__(self, config: Config):
                 socket_keepalive=config.redis_socket_keepalive,
             )
 
-        self.worker_name_base = ""
-        self._uuid = uuid.uuid4()
+        self.hostname = socket.gethostname()
+        self._unique = generate_name(seed=self.hostname)
+        self.worker_name = "UNNAMED WORKER!"  # should never see this
         self.config = config
 
     def worker_cleanup(self):
@@ -59,26 +62,22 @@ def worker_cleanup(self):
         r = self.base_connection.get(self.redis_pinned_store)
         rjson = json.loads(r)
         idex = 0
-        hstname = socket.gethostname()
         for container in rjson:
-            if container["hostname"] == f"{hstname}":
+            if container["hostname"] == self.hostname:
                 rjson.pop(idex)
                 self.base_connection.set(self.redis_pinned_store, json.dumps(rjson))
                 break
             idex += 1
         # purge all workers still running on this container
         workers = Worker.all(connection=self.base_connection)
         for worker in workers:
-            if worker.hostname == f"{hstname}":
+            if worker.hostname == self.hostname:
                 worker.register_death()
 
     def pub_sub(self):
         result = self.base_connection.pubsub()
         return result
 
-    @property
-    def worker_name(self):
-        return f"{self.worker_name_base}_{self._uuid}"
 
     def _listen(self, queue_name):
         log.debug(f'This worker name is: {self.worker_name}')
@@ -92,7 +91,7 @@ class RedisFifoWorker(RedisWorker):
     def __init__(self, config: Config, queue_name: str, counter: int):
         super().__init__(config)
         self.queue_name = queue_name
-        self.worker_name_base = f"{queue_name}_{counter}"
+        self.worker_name = f"{self._unique}_{self.queue_name}_{counter}"
 
     def listen(self):
         """fifo worker instance process"""
@@ -104,17 +103,16 @@ class RedisPinnedWorker(RedisWorker):
     def __init__(self, config: Config, queue_name: str):
         super().__init__(config)
         self.queue_name = queue_name
-        self.worker_name_base = queue_name
+        self.worker_name = f"{self._unique}_{self.queue_name}"
 
     def listen(self):
         """pinned worker instance process"""
         with Connection(self.base_connection):
             # update pinned db
             r = self.base_connection.get(self.redis_pinned_store)
             rjson = json.loads(r)
-            hostname = socket.gethostname()
             for container in rjson:
-                if container["hostname"] == hostname:
+                if container["hostname"] == self.hostname:
                     container["count"] += 1
                     break
             self.base_connection.set(self.redis_pinned_store, json.dumps(rjson))
@@ -125,9 +123,8 @@ def listen(self):
 class RedisProcessWorker(RedisWorker):
     def __init__(self, config: Config):
         super().__init__(config)
-        self.hostname = socket.gethostname()
-        self.queue_name = f"{self.hostname}_processworker"
-        self.worker_name_base = self.queue_name
+        self.queue_name = f"{self._unique}_processworker"
+        self.worker_name = self.queue_name
 
     def listen(self):
         """pinned worker master container process"""

netpalm/backend/plugins/calls/dryrun/dryrun.py

@@ -50,6 +50,6 @@ def dryrun(**kwargs):
             exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook)
 
     except Exception as e:
-        write_meta_error(f"{e}")
+        write_meta_error(e)
 
     return result

netpalm/backend/plugins/calls/getconfig/exec_command.py

@@ -8,6 +8,7 @@
 from netpalm.backend.plugins.drivers.puresnmp.puresnmp_drvr import pursnmp
 from netpalm.backend.plugins.drivers.restconf.restconf import restconf
 from netpalm.backend.plugins.utilities.webhook.webhook import exec_webhook_func
+from netpalm.exceptions import NetpalmCheckError
 
 log = logging.getLogger(__name__)
 
@@ -71,10 +72,11 @@ def exec_command(**kwargs):
                         post_check_result = netmik.sendcommand(sesh, [command])
                         for matchstr in postcheck["match_str"]:
                             if postcheck["match_type"] == "include" and matchstr not in str(post_check_result):
-                                write_meta_error(f"PostCheck Failed: {matchstr} not found in {post_check_result}")
+                                raise NetpalmCheckError(f"PostCheck Failed: {matchstr} not found in {post_check_result}")
                             if postcheck["match_type"] == "exclude" and matchstr in str(post_check_result):
-                                write_meta_error(f"PostCheck Failed: {matchstr} found in {post_check_result}")
+                                raise NetpalmCheckError(f"PostCheck Failed: {matchstr} found in {post_check_result}")
                 netmik.logout(sesh)
+
             elif lib == "napalm":
                 napl = naplm(**kwargs)
                 sesh = napl.connect()
@@ -86,10 +88,11 @@ def exec_command(**kwargs):
                         post_check_result = napl.sendcommand(sesh, [command])
                         for matchstr in postcheck["match_str"]:
                             if postcheck["match_type"] == "include" and matchstr not in str(post_check_result):
-                                write_meta_error(f"PostCheck Failed: {matchstr} not found in {post_check_result}")
+                                raise NetpalmCheckError(f"PostCheck Failed: {matchstr} not found in {post_check_result}")
                             if postcheck["match_type"] == "exclude" and matchstr in str(post_check_result):
-                                write_meta_error(f"PostCheck Failed: {matchstr} found in {post_check_result}")
+                                raise NetpalmCheckError(f"PostCheck Failed: {matchstr} found in {post_check_result}")
                 napl.logout(sesh)
+
             elif lib == "ncclient":
                 ncc = ncclien(**kwargs)
                 sesh = ncc.connect()
@@ -105,6 +108,6 @@ def exec_command(**kwargs):
             current_jobdata = render_netpalm_payload(job_result=result)
             exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook)
     except Exception as e:
-        write_meta_error(f"{e}")
+        write_meta_error(e)
 
     return result

netpalm/backend/plugins/calls/getconfig/ncclient_get.py

@@ -23,6 +23,6 @@ def ncclient_get(**kwargs):
         else:
             raise NotImplementedError(f"unknown 'library' parameter {lib}")
     except Exception as e:
-        write_meta_error(f"{e}")
+        write_meta_error(e)
 
     return result

netpalm/backend/plugins/calls/scriptrunner/script.py

@@ -34,6 +34,6 @@ def script_exec(**kwargs):
             current_jobdata = render_netpalm_payload(job_result=result)
             exec_webhook_func(jobdata=current_jobdata, webhook_payload=webhook)
     except Exception as e:
-        write_meta_error(f"{e}")
+        write_meta_error(e)
 
     return result