From 2121162c6917e853b71590c14b343b30979e42f4 Mon Sep 17 00:00:00 2001 From: Vasu Lakkaraju Date: Thu, 1 Mar 2018 14:47:53 +0530 Subject: [PATCH] Revert "dep ingester flow addeed" --- .../dispatcher/flows/osioAnalysisFlow.yml | 28 ------ f8a_worker/dispatcher/foreach.py | 17 ---- f8a_worker/dispatcher/nodes.yml | 14 --- f8a_worker/workers/DependencyParserTask.py | 54 ------------ .../workers/UnknownDependencyFetcherTask.py | 86 ------------------- 5 files changed, 199 deletions(-) delete mode 100644 f8a_worker/dispatcher/flows/osioAnalysisFlow.yml delete mode 100644 f8a_worker/workers/DependencyParserTask.py delete mode 100644 f8a_worker/workers/UnknownDependencyFetcherTask.py diff --git a/f8a_worker/dispatcher/flows/osioAnalysisFlow.yml b/f8a_worker/dispatcher/flows/osioAnalysisFlow.yml deleted file mode 100644 index b0e073926..000000000 --- a/f8a_worker/dispatcher/flows/osioAnalysisFlow.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- - flow-definitions: - - name: 'osioAnalysisFlow' - queue: '{DEPLOYMENT_PREFIX}_api_osioAnalysisFlow_v0' - edges: - - from: - to: 'DependencyParserTask' - - from: 'DependencyParserTask' - to: 'UnknownDependencyFetcherTask' - - from: 'UnknownDependencyFetcherTask' - to: 'bayesianApiFlow' - foreach: - import: 'f8a_worker.dispatcher.foreach' - function: 'iter_unknown_dependencies' - # result of the function is used as sub-flow arguments - propagate_result: true - # condition: - # and: - # - <<: *recursiveAnalysisCheck - # - name: 'envEqual' - # args: - #env: 'WORKER_ADMINISTRATION_REGION' - #value: 'api' - #- from: 'bayesianApiFlow' - # to: 'OSIOReportGenerationTask' - #- from: 'OSIOReportGenerationTask' - # to: 'OSIOEmailNotificationTask' - diff --git a/f8a_worker/dispatcher/foreach.py b/f8a_worker/dispatcher/foreach.py index 58b054c75..75481e58e 100644 --- a/f8a_worker/dispatcher/foreach.py +++ b/f8a_worker/dispatcher/foreach.py @@ -22,24 +22,7 @@ def _is_url_dependency(dep): return True return False -def iter_unknown_dependencies(storage_pool,node_args): - # Be safe here as fatal errors will cause errors in Dispatcher - try: - aggregated = storage_pool.get('UnknownDependencyFetcherTask') - postgres = storage_pool.get_connected_storage('BayesianPostgres') - arguments = [] - for element in aggregated['result']: - list=element.split(":") - ecosystem = list[0] - name = list[1]+list[2] - version = list[3] - arguments.append(_create_analysis_arguments(ecosystem, name, version)) - logger.info("Arguments for next flows: %s" % str(arguments)) - return arguments - except Exception: - logger.exception("Failed to collect unknown dependencies") - return [] def iter_dependencies_analysis(storage_pool, node_args): # Be safe here as fatal errors will cause errors in Dispatcher diff --git a/f8a_worker/dispatcher/nodes.yml b/f8a_worker/dispatcher/nodes.yml index db4b929f9..d8c177c1e 100644 --- a/f8a_worker/dispatcher/nodes.yml +++ b/f8a_worker/dispatcher/nodes.yml @@ -280,22 +280,8 @@ max_retry: 0 queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_KeywordsSummaryTask_tagging_v0' storage: 'S3KeywordsSummary' - - name: 'UnknownDependencyFetcherTask' - classname: 'UnknownDependencyFetcherTask' - import: 'f8a_worker.workers' - max_retry: 0 - queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_UnknownDependencyFetcher_v0' - storage: 'BayesianPostgres' - - name: 'DependencyParserTask' - classname: 'DependencyParserTask' - import: 'f8a_worker.workers' - max_retry: 0 - queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_DependencyParser_v0' - storage: 'BayesianPostgres' - flows: - - 'osioAnalysisFlow' - 'bayesianFlow' - 'bayesianApiFlow' - 'bayesianPriorityFlow' diff --git a/f8a_worker/workers/DependencyParserTask.py b/f8a_worker/workers/DependencyParserTask.py deleted file mode 100644 index d68ca7803..000000000 --- a/f8a_worker/workers/DependencyParserTask.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Tue Feb 27 19:22:36 2018 - -@author: slakkara -""" -from f8a_worker.base import BaseTask - -def parser(): - pkg_list=['org.apache.maven.resolver:maven-resolver-transport-wagon:1.0.3', - 'org.apache.maven:maven-repository-metadata:3.5.0', - 'org.apache.maven:maven-model-builder:3.2.5', - 'org.jboss.spec.javax.enterprise.concurrent:jboss-concurrency-api_1.0_spec:1.0.0.Final', - 'org.hibernate:hibernate-validator:5.4.2.Final', - 'org.hibernate:hibernate-validator-cdi:5.4.2.Final', - 'org.jboss.forge.addon:bean-validation:3.8.1.Final', - 'org.apache.commons:commons-lang3:3.5', - 'org.jboss.shrinkwrap:shrinkwrap-impl-base:1.2.6', - 'org.apache.maven:maven-settings:3.2.5', - 'org.jboss.shrinkwrap:shrinkwrap-api:1.2.6', - 'org.apache.maven:maven-model-builder:3.5.0', - 'org.jboss.forge.addon:facets-api:3.8.1.Final', - 'com.github.mifmif:generex:1.0.1', - 'org.jboss.forge.addon:ui-impl:3.8.1.Final', - 'org.jboss.narayana.jts:narayana-jts-idlj:5.3.3.Final', - 'org.slf4j:slf4j-api:1.7.2', - 'org.apache.maven:maven-builder-support:3.5.0', - 'org.infinispan:infinispan-commons:8.2.4.Final', - 'org.wildfly.swarm:msc:2017.11.0', - 'org.apache.maven:maven-settings-builder:3.5.0', - 'org.jboss.forge.roaster:roaster-jdt:2.20.1.Final', - 'org.jboss.arquillian.config:arquillian-config-impl-base:1.1.12.Final', - 'io.fabric8:zjsonpatch:0.3.0', - 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.4', - 'commons-lang:commons-lang:2.6', - 'org.wildfly.swarm:fraction-metadata:2017.11.0', - 'io.fabric8:kubernetes-api:3.0.8', - 'javax:javaee-api:7.0', - 'org.eclipse.sisu:org.eclipse.sisu.inject:0.3.3', - 'org.yaml:snakeyaml:1.19', - 'ch.qos.logback:logback-classic:1.1.7'] - - return(pkg_list) - - - -class DependencyParserTask(BaseTask): - - def execute(self, arguments): - result = parser() - return { - "result": result - } \ No newline at end of file diff --git a/f8a_worker/workers/UnknownDependencyFetcherTask.py b/f8a_worker/workers/UnknownDependencyFetcherTask.py deleted file mode 100644 index 4849cff08..000000000 --- a/f8a_worker/workers/UnknownDependencyFetcherTask.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Feb 28 10:46:40 2018 - -@author: slakkara -""" - -from __future__ import division -import os -import json -import traceback -import datetime - -import requests -import os -from collections import Counter, defaultdict -import re -import logging -import semantic_version as sv - -from f8a_worker.graphutils import (GREMLIN_SERVER_URL_REST, create_package_dict,select_latest_version, LICENSE_SCORING_URL_REST) -from f8a_worker.base import BaseTask -from f8a_worker.utils import get_session_retry - - - - - - -def get_dependency_data(self,list): - #LIst is list of dependencies from parser - #for item in list: - # Hardcoded ecosystem - #str1=("org.apache.maven.resolver:io.vertx:vertx-core:3.4.1") - GREMLIN_SERVER_URL_REST="http://bayesian-gremlin-http-preview-b6ff-bayesian-preview.b6ff.rh-idev.openshiftapps.com" - ecosystem="maven" - dep_pkg_list_unknown=[] - dep_pkg_list_known=[] - - - #list=str1.split(":") - #list=["io.vertx","vertx-core","3.4.1"] - for item in list: - list=item.split(":") - #list=["org.apache.maven.resolver","maven-resolver-transport-wagon","1.0.3"] - result = [] - name=list[0]+":"+list[1] - version=list[2] - #ecosystem="maven" - qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + \ - name+ "').has('version','" + version + "').tryNext()") - payload = {'gremlin': qstring} - try: - graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,data=json.dumps(payload)) - #json.dumps(payload)) - if graph_req.status_code == 200: - graph_resp = graph_req.json() - #graph_resp={} - if graph_resp.get('result', {}).get('data'): - result.append(graph_resp["result"]) - if (result[0]['data'][0]['present']): - dep_pkg_list_known.append(ecosystem+":"+name+":"+version) - elif not (result[0]['data'][0]['present']): - dep_pkg_list_unknown.append(ecosystem+":"+name+":"+version) - else: - continue - - #store known_dependency_info in a list - else: - #self.dep_pkg_list_unknown.append(ecosystem+":"+name+":"+version) #store unknown_dependency_info in a list - #log.error("Failed retrieving dependency data.") - continue - except Exception: - #self.log.exception("Error retrieving dependency data.") - #print("Error connecting server") - #print(item) - continue - return (dep_pkg_list_unknown) - - -class UnknownDependencyFetcherTask(BaseTask): - def execute(self, arguments): - result = get_dependency_data(arguments['result']) - return {"result":result} -