diff --git a/gcloud/apigw/views/import_common_template.py b/gcloud/apigw/views/import_common_template.py index 807046d767..5eb984a52a 100644 --- a/gcloud/apigw/views/import_common_template.py +++ b/gcloud/apigw/views/import_common_template.py @@ -13,17 +13,20 @@ import ujson as json +from apigw_manager.apigw.decorators import apigw_require +from blueapps.account.decorators import login_exempt from django.forms.fields import BooleanField from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST -from blueapps.account.decorators import login_exempt from gcloud import err_code from gcloud.apigw.decorators import mark_request_whether_is_trust, return_json_response -from gcloud.common_template.models import CommonTemplate -from gcloud.template_base.utils import read_encoded_template_data from gcloud.apigw.views.utils import logger -from apigw_manager.apigw.decorators import apigw_require +from gcloud.common_template.models import CommonTemplate +from gcloud.template_base.utils import ( + format_import_result_to_response_data, + read_encoded_template_data, +) @login_exempt @@ -70,4 +73,4 @@ def import_common_template(request): "code": err_code.UNKNOWN_ERROR.code, } - return import_result + return format_import_result_to_response_data(import_result) diff --git a/gcloud/apigw/views/import_project_template.py b/gcloud/apigw/views/import_project_template.py index 4f9df4436f..6654c19bd4 100644 --- a/gcloud/apigw/views/import_project_template.py +++ b/gcloud/apigw/views/import_project_template.py @@ -13,17 +13,23 @@ import ujson as json +from apigw_manager.apigw.decorators import apigw_require +from blueapps.account.decorators import login_exempt from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST -from blueapps.account.decorators import login_exempt from gcloud import err_code -from gcloud.apigw.decorators import project_inject -from gcloud.apigw.decorators import mark_request_whether_is_trust, return_json_response -from gcloud.template_base.utils import read_encoded_template_data -from gcloud.tasktmpl3.models import TaskTemplate +from gcloud.apigw.decorators import ( + mark_request_whether_is_trust, + project_inject, + return_json_response, +) from gcloud.apigw.views.utils import logger -from apigw_manager.apigw.decorators import apigw_require +from gcloud.tasktmpl3.models import TaskTemplate +from gcloud.template_base.utils import ( + format_import_result_to_response_data, + read_encoded_template_data, +) @login_exempt @@ -72,4 +78,4 @@ def import_project_template(request, project_id): "code": err_code.UNKNOWN_ERROR.code, } - return import_result + return format_import_result_to_response_data(import_result) diff --git a/gcloud/clocked_task/models.py b/gcloud/clocked_task/models.py index 5dfc0533a1..4c24cdaf0c 100644 --- a/gcloud/clocked_task/models.py +++ b/gcloud/clocked_task/models.py @@ -15,15 +15,18 @@ from django.apps import apps from django.db import models, transaction -from django_celery_beat.models import ( - PeriodicTask as DjangoCeleryBeatPeriodicTask, - ClockedSchedule as DjangoCeleryBeatClockedSchedule, +from django_celery_beat.models import ClockedSchedule as DjangoCeleryBeatClockedSchedule +from django_celery_beat.models import PeriodicTask as DjangoCeleryBeatPeriodicTask + +from gcloud.constants import ( + CLOCKED_TASK_NOT_STARTED, + CLOCKED_TASK_STATE, + PROJECT, + TEMPLATE_SOURCE, ) - -from gcloud.constants import TEMPLATE_SOURCE, PROJECT, CLOCKED_TASK_STATE, CLOCKED_TASK_NOT_STARTED -from gcloud.utils.unique import uniqid -from gcloud.core.models import StaffGroupSet, Project +from gcloud.core.models import Project, StaffGroupSet from gcloud.shortcuts.cmdb import get_business_group_members +from gcloud.utils.unique import uniqid logger = logging.getLogger("root") @@ -39,6 +42,11 @@ def create_task(self, **kwargs): template_name = kwargs["template_name"] notify_type = kwargs.get("notify_type", "[]") notify_receivers = kwargs.get("notify_receivers", "{}") + + optional_keys = ["editor", "edit_time", "create_time"] + # 过滤 optional_keys 中不存在于 kwargs 的属性 + extra_data = {key: kwargs[key] for key in filter(lambda x: x in kwargs, optional_keys)} + with transaction.atomic(): clocked, _ = DjangoCeleryBeatClockedSchedule.objects.get_or_create(clocked_time=plan_start_time) task = ClockedTask.objects.create( @@ -51,6 +59,7 @@ def create_task(self, **kwargs): task_params=task_params, notify_type=notify_type, notify_receivers=notify_receivers, + **extra_data, ) clocked_task_kwargs = {"clocked_task_id": task.id} clocked_task = DjangoCeleryBeatPeriodicTask.objects.create( diff --git a/gcloud/common_template/apis/django/api.py b/gcloud/common_template/apis/django/api.py index d1bd31fbee..8519429cd3 100644 --- a/gcloud/common_template/apis/django/api.py +++ b/gcloud/common_template/apis/django/api.py @@ -17,38 +17,40 @@ from rest_framework.decorators import api_view from gcloud.common_template.models import CommonTemplate +from gcloud.iam_auth.intercept import iam_intercept +from gcloud.iam_auth.view_interceptors.common_template import ( + ExportInterceptor, + FormInterceptor, + ImportInterceptor, + ParentsInterceptor, +) from gcloud.iam_auth.view_interceptors.template import BatchFormInterceptor from gcloud.openapi.schema import AnnotationAutoSchema from gcloud.template_base.apis.django.api import ( base_batch_form, - base_form, base_check_before_import, base_export_templates, + base_form, base_import_templates, base_template_parents, is_full_param_process, ) from gcloud.template_base.apis.django.validators import ( BatchFormValidator, + ExportTemplateApiViewValidator, FormValidator, TemplateParentsValidator, - ExportTemplateApiViewValidator, ) from gcloud.utils.decorators import request_validate -from gcloud.iam_auth.intercept import iam_intercept -from gcloud.iam_auth.view_interceptors.common_template import ( - FormInterceptor, - ExportInterceptor, - ImportInterceptor, - ParentsInterceptor, -) -from .validators import ImportValidator, CheckBeforeImportValidator + +from .validators import CheckBeforeImportValidator, ImportValidator logger = logging.getLogger("root") @swagger_auto_schema( - methods=["get"], auto_schema=AnnotationAutoSchema, + methods=["get"], + auto_schema=AnnotationAutoSchema, ) @api_view(["GET"]) @request_validate(FormValidator) @@ -73,7 +75,8 @@ def form(request): @swagger_auto_schema( - methods=["post"], auto_schema=AnnotationAutoSchema, + methods=["post"], + auto_schema=AnnotationAutoSchema, ) @api_view(["POST"]) @request_validate(BatchFormValidator) @@ -108,7 +111,8 @@ def batch_form(request): @swagger_auto_schema( - methods=["post"], auto_schema=AnnotationAutoSchema, + methods=["post"], + auto_schema=AnnotationAutoSchema, ) @api_view(["POST"]) @request_validate(ExportTemplateApiViewValidator) @@ -129,11 +133,12 @@ def export_templates(request): return: DAT 文件 {} """ - return base_export_templates(request, CommonTemplate, "common", []) + return base_export_templates(request, CommonTemplate, "common") @swagger_auto_schema( - methods=["post"], auto_schema=AnnotationAutoSchema, + methods=["post"], + auto_schema=AnnotationAutoSchema, ) @api_view(["POST"]) @request_validate(ImportValidator) @@ -159,7 +164,8 @@ def import_templates(request): @swagger_auto_schema( - methods=["post"], auto_schema=AnnotationAutoSchema, + methods=["post"], + auto_schema=AnnotationAutoSchema, ) @api_view(["POST"]) @request_validate(CheckBeforeImportValidator) @@ -194,7 +200,8 @@ def check_before_import(request): @swagger_auto_schema( - methods=["get"], auto_schema=AnnotationAutoSchema, + methods=["get"], + auto_schema=AnnotationAutoSchema, ) @api_view(["GET"]) @request_validate(TemplateParentsValidator) diff --git a/gcloud/common_template/models.py b/gcloud/common_template/models.py index 33c2bb004d..6ae4d07688 100644 --- a/gcloud/common_template/models.py +++ b/gcloud/common_template/models.py @@ -11,11 +11,13 @@ specific language governing permissions and limitations under the License. """ +import logging + from django.utils.translation import ugettext_lazy as _ from gcloud import err_code -from gcloud.template_base.models import BaseTemplateManager, BaseTemplate -import logging +from gcloud.template_base.models import BaseTemplate, BaseTemplateManager +from gcloud.template_base.utils import fill_default_version_to_service_activities logger = logging.getLogger("root") @@ -33,19 +35,22 @@ def import_operation_check(self, template_data): data["override_template"] = [] return data + def export_templates(self, template_id_list, **kwargs): + if kwargs.get("is_full"): + template_id_list = list(self.all().values_list("id", flat=True)) + super().export_templates(template_id_list, **kwargs) + def import_templates(self, template_data, override, operator=None): check_info = self.import_operation_check(template_data) + for template in template_data["pipeline_template_data"]["template"].values(): + fill_default_version_to_service_activities(template["tree"]) + # operation validation check if override and (not check_info["can_override"]): message = _("流程导入失败, 不能使用项目流程覆盖公共流程, 请检查后重试 | import_templates") logger.error(message) - return { - "result": False, - "message": message, - "data": 0, - "code": err_code.INVALID_OPERATION.code, - } + return {"result": False, "message": message, "data": 0, "code": err_code.INVALID_OPERATION.code} def defaults_getter(template_dict): return { diff --git a/gcloud/contrib/admin/migration_api/app_maker.py b/gcloud/contrib/admin/migration_api/app_maker.py index 7b05e3f18a..b2172453cb 100644 --- a/gcloud/contrib/admin/migration_api/app_maker.py +++ b/gcloud/contrib/admin/migration_api/app_maker.py @@ -12,60 +12,28 @@ """ import json +import logging import traceback from blueapps.account.decorators import login_exempt -from gcloud.conf import settings from django.http.response import JsonResponse +from django.utils.translation import ugettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST +from pipeline.models import TemplateScheme from gcloud import err_code -from gcloud.core.models import Project +from gcloud.conf import settings from gcloud.contrib.appmaker.models import AppMaker +from gcloud.core.models import Project from gcloud.tasktmpl3.models import TaskTemplate -from pipeline.models import TemplateScheme from .decorators import require_migrate_token -from django.utils.translation import ugettext_lazy as _ -import logging logger = logging.getLogger("root") -@login_exempt -@csrf_exempt -@require_migrate_token -@require_POST -def migrate_app_maker(request): - - try: - params = json.loads(request.body) - except Exception as e: - message = _(f"非法请求: 数据错误, 请求不是合法的Json格式, {e} | migrate_app_maker") - logger.error(message) - return JsonResponse( - { - "result": False, - "message": message, - "code": err_code.REQUEST_PARAM_INVALID.code, - } - ) - - bk_biz_id = params.get("bk_biz_id") - - try: - project = Project.objects.get(bk_biz_id=bk_biz_id) - except Project.DoesNotExist: - return JsonResponse( - { - "result": False, - "message": "can not find project for bk_biz_id: {}".format(bk_biz_id), - "code": err_code.REQUEST_PARAM_INVALID.code, - } - ) - - app_makers = params.get("app_makers", []) +def do_migrate_app_maker(project: Project, app_makers): migrate_result = [] for app_maker in app_makers: # 尝试获取存在的轻应用 @@ -135,4 +103,41 @@ def migrate_app_maker(request): else: migrate_result.append({"name": app_maker["name"], "success": True, "error": None}) + return migrate_result + + +@login_exempt +@csrf_exempt +@require_migrate_token +@require_POST +def migrate_app_maker(request): + + try: + params = json.loads(request.body) + except Exception as e: + message = _(f"非法请求: 数据错误, 请求不是合法的Json格式, {e} | migrate_app_maker") + logger.error(message) + return JsonResponse( + { + "result": False, + "message": message, + "code": err_code.REQUEST_PARAM_INVALID.code, + } + ) + + bk_biz_id = params.get("bk_biz_id") + + try: + project = Project.objects.get(bk_biz_id=bk_biz_id) + except Project.DoesNotExist: + return JsonResponse( + { + "result": False, + "message": "can not find project for bk_biz_id: {}".format(bk_biz_id), + "code": err_code.REQUEST_PARAM_INVALID.code, + } + ) + + migrate_result = do_migrate_app_maker(project, params.get("app_makers", [])) + return JsonResponse({"result": True, "data": migrate_result}) diff --git a/gcloud/tasktmpl3/apis/django/api.py b/gcloud/tasktmpl3/apis/django/api.py index 46b3c567b7..3f13dbafe1 100644 --- a/gcloud/tasktmpl3/apis/django/api.py +++ b/gcloud/tasktmpl3/apis/django/api.py @@ -15,52 +15,54 @@ import ujson as json from django.http import HttpResponseForbidden, JsonResponse +from django.utils.translation import ugettext_lazy as _ from django.views.decorators.http import require_GET, require_POST from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import api_view -from pipeline_web.drawing_new.constants import CANVAS_WIDTH, POSITION -from pipeline_web.drawing_new.drawing import draw_pipeline as draw_pipeline_tree - from gcloud import err_code -from gcloud.utils.strings import check_and_rename_params -from gcloud.utils.decorators import request_validate -from gcloud.tasktmpl3.models import TaskTemplate -from gcloud.tasktmpl3.domains.constants import analysis_pipeline_constants_ref from gcloud.contrib.analysis.analyse_items import task_template from gcloud.iam_auth.intercept import iam_intercept from gcloud.iam_auth.view_interceptors.template import ( - FormInterceptor, + BatchFormInterceptor, ExportInterceptor, + FormInterceptor, ImportInterceptor, - BatchFormInterceptor, ParentsInterceptor, ) from gcloud.openapi.schema import AnnotationAutoSchema -from gcloud.tasktmpl3.domains.constants import get_constant_values -from .validators import ( - ImportValidator, - GetTemplateCountValidator, - DrawPipelineValidator, - AnalysisConstantsRefValidator, - CheckBeforeImportValidator, +from gcloud.tasktmpl3.domains.constants import ( + analysis_pipeline_constants_ref, + get_constant_values, ) +from gcloud.tasktmpl3.models import TaskTemplate from gcloud.template_base.apis.django.api import ( base_batch_form, - base_form, base_check_before_import, base_export_templates, + base_form, base_import_templates, base_template_parents, is_full_param_process, ) from gcloud.template_base.apis.django.validators import ( BatchFormValidator, - FormValidator, ExportTemplateApiViewValidator, + FormValidator, TemplateParentsValidator, ) -from django.utils.translation import ugettext_lazy as _ +from gcloud.utils.decorators import request_validate +from gcloud.utils.strings import check_and_rename_params +from pipeline_web.drawing_new.constants import CANVAS_WIDTH, POSITION +from pipeline_web.drawing_new.drawing import draw_pipeline as draw_pipeline_tree + +from .validators import ( + AnalysisConstantsRefValidator, + CheckBeforeImportValidator, + DrawPipelineValidator, + GetTemplateCountValidator, + ImportValidator, +) logger = logging.getLogger("root") @@ -149,7 +151,7 @@ def export_templates(request, project_id): return: DAT 文件 {} """ - return base_export_templates(request, TaskTemplate, project_id, [project_id]) + return base_export_templates(request, TaskTemplate, project_id, project_id=int(project_id)) @swagger_auto_schema( @@ -178,7 +180,7 @@ def import_templates(request, project_id): } } """ - return base_import_templates(request, TaskTemplate, {"project_id": project_id}) + return base_import_templates(request, TaskTemplate, {"project_id": int(project_id)}) @swagger_auto_schema( diff --git a/gcloud/tasktmpl3/models.py b/gcloud/tasktmpl3/models.py index cfe6d29478..c9677b3727 100644 --- a/gcloud/tasktmpl3/models.py +++ b/gcloud/tasktmpl3/models.py @@ -12,30 +12,37 @@ """ import logging - from collections import Counter -from django.db import models -from django.utils.translation import ugettext_lazy as _ + +from django.apps import apps from django.contrib.auth import get_user_model +from django.db import models from django.db.models import Count - -from pipeline.parser.utils import replace_all_id +from django.utils.translation import ugettext_lazy as _ from pipeline.component_framework.models import ComponentModel from pipeline.contrib.periodic_task.models import PeriodicTask -from pipeline.models import PipelineInstance, TemplateRelationship, PipelineTemplate +from pipeline.models import PipelineInstance, PipelineTemplate, TemplateRelationship +from pipeline.parser.utils import replace_all_id from gcloud import err_code -from gcloud.constants import TASK_FLOW_TYPE, TASK_CATEGORY -from gcloud.constants import AE -from gcloud.template_base.models import BaseTemplate, BaseTemplateManager +from gcloud.analysis_statistics.models import ( + ProjectStatisticsDimension, + TaskflowStatistics, + TaskTmplExecuteTopN, + TemplateNodeStatistics, + TemplateStatistics, +) +from gcloud.constants import AE, TASK_CATEGORY, TASK_FLOW_TYPE from gcloud.core.models import Project -from gcloud.template_base.utils import replace_biz_id_value -from gcloud.utils.managermixins import ClassificationCountMixin -from gcloud.utils.dates import format_datetime -from gcloud.analysis_statistics.models import TemplateStatistics, TemplateNodeStatistics, TaskflowStatistics -from gcloud.utils.components import format_component_name_with_remote -from gcloud.analysis_statistics.models import ProjectStatisticsDimension, TaskTmplExecuteTopN from gcloud.shortcuts.cmdb import get_business_attrinfo +from gcloud.template_base.models import BaseTemplate, BaseTemplateManager +from gcloud.template_base.utils import ( + fill_default_version_to_service_activities, + replace_biz_id_value, +) +from gcloud.utils.components import format_component_name_with_remote +from gcloud.utils.dates import format_datetime +from gcloud.utils.managermixins import ClassificationCountMixin logger = logging.getLogger("root") @@ -403,10 +410,35 @@ def general_group_by(self, prefix_filters, group_by): message = "query_task_list params conditions[%s] have invalid key or value: %s" % (prefix_filters, e) return False, message, None, None - def export_templates(self, template_id_list, project_id): - if self.filter(id__in=template_id_list, project_id=project_id).count() != len(template_id_list): - raise self.model.DoesNotExist("{}(id={}) does not exist.".format(self.model.__name__, template_id_list)) - return super(TaskTemplateManager, self).export_templates(template_id_list) + def export_templates(self, template_id_list, **kwargs): + query_params = {"project_id": kwargs["project_id"]} + if kwargs.get("is_full"): + template_id_list = list(self.filter(**query_params).values_list("id", flat=True)) + else: + query_params["id__in"] = template_id_list + if self.filter(**query_params).count() != len(template_id_list): + raise self.model.DoesNotExist("{}(id={}) does not exist.".format(self.model.__name__, template_id_list)) + + export_data = super(TaskTemplateManager, self).export_templates(template_id_list, **kwargs) + + if kwargs.get("export_app_maker"): + + # 导出任务流程关联的轻应用 + from gcloud.contrib.appmaker.models import AppMaker + + app_maker_cls: AppMaker = apps.get_model("appmaker", "AppMaker") + app_makers = list( + app_maker_cls.objects.filter(task_template_id__in=template_id_list).values( + "id", "name", "desc", "creator", "task_template_id", "template_scheme_id", "project_id" + ) + ) + # formatter + for app_maker in app_makers: + app_maker["username"] = app_maker.pop("creator") + app_maker["template_id"] = app_maker.pop("task_template_id") + export_data["app_makers"] = app_makers + + return export_data def import_operation_check(self, template_data, project_id): data = super(TaskTemplateManager, self).import_operation_check(template_data) @@ -440,22 +472,28 @@ def _reset_biz_selector_value(self, templates_data, bk_biz_id): for template in templates_data["pipeline_template_data"]["template"].values(): replace_biz_id_value(template["tree"], bk_biz_id) + def _reset_project_id(self, templates_data, project_id): + for app_maker in templates_data.get("app_makers") or []: + app_maker["project_id"] = project_id + + for clocked_task in templates_data.get("clocked_tasks") or []: + clocked_task["project_id"] = project_id + def import_templates(self, template_data, override, project_id, operator=None): project = Project.objects.get(id=project_id) check_info = self.import_operation_check(template_data, project_id) # reset biz_cc_id select in templates self._reset_biz_selector_value(template_data, project.bk_biz_id) + # 替换导出数据中的 project_id + self._reset_project_id(template_data, project_id) + for template in template_data["pipeline_template_data"]["template"].values(): + fill_default_version_to_service_activities(template["tree"]) # operation validation check if override and (not check_info["can_override"]): message = _("流程导入失败: 跨业务导入流程不支持覆盖相同ID, 请检查配置 | import_templates") logger.error(message) - return { - "result": False, - "message": message, - "data": 0, - "code": err_code.INVALID_OPERATION.code, - } + return {"result": False, "message": message, "data": 0, "code": err_code.INVALID_OPERATION.code} def defaults_getter(template_dict): return { diff --git a/gcloud/template_base/apis/django/api.py b/gcloud/template_base/apis/django/api.py index fb367c7807..938b3275df 100644 --- a/gcloud/template_base/apis/django/api.py +++ b/gcloud/template_base/apis/django/api.py @@ -11,43 +11,48 @@ specific language governing permissions and limitations under the License. """ -import logging - -import ujson as json -import hashlib import base64 +import hashlib +import logging import traceback from functools import wraps +import ujson as json import yaml from django.db.models import Model -from django.http import JsonResponse, HttpResponse +from django.http import HttpResponse, JsonResponse +from django.utils.translation import ugettext_lazy as _ from drf_yasg.utils import swagger_auto_schema +from pipeline.models import TemplateRelationship from rest_framework.decorators import api_view from rest_framework.request import Request -from pipeline.models import TemplateRelationship from gcloud import err_code from gcloud.conf import settings from gcloud.core.models import Project +from gcloud.exceptions import FlowExportError from gcloud.iam_auth.intercept import iam_intercept -from gcloud.iam_auth.view_interceptors.base_template import YamlExportInterceptor, YamlImportInterceptor +from gcloud.iam_auth.view_interceptors.base_template import ( + YamlExportInterceptor, + YamlImportInterceptor, +) from gcloud.openapi.schema import AnnotationAutoSchema -from gcloud.template_base.domains import TEMPLATE_TYPE_MODEL from gcloud.template_base.apis.django.validators import ( FileValidator, - YamlTemplateImportValidator, YamlTemplateExportValidator, + YamlTemplateImportValidator, ) +from gcloud.template_base.domains import TEMPLATE_TYPE_MODEL from gcloud.template_base.domains.converter_handler import YamlSchemaConverterHandler from gcloud.template_base.domains.importer import TemplateImporter +from gcloud.template_base.utils import ( + format_import_result_to_response_data, + read_template_data_file, +) from gcloud.utils.dates import time_now_str from gcloud.utils.decorators import request_validate from gcloud.utils.strings import string_to_boolean -from gcloud.exceptions import FlowExportError -from gcloud.template_base.utils import read_template_data_file from gcloud.utils.yaml import NoAliasSafeDumper -from django.utils.translation import ugettext_lazy as _ logger = logging.getLogger("root") @@ -134,14 +139,19 @@ def wrapped_view(request, *args, **kwargs): return decorator -def base_export_templates(request: Request, template_model_cls: object, file_prefix: str, export_args: list): +def base_export_templates(request: Request, template_model_cls: object, file_prefix: str, **kwargs): data = request.data template_id_list = data["template_id_list"] # wash try: templates_data = json.loads( - json.dumps(template_model_cls.objects.export_templates(template_id_list, *export_args), sort_keys=True) + json.dumps( + template_model_cls.objects.export_templates( + template_id_list, is_full=request.data.get("is_full") or False, **kwargs + ), + sort_keys=True, + ) ) except FlowExportError as e: return JsonResponse({"result": False, "message": str(e), "code": err_code.UNKNOWN_ERROR.code, "data": None}) @@ -169,7 +179,7 @@ def base_import_templates(request: Request, template_model_cls: object, import_k templates_data = r["data"]["template_data"] try: - result = template_model_cls.objects.import_templates( + import_result = template_model_cls.objects.import_templates( template_data=templates_data, override=override, operator=request.user.username, **import_kwargs ) except Exception: @@ -185,7 +195,7 @@ def base_import_templates(request: Request, template_model_cls: object, import_k } ) - return JsonResponse(result) + return JsonResponse(format_import_result_to_response_data(import_result)) @swagger_auto_schema(methods=["post"], auto_schema=AnnotationAutoSchema) @@ -376,7 +386,6 @@ def export_yaml_templates(request: Request): template_ids = request.data["template_id_list"] template_type = request.data["template_type"] project_id = request.data["project_id"] if template_type == "project" else None - export_args = [project_id] if project_id else [] is_full = request.data["is_full"] if is_full: template_filters = {"is_deleted": False} @@ -388,7 +397,9 @@ def export_yaml_templates(request: Request): converter_handler = YamlSchemaConverterHandler("v1") try: - templates_data = TEMPLATE_TYPE_MODEL[template_type].objects.export_templates(template_ids, *export_args) + templates_data = TEMPLATE_TYPE_MODEL[template_type].objects.export_templates( + template_ids, project_id=project_id + ) convert_result = converter_handler.convert(templates_data) except FlowExportError as e: logger.exception("[export_yaml_templates] convert yaml file error: {}".format(e)) diff --git a/gcloud/template_base/domains/dat_import_helper.py b/gcloud/template_base/domains/dat_import_helper.py new file mode 100644 index 0000000000..80961ad3a8 --- /dev/null +++ b/gcloud/template_base/domains/dat_import_helper.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific lan +""" + +import csv +import json +import logging +import typing +import zlib +from collections import defaultdict + +from MySQLdb import Connection +from pipeline.models import PipelineTemplate + +from gcloud.constants import COMMON +from pipeline_plugins.resource_replacement import base, suites +from pipeline_web.constants import PWE +from pipeline_web.wrapper import PipelineTemplateWebWrapper + +logger = logging.getLogger("root") + + +def blob_data_to_string(blob_string: str) -> str: + """ + 将数据库中 16 进制格式字符串转为原始数据 + :param blob_string: 0x78.... + :return: 解压后的数据 + """ + data = bytes.fromhex(blob_string[2:]) + return zlib.decompress(data).decode("utf-8") + + +def get_old_biz_id__new_biz_info_map( + mysql_config: typing.Dict[str, typing.Any], source_env: str +) -> typing.Dict[int, typing.Dict]: + """ + 获取指定环境老业务 ID - 新业务信息映射关系 + :param mysql_config: + :param source_env: 导出环境代号 + :return: + """ + with Connection(**mysql_config) as conn: + with conn.cursor() as cursor: + cursor.execute(f'SELECT * FROM cc_EnvBizMap WHERE bk_env="{source_env}"') + columns = [desc[0] for desc in cursor.description] + cc_env_biz_infos = [dict(zip(columns, row)) for row in cursor.fetchall()] + return {cc_env_biz_info["bk_old_biz_id"]: cc_env_biz_info for cc_env_biz_info in cc_env_biz_infos} + + +def get_cc_offset(mysql_config: typing.Dict[str, typing.Any], source_env: str) -> int: + """ + 获取指定环境的 CC 资源自增 ID 偏移量 + :param mysql_config: + :param source_env: 导出环境代号 + :return: + """ + with Connection(**mysql_config) as conn: + with conn.cursor() as cursor: + cursor.execute(f'SELECT * FROM cc_EnvIDOffset WHERE env="{source_env}"') + columns = [desc[0] for desc in cursor.description] + cc_env_offset_infos = [dict(zip(columns, row)) for row in cursor.fetchall()] + + return cc_env_offset_infos[0]["offset"] + + +def reuse_imported_common_template( + common_template_id_map: typing.Dict[str, typing.Any], template_data: typing.Dict[str, typing.Any] +): + """ + 复用已导入的公共流程 + :param common_template_id_map: 公共流程导入新应用环境后的 ID 映射关系 + :param template_data: 流程导出数据 + :return: + """ + scheme_id_old_to_new: typing.Dict[int, int] = common_template_id_map["scheme_id_old_to_new"] + temp_id_old_to_new: typing.Dict[str, str] = common_template_id_map[PipelineTemplateWebWrapper.ID_MAP_KEY] + old_tid__template_create_info_map: typing.Dict[str, typing.Dict] = ( + common_template_id_map["template_recreated_info"].get(COMMON) or {} + ) + new_temp_id__pipeline_obj_map: typing.Dict[str, PipelineTemplate] = { + obj.template_id: obj for obj in PipelineTemplate.objects.filter(template_id__in=temp_id_old_to_new.values()) + } + + removed_tid_list: typing.List[str] = [] + removed_pipeline_tid_list: typing.List[str] = [] + old_tid__template_map: typing.Dict[str, typing.Dict[str, typing.Any]] = template_data["template"] + # 将已导入的公共流程，从待导入的流程中移除 + for old_tid, template_create_info in old_tid__template_create_info_map.items(): + if old_tid not in old_tid__template_map: + continue + assert ( + template_create_info["export_data"]["pipeline_template_str_id"] + == old_tid__template_map[old_tid]["pipeline_template_str_id"] + ) + removed_tid_list.append(old_tid) + removed_pipeline_tid_list.append(template_create_info["export_data"]["pipeline_template_str_id"]) + old_tid__template_map.pop(old_tid) + + refs: typing.Dict[str, typing.Dict[str, typing.List[str]]] = template_data["pipeline_template_data"]["refs"] + old_pipeline_tid__info_map: typing.Dict[str, typing.Dict[str, typing.Any]] = template_data[ + "pipeline_template_data" + ]["template"] + + for removed_pipeline_tid in removed_pipeline_tid_list: + # 移除公共流程树 + old_pipeline_tid__info_map.pop(removed_pipeline_tid, None) + + # 其余引用到公共流程的地方替换为新创建的树 ID 及版本 + for ref_pipline_tid, act_ids in refs.pop(removed_pipeline_tid, {}).items(): + if ref_pipline_tid not in old_pipeline_tid__info_map: + continue + # 移除流程间的相互引用无需处理 + for act_id in act_ids: + act = old_pipeline_tid__info_map[ref_pipline_tid]["tree"][PWE.activities][act_id] + if act.get("scheme_id_list"): + act["scheme_id_list"] = [ + scheme_id_old_to_new.get(old_scheme_id, old_scheme_id) + for old_scheme_id in act["scheme_id_list"] + ] + + act["template_id"] = temp_id_old_to_new[removed_pipeline_tid] + act["version"] = new_temp_id__pipeline_obj_map[temp_id_old_to_new[removed_pipeline_tid]].version + + +def add_app_makers(original_project_id: int, template_data: typing.Dict[str, typing.Any], app_maker_csv_filepath: str): + """ + 向已导出的数据中，注入轻应用数据 + :param original_project_id: 原项目 ID + :param template_data: 导出数据 + :param app_maker_csv_filepath: 轻应用导出数据文件 + :return: + """ + template_data["app_makers"] = [] + task_template_ids: typing.Set[str] = set(template_data["template"].keys()) + + with open(app_maker_csv_filepath) as csvfile: + # 创建 CSV 文件的读取器 + app_maker_infos_reader = csv.DictReader(csvfile) + + # 遍历每一行数据 + for app_maker_info in app_maker_infos_reader: + app_maker_info: typing.Dict[str, typing.Any] = app_maker_info + # 已删除数据 / 不在本项目下的不导入 + if any( + [ + not app_maker_info["id"], + # csv 读到的文件都是字符串，需要转为实际类型 + int(app_maker_info["is_deleted"]) == 1, + int(app_maker_info["project_id"]) != original_project_id, + ] + ): + continue + + # 不属于导入流程模板的不导入 + if app_maker_info["task_template_id"] not in task_template_ids: + continue + + template_data["app_makers"].append( + { + "id": int(app_maker_info["id"]), + "name": app_maker_info["name"], + "desc": app_maker_info["desc"], + "username": app_maker_info["creator"], + "project_id": original_project_id, + "template_id": int(app_maker_info["task_template_id"]), + "template_scheme_id": int(app_maker_info["template_scheme_id"]) + if app_maker_info["template_scheme_id"] + else None, + } + ) + + +def add_template_schemes(template_data: typing.Dict[str, typing.Any], pipeline_template_scheme_csv_filepath: str): + """ + 向已导出的数据中添加执行方案 + :param template_data: 导出数据 + :param pipeline_template_scheme_csv_filepath: 执行方案导出的数据文件 + :return: + """ + recorded_ids: typing.Set[int] = set() + id__pipeline_template_info_map: typing.Dict[int, typing.Dict[str, typing.Any]] = { + pipeline_template_info["id"]: pipeline_template_info + for pipeline_template_info in template_data["pipeline_template_data"]["template"].values() + if pipeline_template_info.get("id") + } + pipeline_template_db_ids: typing.Set[int] = set(id__pipeline_template_info_map.keys()) + + with open(pipeline_template_scheme_csv_filepath) as csvfile: + # 创建 CSV 文件的读取器 + template_scheme_info_reader = csv.DictReader(csvfile) + + # 遍历每一行数据 + for template_scheme_info in template_scheme_info_reader: + template_scheme_info: typing.Dict[str, typing.Any] = template_scheme_info + # 没有 ID 或没有绑定模板的情况下直接跳过 + if not template_scheme_info["id"] or not template_scheme_info["template_id"]: + continue + + # 过滤掉不属于导出流程的执行方案 + pipeline_template_db_id = int(template_scheme_info["template_id"]) + if pipeline_template_db_id not in pipeline_template_db_ids: + continue + + recorded_ids.add(pipeline_template_db_id) + + pipeline_template_info: typing.Dict[str, typing.Any] = id__pipeline_template_info_map[ + pipeline_template_db_id + ] + # 在导出流程中，初始化执行方案列表 + if "schemes" not in pipeline_template_info: + pipeline_template_info["schemes"] = [] + + pipeline_template_info["schemes"].append( + { + "id": int(template_scheme_info["id"]), + "name": template_scheme_info["name"], + "unique_id": template_scheme_info["unique_id"], + "template_id": pipeline_template_db_id, + # 解压 DB 二进制数据 + "data": json.loads(blob_data_to_string(template_scheme_info["data"])), + } + ) + + logging.info(f"[add_template_schemes] pipeline template with schemes count -> {len(recorded_ids)}") + + # 按执行方案自增 ID 对数据进行去重 + for pipeline_template_db_id in recorded_ids: + pipeline_template_info = id__pipeline_template_info_map[pipeline_template_db_id] + schemes = list({scheme["id"]: scheme for scheme in pipeline_template_info["schemes"]}.values()) + logging.info( + f"[add_template_schemes] schemes count -> {len(schemes)} to " + f"pipeline_template({pipeline_template_db_id})" + ) + pipeline_template_info["schemes"] = schemes + + +def inject_pipeline_db_id(template_data: typing.Dict[str, typing.Any], pipeline_template_csv_filepath: str): + """ + 注入 PipelineTemplate 自增 ID 到导出数据 + :param template_data: 导出数据 + :param pipeline_template_csv_filepath: 流程模板的数据文件 + :return: + """ + pipeline_template_ids = set(template_data["pipeline_template_data"]["template"].keys()) + + with open(pipeline_template_csv_filepath) as csvfile: + # 创建 CSV 文件的读取器 + pipeline_template_infos_reader = csv.DictReader(csvfile) + + # 遍历每一行数据 + for pipeline_template_info in pipeline_template_infos_reader: + pipeline_template_info: typing.Dict[str, typing.Any] = pipeline_template_info + if not pipeline_template_info["id"]: + continue + + # 过滤掉不属于导出流程的数据 + if pipeline_template_info["template_id"] not in pipeline_template_ids: + continue + + # 根据 template_id 索引到导出数据中的流程数据，并将自增 ID 信息注入 + template_data["pipeline_template_data"]["template"][pipeline_template_info["template_id"]]["id"] = int( + pipeline_template_info["id"] + ) + + +def pipeline_resource_replacement( + pipeline_tree: typing.Dict[str, typing.Any], + cc_offset: int, + old_biz_id__new_biz_info_map: typing.Dict[int, typing.Dict[str, typing.Any]], + db_helper: base.DBHelper, +): + type__code__suite_cls_map: typing.Dict[str, typing.Dict[str, typing.Type[base.Suite]]] = defaultdict(dict) + for suite_cls in suites.SUITES: + type__code__suite_cls_map[suite_cls.TYPE][suite_cls.CODE] = suite_cls + + suite_meta: base.SuiteMeta = base.SuiteMeta( + pipeline_tree=pipeline_tree, offset=cc_offset, old_biz_id__new_biz_info_map=old_biz_id__new_biz_info_map + ) + + logging.info("[pipeline_resource_replacement] start to replace source in activities") + for node_id, service_act in pipeline_tree["activities"].items(): + code: typing.Optional[str] = service_act.get("component", {}).get("code") + if code not in type__code__suite_cls_map["component"]: + continue + suite: base.Suite = type__code__suite_cls_map["component"][code](suite_meta, db_helper) + logging.info(f"[pipeline_resource_replacement] node_id -> {node_id}, suite -> {suite.CODE}") + try: + suite.do(node_id, service_act["component"]) + except Exception: + logging.exception(f"[pipeline_resource_replacement] node_id -> {node_id}, suite -> {suite.CODE} failed") + + logging.info("[pipeline_resource_replacement] start to replace source in constants") + for var_id, constant in pipeline_tree["constants"].items(): + custom_type: typing.Optional[str] = constant.get("custom_type") + if custom_type not in type__code__suite_cls_map["var"]: + continue + suite: base.Suite = type__code__suite_cls_map["var"][custom_type](suite_meta, db_helper) + logging.info(f"[pipeline_resource_replacement] node_id -> {var_id}, suite -> {suite.CODE}") + try: + suite.do(var_id, constant) + except Exception: + pass + + for constant in pipeline_tree["constants"].values(): + constant.pop("resource_replaced", None) + + +def save_resource_mapping( + db_helper: base.DBHelper, table_name: str, template_source_type: str, id_map: typing.Dict[str, typing.Any] +): + """ + 保存已导入的资源映射关系 + :param db_helper: + :param table_name: + :param template_source_type: + :param id_map: + :return: + """ + tid_old_to_new: typing.Dict[int, int] = { + int(old_tid): int(template_create_info["id"]) + for old_tid, template_create_info in id_map["template_recreated_info"].get(template_source_type, {}).items() + } + db_helper.insert_resource_mapping( + table_name=table_name, + resource_type=("template_id", "common_template_id")[template_source_type == COMMON], + source_data_target_data_map=tid_old_to_new, + source_data_type=int, + ) + + scheme_id_old_to_new: typing.Dict[int, int] = id_map["scheme_id_old_to_new"] + db_helper.insert_resource_mapping( + table_name=table_name, + resource_type="scheme_id", + source_data_target_data_map=scheme_id_old_to_new, + source_data_type=str, + ) + + temp_id_old_to_new: typing.Dict[str, str] = id_map[PipelineTemplateWebWrapper.ID_MAP_KEY] + db_helper.insert_resource_mapping( + table_name=table_name, + resource_type="pipeline_template_id", + source_data_target_data_map=temp_id_old_to_new, + source_data_type=str, + ) diff --git a/gcloud/template_base/domains/reference_scene_data_importer.py b/gcloud/template_base/domains/reference_scene_data_importer.py new file mode 100644 index 0000000000..dd9c3ed682 --- /dev/null +++ b/gcloud/template_base/domains/reference_scene_data_importer.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific lan +""" + +import json +import typing +from datetime import datetime, timezone + +from pipeline.models import PipelineTemplate + +from gcloud.clocked_task.models import ClockedTask +from gcloud.conf import settings +from gcloud.constants import PROJECT +from gcloud.contrib.appmaker.models import AppMaker +from pipeline_web.wrapper import PipelineTemplateWebWrapper + + +def import_clocked_tasks( + project_id: int, clocked_tasks: typing.List[typing.Dict[str, typing.Any]], id_map: typing.Dict[str, typing.Any] +) -> typing.Dict[int, typing.Dict[str, typing.Any]]: + """ + 导入计划任务 + :param project_id: + :param clocked_tasks: 导出的计划任务数据 + :param id_map: + :return: + """ + + old_id__new_clocked_task_info_map: typing.Dict[int, typing.Dict[str, typing.Any]] = {} + new_temp_id__pipeline_obj_map: typing.Dict[str, PipelineTemplate] = { + obj.template_id: obj + for obj in PipelineTemplate.objects.filter( + template_id__in=id_map[PipelineTemplateWebWrapper.ID_MAP_KEY].values() + ) + } + + for clocked_task in clocked_tasks: + + # replace project id + clocked_task["project_id"] = project_id + + # replace template_id & template_name + old_tid: int = clocked_task["template_id"] + template_recreated_info: typing.Dict[str, typing.Any] = id_map["template_recreated_info"][ + clocked_task["template_source"] + ][str(old_tid)] + new_tid: int = template_recreated_info["id"] + new_pipeline_template_id: str = template_recreated_info["import_data"]["pipeline_template_id"] + clocked_task["template_id"] = new_tid + clocked_task["template_name"] = new_temp_id__pipeline_obj_map[new_pipeline_template_id].name + + # transfer plan_start_time + dt: datetime = datetime.strptime(clocked_task["plan_start_time"], "%Y-%m-%d %H:%M:%S %Z") + clocked_task["plan_start_time"] = dt.replace(tzinfo=timezone.utc) + + # replace task_params template_schemes_id + clocked_task["task_params"]["template_schemes_id"] = [ + id_map["scheme_id_old_to_new"].get(old_scheme_id, old_scheme_id) + for old_scheme_id in clocked_task["task_params"]["template_schemes_id"] + ] + + # transfer task_params to json string + clocked_task["task_params"] = json.dumps(clocked_task["task_params"]) + + # remove increase id + old_clocked_task_id: int = clocked_task.pop("id", None) + + # import + new_clocked_task_obj: ClockedTask = ClockedTask.objects.create_task(**clocked_task) + old_id__new_clocked_task_info_map[old_clocked_task_id] = {"id": new_clocked_task_obj.id} + + return old_id__new_clocked_task_info_map + + +def import_app_makers( + app_makers: typing.List[typing.Dict[str, typing.Any]], id_map: typing.Dict[str, typing.Any] +) -> typing.Dict[int, typing.Dict[str, typing.Any]]: + """ + 导入轻应用 + :param app_makers: 导出的轻应用任务数据 + :param id_map: + :return: + """ + + old_id__new_app_maker_info_map: typing.Dict[int, typing.Dict[str, typing.Any]] = {} + for app_maker in app_makers: + # 1. replace template_id + old_tid = app_maker["template_id"] + # 轻应用仅引用项目流程 + template_recreated_info = id_map["template_recreated_info"][PROJECT][str(old_tid)] + app_maker["template_id"] = template_recreated_info["id"] + + # 2. replace template_scheme_id + old_scheme_id = app_maker["template_scheme_id"] + app_maker["template_scheme_id"] = id_map["scheme_id_old_to_new"].get(old_scheme_id) + + app_maker["logo_content"] = None + old_id = app_maker["id"] + # 重置 id，在 save_app_maker 进行重建 + app_maker["id"] = None + + if settings.IS_LOCAL: + app_maker["link_prefix"] = "http://localhost/appmaker/" + fake = True + else: + app_maker["link_prefix"] = "%sappmaker/" % settings.APP_HOST + fake = False + + result, app_maker_obj_or_message = AppMaker.objects.save_app_maker( + app_maker["project_id"], app_maker, fake=fake + ) + + if not result: + print(app_maker_obj_or_message) + + new_app_maker_obj: AppMaker = app_maker_obj_or_message + old_id__new_app_maker_info_map[old_id] = {"id": new_app_maker_obj.id} + + return old_id__new_app_maker_info_map diff --git a/gcloud/template_base/management/__init__.py b/gcloud/template_base/management/__init__.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/gcloud/template_base/management/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/gcloud/template_base/management/commands/__init__.py b/gcloud/template_base/management/commands/__init__.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/gcloud/template_base/management/commands/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/gcloud/template_base/management/commands/import_data.py b/gcloud/template_base/management/commands/import_data.py new file mode 100644 index 0000000000..bb8dd7f4a9 --- /dev/null +++ b/gcloud/template_base/management/commands/import_data.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +import json +import logging +import os +import typing + +from django.core.management.base import BaseCommand +from django.db import transaction +from MySQLdb import Connection + +from gcloud.common_template.models import CommonTemplate +from gcloud.constants import COMMON, PROJECT +from gcloud.core.models import Project +from gcloud.tasktmpl3.models import TaskTemplate +from gcloud.template_base.domains import ( + dat_import_helper, + reference_scene_data_importer, +) +from gcloud.template_base.utils import read_template_data_file +from pipeline_plugins.resource_replacement.base import DBHelper + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("root") + + +@transaction.atomic +def do_import( + template_source_type: str, + data_file_path: str, + export_data_dir: str, + cc_offset: int, + old_biz_id__new_biz_info_map: typing.Dict[int, typing.Dict[str, typing.Any]], + db_helper: DBHelper, + target_project: Project = None, + original_project_id: int = None, + import_app_maker: bool = False, + import_clocked_task: bool = False, + reuse_common_template: bool = False, +) -> typing.Dict[str, typing.Any]: + """ + 执行导入操作 + :param template_source_type: + :param data_file_path: + :param export_data_dir: + :param cc_offset: + :param old_biz_id__new_biz_info_map: + :param db_helper: + :param target_project: + :param original_project_id: + :param import_app_maker: + :param import_clocked_task: + :param reuse_common_template: + :return: + """ + template_model_cls: TaskTemplate = (TaskTemplate, CommonTemplate)[template_source_type == COMMON] + + with open(data_file_path, mode="r") as data_file: + r = read_template_data_file(data_file) + template_data: typing.Dict[str, typing.Any] = r["data"]["template_data"] + + if reuse_common_template: + # TODO read from export_data_dir + common_template_id_map = {} + dat_import_helper.reuse_imported_common_template(common_template_id_map, template_data) + + # 注入执行方案｜依赖 PipelineTemplate 的自增 ID，先行注入 + logger.info("[import] inject pipeline template db id to export pipeline template info") + dat_import_helper.inject_pipeline_db_id( + template_data, os.path.join(export_data_dir, "pipeline_pipelinetemplate.csv") + ) + # 注入执行方案 + logger.info("[import] add template schemes to export pipeline template info") + dat_import_helper.add_template_schemes(template_data, os.path.join(export_data_dir, "pipeline_templatescheme.csv")) + + if import_app_maker: + logger.info("[import] add app markers to export data") + dat_import_helper.add_app_makers( + original_project_id, template_data, os.path.join(export_data_dir, "appmaker_appmaker.csv") + ) + + extra_params: typing.Dict[str, typing.Any] = {} + if template_source_type == PROJECT: + extra_params.update({"project_id": target_project.id}) + + for tid, pipeline_template_dict in template_data["pipeline_template_data"]["template"].items(): + logging.info( + f"[import] start to pipeline_resource_replacement: tid -> {tid}, name -> {pipeline_template_dict['name']}" + ) + dat_import_helper.pipeline_resource_replacement( + pipeline_template_dict["tree"], cc_offset, old_biz_id__new_biz_info_map, db_helper + ) + + logger.info("[import] start to import templates") + import_result: typing.Dict[str, typing.Any] = template_model_cls.objects.import_templates( + template_data=template_data, override=False, operator="admin", **extra_params + ) + + if not import_result["result"]: + raise Exception(f"[import] import templates failed, message: {import_result['message']}") + else: + logging.info(f"[import] import templates success, message: {import_result['message']}") + + if import_app_maker: + app_makers: typing.List[typing.Dict[str, typing.Any]] = template_data.get("app_makers") or [] + logging.info(f"[import] import app makers, count -> {len(app_makers)}") + import_result["old_id__new_app_maker_info_map"] = reference_scene_data_importer.import_app_makers( + template_data.get("app_makers") or [], import_result["id_map"] + ) + + if import_clocked_task: + clocked_tasks: typing.List[typing.Dict[str, typing.Any]] = template_data.get("clocked_task") or [] + logging.info(f"[import] import clocked tasks, count -> {len(clocked_tasks)}") + import_result["old_id__new_clocked_task_info_map"] = reference_scene_data_importer.import_clocked_tasks( + target_project.id, clocked_tasks, import_result["id_map"] + ) + + return import_result + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("-t", "--template-source-type", help="Template source type", type=str) + parser.add_argument("-f", "--data-file-path", help=".dat file path", type=str) + # 对于老版本的标准运维，app-maker / clocked-task 可以采用 .csv 的方式注入到 .dat 文件，此处预留 .csv / json 目录 + parser.add_argument("-d", "--export-data-dir", help="Export data dir", type=str) + parser.add_argument("-a", "--app-maker", action="store_true", help="Import app makers") + parser.add_argument("-c", "--clocked-task", action="store_true", help="Import clocked task") + # 期望达到的效果：公共流程先导入，获取 ID 映射数据，存在 + # 对于项目流程的导入，将不再把被引的公共流程作为项目流程导入，而是复用已导入的公共流程 + parser.add_argument("-r", "--reuse-common-template", action="store_true", help="Reuse common template") + + # 环境相关 + parser.add_argument("-e", "--source-env", help="Source env", type=str) + parser.add_argument("-g", "--target-env", help="Target env", type=str) + parser.add_argument("-b", "--original-biz-id", help="Original biz id", type=int) + parser.add_argument("-o", "--original-project-id", help="Original project id", type=int) + + # 中转 DB 相关 + parser.add_argument("-H", "--host", help="Migration DB host", type=str) + parser.add_argument("-P", "--port", help="Migration DB port", type=int) + parser.add_argument("-u", "--user", help="Migration DB user", type=str) + parser.add_argument("-p", "--password", help="Migration DB password", type=str) + parser.add_argument("-D", "--common-migration-database", help="Migration DB database name", type=str) + parser.add_argument( + "-n", + "--self-migration-table-name", + help="Migration table name of sops", + type=str, + default="bk_sops_resource_mapping", + ) + + def handle(self, *args, **options): + + logger.info(f"[import] options: \n{json.dumps(options, indent=2)}") + + template_source_type: str = options["template_source_type"] + data_file_path: str = options["data_file_path"] + export_data_dir: str = options["export_data_dir"] + + # source_env & target_env + source_env: str = options["source_env"] + target_env: str = options["target_env"] + original_biz_id: typing.Optional[int] = options.get("original_biz_id") + original_project_id: typing.Optional[int] = options.get("original_project_id") + + import_app_maker: bool = options.get("app_maker", False) + import_clocked_task: bool = options.get("clocked_task", False) + reuse_common_template: bool = options.get("reuse_common_template", False) + + common_migration_database: str = options["common_migration_database"] + self_migration_table_name: str = options["self_migration_table_name"] + db_config: typing.Dict[str, typing.Any] = { + "host": options["host"], + "port": options["port"], + "user": options["user"], + "password": options["password"], + } + + if template_source_type == PROJECT and not original_project_id: + raise ValueError(f"Original project id required when source type is {PROJECT}") + + cc_offset: int = dat_import_helper.get_cc_offset( + {**db_config, "db": common_migration_database}, source_env=source_env + ) + + logging.info(f"[import] source_env -> {source_env}, cc_offset -> {cc_offset}") + + old_biz_id__new_biz_info_map: typing.Dict[ + int, typing.Dict[str, typing.Any] + ] = dat_import_helper.get_old_biz_id__new_biz_info_map( + {**db_config, "db": common_migration_database}, source_env=source_env + ) + db_helper: DBHelper = DBHelper( + conn=Connection(**{**db_config, "db": common_migration_database}), + source_env=source_env, + target_env=target_env, + ) + + target_project: typing.Optional[Project] = None + if template_source_type == PROJECT: + target_project: Project = Project.objects.get( + bk_biz_id=old_biz_id__new_biz_info_map[original_biz_id]["bk_new_biz_id"] + ) + logging.info( + f"[import] target_project_id -> {target_project.id}, target_biz_id -> {target_project.bk_biz_id}" + ) + + import_result: typing.Dict[str, typing.Any] = do_import( + template_source_type, + data_file_path, + export_data_dir, + cc_offset, + old_biz_id__new_biz_info_map, + db_helper, + target_project=target_project, + original_project_id=original_project_id, + import_app_maker=import_app_maker, + import_clocked_task=import_clocked_task and template_source_type == PROJECT, + reuse_common_template=reuse_common_template, + ) + + dat_import_helper.save_resource_mapping( + db_helper=db_helper, + table_name=self_migration_table_name, + template_source_type=template_source_type, + id_map=import_result["id_map"], + ) + + db_helper.conn.close() diff --git a/gcloud/template_base/models.py b/gcloud/template_base/models.py index ee59fbce2f..008e96f818 100644 --- a/gcloud/template_base/models.py +++ b/gcloud/template_base/models.py @@ -10,29 +10,40 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import json +from collections import defaultdict + from django.apps import apps from django.contrib.auth import get_user_model from django.db import models from django.utils.translation import ugettext_lazy as _ +from pipeline.exceptions import SubprocessExpiredError +from pipeline.models import ( + PipelineTemplate, + TemplateCurrentVersion, + TemplateRelationship, +) +from gcloud import err_code from gcloud.clocked_task.models import ClockedTask +from gcloud.conf import settings +from gcloud.constants import ( + CLOCKED_TASK_NOT_STARTED, + COMMON, + PROJECT, + TASK_CATEGORY, + TEMPLATE_EXPORTER_VERSION, +) +from gcloud.core.utils import convert_readable_username +from gcloud.exceptions import FlowExportError +from gcloud.iam_auth.resource_creator_action.signals import batch_create +from gcloud.template_base.utils import fetch_templates_info, replace_template_id from gcloud.utils import managermixins -from pipeline.exceptions import SubprocessExpiredError from pipeline_web.core.abstract import NodeAttr from pipeline_web.core.models import NodeInTemplate from pipeline_web.parser.clean import PipelineWebTreeCleaner -from pipeline.models import PipelineTemplate, TemplateRelationship, TemplateCurrentVersion from pipeline_web.wrapper import PipelineTemplateWebWrapper -from gcloud import err_code -from gcloud.constants import TEMPLATE_EXPORTER_VERSION, COMMON, PROJECT, CLOCKED_TASK_NOT_STARTED -from gcloud.exceptions import FlowExportError -from gcloud.conf import settings -from gcloud.constants import TASK_CATEGORY -from gcloud.core.utils import convert_readable_username -from gcloud.template_base.utils import replace_template_id, fetch_templates_info -from gcloud.iam_auth.resource_creator_action.signals import batch_create - class BaseTemplateManager(models.Manager, managermixins.ClassificationCountMixin): def fetch_values(self, id, *values): @@ -69,7 +80,7 @@ def create_pipeline_template(self, **kwargs): NodeInTemplate.objects.create_nodes_in_template(pipeline_template, pipeline_web_tree.origin_data) return pipeline_template - def export_templates(self, template_id_list): + def export_templates(self, template_id_list, **kwargs): template_source_type = PROJECT if self.model.__name__ == "TaskTemplate" else COMMON templates = list(self.filter(id__in=template_id_list).select_related("pipeline_template").values()) pipeline_template_id_list = [] @@ -110,6 +121,43 @@ def export_templates(self, template_id_list): "exporter_version": TEMPLATE_EXPORTER_VERSION, "template_source": template_source_type, } + + if kwargs.get("export_clocked_task"): + # 导出计划任务 + clocked_task_cls = apps.get_model("clocked_task", "ClockedTask") + clocked_tasks = [] + for clocked_task_obj in clocked_task_cls.objects.filter( + template_source=template_source_type, template_id__in=template_id_list + ): + clocked_tasks.append( + { + "id": clocked_task_obj.id, + "project_id": clocked_task_obj.project_id, + "template_id": clocked_task_obj.template_id, + "template_name": clocked_task_obj.template_name, + "template_source": clocked_task_obj.template_source, + "task_name": clocked_task_obj.task_name, + "task_params": json.loads(clocked_task_obj.task_params), + "plan_start_time": clocked_task_obj.plan_start_time.strftime( + PipelineTemplateWebWrapper.SERIALIZE_DATE_FORMAT + ), + "edit_time": clocked_task_obj.edit_time.strftime( + PipelineTemplateWebWrapper.SERIALIZE_DATE_FORMAT + ), + "create_time": clocked_task_obj.create_time.strftime( + PipelineTemplateWebWrapper.SERIALIZE_DATE_FORMAT + ), + "editor": clocked_task_obj.editor, + "creator": clocked_task_obj.creator, + } + ) + result["clocked_task"] = clocked_tasks + + if not kwargs.get("export_template_scheme"): + # 将不需要导出的执行方案移除 + for pipeline_template_info in pipeline_temp_data["template"].values(): + pipeline_template_info.pop("schemes", None) + return result def import_operation_check(self, template_data): @@ -143,6 +191,7 @@ def _perform_import(self, template_data, check_info, override, defaults_getter, # import_id -> reuse_id for template_to_be_replaced in check_info["override_template"]: task_template_id = template_to_be_replaced["id"] + # pipeline_template_id template_id = template_data["template"][str(task_template_id)]["pipeline_template_str_id"] tid_to_reuse[template_id] = template_to_be_replaced["template_id"] @@ -163,8 +212,11 @@ def _perform_import(self, template_data, check_info, override, defaults_getter, ) ) + new_pipeline_template_id__old_tid_map = {} for tid, template_dict in list(template.items()): - template_dict["pipeline_template_id"] = old_id_to_new_id[template_dict["pipeline_template_str_id"]] + new_pipeline_template_id = old_id_to_new_id[template_dict["pipeline_template_str_id"]] + new_pipeline_template_id__old_tid_map[new_pipeline_template_id] = tid + template_dict["pipeline_template_id"] = new_pipeline_template_id defaults = defaults_getter(template_dict) # use update or create to avoid id conflict if override: @@ -193,9 +245,33 @@ def _perform_import(self, template_data, check_info, override, defaults_getter, if create_templates: batch_create.send(self.model, instance=create_templates, creator=operator) + template_recreated_info = defaultdict(dict) + template_source_type = PROJECT if self.model.__name__ == "TaskTemplate" else COMMON + created_template_objs = self.model.objects.filter( + pipeline_template_id__in=new_pipeline_template_id__old_tid_map.keys() + ) + + # 建立新老流程模板的映射关系 + # 保留 template_source_type 的原因：后续导入项目流程所依赖的「公共流程」，可能不再以项目流程的方式导入，预留区分 + for created_template_obj in created_template_objs: + new_tid = created_template_obj.id + old_tid = new_pipeline_template_id__old_tid_map[created_template_obj.pipeline_template_id] + template_recreated_info[template_source_type][old_tid] = { + "id": new_tid, + "export_data": template[old_tid], + "import_data": { + "id": new_tid, + "name": created_template_obj.name, + "pipeline_template_id": created_template_obj.pipeline_template_id, + }, + } + + id_map["template_recreated_info"] = template_recreated_info + return { "result": True, "data": {"count": len(template), "flows": flows}, + "id_map": id_map, "message": "Successfully imported %s flows" % len(template), "code": err_code.SUCCESS.code, } diff --git a/gcloud/template_base/utils.py b/gcloud/template_base/utils.py index 120c0cb3ae..6d0838fe18 100644 --- a/gcloud/template_base/utils.py +++ b/gcloud/template_base/utils.py @@ -15,16 +15,16 @@ import hashlib import logging from functools import partial -from typing import Tuple, List, Optional, Dict +from typing import Any, Dict, List, Optional, Tuple import ujson as json from django.apps import apps - -from gcloud.constants import COMMON, PROJECT +from django.utils.translation import ugettext_lazy as _ from pipeline.core.constants import PE + from gcloud import err_code from gcloud.conf import settings -from django.utils.translation import ugettext_lazy as _ +from gcloud.constants import COMMON, PROJECT logger = logging.getLogger("root") @@ -105,6 +105,20 @@ def replace_biz_id_value(pipeline_tree: dict, bk_biz_id: int): constant["value"] = bk_biz_id +def fill_default_version_to_service_activities(pipeline_tree): + """ + 填充默认版本到 ServiceActivity 类型的节点，避免因导出数据版本丢失导致流程导入后无法正常执行 + :param pipeline_tree: + :return: + """ + service_acts = [act for act in pipeline_tree["activities"].values() if act["type"] == "ServiceActivity"] + for act in service_acts: + if not act.get("version"): + act["version"] = "legacy" + if not act["component"].get("version"): + act["component"]["version"] = "legacy" + + def fetch_templates_info( pipeline_template_ids: List, fetch_fields: Tuple, @@ -136,3 +150,17 @@ def get_templates(template_model): ) templates = task_templates + common_templates return templates + + +def format_import_result_to_response_data(import_result: Dict[str, Any]) -> Dict[str, Any]: + """ + 将模板导出结果解析为接口返回数据 + :param import_result: + :return: + """ + return { + "result": import_result["result"], + "message": import_result["message"], + "code": import_result["code"], + "data": import_result["data"], + } diff --git a/gcloud/tests/apigw/views/test_import_common_template.py b/gcloud/tests/apigw/views/test_import_common_template.py index cf9a7d3d91..738e6e4801 100644 --- a/gcloud/tests/apigw/views/test_import_common_template.py +++ b/gcloud/tests/apigw/views/test_import_common_template.py @@ -14,14 +14,13 @@ import ujson as json - +from gcloud import err_code from gcloud.common_template.models import CommonTemplate from gcloud.tests.mock import * # noqa from gcloud.tests.mock_settings import * # noqa from .utils import APITest - TEST_PROJECT_ID = "123" TEST_PROJECT_NAME = "biz name" TEST_BIZ_CC_ID = "123" @@ -82,7 +81,10 @@ def test_import_common_template__import_templates_error(self): MagicMock(return_value={"result": True, "data": {"template_data": "token"}}), ) @mock.patch( - COMMONTEMPLATE_IMPORT_TEMPLATES, MagicMock(return_value={"result": False, "message": "token"}), + COMMONTEMPLATE_IMPORT_TEMPLATES, + MagicMock( + return_value={"result": False, "message": "token", "code": err_code.INVALID_OPERATION.code, "data": 0} + ), ) def test_import_common_template__import_templates_fail(self): response = self.client.post( @@ -102,7 +104,8 @@ def test_import_common_template__import_templates_fail(self): MagicMock(return_value={"result": True, "data": {"template_data": "token"}}), ) @mock.patch( - COMMONTEMPLATE_IMPORT_TEMPLATES, MagicMock(return_value={"result": True, "message": "token"}), + COMMONTEMPLATE_IMPORT_TEMPLATES, + MagicMock(return_value={"result": True, "message": "token", "code": 0, "data": {}}), ) def test_import_common_template__success(self): response = self.client.post( diff --git a/gcloud/tests/template_base/models/base_template_manager/test__perform_import.py b/gcloud/tests/template_base/models/base_template_manager/test__perform_import.py index 9c0919c4a7..5df8c1b5c8 100644 --- a/gcloud/tests/template_base/models/base_template_manager/test__perform_import.py +++ b/gcloud/tests/template_base/models/base_template_manager/test__perform_import.py @@ -12,9 +12,10 @@ """ from django.test import TestCase + +from gcloud.template_base.models import BaseTemplateManager from gcloud.tests.mock import * # noqa from gcloud.tests.mock_settings import * # noqa -from gcloud.template_base.models import BaseTemplateManager from pipeline_web.wrapper import PipelineTemplateWebWrapper @@ -26,21 +27,20 @@ class MockArgs(object): def __init__(self): self.template_data = { "exporter_version": 1, - "pipeline_template_data": { - }, + "pipeline_template_data": {}, "template": { "3": { - 'category': 'Default', - 'executor_proxy': '', - 'id': 3, - 'notify_type': '{"success":[],"fail":[]}', - 'pipeline_template_id': 'nb35afe8b9f335cb9f36f6c55906fcd9', - 'pipeline_template_str_id': 'nb35afe8b9f335cb9f36f6c55906fcd9', - 'project_id': 1, - 'time_out': 20 + "category": "Default", + "executor_proxy": "", + "id": 3, + "notify_type": '{"success":[],"fail":[]}', + "pipeline_template_id": "nb35afe8b9f335cb9f36f6c55906fcd9", + "pipeline_template_str_id": "nb35afe8b9f335cb9f36f6c55906fcd9", + "project_id": 1, + "time_out": 20, } }, - "template_source": "project" + "template_source": "project", } self.check_info = { @@ -50,7 +50,7 @@ def __init__(self): ], "override_template": [ {"id": 3, "name": "定时器1", "template_id": "b7ccd053a6dc39959a9e585dfac9811b"}, - ] + ], } self.operator = "admin" @@ -79,6 +79,7 @@ def filter(self, id__in=None, is_deleted=None, template_id__in=None, pipeline_te template = MagicMock() template.id = 4 template.name = "定时器1" + template.pipeline_template_id = list(pipeline_template_id__in)[0] return [template] return self @@ -94,21 +95,18 @@ def update(self, creator): def bulk_create(self, new_objects): return True + def __name__(self): + return "TaskTemplate" + mock_id_map = { - PipelineTemplateWebWrapper.ID_MAP_KEY: { - 'nb35afe8b9f335cb9f36f6c55906fcd9': 'nb35afe8b9f335cb9f36f6c55906fcd9' - } + PipelineTemplateWebWrapper.ID_MAP_KEY: {"nb35afe8b9f335cb9f36f6c55906fcd9": "nb35afe8b9f335cb9f36f6c55906fcd9"} } class PerformImportTest(TestCase): - @mock.patch( - 'pipeline_web.wrapper.PipelineTemplateWebWrapper.import_templates', - mock.Mock(return_value=mock_id_map) - ) - @mock.patch('gcloud.iam_auth.resource_creator_action.signals.batch_create.send', - mock.Mock(return_value=True)) + @mock.patch("pipeline_web.wrapper.PipelineTemplateWebWrapper.import_templates", mock.Mock(return_value=mock_id_map)) + @mock.patch("gcloud.iam_auth.resource_creator_action.signals.batch_create.send", mock.Mock(return_value=True)) def mock_perform_import(self, override): """ @param override: 是否覆盖 @@ -122,7 +120,7 @@ def mock_perform_import(self, override): check_info=mock_args.check_info, override=override, defaults_getter=mock_args.defaults_getter, - operator=mock_args.operator + operator=mock_args.operator, ) def test_override(self): diff --git a/pipeline_plugins/components/utils/sites/open/utils.py b/pipeline_plugins/components/utils/sites/open/utils.py index 02b2022dfd..4296e55388 100644 --- a/pipeline_plugins/components/utils/sites/open/utils.py +++ b/pipeline_plugins/components/utils/sites/open/utils.py @@ -11,28 +11,31 @@ specific language governing permissions and limitations under the License. """ -import re import logging +import re from collections import Counter from cryptography.fernet import Fernet - -import env from django.utils.translation import ugettext_lazy as _ -from pipeline_plugins.base.utils.inject import supplier_account_for_business -from pipeline_plugins.variables.utils import find_module_with_relation - -from gcloud.utils import cmdb -from gcloud.utils.ip import get_ip_by_regex, extract_ip_from_ip_str, get_ipv6_and_cloud_id_from_ipv6_cloud_str +import env from gcloud.conf import settings from gcloud.core.models import EngineConfig +from gcloud.utils import cmdb +from gcloud.utils.ip import ( + extract_ip_from_ip_str, + get_ip_by_regex, + get_ipv6_and_cloud_id_from_ipv6_cloud_str, +) +from pipeline_plugins.base.utils.inject import supplier_account_for_business +from pipeline_plugins.variables.utils import find_module_with_relation __all__ = [ "cc_get_ips_info_by_str", "get_job_instance_url", "get_node_callback_url", "plat_ip_reg", + "ip_pattern", "get_nodeman_job_url", "get_difference_ip_list", "get_biz_ip_from_frontend", diff --git a/pipeline_plugins/resource_replacement/__init__.py b/pipeline_plugins/resource_replacement/__init__.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/pipeline_plugins/resource_replacement/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline_plugins/resource_replacement/base.py b/pipeline_plugins/resource_replacement/base.py new file mode 100644 index 0000000000..8dd7bd687f --- /dev/null +++ b/pipeline_plugins/resource_replacement/base.py @@ -0,0 +1,473 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import abc +import datetime +import logging +import re +import typing + +from MySQLdb import Connection + +from pipeline_plugins.components.collections.sites.open.cc.base import ( + cc_parse_path_text, +) +from pipeline_plugins.components.utils.sites.open.utils import ip_pattern + +logger = logging.getLogger("root") + + +class DBHelper: + def __init__(self, conn: typing.Optional[Connection], source_env: str, target_env: str): + self.conn = conn + self.source_env = source_env + self.target_env = target_env + + def fetch_resource_id_map( + self, resource_type: str, source_data: typing.List[typing.Union[int, str]], source_data_type: type + ) -> typing.Dict[typing.Union[int, str], typing.Union[int, str]]: + """ + 获取资源新老关系映射 + :param resource_type: + :param source_data: + :param source_data_type: + :return: + """ + source_str_data: typing.List[str] = [f'"{str(source_id)}"' for source_id in source_data] + with self.conn.cursor() as cursor: + cursor.execute( + f"SELECT * FROM bk_job_resource_mapping " + f'WHERE source_env="{self.source_env}" and target_env="{self.target_env}" ' + f'and resource_type="{resource_type}" and source_data in ({",".join(source_str_data)})' + ) + columns = [desc[0] for desc in cursor.description] + job_resource_mapping_infos: typing.List[typing.Dict] = [ + dict(zip(columns, row)) for row in cursor.fetchall() + ] + + resource_id_map: typing.Dict[typing.Union[int, str], typing.Union[int, str]] = {} + for job_resource_mapping_info in job_resource_mapping_infos: + resource_id_map[source_data_type(job_resource_mapping_info["source_data"])] = source_data_type( + job_resource_mapping_info["target_data"] + ) + + data_not_in_mapping: typing.Set[typing.Union[int, str]] = set(source_data) - set(resource_id_map.keys()) + if data_not_in_mapping: + logger.warning(f"[fetch_resource_id_map] data_not_in_mapping -> {data_not_in_mapping}") + logger.info(f"[fetch_resource_id_map] resource_id_map -> {resource_id_map}") + return resource_id_map + + def insert_resource_mapping( + self, + table_name: str, + resource_type: str, + source_data_target_data_map: typing.Dict[typing.Union[str, int], typing.Union[str, int]], + source_data_type: type, + ): + """ + 插入新的映射关系 + :param table_name: + :param resource_type: + :param source_data_target_data_map: + :param source_data_type: + :return: + """ + if not source_data_target_data_map: + return + + data: typing.List[typing.Tuple] = [] + + # 采取先删后增的策略 + source_str_data: typing.List[str] = [f'"{str(source_id)}"' for source_id in source_data_target_data_map.keys()] + with self.conn.cursor() as cursor: + cursor.execute( + f"DELETE FROM {table_name} " + f'WHERE source_env="{self.source_env}" and target_env="{self.target_env}" ' + f'and resource_type="{resource_type}" and source_data in ({",".join(source_str_data)})' + ) + + for source_data, target_data in source_data_target_data_map.items(): + data.append( + ( + resource_type, + self.source_env, + str(source_data), + ("Long", "String")[source_data_type == str], + self.target_env, + str(target_data), + ("Long", "String")[source_data_type == str], + "sops-migration-tool", + int(datetime.datetime.now().timestamp() * 1000), + "sops-migration-tool", + int(datetime.datetime.now().timestamp() * 1000), + "Auto created by sops-migration-tool", + ) + ) + + with self.conn.cursor() as cursor: + insert_sql: str = ( + f"INSERT INTO {table_name} " f"VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + ) + cursor.executemany(insert_sql, data) + self.conn.commit() + + +class SuiteMeta: + pipeline_tree: typing.Dict[str, typing.Any] + offset: int + old_biz_id__new_biz_info_map: typing.Dict[int, typing.Dict[str, typing.Any]] + + def __init__( + self, + pipeline_tree: typing.Dict[str, typing.Any], + offset: int, + old_biz_id__new_biz_info_map: typing.Dict[int, typing.Dict[str, typing.Any]], + ): + """ + :param offset: 自增 ID 相关资源（除业务 ID）外的环境偏移量 + :param old_biz_id__new_biz_info_map: 老业务 ID - 新业务信息映射关系 + """ + self.pipeline_tree = pipeline_tree + self.offset = offset + self.old_biz_id__new_biz_info_map = old_biz_id__new_biz_info_map + + +class Suite(abc.ABC): + TYPE: str = "" + CODE: str = "" + + suite_meta: SuiteMeta + + def __init__(self, suite_meta: SuiteMeta, db_helper: DBHelper): + self.suite_meta = suite_meta + self.db_helper = db_helper + + @abc.abstractmethod + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + raise NotImplementedError + + +class CmdbSuite(Suite, abc.ABC): + def __init__(self, suite_meta: SuiteMeta, db_helper: DBHelper): + super().__init__(suite_meta, db_helper) + + self.biz_old_name__new_name_map: typing.Dict[str, str] = { + new_biz_info["bk_old_biz_name"]: new_biz_info["bk_new_biz_name"] + for new_biz_info in self.suite_meta.old_biz_id__new_biz_info_map.values() + } + + def to_new_topo_select(self, old_topo_select: str) -> str: + """ + 将拓扑节点选择目标替换为新的目标 + :param old_topo_select: + :return: + """ + + # handle {bk_inst_id}_{ip} + try: + ip_or_bk_inst_id = old_topo_select.split("_")[-1] + except Exception: + logger.warning( + f"[to_new_topo_select] {old_topo_select} not matching /bk_obj_id/_/bk_inst_id/ pattern, skip" + ) + return old_topo_select + + if ip_pattern.match(ip_or_bk_inst_id): + bk_inst_id, ip = old_topo_select.rsplit("_", 1) + bk_inst_id = int(bk_inst_id) + new_topo_select: str = f"{bk_inst_id + self.suite_meta.offset}_{ip}" + logger.info(f"[to_new_topo_select] {old_topo_select} -> {new_topo_select}") + return new_topo_select + + try: + # handle {bk_obj_id}_{bk_inst_id} + bk_obj_id, bk_inst_id = old_topo_select.rsplit("_", 1) + bk_inst_id = int(bk_inst_id) + except Exception: + logger.warning( + f"[to_new_topo_select] {old_topo_select} not matching /bk_obj_id/_/bk_inst_id/ pattern, skip" + ) + return old_topo_select + + if bk_obj_id == "biz": + # 业务和其他实例的偏移不同，需要单独处理 + try: + bk_new_inst_id: int = self.suite_meta.old_biz_id__new_biz_info_map[bk_inst_id]["bk_new_biz_id"] + new_topo_select: str = f"{bk_obj_id}_{bk_new_inst_id}" + except KeyError: + logger.warning( + f"[to_new_topo_select] cannot find new business to old biz[{bk_inst_id}] by {old_topo_select}" + ) + return old_topo_select + else: + new_topo_select: str = f"{bk_obj_id}_{bk_inst_id + self.suite_meta.offset}" + + logger.info(f"[to_new_topo_select] {old_topo_select} -> {new_topo_select}") + + return new_topo_select + + def to_new_cloud_id(self, old_cloud_id: int) -> int: + """ + 获得迁移后的云区域 ID + 规则：除直连区域（0）外，其他云区域 ID 按规定量偏移 + :param old_cloud_id: + :return: + """ + + if old_cloud_id == 0: + logger.info("[to_new_cloud_id] skip default area") + return old_cloud_id + + new_cloud_id: int = old_cloud_id + self.suite_meta.offset + logger.info(f"[to_new_cloud_id] {old_cloud_id} -> {new_cloud_id}") + return new_cloud_id + + def to_new_ip_list_str_or_raise(self, old_ip_list_str: str) -> str: + # 匹配出所有格式为 云区域:IP 的输入 + local_ip_pattern = re.compile(r"(\d+:)?((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)(?!\d)") + + cloud_ip_list: typing.List[typing.List] = [ + match.group().split(":") for match in local_ip_pattern.finditer(old_ip_list_str) + ] + + plat_ip_list: typing.List[str] = [] + without_plat_ip_list: typing.List[str] = [] + for cloud_ip in cloud_ip_list: + if len(cloud_ip) == 1: + without_plat_ip_list.append(cloud_ip[0]) + else: + _cloud, _ip = cloud_ip + plat_ip_list.append(f"{self.to_new_cloud_id(int(_cloud))}:{_ip}") + + # 最小处理原则，如果填写的 IP 不包含云区域，则不做处理，尽可能不修改用户数据 + if not plat_ip_list: + logger.info(f"[to_new_ip_list_str_or_raise] {old_ip_list_str} not hit plat_ip, skip") + return old_ip_list_str + + # 没有匹配到任何 IP，跳过 + if not (plat_ip_list or without_plat_ip_list): + logger.info(f"[to_new_ip_list_str_or_raise] {old_ip_list_str} not hit any ip patterns, skip") + return old_ip_list_str + + # 使用换行符重新整合处理后的数据 + new_ip_list_str: str = "\n".join(without_plat_ip_list + plat_ip_list) + logger.info(f"[to_new_ip_list_str_or_raise] {old_ip_list_str} -> {new_ip_list_str}") + return new_ip_list_str + + def get_attr_data_or_raise(self, schema_attr_data: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + """ + 获取表单值的实际引用处 + :param schema_attr_data: + :return: + """ + + if not schema_attr_data["value"]: + raise ValueError + + if isinstance(schema_attr_data["value"], str): + # 尝试找出引用的常量 + attr_data_from_constants: typing.Optional[typing.Dict[str, typing.Any]] = self.suite_meta.pipeline_tree.get( + "constants", {} + ).get(schema_attr_data["value"]) + + if attr_data_from_constants: + # 如果变量已被处理，本轮直接跳过 + if attr_data_from_constants.get("resource_replaced"): + raise ValueError + # 标记已处理节点，此处认为取出则一定会被处理，避免变量复用场景被多次处理 + attr_data_from_constants["resource_replaced"] = True + return attr_data_from_constants + else: + attr_data_from_constants = None + + # 如果常量存在，将作为替换目标，否则认为值存在于 schema_attr_data + attr_data: typing.Dict[str, typing.Any] = attr_data_from_constants or schema_attr_data + + if not attr_data["value"]: + raise ValueError + + return attr_data + + def to_new_cmdb_id_form(self, component: typing.Dict[str, typing.Any], key: str, source_key: str): + """ + CMDB 自增 ID 替换 + :param component: + :param key: + :param source_key: + :return: + """ + if key not in component["data"]: + return + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"][key]) + except Exception: + return + + if not isinstance(attr_data["value"], list): + return + + for item in attr_data["value"]: + try: + if isinstance(item[source_key], str): + if source_key in ["bk_cloud_id", "nodeman_bk_cloud_id"]: + item[source_key] = str(self.to_new_cloud_id(int(item[source_key]))) + else: + item[source_key] = str(int(item[source_key]) + self.suite_meta.offset) + else: + if source_key in ["bk_cloud_id", "nodeman_bk_cloud_id"]: + item[source_key] = self.to_new_cloud_id(item[source_key]) + else: + item[source_key] = item[source_key] + self.suite_meta.offset + except Exception: + pass + + def process_ip_list_str(self, node_id: str, schema_attr_data: typing.Dict[str, typing.Any]): + """ + 处理 IP 列表字符串，将其中可能存在的云区域 ID 按规则偏移 + :param node_id: + :param schema_attr_data: + :return: + """ + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(schema_attr_data) + except ValueError: + return + + # 类型不符合预期时不做处理 + if not isinstance(attr_data["value"], str): + return + + try: + attr_data["value"] = self.to_new_ip_list_str_or_raise(attr_data["value"]) + except ValueError: + pass + + def process_cc_id(self, node_id: str, schema_attr_data: typing.Dict[str, typing.Any]): + """ + 处理业务 ID + :param node_id: + :param schema_attr_data: + :return: + """ + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(schema_attr_data) + except ValueError: + return + + # 类型不符合预期时不做处理 + if not isinstance(attr_data["value"], int): + return + + try: + new_biz_info: typing.Dict[str, typing.Any] = self.suite_meta.old_biz_id__new_biz_info_map[ + attr_data["value"] + ] + except KeyError: + return + + schema_attr_data["value"] = new_biz_info["bk_new_biz_id"] + + def process_topo_select_text(self, node_id: str, schema_attr_data: typing.Dict[str, typing.Any]): + """ + 处理拓扑文本路径 + :param node_id: + :param schema_attr_data: + :return: + """ + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(schema_attr_data) + except ValueError: + return + + # 类型不符合预期时不做处理 + if not isinstance(attr_data["value"], str): + return + + path_list = cc_parse_path_text(attr_data["value"]) + + new_path_str_list: typing.List[str] = [] + for path in path_list: + try: + if path[0] in self.biz_old_name__new_name_map: + path[0] = self.biz_old_name__new_name_map[path[0]] + new_path_str_list.append(" > ".join(path)) + except IndexError: + pass + + attr_data["value"] = "\n".join(new_path_str_list) + + def process_topo_select(self, node_id: str, schema_attr_data: typing.Dict[str, typing.Any]): + """ + 处理拓扑节点选择 + :param node_id: + :param schema_attr_data: + :return: + """ + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(schema_attr_data) + except ValueError: + return + + # 类型不符合预期时不做处理 + if not isinstance(attr_data["value"], list): + return + + new_topo_select_list: typing.List[str] = [] + for topo_select in attr_data["value"]: + new_topo_select_list.append(self.to_new_topo_select(topo_select)) + + attr_data["value"] = new_topo_select_list + + +class JobSuite(CmdbSuite, abc.ABC): + def to_new_ip_form(self, component: typing.Dict[str, typing.Any], key: str, ip_key: str): + """ + IP 表单替换 + :param component: + :param key: + :param ip_key: + :return: + """ + if key not in component["data"]: + return + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"][key]) + except Exception: + return + + if not isinstance(attr_data["value"], list): + return + + for item in attr_data["value"]: + try: + item[ip_key] = self.to_new_ip_list_str_or_raise(item[ip_key]) + except Exception: + pass + + def to_new_job_id( + self, component: typing.Dict[str, typing.Any], key: str, resource_type: str, source_data_type: type + ): + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"][key]) + + if isinstance(attr_data["value"], int): + resource_id_map: typing.Dict[int, int] = self.db_helper.fetch_resource_id_map( + resource_type=resource_type, source_data=[attr_data["value"]], source_data_type=source_data_type + ) + attr_data["value"] = resource_id_map.get(attr_data["value"], attr_data["value"]) + + except Exception: + pass diff --git a/pipeline_plugins/resource_replacement/cc.py b/pipeline_plugins/resource_replacement/cc.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/pipeline_plugins/resource_replacement/cc.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline_plugins/resource_replacement/suites.py b/pipeline_plugins/resource_replacement/suites.py new file mode 100644 index 0000000000..6a76270911 --- /dev/null +++ b/pipeline_plugins/resource_replacement/suites.py @@ -0,0 +1,518 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging +import typing + +from pipeline_plugins.components.collections.sites.open.cc.base import ( + cc_get_name_id_from_combine_value, +) + +from . import base + +logger = logging.getLogger("root") + + +class CCHostCustomPropertyChangeSuite(base.CmdbSuite): + CODE = "cc_host_custom_property_change" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "cc_ip_list" in component["data"]: + self.process_ip_list_str(node_id, component["data"]["cc_ip_list"]) + + +class CCCreateSetSuite(base.CmdbSuite): + CODE = "cc_create_set" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + if "cc_set_parent_select" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_set_parent_select"]) + if "cc_set_parent_select_topo" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_set_parent_select_topo"]) + if "cc_set_parent_select_text" in dict(component["data"]): + self.process_topo_select_text(node_id, component["data"]["cc_set_parent_select_text"]) + + +class CCCreateModuleSuite(base.CmdbSuite): + CODE = "cc_create_module" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + if "cc_set_select_topo" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_set_select_topo"]) + if "cc_set_select_text" in component["data"]: + self.process_topo_select_text(node_id, component["data"]["cc_set_select_text"]) + + if "cc_module_infos_template" not in component["data"]: + return + + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise( + component["data"]["cc_module_infos_template"] + ) + if not isinstance(attr_data["value"], list): + return + + for template_info in attr_data["value"]: + # 避免引用变量导致的解析失败 + try: + name, inst_id = cc_get_name_id_from_combine_value(template_info["cc_service_template"]) + template_info["cc_service_template"] = f"{name}_{inst_id + self.suite_meta.offset}" + except Exception: + continue + + +class CCCreateSetBySetTemplateSuite(CCCreateSetSuite): + CODE = "cc_create_set_by_template" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + super().do(node_id, component) + + if "cc_set_template" not in component["data"]: + return + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"]["cc_set_template"]) + except ValueError: + return + + if not isinstance(attr_data["value"], int): + return + + attr_data["value"] = attr_data["value"] + self.suite_meta.offset + + +class CCUpdateModuleSuite(base.CmdbSuite): + CODE = "cc_update_module" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + if "cc_module_select" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_module_select"]) + if "cc_module_select_topo" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_module_select_topo"]) + if "cc_module_select_text" in component["data"]: + self.process_topo_select_text(node_id, component["data"]["cc_module_select_text"]) + + +class CCEmptySetHostsSuite(base.CmdbSuite): + CODE = "cc_empty_set_hosts" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + if "cc_set_select" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_set_select"]) + if "cc_set_select_topo" in component["data"]: + self.process_topo_select(node_id, component["data"]["cc_set_select_topo"]) + if "cc_set_select_text" in component["data"]: + self.process_topo_select_text(node_id, component["data"]["cc_set_select_text"]) + + +class CCBatchDeleteSetSuite(CCEmptySetHostsSuite): + CODE = "cc_batch_delete_set" + TYPE = "component" + + +class CCUpdateSetSuite(CCEmptySetHostsSuite): + CODE = "cc_update_set" + TYPE = "component" + + +class CCUpdateSetServiceStatusSuite(CCEmptySetHostsSuite): + CODE = "cc_update_set_service_status" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + super().do(node_id, component) + + if "set_list" not in component["data"]: + return + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"]["set_list"]) + except ValueError: + return + + if not isinstance(attr_data["value"], str): + return + + set_new_str_ids: typing.List[str] = [] + try: + for set_str_id in attr_data["value"].split(","): + set_new_str_ids.append(str(int(set_str_id) + self.suite_meta.offset)) + except Exception: + return + + attr_data["value"] = ",".join(set_new_str_ids) + + +class CCVarCmdbSetAllocationSuite(base.CmdbSuite): + CODE = "set_allocation" + TYPE = "var" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + config: typing.Dict[str, typing.Any] = component["value"]["config"] + + try: + if config.get("set_template_id"): + config["set_template_id"] = self.to_new_topo_select(config["set_template_id"]) + except Exception: + pass + + try: + for host_resource in config.get("host_resources") or []: + host_resource["id"] = self.to_new_topo_select(host_resource["id"]) + except Exception: + pass + + try: + for module_detail in config.get("module_detail") or []: + module_detail["id"] = module_detail["id"] + self.suite_meta.offset + except Exception: + pass + + +class CCVarIpPickerVariableSuite(base.CmdbSuite): + CODE = "ip" + TYPE = "var" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + new_var_ip_tree: typing.List[str] = [] + try: + for topo_select in component["value"].get("var_ip_tree") or []: + new_var_ip_tree.append(self.to_new_topo_select(topo_select)) + except Exception: + pass + + if new_var_ip_tree: + component["value"]["var_ip_tree"] = new_var_ip_tree + + try: + component["value"]["var_ip_custom_value"] = self.to_new_ip_list_str_or_raise( + component["value"]["var_ip_custom_value"] + ) + except Exception: + pass + + +class CCVarCmdbIpSelectorSuite(base.CmdbSuite): + CODE = "ip_selector" + TYPE = "var" + + def to_new_conditions(self, conditions: typing.List[typing.Dict[str, typing.Union[str, typing.List[str]]]]): + for cond in conditions: + if cond["field"] == "biz": + cond["value"] = [ + self.biz_old_name__new_name_map.get(biz_old_name, biz_old_name) for biz_old_name in cond["value"] + ] + elif cond["field"] == "host": + try: + cond["value"] = [self.to_new_ip_list_str_or_raise(ip) for ip in cond["value"]] + except Exception: + pass + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + for topo in component["value"].get("topo") or []: + topo["bk_inst_id"] = topo["bk_inst_id"] + self.suite_meta.offset + + for host in component["value"].get("ip") or []: + host["bk_host_id"] = host["bk_host_id"] + self.suite_meta.offset + host["bk_cloud_id"] = self.to_new_cloud_id(host["bk_cloud_id"]) + if "cloud" in host: + for cloud in host["cloud"]: + cloud["id"] = str(self.to_new_cloud_id(int(cloud["id"]))) + + self.to_new_conditions(component["value"].get("filters") or []) + self.to_new_conditions(component["value"].get("excludes") or []) + + +class CCVarCmdbIpFilterSuite(base.CmdbSuite): + CODE = "ip_filter" + TYPE = "var" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + try: + component["value"]["origin_ips"] = self.to_new_ip_list_str_or_raise(component["value"]["origin_ips"]) + except Exception: + pass + + +class CCVarSetModuleIpSelectorSuite(base.CmdbSuite): + CODE = "set_module_ip_selector" + TYPE = "var" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + try: + component["value"]["var_ip_custom_value"] = self.to_new_ip_list_str_or_raise( + component["value"]["var_ip_custom_value"] + ) + except Exception: + pass + + +class CCVarSetModuleSelectorSuite(base.CmdbSuite): + CODE = "set_module_selector" + TYPE = "var" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + try: + component["value"]["bk_set_id"] = component["value"]["bk_set_id"] + self.suite_meta.offset + except Exception: + pass + + try: + new_module_ids: typing.List[int] = [] + for module_id in component["value"]["bk_module_id"] or []: + new_module_ids.append(module_id + self.suite_meta.offset) + if new_module_ids: + component["value"]["bk_module_id"] = new_module_ids + except Exception: + pass + + +class JobLocalContentUploadSuite(base.JobSuite): + CODE = "job_local_content_upload" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "job_ip_list" in component["data"]: + self.process_ip_list_str(node_id, component["data"]["job_ip_list"]) + + +class JobPushLocalFilesSuite(base.JobSuite): + CODE = "job_push_local_files" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + if "job_target_ip_list" in component["data"]: + self.process_ip_list_str(node_id, component["data"]["job_target_ip_list"]) + + +class JobFastPushFileSuite(JobLocalContentUploadSuite): + CODE = "job_fast_push_file" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + super().do(node_id, component) + + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + + self.to_new_ip_form(component, "job_source_files", "ip") + self.to_new_ip_form(component, "job_dispatch_attr", "job_ip_list") + + +class JobFastExecuteScriptSuite(JobLocalContentUploadSuite): + CODE = "job_fast_execute_script" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + super().do(node_id, component) + + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + + +class JobCronTaskSuite(base.JobSuite): + CODE = "job_cron_task" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + + self.to_new_job_id(component, key="job_cron_job_id", resource_type="cron_job_id", source_data_type=int) + + +class JobExecuteTaskSuite(base.JobSuite): + + CODE = "job_execute_task" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + + self.to_new_job_id(component, key="job_task_id", resource_type="task_plan_id", source_data_type=int) + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise(component["data"]["job_global_var"]) + + task_plan_variable_ids: typing.List[int] = [ + job_global_var["id"] for job_global_var in attr_data["value"] or [] + ] + task_plan_variable_id_map: typing.Dict[int, int] = self.db_helper.fetch_resource_id_map( + resource_type="task_plan_variable_id", source_data=task_plan_variable_ids, source_data_type=int + ) + + for job_global_var in attr_data["value"]: + job_global_var["id"] = task_plan_variable_id_map.get(job_global_var["id"], job_global_var["id"]) + # IP 替换 + if job_global_var.get("category") == 3: + job_global_var["value"] = self.to_new_ip_list_str_or_raise(job_global_var["value"]) + + except Exception: + pass + + +class JobAllBizJobFastPushFileSuite(base.JobSuite): + + CODE = "all_biz_job_fast_push_file" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + self.to_new_cmdb_id_form(component, "job_dispatch_attr", "bk_cloud_id") + self.to_new_cmdb_id_form(component, "job_source_files", "bk_cloud_id") + + +class JobAllBizJobFastExecuteScriptSuite(base.JobSuite): + + CODE = "all_biz_job_fast_execute_script" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + self.to_new_cmdb_id_form(component, "job_target_ip_table", "bk_cloud_id") + + +class JobAllBizJobExecuteJobPlanSuite(base.JobSuite): + + CODE = "all_biz_execute_job_plan" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + + if "all_biz_job_config" not in component["data"]: + return + + try: + attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise( + component["data"]["all_biz_job_config"] + ) + except ValueError: + return + + task_template_id_map: typing.Dict[int, int] = self.db_helper.fetch_resource_id_map( + resource_type="task_template_id", source_data=[attr_data["value"]["job_template_id"]], source_data_type=int + ) + attr_data["value"]["job_template_id"] = task_template_id_map.get( + attr_data["value"]["job_template_id"], attr_data["value"]["job_template_id"] + ) + + task_plan_id_map: typing.Dict[int, int] = self.db_helper.fetch_resource_id_map( + resource_type="task_plan_id", source_data=[attr_data["value"]["job_plan_id"]], source_data_type=int + ) + attr_data["value"]["job_plan_id"] = task_plan_id_map.get( + attr_data["value"]["job_plan_id"], attr_data["value"]["job_plan_id"] + ) + + task_plan_variable_ids: typing.List[int] = [ + job_global_var["id"] for job_global_var in attr_data["value"].get("job_global_var") or [] + ] + task_plan_variable_id_map: typing.Dict[int, int] = self.db_helper.fetch_resource_id_map( + resource_type="task_plan_variable_id", source_data=task_plan_variable_ids, source_data_type=int + ) + + for job_global_var in attr_data["value"].get("job_global_var") or []: + job_global_var["id"] = task_plan_variable_id_map.get(job_global_var["id"], job_global_var["id"]) + # IP 替换 + if job_global_var.get("category") == 3: + job_global_var["value"] = self.to_new_ip_list_str_or_raise(job_global_var["value"]) + + +class NodemanPluginOperateSuite(base.CmdbSuite): + CODE = "nodeman_plugin_operate" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "biz_cc_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["biz_cc_id"]) + + try: + nodeman_host_info_attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise( + component["data"]["nodeman_host_info"] + ) + nodeman_host_info_attr_data["value"]["nodeman_bk_cloud_id"] = self.to_new_cloud_id( + nodeman_host_info_attr_data["value"]["nodeman_bk_cloud_id"] + ) + except Exception: + pass + + +class NodemanCreateTaskSuite(base.CmdbSuite): + CODE = "nodeman_create_task" + TYPE = "component" + + def do(self, node_id: str, component: typing.Dict[str, typing.Any]): + if "bk_biz_id" in component["data"]: + self.process_cc_id(node_id, component["data"]["bk_biz_id"]) + + try: + nodeman_op_target_attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise( + component["data"]["nodeman_op_target"] + ) + nodeman_op_target_attr_data["value"]["nodeman_bk_cloud_id"] = self.to_new_cloud_id( + nodeman_op_target_attr_data["value"]["nodeman_bk_cloud_id"] + ) + except Exception: + pass + + try: + nodeman_op_info_attr_data: typing.Dict[str, typing.Any] = self.get_attr_data_or_raise( + component["data"]["nodeman_op_info"] + ) + for host in nodeman_op_info_attr_data["value"]["nodeman_hosts"]: + host["nodeman_bk_cloud_id"] = self.to_new_cloud_id(host["nodeman_bk_cloud_id"]) + except Exception: + pass + + +SUITES = [ + CCHostCustomPropertyChangeSuite, + CCCreateSetSuite, + CCCreateModuleSuite, + CCCreateSetBySetTemplateSuite, + CCUpdateModuleSuite, + CCEmptySetHostsSuite, + CCBatchDeleteSetSuite, + CCUpdateSetSuite, + CCUpdateSetServiceStatusSuite, + CCVarCmdbSetAllocationSuite, + CCVarIpPickerVariableSuite, + CCVarCmdbIpSelectorSuite, + CCVarCmdbIpFilterSuite, + CCVarSetModuleIpSelectorSuite, + CCVarSetModuleSelectorSuite, + JobLocalContentUploadSuite, + JobPushLocalFilesSuite, + JobFastPushFileSuite, + JobFastExecuteScriptSuite, + JobCronTaskSuite, + JobExecuteTaskSuite, + JobAllBizJobFastPushFileSuite, + JobAllBizJobFastExecuteScriptSuite, + JobAllBizJobExecuteJobPlanSuite, + NodemanPluginOperateSuite, + NodemanCreateTaskSuite, +] diff --git a/pipeline_plugins/tests/resource_replacement/__init__.py b/pipeline_plugins/tests/resource_replacement/__init__.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/pipeline_plugins/tests/resource_replacement/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline_plugins/tests/resource_replacement/base.py b/pipeline_plugins/tests/resource_replacement/base.py new file mode 100644 index 0000000000..61ded04da9 --- /dev/null +++ b/pipeline_plugins/tests/resource_replacement/base.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import typing + +from pipeline_plugins.resource_replacement import base + +FIRST_ACT_ID: str = "nodec5d2a1ba8df259c8dd4deeb0fc66" + +PIPELINE_TREE_MOCK_DATA: typing.Dict[str, typing.Any] = { + "name": "new20230418143711", + "activities": { + FIRST_ACT_ID: { + "id": FIRST_ACT_ID, + "incoming": ["line44cf9510e757d159328d64819435"], + "name": "测试原子", + "outgoing": "line7824dc9c613f9f4b82ae20f76e33", + "type": "ServiceActivity", + } + }, + "end_event": { + "id": "nodecf1085a27fe0fbf3020aca8ddc6f", + "incoming": ["line7824dc9c613f9f4b82ae20f76e33"], + "name": "", + "outgoing": "", + "type": "EmptyEndEvent", + }, + "flows": { + "line44cf9510e757d159328d64819435": { + "id": "line44cf9510e757d159328d64819435", + "is_default": False, + "source": "node332f956ed5069aa5b316ea1a27e8", + "target": FIRST_ACT_ID, + }, + "line7824dc9c613f9f4b82ae20f76e33": { + "id": "line7824dc9c613f9f4b82ae20f76e33", + "is_default": False, + "source": FIRST_ACT_ID, + "target": "nodecf1085a27fe0fbf3020aca8ddc6f", + }, + }, + "gateways": {}, + "line": [ + { + "id": "line44cf9510e757d159328d64819435", + "source": {"arrow": "Right", "id": "node332f956ed5069aa5b316ea1a27e8"}, + "target": {"arrow": "Left", "id": FIRST_ACT_ID}, + }, + { + "id": "line7824dc9c613f9f4b82ae20f76e33", + "source": {"arrow": "Right", "id": FIRST_ACT_ID}, + "target": {"arrow": "Left", "id": "nodecf1085a27fe0fbf3020aca8ddc6f"}, + }, + ], + "outputs": [], + "start_event": { + "id": "node332f956ed5069aa5b316ea1a27e8", + "incoming": "", + "name": "", + "outgoing": "line44cf9510e757d159328d64819435", + "type": "EmptyStartEvent", + }, + "template_id": "", + "constants": {}, + "default_flow_type": "common", +} + +OLD_BIZ_ID__NEW_BIZ_INFO_MAP = { + 2: { + "bk_old_biz_name": "测试业务", + "bk_old_biz_id": 2, + "bk_new_biz_name": "测试业务_new", + "bk_new_biz_id": 1002, + "bk_env": "o", + } +} + + +class DBMockHelper(base.DBHelper): + def fetch_resource_id_map( + self, resource_type: str, source_data: typing.List[typing.Union[int, str]], source_data_type: type + ) -> typing.Dict[typing.Union[int, str], typing.Union[int, str]]: + """ + 获取资源新老关系映射 + :param resource_type: + :param source_data: + :param source_data_type: + :return: + """ + if source_data_type == str: + return {source_id: f"{source_id}_new" for source_id in source_data} + else: + return {int(source_id): int(source_id) + 100000 for source_id in source_data} diff --git a/pipeline_plugins/tests/resource_replacement/test_cc_suites.py b/pipeline_plugins/tests/resource_replacement/test_cc_suites.py new file mode 100644 index 0000000000..1c68c50142 --- /dev/null +++ b/pipeline_plugins/tests/resource_replacement/test_cc_suites.py @@ -0,0 +1,765 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from copy import deepcopy + +from django.test import TestCase + +from pipeline_plugins.resource_replacement import base, suites + +from . import base as local_base + + +class CCHostCustomPropertyChangeSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_host_custom_property_change", + "data": { + "cc_ip_list": { + "hook": False, + "need_render": True, + "value": "127.0.0.1\n2:127.0.0.1,3:127.0.0.1\n0:127.0.0.3", + }, + "cc_custom_property": {"hook": False, "need_render": True, "value": "bk_bak_operator"}, + "cc_hostname_rule": {"hook": False, "need_render": True, "value": []}, + "cc_custom_rule": {"hook": False, "need_render": True, "value": []}, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCHostCustomPropertyChangeSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + "127.0.0.1\n10002:127.0.0.1\n10003:127.0.0.1\n0:127.0.0.3", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_ip_list"]["value"], + ) + + def test_do_by_hook(self): + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_ip_list"]["hook"] = True + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_ip_list"][ + "value" + ] = "${cc_ip_list}" + + self.PIPELINE_TREE["constants"]["${cc_ip_list}"] = { + "key": "${cc_ip_list}", + "desc": "", + "custom_type": "", + "source_info": { + local_base.FIRST_ACT_ID: ["cc_ip_list"], + }, + "value": "2:127.0.0.1, 3:127.0.0.1", + "show_type": "show", + "source_type": "component_inputs", + "validation": "", + "index": 1, + "version": "legacy", + "plugin_code": "", + } + + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCHostCustomPropertyChangeSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual("10002:127.0.0.1\n10003:127.0.0.1", self.PIPELINE_TREE["constants"]["${cc_ip_list}"]["value"]) + self.assertTrue(self.PIPELINE_TREE["constants"]["${cc_ip_list}"]["resource_replaced"]) + + # 验证不重复更新 + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual("10002:127.0.0.1\n10003:127.0.0.1", self.PIPELINE_TREE["constants"]["${cc_ip_list}"]["value"]) + + def test_do_by_constants(self): + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_ip_list"][ + "value" + ] = "${ip_list}" + self.PIPELINE_TREE["constants"]["${ip_list}"] = { + "custom_type": "textarea", + "desc": "", + "index": 1, + "key": "${ip_list}", + "name": "ip_list", + "show_type": "show", + "source_info": {}, + "source_tag": "textarea.textarea", + "source_type": "custom", + "value": "127.0.0.1\n1:127.0.0.2", + "version": "legacy", + } + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCHostCustomPropertyChangeSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + + self.assertTrue(self.PIPELINE_TREE["constants"]["${ip_list}"]["resource_replaced"]) + self.assertEqual("127.0.0.1\n10001:127.0.0.2", self.PIPELINE_TREE["constants"]["${ip_list}"]["value"]) + + +class CCCreateSetSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_create_set", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "cc_select_set_parent_method": {"hook": False, "need_render": True, "value": "text"}, + "cc_set_parent_select_topo": {"hook": False, "need_render": True, "value": ["biz_2"]}, + "cc_set_parent_select_text": {"hook": False, "need_render": True, "value": "测试业务"}, + }, + "version": "v2.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCCreateSetSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]["bk_new_biz_id"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + [f"biz_{local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]['bk_new_biz_id']}"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_parent_select_topo"][ + "value" + ], + ) + self.assertEqual( + "测试业务_new", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_parent_select_text"][ + "value" + ], + ) + + +class CCCreateModuleSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_create_module", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "cc_set_select_method": {"hook": False, "need_render": True, "value": "topo"}, + "cc_set_select_topo": { + "hook": False, + "need_render": True, + "value": [ + "set_97", + "set_98", + ], + }, + "cc_set_select_text": {"hook": False, "need_render": True, "value": "测试业务>集群A\n测试业务>集群B"}, + "cc_create_method": {"hook": False, "need_render": True, "value": "category"}, + "cc_module_infos_template": { + "hook": False, + "need_render": True, + "value": [ + {"cc_service_template": "test_40", "bk_module_type": "普通", "operator": "", "bk_bak_operator": ""} + ], + }, + }, + "version": "legacy", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCCreateModuleSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]["bk_new_biz_id"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + ["set_10097", "set_10098"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_select_topo"][ + "value" + ], + ) + self.assertEqual( + "测试业务_new > 集群A\n测试业务_new > 集群B", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_select_text"][ + "value" + ], + ) + self.assertEqual( + "test_10040", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_module_infos_template"][ + "value" + ][0]["cc_service_template"], + ) + + +class CCCreateSetBySetTemplateSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_create_set_by_template", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "cc_select_set_parent_method": {"hook": False, "need_render": True, "value": "topo"}, + "cc_set_parent_select_topo": {"hook": False, "need_render": True, "value": ["biz_2", "set_98"]}, + "cc_set_parent_select_text": {"hook": False, "need_render": True, "value": "测试业务>B"}, + "cc_set_name": {"hook": False, "need_render": True, "value": "A,B"}, + "cc_set_template": {"hook": False, "need_render": True, "value": 25}, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCCreateSetBySetTemplateSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]["bk_new_biz_id"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + ["biz_1002", "set_10098"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_parent_select_topo"][ + "value" + ], + ) + self.assertEqual( + "测试业务_new > B", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_parent_select_text"][ + "value" + ], + ) + self.assertEqual( + 10025, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_template"]["value"], + ) + + +class CCUpdateModuleSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_update_module", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "cc_module_select_method": {"hook": False, "need_render": True, "value": "text"}, + "cc_module_select_topo": { + "hook": False, + "need_render": True, + "value": ["set_102", "module_232", "module_217"], + }, + "cc_module_select": {"hook": False, "need_render": True, "value": ["set_102", "module_232", "module_217"]}, + "cc_module_select_text": {"hook": False, "need_render": True, "value": "测试业务>A>B\n测试业务>A1>B1"}, + "cc_module_property": {"hook": False, "need_render": True, "value": "bk_module_name"}, + "cc_module_prop_value": {"hook": False, "need_render": True, "value": "111"}, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCUpdateModuleSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]["bk_new_biz_id"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + ["set_10102", "module_10232", "module_10217"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_module_select_topo"][ + "value" + ], + ) + self.assertEqual( + ["set_10102", "module_10232", "module_10217"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_module_select"]["value"], + ) + self.assertEqual( + "测试业务_new > A > B\n测试业务_new > A1 > B1", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_module_select_text"][ + "value" + ], + ) + + +class CCEmptySetHostsSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_empty_set_hosts", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "cc_set_select_method": {"hook": False, "need_render": True, "value": "text"}, + "cc_set_select_topo": {"hook": False, "need_render": True, "value": ["set_102", 1, "hahaha"]}, + "cc_set_select": {"hook": False, "need_render": True, "value": ["set_1", "custom_102", "error_xxxx"]}, + "cc_set_select_text": {"hook": False, "need_render": True, "value": "测试业务>集群"}, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCEmptySetHostsSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP[2]["bk_new_biz_id"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + ["set_10102", 1, "hahaha"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_select_topo"][ + "value" + ], + ) + self.assertEqual( + ["set_10001", "custom_10102", "error_xxxx"], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_select"]["value"], + ) + self.assertEqual( + "测试业务_new > 集群", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["cc_set_select_text"][ + "value" + ], + ) + + +class CCUpdateSetServiceStatusSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "cc_update_set_service_status", + "data": { + "set_select_method": {"hook": False, "need_render": True, "value": "id"}, + "set_attr_id": {"hook": False, "need_render": True, "value": ""}, + "set_list": {"hook": False, "need_render": True, "value": "1,2,3"}, + "set_status": {"hook": False, "need_render": True, "value": "1"}, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCUpdateSetServiceStatusSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + "10001,10002,10003", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["set_list"]["value"], + ) + + +class CCVarCmdbSetAllocationSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "set_allocation", + "desc": "", + "index": 1, + "key": "${VarCmdbSetAllocation}", + "name": "VarCmdbSetAllocation", + "source_info": {}, + "source_tag": "var_cmdb_resource_allocation.set_allocation", + "source_type": "custom", + "value": { + "config": { + "set_count": "1", + "set_template_id": "set_102", + "set_template_name": "test1207", + "host_resources": [ + {"id": "set_101", "label": "标准运维"}, + {"id": "set_102", "label": "test1207"}, + ], + "mute_attribute": "", + "filter_lock": False, + "shareEqually": "", + "module_detail": [ + { + "id": 232, + "name": "test", + "host_count": "1", + "reuse_module": "", + } + ], + }, + "separator": ",", + }, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarCmdbSetAllocation}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarCmdbSetAllocationSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarCmdbSetAllocation}", self.PIPELINE_TREE["constants"]["${VarCmdbSetAllocation}"]) + self.assertEqual( + [ + {"id": "set_10101", "label": "标准运维"}, + {"id": "set_10102", "label": "test1207"}, + ], + self.PIPELINE_TREE["constants"]["${VarCmdbSetAllocation}"]["value"]["config"]["host_resources"], + ) + self.assertEqual( + "set_10102", + self.PIPELINE_TREE["constants"]["${VarCmdbSetAllocation}"]["value"]["config"]["set_template_id"], + ) + self.assertEqual( + [ + { + "id": 10232, + "name": "test", + "host_count": "1", + "reuse_module": "", + } + ], + self.PIPELINE_TREE["constants"]["${VarCmdbSetAllocation}"]["value"]["config"]["module_detail"], + ) + + +class CCVarIpPickerVariableSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "ip", + "desc": "", + "index": 3, + "key": "${VarIpPickerVariable}", + "name": "VarIpPickerVariable", + "source_info": {}, + "source_tag": "var_ip_picker.ip_picker", + "source_type": "custom", + "validation": "", + "is_condition_hide": "false", + "pre_render_mako": False, + "value": { + "var_ip_method": "custom", + "var_ip_custom_value": "127.0.0.1\n1:127.0.0.1", + "var_ip_tree": ["232_127.0.0.1", "module_217", "set_96", "module_218", "module_219"], + }, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarIpPickerVariable}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarIpPickerVariableSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarIpPickerVariable}", self.PIPELINE_TREE["constants"]["${VarIpPickerVariable}"]) + self.assertEqual( + ["10232_127.0.0.1", "module_10217", "set_10096", "module_10218", "module_10219"], + self.PIPELINE_TREE["constants"]["${VarIpPickerVariable}"]["value"]["var_ip_tree"], + ) + self.assertEqual( + "127.0.0.1\n10001:127.0.0.1", + self.PIPELINE_TREE["constants"]["${VarIpPickerVariable}"]["value"]["var_ip_custom_value"], + ) + + +class CCVarCmdbIpSelectorSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "ip_selector", + "desc": "", + "index": 1, + "key": "${a}", + "name": "1", + "show_type": "show", + "source_info": {}, + "source_tag": "var_cmdb_ip_selector.ip_selector", + "source_type": "custom", + "value": { + "selectors": ["ip"], + "topo": [{"bk_inst_id": 20, "bk_obj_id": "module"}, {"bk_inst_id": 39, "bk_obj_id": "set"}], + "ip": [ + { + "bk_cloud_id": 1, + "bk_host_innerip": "127.0.0.1", + "bk_host_id": 1, + "bk_host_name": "xxxxx", + "cloud": [{"id": "1", "bk_inst_name": "default area"}], + "agent": 1, + } + ], + "filters": [{"field": "host", "value": ["0:127.0.0.1", "127.0.0.2", "1:127.0.0.1"]}], + "excludes": [{"field": "biz", "value": ["测试业务"]}], + "with_cloud_id": False, + "separator": ",", + }, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarCmdbIpSelectorSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarCmdbIpSelector}", self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"]) + self.assertEqual( + [{"bk_inst_id": 10020, "bk_obj_id": "module"}, {"bk_inst_id": 10039, "bk_obj_id": "set"}], + self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"]["value"]["topo"], + ) + self.assertEqual( + [ + { + "bk_cloud_id": 10001, + "bk_host_innerip": "127.0.0.1", + "bk_host_id": 10001, + "bk_host_name": "xxxxx", + "cloud": [{"id": "10001", "bk_inst_name": "default area"}], + "agent": 1, + } + ], + self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"]["value"]["ip"], + ) + self.assertEqual( + [{"field": "host", "value": ["0:127.0.0.1", "127.0.0.2", "10001:127.0.0.1"]}], + self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"]["value"]["filters"], + ) + + self.assertEqual( + [{"field": "biz", "value": ["测试业务_new"]}], + self.PIPELINE_TREE["constants"]["${VarCmdbIpSelector}"]["value"]["excludes"], + ) + + +class CCVarCmdbIpFilterSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "ip_filter", + "desc": "", + "index": 1, + "key": "${a}", + "name": "1", + "show_type": "show", + "source_info": {}, + "source_tag": "var_cmdb_ip_filter.ip_filter", + "source_type": "custom", + "validation": "", + "is_condition_hide": "false", + "pre_render_mako": False, + "value": { + "origin_ips": "127.0.0.1\n0:127.0.0.1\n2:127.0.0.1", + "gse_agent_status": 1, + "ip_cloud": False, + "ip_separator": ",", + }, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarCmdbIpFilter}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarCmdbIpFilterSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarCmdbIpFilter}", self.PIPELINE_TREE["constants"]["${VarCmdbIpFilter}"]) + self.assertEqual( + "127.0.0.1\n0:127.0.0.1\n10002:127.0.0.1", + self.PIPELINE_TREE["constants"]["${VarCmdbIpFilter}"]["value"]["origin_ips"], + ) + + +class CCVarSetModuleIpSelectorSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "set_module_ip_selector", + "desc": "", + "index": 1, + "key": "${a}", + "name": "1", + "show_type": "show", + "source_info": {}, + "source_tag": "set_module_ip_selector.ip_selector", + "source_type": "custom", + "validation": "", + "is_condition_hide": "false", + "pre_render_mako": False, + "value": { + "var_ip_method": "manual", + "var_ip_custom_value": "1:127.0.0.1\n2:127.0.0.1", + "var_ip_select_value": {"var_set": ["直连区域", "云区域"], "var_module": ["空闲机", "故障机"], "var_module_name": ""}, + "var_ip_manual_value": {"var_manual_set": "all", "var_manual_module": "all", "var_module_name": ""}, + "var_filter_set": "test", + "var_filter_module": "test", + }, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarSetModuleIpSelector}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarSetModuleIpSelectorSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarSetModuleIpSelector}", self.PIPELINE_TREE["constants"]["${VarSetModuleIpSelector}"]) + self.assertEqual( + "10001:127.0.0.1\n10002:127.0.0.1", + self.PIPELINE_TREE["constants"]["${VarSetModuleIpSelector}"]["value"]["var_ip_custom_value"], + ) + + +class CCVarSetModuleSelectorSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "custom_type": "set_module_selector", + "desc": "", + "index": 1, + "key": "${a}", + "name": "1", + "show_type": "show", + "source_info": {}, + "source_tag": "var_set_module_selector.set_module_selector", + "source_type": "custom", + "validation": "", + "is_condition_hide": "false", + "pre_render_mako": False, + "value": {"bk_set_id": 96, "bk_module_id": [219, 218]}, + "version": "legacy", + "is_meta": False, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["constants"]["${VarSetModuleSelector}"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.CCVarSetModuleSelectorSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do("${VarSetModuleSelector}", self.PIPELINE_TREE["constants"]["${VarSetModuleSelector}"]) + self.assertEqual( + 10096, + self.PIPELINE_TREE["constants"]["${VarSetModuleSelector}"]["value"]["bk_set_id"], + ) + self.assertEqual( + [10219, 10218], + self.PIPELINE_TREE["constants"]["${VarSetModuleSelector}"]["value"]["bk_module_id"], + ) diff --git a/pipeline_plugins/tests/resource_replacement/test_job_suites.py b/pipeline_plugins/tests/resource_replacement/test_job_suites.py new file mode 100644 index 0000000000..437e19a4f1 --- /dev/null +++ b/pipeline_plugins/tests/resource_replacement/test_job_suites.py @@ -0,0 +1,488 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy + +from django.test import TestCase + +from pipeline_plugins.resource_replacement import base, suites + +from . import base as local_base + + +class JobLocalContentUploadSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "job_local_content_upload", + "data": { + "local_name": {"hook": False, "need_render": True, "value": ".conf"}, + "local_content": {"hook": False, "need_render": True, "value": "111"}, + "job_ip_list": {"hook": False, "need_render": True, "value": "1:127.0.0.1\n127.0.0.1"}, + "file_account": {"hook": False, "need_render": True, "value": "administrator"}, + "file_path": {"hook": False, "need_render": True, "value": "/tmp/"}, + "job_rolling_config": { + "hook": False, + "need_render": True, + "value": {"job_rolling_execute": [], "job_rolling_expression": "", "job_rolling_mode": 1}, + }, + }, + "version": "v1.1", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobLocalContentUploadSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + "127.0.0.1\n10001:127.0.0.1", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_ip_list"]["value"], + ) + + +class JobPushLocalFilesSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "job_push_local_files", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "job_local_files_info": { + "hook": False, + "need_render": True, + "value": { + "job_local_files": [], + "job_target_path": "/tmp/", + "add_files": "", + "job_push_multi_local_files_table": [ + { + "show_file": "xxx.png", + "file_info": [ + { + "status": "success", + "name": "xxx.png", + "size": 286460, + "percentage": 100, + "uid": 1681437569692, + "raw": {"uid": 1681437569692}, + "response": { + "result": True, + "tag": { + "type": "job_repo", + "tags": {"file_path": "xxx.png", "name": "xxx.png"}, + }, + "md5": "59565321355a381aab43dbcd975445e3", + }, + } + ], + "target_path": "/tmp/", + "md5": "59565321355a381aab43dbcd975445e3", + } + ], + }, + }, + "job_across_biz": {"hook": False, "need_render": True, "value": False}, + "job_target_ip_list": {"hook": True, "need_render": True, "value": "2:127.0.0.1"}, + "job_target_account": {"hook": False, "need_render": True, "value": "root"}, + "job_timeout": {"hook": False, "need_render": True, "value": ""}, + }, + "version": "2.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobPushLocalFilesSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + "10002:127.0.0.1", + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_target_ip_list"][ + "value" + ], + ) + + +class JobFastPushFileSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "job_fast_push_file", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "job_source_files": { + "hook": False, + "need_render": True, + "value": [ + {"ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + {"ip": "1:127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + ], + }, + "job_ip_list": {"hook": False, "need_render": True, "value": "127.0.0.1\n127.0.0.2"}, + "job_account": {"hook": False, "need_render": True, "value": "root"}, + "job_target_path": {"hook": False, "need_render": True, "value": "/tmp/tmp.conf"}, + "job_timeout": {"hook": False, "need_render": True, "value": ""}, + # v2 + "job_dispatch_attr": { + "hook": False, + "need_render": True, + "value": [ + {"job_ip_list": "1:127.0.0.1\n2:127.0.0.1", "job_target_path": "/tmp/", "job_account": "root"} + ], + }, + }, + "version": "legacy", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobFastPushFileSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + [ + {"ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + {"ip": "10001:127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_source_files"]["value"], + ) + self.assertEqual( + [{"job_ip_list": "10001:127.0.0.1\n10002:127.0.0.1", "job_target_path": "/tmp/", "job_account": "root"}], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_dispatch_attr"][ + "value" + ], + ) + + +class JobCronTaskSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "job_cron_task", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "job_cron_job_id": {"hook": False, "need_render": True, "value": 35}, + "job_cron_name": {"hook": False, "need_render": True, "value": "sops_task"}, + "job_cron_expression": {"hook": False, "need_render": True, "value": "1111"}, + "job_cron_status": {"hook": False, "need_render": True, "value": 2}, + }, + "version": "legacy", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobCronTaskSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + 100035, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_cron_job_id"]["value"], + ) + + +class JobExecuteTaskSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "job_execute_task", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "job_task_id": {"hook": False, "need_render": True, "value": 35}, + "button_refresh": {"hook": False, "need_render": True, "value": ""}, + "job_global_var": { + "hook": False, + "need_render": True, + "value": [ + {"id": 51, "category": 3, "name": "ip_list", "value": '"30308"', "description": ""}, + {"id": 52, "category": 3, "name": "ip_list", "value": "2:127.0.0.1", "description": ""}, + ], + }, + "job_success_id": {"hook": False, "need_render": True, "value": ""}, + "button_refresh_2": {"hook": False, "need_render": True, "value": ""}, + }, + "version": "1.2", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobExecuteTaskSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + 100035, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_task_id"]["value"], + ) + self.assertEqual( + [ + {"id": 100051, "category": 3, "name": "ip_list", "value": '"30308"', "description": ""}, + {"id": 100052, "category": 3, "name": "ip_list", "value": "10002:127.0.0.1", "description": ""}, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_global_var"]["value"], + ) + + +class JobAllBizJobFastPushFileSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "all_biz_job_fast_push_file", + "data": { + "all_biz_cc_id": {"hook": False, "need_render": True, "value": 9991001}, + "job_source_files": { + "hook": False, + "need_render": True, + "value": [ + {"bk_cloud_id": "0", "ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + {"bk_cloud_id": "2", "ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + ], + }, + "upload_speed_limit": {"hook": False, "need_render": True, "value": ""}, + "download_speed_limit": {"hook": False, "need_render": True, "value": ""}, + "job_dispatch_attr": { + "hook": False, + "need_render": True, + "value": [ + { + "bk_cloud_id": "0", + "job_ip_list": "127.0.0.1", + "job_target_path": "/tmp/", + "job_target_account": "root", + }, + { + "bk_cloud_id": "1", + "job_ip_list": "127.0.0.1", + "job_target_path": "/tmp/", + "job_target_account": "root", + }, + ], + }, + "job_timeout": {"hook": False, "need_render": True, "value": ""}, + "job_rolling_config": { + "hook": False, + "need_render": True, + "value": {"job_rolling_execute": [], "job_rolling_expression": "", "job_rolling_mode": 1}, + }, + }, + "version": "v1.1", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobAllBizJobFastPushFileSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + [ + {"bk_cloud_id": "0", "ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + {"bk_cloud_id": "10002", "ip": "127.0.0.1", "files": "/tmp/tmp.conf", "account": "root"}, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_source_files"]["value"], + ) + self.assertEqual( + [ + { + "bk_cloud_id": "0", + "job_ip_list": "127.0.0.1", + "job_target_path": "/tmp/", + "job_target_account": "root", + }, + { + "bk_cloud_id": "10001", + "job_ip_list": "127.0.0.1", + "job_target_path": "/tmp/", + "job_target_account": "root", + }, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_dispatch_attr"][ + "value" + ], + ) + + +class JobAllBizJobFastExecuteScriptSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "all_biz_job_fast_execute_script", + "data": { + "all_biz_cc_id": {"hook": False, "need_render": True, "value": 9991001}, + "job_script_type": {"hook": False, "need_render": True, "value": "1"}, + "job_content": {"hook": False, "need_render": True, "value": "ls -al\n"}, + "job_script_param": {"hook": False, "need_render": True, "value": ""}, + "job_script_timeout": {"hook": False, "need_render": True, "value": ""}, + "job_target_account": {"hook": False, "need_render": True, "value": "root"}, + "job_target_ip_table": { + "hook": False, + "need_render": True, + "value": [ + {"bk_cloud_id": "0", "ip": "127.0.0.1\n127.0.0.2"}, + {"bk_cloud_id": "999", "ip": "127.0.0.1\n127.0.0.2"}, + ], + }, + "job_rolling_config": { + "hook": False, + "need_render": True, + "value": {"job_rolling_execute": [], "job_rolling_expression": "", "job_rolling_mode": 1}, + }, + }, + "version": "v1.1", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobAllBizJobFastExecuteScriptSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + [ + {"bk_cloud_id": "0", "ip": "127.0.0.1\n127.0.0.2"}, + {"bk_cloud_id": "10999", "ip": "127.0.0.1\n127.0.0.2"}, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["job_target_ip_table"][ + "value" + ], + ) + + +class JobAllBizJobExecuteJobPlanSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "all_biz_execute_job_plan", + "data": { + "all_biz_job_config": { + "hook": False, + "need_render": True, + "value": { + "all_biz_cc_id": 9991001, + "pull_job_template_list": "", + "job_template_id": 41, + "job_plan_id": 46, + "job_global_var": [ + {"id": 62, "type": 2, "name": "ipv6", "description": ""}, + {"id": 63, "type": 3, "name": "iplist", "value": "587,612,613", "description": ""}, + ], + }, + } + }, + "version": "v1.1", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.JobAllBizJobExecuteJobPlanSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + { + "all_biz_cc_id": 9991001, + "pull_job_template_list": "", + "job_template_id": 100041, + "job_plan_id": 100046, + "job_global_var": [ + {"id": 100062, "type": 2, "name": "ipv6", "description": ""}, + {"id": 100063, "type": 3, "name": "iplist", "value": "587,612,613", "description": ""}, + ], + }, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["all_biz_job_config"][ + "value" + ], + ) diff --git a/pipeline_plugins/tests/resource_replacement/test_nodeman_suites.py b/pipeline_plugins/tests/resource_replacement/test_nodeman_suites.py new file mode 100644 index 0000000000..57e725de68 --- /dev/null +++ b/pipeline_plugins/tests/resource_replacement/test_nodeman_suites.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy + +from django.test import TestCase + +from pipeline_plugins.resource_replacement import base, suites + +from . import base as local_base + + +class NodemanPluginOperateSuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "nodeman_plugin_operate", + "data": { + "biz_cc_id": {"hook": False, "need_render": True, "value": 2}, + "nodeman_host_os_type": {"hook": False, "need_render": True, "value": "linux"}, + "nodeman_host_info": { + "hook": False, + "need_render": True, + "value": { + "nodeman_host_input_type": "host_ip", + "nodeman_bk_cloud_id": 3, + "nodeman_host_ip": "127.0.0.1", + "nodeman_host_id": "", + }, + }, + "nodeman_plugin_operate": { + "hook": False, + "need_render": True, + "value": { + "nodeman_op_type": "MAIN_INSTALL_PLUGIN", + "nodeman_plugin_type": "official", + "nodeman_plugin": "basereport", + "nodeman_plugin_version": "10.8.51", + "install_config": [], + }, + }, + }, + "version": "v1.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.NodemanPluginOperateSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["biz_cc_id"]["value"], + ) + self.assertEqual( + 10003, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["nodeman_host_info"][ + "value" + ]["nodeman_bk_cloud_id"], + ) + + +class NodemanCreateTaskV1SuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "nodeman_create_task", + "data": { + "bk_biz_id": {"hook": False, "need_render": True, "value": 2}, + "nodeman_op_target": { + "hook": False, + "need_render": True, + "value": {"nodeman_bk_cloud_id": 3, "nodeman_node_type": "AGENT"}, + }, + "nodeman_op_info": { + "hook": False, + "need_render": True, + "value": { + "nodeman_op_type": "INSTALL", + "nodeman_ap_id": 2, + "nodeman_hosts": [ + { + "inner_ip": "127.0.0.1", + } + ], + "nodeman_ip_str": "", + }, + }, + }, + "version": "v2.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.NodemanCreateTaskSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["bk_biz_id"]["value"], + ) + self.assertEqual( + 10003, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["nodeman_op_target"][ + "value" + ]["nodeman_bk_cloud_id"], + ) + + +class NodemanCreateTaskV3SuiteTestCase(TestCase): + PIPELINE_TREE = None + COMPONENT = { + "code": "nodeman_create_task", + "data": { + "bk_biz_id": {"hook": False, "need_render": True, "value": 2}, + "nodeman_node_type": {"hook": False, "need_render": True, "value": "AGENT"}, + "nodeman_op_info": { + "hook": False, + "need_render": True, + "value": { + "nodeman_op_type": "INSTALL", + "nodeman_hosts": [ + { + "nodeman_bk_cloud_id": 2, + "nodeman_ap_id": 3, + "inner_ip": "127.0.0.1", + }, + { + "nodeman_bk_cloud_id": 0, + "nodeman_ap_id": 3, + "inner_ip": "127.0.0.1", + }, + ], + "nodeman_other_hosts": [], + }, + }, + }, + "version": "v3.0", + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self) -> None: + self.PIPELINE_TREE = deepcopy(local_base.PIPELINE_TREE_MOCK_DATA) + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"] = self.COMPONENT + super().setUp() + + def test_do(self): + suite_meta: base.SuiteMeta = base.SuiteMeta( + self.PIPELINE_TREE, offset=10000, old_biz_id__new_biz_info_map=local_base.OLD_BIZ_ID__NEW_BIZ_INFO_MAP + ) + suite: base.Suite = suites.NodemanCreateTaskSuite(suite_meta, local_base.DBMockHelper(None, "", "")) + suite.do(local_base.FIRST_ACT_ID, self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]) + self.assertEqual( + 1002, + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["bk_biz_id"]["value"], + ) + self.assertEqual( + [ + { + "nodeman_bk_cloud_id": 10002, + "nodeman_ap_id": 3, + "inner_ip": "127.0.0.1", + }, + { + "nodeman_bk_cloud_id": 0, + "nodeman_ap_id": 3, + "inner_ip": "127.0.0.1", + }, + ], + self.PIPELINE_TREE["activities"][local_base.FIRST_ACT_ID]["component"]["data"]["nodeman_op_info"]["value"][ + "nodeman_hosts" + ], + ) diff --git a/pipeline_plugins/variables/collections/sites/open/cc.py b/pipeline_plugins/variables/collections/sites/open/cc.py index a9592e5e6d..ed4a64dec3 100644 --- a/pipeline_plugins/variables/collections/sites/open/cc.py +++ b/pipeline_plugins/variables/collections/sites/open/cc.py @@ -18,25 +18,30 @@ from django.conf import settings from django.contrib.admin.utils import flatten from django.utils.translation import ugettext_lazy as _ +from pipeline.core.data.var import LazyVariable +from gcloud.conf import settings as gcloud_settings from gcloud.constants import Type from gcloud.core.models import Project from gcloud.utils.cmdb import get_business_host, get_business_host_by_hosts_ids - -from gcloud.utils.ip import get_ip_by_regex, get_plat_ip_by_regex, extract_ip_from_ip_str -from gcloud.conf import settings as gcloud_settings -from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6 -from pipeline_plugins.variables.collections.sites.open.ip_filter_base import ( - GseAgentStatusIpFilter, - GseAgentStatusIpV6Filter, +from gcloud.utils.ip import ( + extract_ip_from_ip_str, + get_ip_by_regex, + get_plat_ip_by_regex, ) -from pipeline.core.data.var import LazyVariable -from pipeline_plugins.cmdb_ip_picker.utils import get_ip_picker_result -from pipeline_plugins.base.utils.inject import supplier_account_for_project from pipeline_plugins.base.utils.adapter import cc_get_inner_ip_by_module_id +from pipeline_plugins.base.utils.inject import supplier_account_for_project +from pipeline_plugins.cmdb_ip_picker.utils import get_ip_picker_result +from pipeline_plugins.components.collections.sites.open.cc.base import ( + cc_get_host_by_innerip_with_ipv6, +) from pipeline_plugins.components.utils import cc_get_ips_info_by_str from pipeline_plugins.components.utils.common import ip_re -from pipeline_plugins.variables.base import SelfExplainVariable, FieldExplain +from pipeline_plugins.variables.base import FieldExplain, SelfExplainVariable +from pipeline_plugins.variables.collections.sites.open.ip_filter_base import ( + GseAgentStatusIpFilter, + GseAgentStatusIpV6Filter, +) logger = logging.getLogger("root") get_client_by_user = gcloud_settings.ESB_GET_CLIENT_BY_USER diff --git a/pipeline_web/wrapper.py b/pipeline_web/wrapper.py index 98d284e332..8f1f82d9cc 100644 --- a/pipeline_web/wrapper.py +++ b/pipeline_web/wrapper.py @@ -11,31 +11,29 @@ specific language governing permissions and limitations under the License. """ +import copy import datetime import hashlib -import copy import logging import ujson as json from django.apps import apps from django.conf import settings from django.db.models import Q - -from pipeline.utils.uniqid import uniqid -from pipeline.parser.utils import replace_all_id +from pipeline.exceptions import PipelineException, SubprocessExpiredError from pipeline.models import PipelineTemplate, Snapshot, TemplateScheme -from pipeline.exceptions import SubprocessExpiredError, PipelineException +from pipeline.parser.utils import replace_all_id +from pipeline.utils.uniqid import uniqid +from gcloud.template_base.utils import replace_template_id from gcloud.utils.algorithms import topology_sort from pipeline_web.constants import PWE from pipeline_web.core.abstract import NodeAttr from pipeline_web.core.models import NodeInTemplate -from pipeline_web.parser.clean import PipelineWebTreeCleaner from pipeline_web.drawing_new.drawing import draw_pipeline +from pipeline_web.parser.clean import PipelineWebTreeCleaner from pipeline_web.preview_base import PipelineTemplateWebPreviewer -from gcloud.template_base.utils import replace_template_id - WEB_TREE_FIELDS = {"location", "line"} logger = logging.getLogger("root") @@ -177,7 +175,7 @@ def _unfold_subprocess(pipeline_data, template_model, recursive_limit): def _export_template(cls, template_obj, subprocess, refs, template_versions, root=True): """ 导出模板 wrapper 函数 - @param template_id: 需要导出的模板 id + @param template_obj: 需要导出的模板 @param subprocess: 子流程记录字典 @param refs: 引用关系记录字典: 被引用模板 -> 引用模板 -> 引用节点 @param root: 是否是根模板 @@ -189,6 +187,7 @@ def _export_template(cls, template_obj, subprocess, refs, template_versions, roo "template %s has expired subprocess, please update it before exporting." % template_obj.name ) template = { + "id": template_obj.id, "create_time": template_obj.create_time.strftime(cls.SERIALIZE_DATE_FORMAT), "edit_time": template_obj.edit_time.strftime(cls.SERIALIZE_DATE_FORMAT), "creator": template_obj.creator, @@ -197,6 +196,12 @@ def _export_template(cls, template_obj, subprocess, refs, template_versions, roo "is_deleted": template_obj.is_deleted, "name": template_obj.name, "template_id": template_obj.template_id, + # 执行方案 + "schemes": list( + TemplateScheme.objects.filter(template_id=template_obj.id).values( + "id", "unique_id", "name", "data", "template_id" + ) + ), } tree = template_obj.data @@ -295,21 +300,25 @@ def _update_order_from_refs(cls, refs, id_maps=None): return topology_sort(forward_refs) @classmethod - def _update_or_create_version(cls, template, order): + def _update_or_create_version(cls, tid__template_map, order): """ 根据传入的顺序更新子流程引用模板的版本 - @param template: 模板数据字典 + @param tid__template_map: 模板数据字典 @param order: 更新顺序 @return: """ for tid in order: - for act_id, act in list(template[tid]["tree"][PWE.activities].items()): - if act[PWE.type] == PWE.SubProcess: - subprocess_data = template[act["template_id"]]["tree"] - h = hashlib.md5() - h.update(json.dumps(subprocess_data).encode("utf-8")) - md5sum = h.hexdigest() - act["version"] = md5sum + cls._update_or_create_version_single(tid, tid__template_map) + + @classmethod + def _update_or_create_version_single(cls, tid, tid__template_map): + for act_id, act in list(tid__template_map[tid]["tree"][PWE.activities].items()): + if act[PWE.type] == PWE.SubProcess: + subprocess_data = tid__template_map[act["template_id"]]["tree"] + h = hashlib.md5() + h.update(json.dumps(subprocess_data).encode("utf-8")) + md5sum = h.hexdigest() + act["version"] = md5sum @classmethod def complete_canvas_data(cls, template_data): @@ -394,10 +403,26 @@ def import_templates(cls, template_data, override=False, tid_to_reuse=None): pipeline_web_tree.clean() origin_data[tid] = pipeline_web_tree.origin_data - cls._update_or_create_version(template, cls._update_order_from_refs(refs, temp_id_old_to_new)) - - # import template - for tid, template_dict in list(template.items()): + scheme_id_old_to_new = {} + tid_order = cls._update_order_from_refs(refs, temp_id_old_to_new) + tid_order_set = set(tid_order) + # 计算出没有（被）引用的流程 ID 集合 + single_temp_ids = set(temp_id_old_to_new.values()) - tid_order_set + # 按照关系拓扑顺序创建 Pipeline，从而保证执行方案的再创建引用以及子流程版本更新 + for tid in list(single_temp_ids) + tid_order: + template_dict = template[tid] + + # 替换执行方案 ID + for act_id, act in list(template_dict["tree"][PWE.activities].items()): + if act[PWE.type] == PWE.SubProcess and act.get("scheme_id_list"): + act["scheme_id_list"] = [ + scheme_id_old_to_new.get(old_scheme_id, old_scheme_id) + for old_scheme_id in act["scheme_id_list"] + ] + + # 替换引用子流程版本，仅涉及引用/被引关系的流程需要处理 + if tid in tid_order_set: + cls._update_or_create_version_single(tid, template) defaults = cls._kwargs_for_template_dict(template_dict, include_str_id=True) pipeline_template = PipelineTemplate.objects.create(**defaults) @@ -406,12 +431,14 @@ def import_templates(cls, template_data, override=False, tid_to_reuse=None): NodeInTemplate.objects.create_nodes_in_template(pipeline_template, origin_data[tid]) # import template scheme - schemes = [] + scheme_objs_to_be_created = [] + unique_id__old_scheme_id_map = {} for scheme_data in template_dict.get("schemes", []): scheme_node_data = scheme_data["data"] try: new_scheme_node_ids = [] scheme_node_ids = json.loads(scheme_data["data"]) + # 非覆盖场景需要将执行方案中的 node_id 替换为新生成的 node_id for node_id in scheme_node_ids: new_scheme_node_ids.append( template_node_id_old_to_new[pipeline_template.template_id]["activities"].get( @@ -422,24 +449,35 @@ def import_templates(cls, template_data, override=False, tid_to_reuse=None): except Exception: logger.exception("scheme node id replace error for template(%s)" % pipeline_template.name) - schemes.append( + unique_id = uniqid() + unique_id__old_scheme_id_map[unique_id] = scheme_data["id"] + scheme_objs_to_be_created.append( TemplateScheme( template_id=pipeline_template.id, - unique_id=uniqid(), + unique_id=unique_id, name=scheme_data["name"], data=scheme_node_data, ) ) - if schemes: - TemplateScheme.objects.bulk_create(schemes, batch_size=5000) + if scheme_objs_to_be_created: + TemplateScheme.objects.bulk_create(scheme_objs_to_be_created, batch_size=5000) + # 反查出新创建的执行方案，并建立新老 ID 的映射关系 + for scheme_data in TemplateScheme.objects.filter( + unique_id__in=unique_id__old_scheme_id_map.keys() + ).values("unique_id", "id"): + old_scheme_id = unique_id__old_scheme_id_map[scheme_data["unique_id"]] + scheme_id_old_to_new[old_scheme_id] = scheme_data["id"] else: # 1. replace subprocess template id tid_to_reuse = tid_to_reuse or {} + scheme_id_old_to_new = {} + # pipeline_template_id, template_id for import_id, reuse_id in list(tid_to_reuse.items()): # referenced template -> referencer -> reference act referencer_info_dict = refs.get(import_id, {}) + # 引用到公共流程的 Pipeline 模板 ID，节点 for referencer, nodes in list(referencer_info_dict.items()): for node_id in nodes: template[referencer]["tree"][PWE.activities][node_id]["template_id"] = reuse_id @@ -475,9 +513,7 @@ def import_templates(cls, template_data, override=False, tid_to_reuse=None): # create node in template NodeInTemplate.objects.update_nodes_in_template(pipeline_template, origin_data[tid]) - return { - cls.ID_MAP_KEY: temp_id_old_to_new, - } + return {cls.ID_MAP_KEY: temp_id_old_to_new, "scheme_id_old_to_new": scheme_id_old_to_new} class PipelineInstanceWebWrapper(object):