feature: 导入导出支持 (#6726)

* feature: 导入导出支持

* minor: review 修改

* minor: 修复验证导入过程出现的问题

gcloud/apigw/views/import_common_template.py

@@ -13,17 +13,20 @@
 
 
 import ujson as json
+from apigw_manager.apigw.decorators import apigw_require
+from blueapps.account.decorators import login_exempt
 from django.forms.fields import BooleanField
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
 
-from blueapps.account.decorators import login_exempt
 from gcloud import err_code
 from gcloud.apigw.decorators import mark_request_whether_is_trust, return_json_response
-from gcloud.common_template.models import CommonTemplate
-from gcloud.template_base.utils import read_encoded_template_data
 from gcloud.apigw.views.utils import logger
-from apigw_manager.apigw.decorators import apigw_require
+from gcloud.common_template.models import CommonTemplate
+from gcloud.template_base.utils import (
+    format_import_result_to_response_data,
+    read_encoded_template_data,
+)
 
 
 @login_exempt
@@ -70,4 +73,4 @@ def import_common_template(request):
             "code": err_code.UNKNOWN_ERROR.code,
         }
 
-    return import_result
+    return format_import_result_to_response_data(import_result)

gcloud/apigw/views/import_project_template.py

@@ -13,17 +13,23 @@
 
 
 import ujson as json
+from apigw_manager.apigw.decorators import apigw_require
+from blueapps.account.decorators import login_exempt
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
 
-from blueapps.account.decorators import login_exempt
 from gcloud import err_code
-from gcloud.apigw.decorators import project_inject
-from gcloud.apigw.decorators import mark_request_whether_is_trust, return_json_response
-from gcloud.template_base.utils import read_encoded_template_data
-from gcloud.tasktmpl3.models import TaskTemplate
+from gcloud.apigw.decorators import (
+    mark_request_whether_is_trust,
+    project_inject,
+    return_json_response,
+)
 from gcloud.apigw.views.utils import logger
-from apigw_manager.apigw.decorators import apigw_require
+from gcloud.tasktmpl3.models import TaskTemplate
+from gcloud.template_base.utils import (
+    format_import_result_to_response_data,
+    read_encoded_template_data,
+)
 
 
 @login_exempt
@@ -72,4 +78,4 @@ def import_project_template(request, project_id):
             "code": err_code.UNKNOWN_ERROR.code,
         }
 
-    return import_result
+    return format_import_result_to_response_data(import_result)

gcloud/clocked_task/models.py

@@ -15,15 +15,18 @@
 
 from django.apps import apps
 from django.db import models, transaction
-from django_celery_beat.models import (
-    PeriodicTask as DjangoCeleryBeatPeriodicTask,
-    ClockedSchedule as DjangoCeleryBeatClockedSchedule,
+from django_celery_beat.models import ClockedSchedule as DjangoCeleryBeatClockedSchedule
+from django_celery_beat.models import PeriodicTask as DjangoCeleryBeatPeriodicTask
+
+from gcloud.constants import (
+    CLOCKED_TASK_NOT_STARTED,
+    CLOCKED_TASK_STATE,
+    PROJECT,
+    TEMPLATE_SOURCE,
 )
-
-from gcloud.constants import TEMPLATE_SOURCE, PROJECT, CLOCKED_TASK_STATE, CLOCKED_TASK_NOT_STARTED
-from gcloud.utils.unique import uniqid
-from gcloud.core.models import StaffGroupSet, Project
+from gcloud.core.models import Project, StaffGroupSet
 from gcloud.shortcuts.cmdb import get_business_group_members
+from gcloud.utils.unique import uniqid
 
 logger = logging.getLogger("root")
 
@@ -39,6 +42,11 @@ def create_task(self, **kwargs):
         template_name = kwargs["template_name"]
         notify_type = kwargs.get("notify_type", "[]")
         notify_receivers = kwargs.get("notify_receivers", "{}")
+
+        optional_keys = ["editor", "edit_time", "create_time"]
+        # 过滤 optional_keys 中不存在于 kwargs 的属性
+        extra_data = {key: kwargs[key] for key in filter(lambda x: x in kwargs, optional_keys)}
+
         with transaction.atomic():
             clocked, _ = DjangoCeleryBeatClockedSchedule.objects.get_or_create(clocked_time=plan_start_time)
             task = ClockedTask.objects.create(
@@ -51,6 +59,7 @@ def create_task(self, **kwargs):
                 task_params=task_params,
                 notify_type=notify_type,
                 notify_receivers=notify_receivers,
+                **extra_data,
             )
             clocked_task_kwargs = {"clocked_task_id": task.id}
             clocked_task = DjangoCeleryBeatPeriodicTask.objects.create(

gcloud/common_template/apis/django/api.py

@@ -17,38 +17,40 @@
 from rest_framework.decorators import api_view
 
 from gcloud.common_template.models import CommonTemplate
+from gcloud.iam_auth.intercept import iam_intercept
+from gcloud.iam_auth.view_interceptors.common_template import (
+    ExportInterceptor,
+    FormInterceptor,
+    ImportInterceptor,
+    ParentsInterceptor,
+)
 from gcloud.iam_auth.view_interceptors.template import BatchFormInterceptor
 from gcloud.openapi.schema import AnnotationAutoSchema
 from gcloud.template_base.apis.django.api import (
     base_batch_form,
-    base_form,
     base_check_before_import,
     base_export_templates,
+    base_form,
     base_import_templates,
     base_template_parents,
     is_full_param_process,
 )
 from gcloud.template_base.apis.django.validators import (
     BatchFormValidator,
+    ExportTemplateApiViewValidator,
     FormValidator,
     TemplateParentsValidator,
-    ExportTemplateApiViewValidator,
 )
 from gcloud.utils.decorators import request_validate
-from gcloud.iam_auth.intercept import iam_intercept
-from gcloud.iam_auth.view_interceptors.common_template import (
-    FormInterceptor,
-    ExportInterceptor,
-    ImportInterceptor,
-    ParentsInterceptor,
-)
-from .validators import ImportValidator, CheckBeforeImportValidator
+
+from .validators import CheckBeforeImportValidator, ImportValidator
 
 logger = logging.getLogger("root")
 
 
 @swagger_auto_schema(
-    methods=["get"], auto_schema=AnnotationAutoSchema,
+    methods=["get"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["GET"])
 @request_validate(FormValidator)
@@ -73,7 +75,8 @@ def form(request):
 
 
 @swagger_auto_schema(
-    methods=["post"], auto_schema=AnnotationAutoSchema,
+    methods=["post"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["POST"])
 @request_validate(BatchFormValidator)
@@ -108,7 +111,8 @@ def batch_form(request):
 
 
 @swagger_auto_schema(
-    methods=["post"], auto_schema=AnnotationAutoSchema,
+    methods=["post"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["POST"])
 @request_validate(ExportTemplateApiViewValidator)
@@ -129,11 +133,12 @@ def export_templates(request):
     return: DAT 文件
     {}
     """
-    return base_export_templates(request, CommonTemplate, "common", [])
+    return base_export_templates(request, CommonTemplate, "common")
 
 
 @swagger_auto_schema(
-    methods=["post"], auto_schema=AnnotationAutoSchema,
+    methods=["post"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["POST"])
 @request_validate(ImportValidator)
@@ -159,7 +164,8 @@ def import_templates(request):
 
 
 @swagger_auto_schema(
-    methods=["post"], auto_schema=AnnotationAutoSchema,
+    methods=["post"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["POST"])
 @request_validate(CheckBeforeImportValidator)
@@ -194,7 +200,8 @@ def check_before_import(request):
 
 
 @swagger_auto_schema(
-    methods=["get"], auto_schema=AnnotationAutoSchema,
+    methods=["get"],
+    auto_schema=AnnotationAutoSchema,
 )
 @api_view(["GET"])
 @request_validate(TemplateParentsValidator)

gcloud/common_template/models.py

@@ -11,11 +11,13 @@
 specific language governing permissions and limitations under the License.
 """
 
+import logging
+
 from django.utils.translation import ugettext_lazy as _
 
 from gcloud import err_code
-from gcloud.template_base.models import BaseTemplateManager, BaseTemplate
-import logging
+from gcloud.template_base.models import BaseTemplate, BaseTemplateManager
+from gcloud.template_base.utils import fill_default_version_to_service_activities
 
 logger = logging.getLogger("root")
 
@@ -33,19 +35,22 @@ def import_operation_check(self, template_data):
             data["override_template"] = []
         return data
 
+    def export_templates(self, template_id_list, **kwargs):
+        if kwargs.get("is_full"):
+            template_id_list = list(self.all().values_list("id", flat=True))
+        super().export_templates(template_id_list, **kwargs)
+
     def import_templates(self, template_data, override, operator=None):
         check_info = self.import_operation_check(template_data)
 
+        for template in template_data["pipeline_template_data"]["template"].values():
+            fill_default_version_to_service_activities(template["tree"])
+
         # operation validation check
         if override and (not check_info["can_override"]):
             message = _("流程导入失败, 不能使用项目流程覆盖公共流程, 请检查后重试 | import_templates")
             logger.error(message)
-            return {
-                "result": False,
-                "message": message,
-                "data": 0,
-                "code": err_code.INVALID_OPERATION.code,
-            }
+            return {"result": False, "message": message, "data": 0, "code": err_code.INVALID_OPERATION.code}
 
         def defaults_getter(template_dict):
             return {

gcloud/contrib/admin/migration_api/app_maker.py

@@ -12,60 +12,28 @@
 """
 
 import json
+import logging
 import traceback
 
 from blueapps.account.decorators import login_exempt
-from gcloud.conf import settings
 from django.http.response import JsonResponse
+from django.utils.translation import ugettext_lazy as _
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
+from pipeline.models import TemplateScheme
 
 from gcloud import err_code
-from gcloud.core.models import Project
+from gcloud.conf import settings
 from gcloud.contrib.appmaker.models import AppMaker
+from gcloud.core.models import Project
 from gcloud.tasktmpl3.models import TaskTemplate
-from pipeline.models import TemplateScheme
 
 from .decorators import require_migrate_token
-from django.utils.translation import ugettext_lazy as _
-import logging
 
 logger = logging.getLogger("root")
 
 
-@login_exempt
-@csrf_exempt
-@require_migrate_token
-@require_POST
-def migrate_app_maker(request):
-
-    try:
-        params = json.loads(request.body)
-    except Exception as e:
-        message = _(f"非法请求: 数据错误, 请求不是合法的Json格式, {e} | migrate_app_maker")
-        logger.error(message)
-        return JsonResponse(
-            {
-                "result": False,
-                "message": message,
-                "code": err_code.REQUEST_PARAM_INVALID.code,
-            }
-        )
-
-    bk_biz_id = params.get("bk_biz_id")
-
-    try:
-        project = Project.objects.get(bk_biz_id=bk_biz_id)
-    except Project.DoesNotExist:
-        return JsonResponse(
-            {
-                "result": False,
-                "message": "can not find project for bk_biz_id: {}".format(bk_biz_id),
-                "code": err_code.REQUEST_PARAM_INVALID.code,
-            }
-        )
-
-    app_makers = params.get("app_makers", [])
+def do_migrate_app_maker(project: Project, app_makers):
     migrate_result = []
     for app_maker in app_makers:
         # 尝试获取存在的轻应用
@@ -135,4 +103,41 @@ def migrate_app_maker(request):
         else:
             migrate_result.append({"name": app_maker["name"], "success": True, "error": None})
 
+    return migrate_result
+
+
+@login_exempt
+@csrf_exempt
+@require_migrate_token
+@require_POST
+def migrate_app_maker(request):
+
+    try:
+        params = json.loads(request.body)
+    except Exception as e:
+        message = _(f"非法请求: 数据错误, 请求不是合法的Json格式, {e} | migrate_app_maker")
+        logger.error(message)
+        return JsonResponse(
+            {
+                "result": False,
+                "message": message,
+                "code": err_code.REQUEST_PARAM_INVALID.code,
+            }
+        )
+
+    bk_biz_id = params.get("bk_biz_id")
+
+    try:
+        project = Project.objects.get(bk_biz_id=bk_biz_id)
+    except Project.DoesNotExist:
+        return JsonResponse(
+            {
+                "result": False,
+                "message": "can not find project for bk_biz_id: {}".format(bk_biz_id),
+                "code": err_code.REQUEST_PARAM_INVALID.code,
+            }
+        )
+
+    migrate_result = do_migrate_app_maker(project, params.get("app_makers", []))
+
     return JsonResponse({"result": True, "data": migrate_result})