Skip to content

Commit

Permalink
Merge pull request #2788 from actiontech/backup-dev
Browse files Browse the repository at this point in the history 
merge backup-dev to main
  • Loading branch information
@ColdWaterLW
ColdWaterLW authored Nov 29, 2024
2 parents b68ff29 + 9b80bd1 commit cb0747e
Show file tree
Hide file tree
Showing 37 changed files with 1,339 additions and 603 deletions.
4 changes: 4 additions & 0 deletions sqle/api/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
v1ProjectOpRouter.PUT("/:project_name/workflows/:workflow_name/tasks/:task_id/schedule", DeprecatedBy(apiV2))
v1ProjectOpRouter.PATCH("/:project_name/workflows/:workflow_name/", DeprecatedBy(apiV2))
v1ProjectOpRouter.POST("/:project_name/workflows/:workflow_id/tasks/:task_id/order_file", v1.UpdateSqlFileOrderByWorkflowV1)
v1ProjectOpRouter.GET("/:project_name/workflows/:workflow_id/backup_sqls", v1.GetBackupSqlList)
v1ProjectOpRouter.POST("/:project_name/workflows/:workflow_id/create_rollback_workflow", v1.CreateRollbackWorkflow)

// sql version
v1ProjectOpRouter.POST("/:project_name/sql_versions/:sql_version_id/batch_release_workflows", v1.BatchReleaseWorkflows)
Expand Down Expand Up @@ -448,6 +450,8 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
// task
v1Router.GET("/tasks/audits/:task_id/", v1.GetTask)
v1Router.GET("/tasks/audits/:task_id/sqls", v1.GetTaskSQLs)
v1Router.PATCH("/tasks/audits/:task_id/sqls/:sql_id/backup_strategy", v1.UpdateSqlBackupStrategy)
v1Router.PATCH("/tasks/audits/:task_id/backup_strategy", v1.UpdateTaskBackupStrategy)
v2Router.GET("/tasks/audits/:task_id/sqls", v2.GetTaskSQLs)
v2Router.GET("/tasks/audits/:task_id/files", v2.GetAuditFileList)
v2Router.GET("/tasks/audits/:task_id/files/:file_id/", v2.GetAuditFileExecStatistic)
Expand Down
40 changes: 3 additions & 37 deletions sqle/api/controller/v1/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type UpdateSqlBackupStrategyReq struct {
// @Success 200 {object} controller.BaseRes
// @router /v1/tasks/audits/{task_id}/sqls/{sql_id}/backup_strategy [patch]
func UpdateSqlBackupStrategy(c echo.Context) error {
return nil
return updateSqlBackupStrategy(c)
}

type UpdateTaskBackupStrategyReq struct {
Expand All @@ -43,7 +43,7 @@ type UpdateTaskBackupStrategyReq struct {
// @Success 200 {object} controller.BaseRes
// @router /v1/tasks/audits/{task_id}/backup_strategy [patch]
func UpdateTaskBackupStrategy(c echo.Context) error {
return nil
return updateTaskBackupStrategy(c)
}

// @Summary 下载工单中的SQL备份
Expand Down Expand Up @@ -100,39 +100,5 @@ type BackupSqlListRes struct {
// @Success 200 {object} v1.BackupSqlListRes
// @router /v1/projects/{project_name}/workflows/{workflow_id}/backup_sqls [get]
func GetBackupSqlList(c echo.Context) error {
return nil
}

type CreateRollbackWorkflowReq struct {
Subject string `json:"workflow_subject" form:"workflow_subject" valid:"required,name"`
Desc string `json:"desc" form:"desc"`
SqlVersionID *uint `json:"sql_version_id" form:"sql_version_id"`
TaskIds []uint `json:"task_ids" form:"task_ids" valid:"required"`
RollbackSqlIds []uint `json:"rollback_sql_ids" form:"rollback_sql_ids" valid:"required"`
}

type CreateRollbackWorkflowRes struct {
controller.BaseRes
Data *CreateRollbackWorkflowResData `json:"data"`
}

type CreateRollbackWorkflowResData struct {
WorkflowID string `json:"workflow_id"`
}

// CreateRollbackWorkflow
// @Summary 创建回滚工单
// @Description create rollback workflow
// @Accept json
// @Produce json
// @Tags workflow
// @Id CreateRollbackWorkflow
// @Security ApiKeyAuth
// @Param instance body v1.CreateRollbackWorkflowReq true "create rollback workflow request"
// @Param project_name path string true "project name"
// @Param workflow_id path string true "origin workflow id to rollback"
// @Success 200 {object} CreateRollbackWorkflowRes
// @router /v1/projects/{project_name}/workflows/{workflow_id}/rollback [post]
func CreateRollbackWorkflow(c echo.Context) error {
return nil
return getBackupSqlList(c)
}
27 changes: 27 additions & 0 deletions sqle/api/controller/v1/backup_ce.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//go:build !enterprise
// +build !enterprise

package v1

import (
"github.com/actiontech/sqle/sqle/api/controller"
"github.com/actiontech/sqle/sqle/errors"

"github.com/labstack/echo/v4"
)

func getBackupSqlList(c echo.Context) error {
return controller.JSONBaseErrorReq(c, errors.NewNotSupportBackupErr())
}

func createRollbackWorkflow(c echo.Context) error {
return controller.JSONBaseErrorReq(c, errors.NewNotSupportBackupErr())
}

func updateSqlBackupStrategy(c echo.Context) error {
return controller.JSONBaseErrorReq(c, errors.NewNotSupportBackupErr())
}

func updateTaskBackupStrategy(c echo.Context) error {
return controller.JSONBaseErrorReq(c, errors.NewNotSupportBackupErr())
}
1 change: 1 addition & 0 deletions sqle/api/controller/v1/instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ func GetInstanceTips(c echo.Context) error {
Host: inst.Host,
Port: inst.Port,
WorkflowTemplateId: uint32(template.ID),
EnableBackup: inst.EnableBackup,
}
instanceTipsResV1 = append(instanceTipsResV1, instanceTipRes)
}
Expand Down
115 changes: 67 additions & 48 deletions sqle/api/controller/v1/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ type AuditTaskResV1 struct {
FileOrderMethod string `json:"file_order_method,omitempty"`
ExecMode string `json:"exec_mode,omitempty"`
EnableBackup bool `json:"enable_backup"`
BackupConflictWithInstance bool `json:"backup_conflict_with_instance"`
BackupConflictWithInstance bool `json:"backup_conflict_with_instance"` // 当数据源备份开启，工单备份关闭，则需要提示审核人工单备份策略与数据源备份策略不一致
AuditFiles []AuditFileResp `json:"audit_files,omitempty"`
}

Expand All @@ -74,21 +74,22 @@ type AuditFileResp struct {

func convertTaskToRes(task *model.Task) *AuditTaskResV1 {
return &AuditTaskResV1{
Id: task.ID,
InstanceName: task.InstanceName(),
InstanceDbType: task.DBType,
InstanceSchema: task.Schema,
AuditLevel: task.AuditLevel,
Score: task.Score,
PassRate: task.PassRate,
Status: task.Status,
SQLSource: task.SQLSource,
ExecStartTime: task.ExecStartAt,
ExecEndTime: task.ExecEndAt,
ExecMode: task.ExecMode,
EnableBackup: task.EnableBackup,
FileOrderMethod: task.FileOrderMethod,
AuditFiles: convertToAuditFileResp(task.AuditFiles),
Id: task.ID,
InstanceName: task.InstanceName(),
InstanceDbType: task.DBType,
InstanceSchema: task.Schema,
AuditLevel: task.AuditLevel,
Score: task.Score,
PassRate: task.PassRate,
Status: task.Status,
SQLSource: task.SQLSource,
ExecStartTime: task.ExecStartAt,
ExecEndTime: task.ExecEndAt,
ExecMode: task.ExecMode,
EnableBackup: task.EnableBackup,
BackupConflictWithInstance: server.BackupService{}.IsBackupConflictWithInstance(task.EnableBackup, task.InstanceEnableBackup),
FileOrderMethod: task.FileOrderMethod,
AuditFiles: convertToAuditFileResp(task.AuditFiles),
}
}
func convertToAuditFileResp(files []*model.AuditFile) []AuditFileResp {
Expand Down Expand Up @@ -359,6 +360,7 @@ func CreateAndAuditTask(c echo.Context) error {
}
task.EnableBackup = req.EnableBackup
}
task.InstanceEnableBackup = tmpInst.EnableBackup

err = convertSQLSourceEncodingFromTask(task)
if err != nil {
Expand Down Expand Up @@ -484,15 +486,15 @@ type GetAuditTaskSQLsResV1 struct {
}

type AuditTaskSQLResV1 struct {
Number uint `json:"number"`
ExecSQL string `json:"exec_sql"`
AuditResult string `json:"audit_result"`
AuditLevel string `json:"audit_level"`
AuditStatus string `json:"audit_status"`
ExecResult string `json:"exec_result"`
ExecStatus string `json:"exec_status"`
RollbackSQL string `json:"rollback_sql,omitempty"`
Description string `json:"description"`
Number uint `json:"number"`
ExecSQL string `json:"exec_sql"`
AuditResult string `json:"audit_result"`
AuditLevel string `json:"audit_level"`
AuditStatus string `json:"audit_status"`
ExecResult string `json:"exec_result"`
ExecStatus string `json:"exec_status"`
RollbackSQLs []string `json:"rollback_sqls,omitempty"`
Description string `json:"description"`
}

// @Summary 获取指定扫描任务的SQLs信息
Expand Down Expand Up @@ -542,19 +544,22 @@ func GetTaskSQLs(c echo.Context) error {
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}

rollbackSqlMap, err := server.BackupService{}.GetRollbackSqlsMap(task.ID)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
taskSQLsRes := make([]*AuditTaskSQLResV1, 0, len(taskSQLs))
for _, taskSQL := range taskSQLs {
taskSQLRes := &AuditTaskSQLResV1{
Number: taskSQL.Number,
Description: taskSQL.Description,
ExecSQL: taskSQL.ExecSQL,
AuditResult: taskSQL.GetAuditResults(ctx),
AuditLevel: taskSQL.AuditLevel,
AuditStatus: taskSQL.AuditStatus,
ExecResult: taskSQL.ExecResult,
ExecStatus: taskSQL.ExecStatus,
RollbackSQL: taskSQL.RollbackSQL.String,
Number: taskSQL.Number,
Description: taskSQL.Description,
ExecSQL: taskSQL.ExecSQL,
AuditResult: taskSQL.GetAuditResults(ctx),
AuditLevel: taskSQL.AuditLevel,
AuditStatus: taskSQL.AuditStatus,
ExecResult: taskSQL.ExecResult,
ExecStatus: taskSQL.ExecStatus,
RollbackSQLs: rollbackSqlMap[taskSQL.Id],
}
taskSQLsRes = append(taskSQLsRes, taskSQLRes)
}
Expand Down Expand Up @@ -623,6 +628,10 @@ func DownloadTaskSQLReportFile(c echo.Context) error {
if err != nil {
return controller.JSONBaseErrorReq(c, errors.New(errors.WriteDataToTheFileError, err))
}
rollbackSqlMap, err := server.BackupService{}.GetRollbackSqlsMap(task.ID)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
for _, td := range taskSQLsDetail {
taskSql := &model.ExecuteSQL{
AuditResults: td.AuditResults,
Expand All @@ -636,7 +645,7 @@ func DownloadTaskSQLReportFile(c echo.Context) error {
taskSql.GetAuditResultDesc(ctx),
taskSql.GetExecStatusDesc(ctx),
td.ExecResult,
td.RollbackSQL.String,
strings.Join(rollbackSqlMap[taskSql.ID], "\n"),
td.Description,
})
if err != nil {
Expand Down Expand Up @@ -1043,6 +1052,10 @@ func AuditTaskGroupV1(c echo.Context) error {
return controller.JSONBaseErrorReq(c, err)
}
defer plugin.Close(context.TODO())
instanceMap := make(map[uint64]*model.Instance)
for _, instance := range instances {
instanceMap[instance.ID] = instance
}

for _, task := range tasks {
task.SQLSource = sqls.SourceType
Expand All @@ -1058,6 +1071,11 @@ func AuditTaskGroupV1(c echo.Context) error {
}
task.EnableBackup = req.EnableBackup
}
if instance, exist := instanceMap[task.InstanceId]; exist {
task.InstanceEnableBackup = instance.EnableBackup
} else {
return controller.JSONBaseErrorReq(c, fmt.Errorf("can not find instance in task"))
}
err := addSQLsFromFileToTasks(sqls, task, plugin)
if err != nil {
return controller.JSONBaseErrorReq(c, errors.New(errors.GenericError, fmt.Errorf("add sqls from file to task failed: %v", err)))
Expand Down Expand Up @@ -1104,18 +1122,19 @@ func AuditTaskGroupV1(c echo.Context) error {
tasksRes := make([]*AuditTaskResV1, len(tasks))
for i, task := range tasks {
tasksRes[i] = &AuditTaskResV1{
Id: task.ID,
InstanceName: task.InstanceName(),
InstanceDbType: task.DBType,
InstanceSchema: task.Schema,
AuditLevel: task.AuditLevel,
Score: task.Score,
PassRate: task.PassRate,
Status: task.Status,
EnableBackup: task.EnableBackup,
SQLSource: task.SQLSource,
ExecStartTime: task.ExecStartAt,
ExecEndTime: task.ExecEndAt,
Id: task.ID,
InstanceName: task.InstanceName(),
InstanceDbType: task.DBType,
InstanceSchema: task.Schema,
AuditLevel: task.AuditLevel,
Score: task.Score,
PassRate: task.PassRate,
Status: task.Status,
EnableBackup: task.EnableBackup,
BackupConflictWithInstance: server.BackupService{}.IsBackupConflictWithInstance(task.EnableBackup, task.InstanceEnableBackup),
SQLSource: task.SQLSource,
ExecStartTime: task.ExecStartAt,
ExecEndTime: task.ExecEndAt,
}
}

Expand Down
Loading

0 comments on commit cb0747e

Please sign in to comment.