From 99159a6e1ba0b2e36f18175342526b2309d45a50 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 02:22:04 +0530 Subject: [PATCH 01/29] fix: stage artifact logic (initial commit) --- .../sql/repository/CiArtifactRepository.go | 2 +- .../pipelineConfig/CdWorfkflowRepository.go | 12 +++ .../pipelineConfig/CiWorkflowRepository.go | 15 +++- internal/util/ValidateUtil.go | 3 +- .../devtronApps/PreStageTriggerService.go | 11 +-- pkg/eventProcessor/bean/workflowEventBean.go | 5 +- .../in/WorkflowEventProcessorService.go | 12 ++- pkg/pipeline/CdHandler.go | 32 +++++--- pkg/pipeline/CiHandler.go | 28 +++++-- pkg/pipeline/CiService.go | 7 +- pkg/pipeline/types/CiCdConfig.go | 15 +++- pkg/pipeline/util/CiCdUtil.go | 8 ++ pkg/workflow/cd/CdWorkflowRunnerService.go | 19 +++-- pkg/workflow/cd/adapter/adapter.go | 1 + pkg/workflow/cd/bean/bean.go | 1 + pkg/workflow/dag/WorkflowDagExecutor.go | 75 +++++++++++-------- pkg/workflow/dag/bean/bean.go | 3 +- {internal/util => util/urlUtil}/UrlUtil.go | 2 +- wire_gen.go | 2 +- 19 files changed, 177 insertions(+), 76 deletions(-) create mode 100644 pkg/pipeline/util/CiCdUtil.go rename {internal/util => util/urlUtil}/UrlUtil.go (98%) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index d6297f0781..ebd718c7e6 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -73,7 +73,7 @@ type CiArtifact struct { ScanEnabled bool `sql:"scan_enabled,notnull"` Scanned bool `sql:"scanned,notnull"` ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` - IsArtifactUploaded bool `sql:"is_artifact_uploaded"` + IsArtifactUploaded bool `sql:"is_artifact_uploaded"` // Deprecated; Use pipelineConfig.CiWorkflow instead. CredentialsSourceType string `sql:"credentials_source_type"` CredentialSourceValue string `sql:"credentials_source_value"` ComponentId int `sql:"component_id"` diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 59a4427515..c796d5f5b1 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -43,6 +43,7 @@ type CdWorkflowRepository interface { FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType apiBean.WorkflowType, limit int, runnerStatuses []string) ([]CdWorkflowRunner, error) SaveWorkFlowRunner(wfr *CdWorkflowRunner) (*CdWorkflowRunner, error) UpdateWorkFlowRunner(wfr *CdWorkflowRunner) error + UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error UpdateWorkFlowRunnersWithTxn(wfrs []*CdWorkflowRunner, tx *pg.Tx) error @@ -134,6 +135,8 @@ type CdWorkflowRunner struct { FinishedOn time.Time `sql:"finished_on"` Namespace string `sql:"namespace"` LogLocation string `sql:"log_file_path"` + CdArtifactLocation string `sql:"cd_artifact_location"` + IsArtifactUploaded *bool `sql:"is_artifact_uploaded"` TriggeredBy int32 `sql:"triggered_by"` CdWorkflowId int `sql:"cd_workflow_id"` PodName string `sql:"pod_name"` @@ -191,6 +194,7 @@ type CdWorkflowWithArtifact struct { MaterialInfo string `json:"material_info,omitempty"` DataSource string `json:"data_source,omitempty"` CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` WorkflowType string `json:"workflow_type,omitempty"` ExecutorType string `json:"executor_type,omitempty"` BlobStorageEnabled bool `json:"blobStorageEnabled"` @@ -502,6 +506,14 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunner(wfr *CdWorkflowRunner return err } +func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error { + _, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)). + Set("is_artifact_uploaded = ?", isArtifactUploaded). + Where("id = ?", wfrId). + Update() + return err +} + func (impl *CdWorkflowRepositoryImpl) GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) { var cdWfrs []*CdWorkflowRunner err := impl.dbConnection.Model(&cdWfrs). diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index f54ad9ffcb..436ff5cd65 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -32,6 +32,7 @@ type CiWorkflowRepository interface { SaveWorkFlow(wf *CiWorkflow) error FindLastTriggeredWorkflow(pipelineId int) (*CiWorkflow, error) UpdateWorkFlow(wf *CiWorkflow) error + UpdateArtifactUploaded(id int, isUploaded bool) error FindByStatusesIn(activeStatuses []string) ([]*CiWorkflow, error) FindByPipelineId(pipelineId int, offset int, size int) ([]WorkflowWithArtifact, error) FindById(id int) (*CiWorkflow, error) @@ -72,6 +73,7 @@ type CiWorkflow struct { GitTriggers map[int]GitCommit `sql:"git_triggers"` TriggeredBy int32 `sql:"triggered_by"` CiArtifactLocation string `sql:"ci_artifact_location"` + IsArtifactUploaded *bool `sql:"is_artifact_uploaded"` PodName string `sql:"pod_name"` CiBuildType string `sql:"ci_build_type"` EnvironmentId int `sql:"environment_id"` @@ -111,7 +113,8 @@ type WorkflowWithArtifact struct { CiArtifactId int `json:"ci_artifact_d"` BlobStorageEnabled bool `json:"blobStorageEnabled"` CiBuildType string `json:"ci_build_type"` - IsArtifactUploaded bool `json:"is_artifact_uploaded"` + IsArtifactUploadedV2 *bool `json:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future + IsArtifactUploaded bool `json:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table EnvironmentId int `json:"environmentId"` EnvironmentName string `json:"environmentName"` RefCiWorkflowId int `json:"referenceCiWorkflowId"` @@ -191,7 +194,7 @@ func (impl *CiWorkflowRepositoryImpl) FindByStatusesIn(activeStatuses []string) // FindByPipelineId gets only those workflowWithArtifact whose parent_ci_workflow_id is null, this is done to accommodate multiple ci_artifacts through a single workflow(parent), making child workflows for other ci_artifacts (this has been done due to design understanding and db constraint) single workflow single ci-artifact func (impl *CiWorkflowRepositoryImpl) FindByPipelineId(pipelineId int, offset int, limit int) ([]WorkflowWithArtifact, error) { var wfs []WorkflowWithArtifact - queryTemp := "select cia.id as ci_artifact_id, env.environment_name, cia.image, cia.is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id left join environment env on env.id = wf.environment_id where wf.ci_pipeline_id = ? and parent_ci_workflow_id is null order by wf.started_on desc offset ? limit ?;" + queryTemp := "select cia.id as ci_artifact_id, env.environment_name, cia.image, cia.is_artifact_uploaded as old_is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id left join environment env on env.id = wf.environment_id where wf.ci_pipeline_id = ? and parent_ci_workflow_id is null order by wf.started_on desc offset ? limit ?;" _, err := impl.dbConnection.Query(&wfs, queryTemp, pipelineId, offset, limit) if err != nil { return nil, err @@ -269,6 +272,14 @@ func (impl *CiWorkflowRepositoryImpl) UpdateWorkFlow(wf *CiWorkflow) error { return err } +func (impl *CiWorkflowRepositoryImpl) UpdateArtifactUploaded(id int, isUploaded bool) error { + _, err := impl.dbConnection.Model(&CiWorkflow{}). + Set("is_artifact_uploaded = ?", isUploaded). + Where("id = ?", id). + Update() + return err +} + func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) { err = impl.dbConnection.Model(&ciWorkflow). Column("ci_workflow.*", "CiPipeline"). diff --git a/internal/util/ValidateUtil.go b/internal/util/ValidateUtil.go index 4adaaf6f65..00339c051f 100644 --- a/internal/util/ValidateUtil.go +++ b/internal/util/ValidateUtil.go @@ -18,6 +18,7 @@ package util import ( "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/util/urlUtil" "regexp" "strings" @@ -65,7 +66,7 @@ func validateNonEmptyUrl(fl validator.FieldLevel) bool { if value == "" { return true } - return IsValidUrl(value) + return urlUtil.IsValidUrl(value) } func IntValidator() (*validator.Validate, error) { diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 196110fcd7..29d310b48a 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -51,6 +51,7 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "maps" + "path" "strconv" "strings" "time" @@ -936,23 +937,23 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if cdArtifactLocationFormat == "" { + if len(cdArtifactLocationFormat) == 0 { cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } if cdWorkflowConfig.LogsBucket == "" { cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } - ArtifactLocation := fmt.Sprintf("s3://%s/"+impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWorkflowConfig.LogsBucket, cdWf.Id, runner.Id) - artifactFileName := fmt.Sprintf(impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWf.Id, runner.Id) + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfig.LogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) + artifactFileName := fmt.Sprintf(cdArtifactLocationFormat, cdWf.Id, runner.Id) return ArtifactLocation, cdWorkflowConfig.LogsBucket, artifactFileName } func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if cdArtifactLocationFormat == "" { + if len(cdArtifactLocationFormat) == 0 { cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } - ArtifactLocation := fmt.Sprintf("%s/"+cdArtifactLocationFormat, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, runner.Id) + ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation } diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 37112b5fa2..f9cb2b20cc 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -39,6 +39,7 @@ type CdStageCompleteEvent struct { CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` } type UserDeploymentRequest struct { @@ -77,12 +78,12 @@ type CiCompleteEvent struct { Metrics util.CIMetrics `json:"metrics"` AppName string `json:"appName"` IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` + FailureReason string `json:"failureReason"` // FailureReason is used for notifying the failure reason to the user. Should be short and user-friendly ImageDetailsFromCR json.RawMessage `json:"imageDetailsFromCR"` PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` PluginArtifactStage string `json:"pluginArtifactStage"` pluginImageDetails *registry.ImageDetailsFromCR - PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` + PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` } func (c *CiCompleteEvent) GetPluginImageDetails() *registry.ImageDetailsFromCR { diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 43bd26c219..9889c5007d 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -165,14 +165,20 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { impl.logger.Errorw("could not get wf runner", "err", err) return } - + wfr.IsArtifactUploaded = cdStageCompleteEvent.IsArtifactUploaded if wfr.Status != string(v1alpha1.NodeSucceeded) { impl.logger.Debugw("event received from ci runner, updating workflow runner status as succeeded", "savedWorkflowRunnerId", wfr.Id, "oldStatus", wfr.Status, "podStatus", wfr.PodStatus) - err = impl.cdWorkflowRunnerService.UpdateWfrStatus(wfr, string(v1alpha1.NodeSucceeded), 1) + err = impl.cdWorkflowRunnerService.UpdateWfr(wfr, 1) if err != nil { impl.logger.Errorw("update cd-wf-runner failed for id ", "cdWfrId", wfr.Id, "err", err) return } + } else if cdStageCompleteEvent.IsArtifactUploaded { + err = impl.cdWorkflowRunnerService.UpdateIsArtifactUploaded(wfr.Id, cdStageCompleteEvent.IsArtifactUploaded) + if err != nil { + impl.logger.Errorw("error in updating isArtifactUploaded", "cdWfrId", wfr.Id, "err", err) + return + } } triggerContext := triggerBean.TriggerContext{ @@ -514,7 +520,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error { ReferenceId: pointer.String(msg.MsgId), } - if ciCompleteEvent.FailureReason != "" { + if len(ciCompleteEvent.FailureReason) != 0 { req.FailureReason = ciCompleteEvent.FailureReason err := impl.workflowDagExecutor.HandleCiStepFailedEvent(ciCompleteEvent.PipelineId, req) if err != nil { diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index d0efef7adc..e21d01a34f 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" "os" "path/filepath" "strconv" @@ -216,29 +217,32 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus return 0, "", err } - ciWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(savedWorkflow.CdWorkflow.PipelineId) + cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(savedWorkflow.CdWorkflow.PipelineId) if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("unable to fetch ciWorkflowConfig", "err", err) + impl.Logger.Errorw("unable to fetch cdWorkflowConfig", "err", err) return 0, "", err } - ciArtifactLocationFormat := ciWorkflowConfig.CdArtifactLocationFormat - if ciArtifactLocationFormat == "" { - ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() + cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat + if len(cdArtifactLocationFormat) == 0 { + cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } - + if cdWorkflowConfig.LogsBucket == "" { + cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() + } + cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if savedWorkflow.Status != executors.WorkflowCancel { savedWorkflow.Status = status } + savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus savedWorkflow.Message = message savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName - savedWorkflow.UpdatedOn = time.Now() - savedWorkflow.UpdatedBy = 1 + savedWorkflow.UpdateAuditLog(1) impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(savedWorkflow) if err != nil { @@ -687,7 +691,14 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i BucketName: cdConfig.LogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, } - key := fmt.Sprintf("%s/"+impl.config.GetArtifactLocationFormat(), impl.config.GetDefaultArtifactKeyPrefix(), wfr.CdWorkflow.Id, wfr.Id) + cdArtifactLocationFormat := cdConfig.CdArtifactLocationFormat + if len(cdArtifactLocationFormat) == 0 { + cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() + } + key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) + if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { + key = wfr.CdArtifactLocation + } baseLogLocationPathConfig := impl.config.BaseLogLocationPath blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) @@ -747,6 +758,9 @@ func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pip workflow.Image = wfr.CdWorkflow.CiArtifact.Image workflow.PipelineId = wfr.CdWorkflow.PipelineId workflow.CiArtifactId = wfr.CdWorkflow.CiArtifactId + if wfr.CdWorkflow.CiArtifact != nil { + workflow.IsArtifactUploaded = *wfr.IsArtifactUploaded + } workflow.BlobStorageEnabled = wfr.BlobStorageEnabled workflow.RefCdWorkflowRunnerId = wfr.RefCdWorkflowRunnerId } diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 30ea55aa13..343d044eac 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "io/ioutil" "net/http" "os" @@ -528,6 +529,10 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int var ciWorkLowResponses []types.WorkflowResponse for _, w := range workFlows { + isArtifactUploaded := w.IsArtifactUploaded + if w.IsArtifactUploadedV2 != nil { + isArtifactUploaded = *w.IsArtifactUploadedV2 + } wfResponse := types.WorkflowResponse{ Id: w.Id, Name: w.Name, @@ -546,12 +551,13 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int TriggeredByEmail: w.EmailId, ArtifactId: w.CiArtifactId, BlobStorageEnabled: w.BlobStorageEnabled, - IsArtifactUploaded: w.IsArtifactUploaded, + IsArtifactUploaded: isArtifactUploaded, EnvironmentId: w.EnvironmentId, EnvironmentName: w.EnvironmentName, ReferenceWorkflowId: w.RefCiWorkflowId, PodName: w.PodName, } + if w.Message == bean3.ImageTagUnavailableMessage { customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) if err != nil && err != pg.ErrNoRows { @@ -724,6 +730,10 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build } environmentName = env.Name } + isArtifactUploaded := ciArtifact.IsArtifactUploaded + if workflow.IsArtifactUploaded != nil { + isArtifactUploaded = *workflow.IsArtifactUploaded + } workflowResponse := types.WorkflowResponse{ Id: workflow.Id, Name: workflow.Name, @@ -742,7 +752,7 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build TriggeredByEmail: triggeredByUserEmailId, Artifact: ciArtifact.Image, ArtifactId: ciArtifact.Id, - IsArtifactUploaded: ciArtifact.IsArtifactUploaded, + IsArtifactUploaded: isArtifactUploaded, EnvironmentId: workflow.EnvironmentId, EnvironmentName: environmentName, PipelineType: workflow.CiPipeline.PipelineType, @@ -929,8 +939,14 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, } - key := fmt.Sprintf("%s/"+impl.config.GetArtifactLocationFormat(), impl.config.GetDefaultArtifactKeyPrefix(), ciWorkflow.Id, ciWorkflow.Id) - + ciArtifactLocationFormat := ciConfig.CiArtifactLocationFormat + if len(ciArtifactLocationFormat) == 0 { + ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() + } + key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) + if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { + key = ciWorkflow.CiArtifactLocation + } baseLogLocationPathConfig := impl.config.BaseLogLocationPath blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) @@ -1148,10 +1164,10 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } ciArtifactLocationFormat := ciWorkflowConfig.CiArtifactLocationFormat - if ciArtifactLocationFormat == "" { + if len(ciArtifactLocationFormat) == 0 { ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } - ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflowConfig.LogsBucket, savedWorkflow.Id, savedWorkflow.Id) + ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if savedWorkflow.Status != executors.WorkflowCancel { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 4bf3427ef1..5d3b1f6393 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -32,6 +32,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "maps" + "path" "path/filepath" "strconv" "strings" @@ -492,8 +493,8 @@ func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfig *pipelineCon if ciArtifactLocationFormat == "" { ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } - ArtifactLocation := fmt.Sprintf("s3://%s/%s/"+ciArtifactLocationFormat, ciWorkflowConfig.LogsBucket, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, savedWf.Id) - artifactFileName := fmt.Sprintf(impl.config.GetDefaultArtifactKeyPrefix()+"/"+ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfig.LogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) + artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation, ciWorkflowConfig.LogsBucket, artifactFileName } @@ -502,7 +503,7 @@ func (impl *CiServiceImpl) buildDefaultArtifactLocation(ciWorkflowConfig *pipeli if ciArtifactLocationFormat == "" { ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() } - ArtifactLocation := fmt.Sprintf("%s/"+ciArtifactLocationFormat, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, savedWf.Id) + ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation } diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 1257ccd1d3..90ebf555e4 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "os/user" + "path" "path/filepath" "strconv" "strings" @@ -363,7 +364,7 @@ func (impl *CiCdConfig) GetDefaultBuildLogsKeyPrefix() string { return "" } } -func (impl *CiCdConfig) GetDefaultArtifactKeyPrefix() string { +func (impl *CiCdConfig) getDefaultArtifactKeyPrefix() string { switch impl.Type { case CiConfigType: return impl.CiDefaultArtifactKeyPrefix @@ -388,9 +389,17 @@ func (impl *CiCdConfig) GetWorkflowServiceAccount() string { func (impl *CiCdConfig) GetArtifactLocationFormat() string { switch impl.Type { case CiConfigType: - return impl.CiArtifactLocationFormat + ciArtifactLocationFormat := impl.CiArtifactLocationFormat + if len(impl.getDefaultArtifactKeyPrefix()) != 0 { + ciArtifactLocationFormat = path.Join(impl.getDefaultArtifactKeyPrefix(), ciArtifactLocationFormat) + } + return ciArtifactLocationFormat case CdConfigType: - return impl.CdArtifactLocationFormat + cdArtifactLocationFormat := impl.CdArtifactLocationFormat + if len(impl.getDefaultArtifactKeyPrefix()) != 0 { + cdArtifactLocationFormat = path.Join(impl.getDefaultArtifactKeyPrefix(), cdArtifactLocationFormat) + } + return cdArtifactLocationFormat default: return "" } diff --git a/pkg/pipeline/util/CiCdUtil.go b/pkg/pipeline/util/CiCdUtil.go new file mode 100644 index 0000000000..374f76d5c8 --- /dev/null +++ b/pkg/pipeline/util/CiCdUtil.go @@ -0,0 +1,8 @@ +package util + +import "github.com/devtron-labs/devtron/util/urlUtil" + +func IsValidUrlSubPath(subPath string) bool { + url := "http://127.0.0.1:8080/" + subPath + return urlUtil.IsValidUrl(url) +} diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index 23fc0c177f..2a5028142b 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -22,13 +22,13 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" "github.com/go-pg/pg" "go.uber.org/zap" - "time" ) type CdWorkflowRunnerService interface { FindWorkflowRunnerById(wfrId int) (*bean.CdWorkflowRunnerDto, error) CheckIfWfrLatest(wfrId, pipelineId int) (isLatest bool, err error) - UpdateWfrStatus(dto *bean.CdWorkflowRunnerDto, status string, updatedBy int) error + UpdateWfr(dto *bean.CdWorkflowRunnerDto, updatedBy int) error + UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error } type CdWorkflowRunnerServiceImpl struct { @@ -63,11 +63,9 @@ func (impl *CdWorkflowRunnerServiceImpl) CheckIfWfrLatest(wfrId, pipelineId int) return isLatest, nil } -func (impl *CdWorkflowRunnerServiceImpl) UpdateWfrStatus(dto *bean.CdWorkflowRunnerDto, status string, updatedBy int) error { +func (impl *CdWorkflowRunnerServiceImpl) UpdateWfr(dto *bean.CdWorkflowRunnerDto, updatedBy int) error { runnerDbObj := adapter.ConvertCdWorkflowRunnerDtoToDbObj(dto) - runnerDbObj.Status = status - runnerDbObj.UpdatedBy = int32(updatedBy) - runnerDbObj.UpdatedOn = time.Now() + runnerDbObj.UpdateAuditLog(int32(updatedBy)) err := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runnerDbObj) if err != nil { impl.logger.Errorw("error in updating runner status in db", "runnerId", runnerDbObj.Id, "err", err) @@ -75,3 +73,12 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateWfrStatus(dto *bean.CdWorkflowRun } return nil } + +func (impl *CdWorkflowRunnerServiceImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error { + err := impl.cdWorkflowRepository.UpdateIsArtifactUploaded(wfrId, isArtifactUploaded) + if err != nil { + impl.logger.Errorw("error in updating isArtifactUploaded in db", "wfrId", wfrId, "err", err) + return err + } + return nil +} diff --git a/pkg/workflow/cd/adapter/adapter.go b/pkg/workflow/cd/adapter/adapter.go index 5e2dc81017..92cbbf0d93 100644 --- a/pkg/workflow/cd/adapter/adapter.go +++ b/pkg/workflow/cd/adapter/adapter.go @@ -83,5 +83,6 @@ func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineC RefCdWorkflowRunnerId: dto.RefCdWorkflowRunnerId, ImagePathReservationIds: dto.ImagePathReservationIds, ReferenceId: dto.ReferenceId, + IsArtifactUploaded: dto.IsArtifactUploaded, } } diff --git a/pkg/workflow/cd/bean/bean.go b/pkg/workflow/cd/bean/bean.go index cb18dc397c..314a87107e 100644 --- a/pkg/workflow/cd/bean/bean.go +++ b/pkg/workflow/cd/bean/bean.go @@ -50,4 +50,5 @@ type CdWorkflowRunnerDto struct { RefCdWorkflowRunnerId int `json:"refCdWorkflowRunnerId"` ImagePathReservationIds []int `json:"imagePathReservationIds"` ReferenceId *string `json:"referenceId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` } diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index fa31d5dd83..2038be0185 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/devtron-labs/common-lib/async" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" @@ -118,6 +119,7 @@ type WorkflowDagExecutorImpl struct { manifestCreationService manifest.ManifestCreationService commonArtifactService artifacts.CommonArtifactService deploymentConfigService common2.DeploymentConfigService + asyncRunnable *async.Runnable } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -140,7 +142,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, - deploymentConfigService common2.DeploymentConfigService) *WorkflowDagExecutorImpl { + deploymentConfigService common2.DeploymentConfigService, + asyncRunnable *async.Runnable) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -163,6 +166,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, deploymentConfigService: deploymentConfigService, + asyncRunnable: asyncRunnable, } config, err := types.GetCdConfig() if err != nil { @@ -722,6 +726,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger return 0, err } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) + savedWorkflow.IsArtifactUploaded = &request.IsArtifactUploaded impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) if err != nil { @@ -734,7 +739,6 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger impl.logger.Errorw("error in deactivation images", "err", err) return 0, err } - } pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) @@ -762,16 +766,15 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger createdOn = imagePushedAt } buildArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: pipeline.Id, - WorkflowId: request.WorkflowId, - ScanEnabled: pipeline.ScanEnabled, - Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: pipeline.Id, + WorkflowId: request.WorkflowId, + ScanEnabled: pipeline.ScanEnabled, + Scanned: false, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, } plugin, err := impl.globalPluginRepository.GetPluginByName(bean3.VULNERABILITY_SCANNING_PLUGIN) if err != nil || len(plugin) == 0 { @@ -832,16 +835,15 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var ciArtifactArr []*repository.CiArtifact for _, ci := range childrenCi { ciArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: ci.Id, - ParentCiArtifact: buildArtifact.Id, - ScanEnabled: ci.ScanEnabled, - Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: ci.Id, + ParentCiArtifact: buildArtifact.Id, + ScanEnabled: ci.ScanEnabled, + Scanned: false, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if ci.ScanEnabled { ciArtifact.Scanned = true @@ -947,29 +949,39 @@ func (impl *WorkflowDagExecutorImpl) WriteCiSuccessEvent(request *bean2.CiArtifa } func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, request *bean2.CiArtifactWebhookRequest) (err error) { - + if request == nil || request.WorkflowId == nil { + impl.logger.Errorw("invalid request received", "request", request) + return fmt.Errorf("invalid request received! workflowId is required") + } savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) if err != nil { impl.logger.Errorw("cannot get saved wf", "wf ID: ", *request.WorkflowId, "err", err) return err } - - pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) + if request.IsArtifactUploaded { + dbErr := impl.ciWorkflowRepository.UpdateArtifactUploaded(savedWorkflow.Id, request.IsArtifactUploaded) + if dbErr != nil { + impl.logger.Errorw("update workflow status", "ciWorkflowId", savedWorkflow.Id, "err", dbErr) + } + } + pipelineModel, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) if err != nil { impl.logger.Errorw("unable to find pipeline", "ID", ciPipelineId, "err", err) return err } - - go func() { + customTagServiceRunnableFunc := func() { if len(savedWorkflow.ImagePathReservationIds) > 0 { err = impl.customTagService.DeactivateImagePathReservationByImageIds(savedWorkflow.ImagePathReservationIds) if err != nil { - impl.logger.Errorw("unable to deactivate impage_path_reservation ", err) + impl.logger.Errorw("unable to deactivate ImagePathReservation", "imagePathReservationIds", savedWorkflow.ImagePathReservationIds, "err", err) } } - }() - - go impl.WriteCiStepFailedEvent(pipeline, request, savedWorkflow) + } + impl.asyncRunnable.Execute(customTagServiceRunnableFunc) + notificationServiceRunnableFunc := func() { + impl.WriteCiStepFailedEvent(pipelineModel, request, savedWorkflow) + } + impl.asyncRunnable.Execute(notificationServiceRunnableFunc) return nil } @@ -1022,7 +1034,6 @@ func (impl *WorkflowDagExecutorImpl) HandleExternalCiWebhook(externalCiId int, r ExternalCiPipelineId: externalCiId, ScanEnabled: false, Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if err = impl.ciArtifactRepository.Save(artifact); err != nil { diff --git a/pkg/workflow/dag/bean/bean.go b/pkg/workflow/dag/bean/bean.go index fe4f12ed66..62a70fa611 100644 --- a/pkg/workflow/dag/bean/bean.go +++ b/pkg/workflow/dag/bean/bean.go @@ -30,7 +30,8 @@ type CiArtifactWebhookRequest struct { WorkflowId *int `json:"workflowId"` UserId int32 `json:"userId"` IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` + FailureReason string `json:"failureReason"` // FailureReason is used for notifying the failure reason to the user. Should be short and user-friendly + InternalError string `json:"internalError,omitempty"` // InternalError is used as workflow failure message. Should be more detailed and technical PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") } diff --git a/internal/util/UrlUtil.go b/util/urlUtil/UrlUtil.go similarity index 98% rename from internal/util/UrlUtil.go rename to util/urlUtil/UrlUtil.go index f7668656e6..dc0c106d9c 100644 --- a/internal/util/UrlUtil.go +++ b/util/urlUtil/UrlUtil.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package util +package urlUtil import ( "net/url" diff --git a/wire_gen.go b/wire_gen.go index ac5db2be55..10bb0d2362 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -665,7 +665,7 @@ func InitializeApp() (*App, error) { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From 80a0d95ebee140e6a5479b9cd700e30155c544e1 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 02:24:43 +0530 Subject: [PATCH 02/29] fix: data converter --- pkg/workflow/cd/adapter/adapter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workflow/cd/adapter/adapter.go b/pkg/workflow/cd/adapter/adapter.go index 92cbbf0d93..a782e8333e 100644 --- a/pkg/workflow/cd/adapter/adapter.go +++ b/pkg/workflow/cd/adapter/adapter.go @@ -83,6 +83,6 @@ func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineC RefCdWorkflowRunnerId: dto.RefCdWorkflowRunnerId, ImagePathReservationIds: dto.ImagePathReservationIds, ReferenceId: dto.ReferenceId, - IsArtifactUploaded: dto.IsArtifactUploaded, + IsArtifactUploaded: &dto.IsArtifactUploaded, } } From c3e86227b574d7768b2dadf26ef83a5e2a9c1660 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 11:55:27 +0530 Subject: [PATCH 03/29] added migration script --- scripts/sql/29200000_artifact_uploaded_flag.down.sql | 5 +++++ scripts/sql/29200000_artifact_uploaded_flag.up.sql | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 scripts/sql/29200000_artifact_uploaded_flag.down.sql create mode 100644 scripts/sql/29200000_artifact_uploaded_flag.up.sql diff --git a/scripts/sql/29200000_artifact_uploaded_flag.down.sql b/scripts/sql/29200000_artifact_uploaded_flag.down.sql new file mode 100644 index 0000000000..7018f6d3f4 --- /dev/null +++ b/scripts/sql/29200000_artifact_uploaded_flag.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE public.ci_workflow + DROP COLUMN IF EXISTS is_artifact_uploaded; +ALTER TABLE public.cd_workflow_runner + DROP COLUMN IF EXISTS is_artifact_uploaded, + DROP COLUMN IF EXISTS cd_artifact_location; diff --git a/scripts/sql/29200000_artifact_uploaded_flag.up.sql b/scripts/sql/29200000_artifact_uploaded_flag.up.sql new file mode 100644 index 0000000000..174048f72b --- /dev/null +++ b/scripts/sql/29200000_artifact_uploaded_flag.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE public.ci_workflow + ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN; +ALTER TABLE public.cd_workflow_runner + ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN, + ADD COLUMN IF NOT EXISTS cd_artifact_location varchar(256); From 8468c6623f6f7629b8ac8f81e10606f76114cc0c Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 12:21:34 +0530 Subject: [PATCH 04/29] updated ConvertCdWorkflowRunnerDtoToDbObj data adapter method --- pkg/workflow/cd/adapter/adapter.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/workflow/cd/adapter/adapter.go b/pkg/workflow/cd/adapter/adapter.go index a782e8333e..af6c00adbb 100644 --- a/pkg/workflow/cd/adapter/adapter.go +++ b/pkg/workflow/cd/adapter/adapter.go @@ -84,5 +84,11 @@ func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineC ImagePathReservationIds: dto.ImagePathReservationIds, ReferenceId: dto.ReferenceId, IsArtifactUploaded: &dto.IsArtifactUploaded, + AuditLog: sql.AuditLog{ + CreatedOn: dto.StartedOn, + CreatedBy: dto.TriggeredBy, + UpdatedOn: dto.StartedOn, + UpdatedBy: dto.TriggeredBy, + }, } } From ba6023b88ad7bbe4a5ca678eb0931e7cb9db417c Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 12:29:37 +0530 Subject: [PATCH 05/29] refactored CdWorkflowWithArtifact struct to bean --- .../DeploymentPipelineRestHandler.go | 8 ++--- .../pipelineConfig/CdWorfkflowRepository.go | 30 ---------------- .../history/cdPipelineDeploymentHistory.go | 12 ++++--- .../cdPipeline/DeploymentHistoryService.go | 3 +- pkg/pipeline/CdHandler.go | 28 +++++++-------- pkg/pipeline/bean/CdHandlerBean.go | 36 +++++++++++++++++++ 6 files changed, 63 insertions(+), 54 deletions(-) create mode 100644 pkg/pipeline/bean/CdHandlerBean.go diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 80bffe6275..35bd4355c5 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -51,10 +51,10 @@ import ( ) type DeploymentHistoryResp struct { - CdWorkflows []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"` - TagsEdiatable bool `json:"tagsEditable"` - AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app - HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` + CdWorkflows []pipelineBean.CdWorkflowWithArtifact `json:"cdWorkflows"` + TagsEdiatable bool `json:"tagsEditable"` + AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app + HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` } type DevtronAppDeploymentRestHandler interface { diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index c796d5f5b1..7ba6cadd74 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -23,7 +23,6 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" - repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/sql" @@ -176,35 +175,6 @@ type CiPipelineMaterialResponse struct { Regex string `json:"regex"` } -type CdWorkflowWithArtifact struct { - Id int `json:"id"` - CdWorkflowId int `json:"cd_workflow_id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - PipelineId int `json:"pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - MaterialInfo string `json:"material_info,omitempty"` - DataSource string `json:"data_source,omitempty"` - CiArtifactId int `json:"ci_artifact_id,omitempty"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - WorkflowType string `json:"workflow_type,omitempty"` - ExecutorType string `json:"executor_type,omitempty"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]GitCommit `json:"gitTriggers"` - CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` - ImageReleaseTags []*repository2.ImageTag `json:"imageReleaseTags"` - ImageComment *repository2.ImageComment `json:"imageComment"` - RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` -} - type TriggerWorkflowStatus struct { CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"` CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"` diff --git a/pkg/devtronResource/bean/history/cdPipelineDeploymentHistory.go b/pkg/devtronResource/bean/history/cdPipelineDeploymentHistory.go index 527b533948..3578370b77 100644 --- a/pkg/devtronResource/bean/history/cdPipelineDeploymentHistory.go +++ b/pkg/devtronResource/bean/history/cdPipelineDeploymentHistory.go @@ -1,6 +1,8 @@ package history -import "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" +import ( + "github.com/devtron-labs/devtron/pkg/pipeline/bean" +) type CdPipelineDeploymentHistoryListReq struct { PipelineId int @@ -18,8 +20,8 @@ type CdPipelineDeploymentHistoryConfigListReq struct { } type DeploymentHistoryResp struct { - CdWorkflows []pipelineConfig.CdWorkflowWithArtifact `json:"cdWorkflows"` - TagsEditable bool `json:"tagsEditable"` - AppReleaseTagNames []string `json:"appReleaseTagNames"` // unique list of tags exists in the app - HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` + CdWorkflows []bean.CdWorkflowWithArtifact `json:"cdWorkflows"` + TagsEditable bool `json:"tagsEditable"` + AppReleaseTagNames []string `json:"appReleaseTagNames"` // unique list of tags exists in the app + HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` } diff --git a/pkg/devtronResource/history/deployment/cdPipeline/DeploymentHistoryService.go b/pkg/devtronResource/history/deployment/cdPipeline/DeploymentHistoryService.go index dccbf40f7e..31090d42b8 100644 --- a/pkg/devtronResource/history/deployment/cdPipeline/DeploymentHistoryService.go +++ b/pkg/devtronResource/history/deployment/cdPipeline/DeploymentHistoryService.go @@ -4,6 +4,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" historyBean "github.com/devtron-labs/devtron/pkg/devtronResource/bean/history" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "go.uber.org/zap" ) @@ -36,7 +37,7 @@ func NewDeploymentHistoryServiceImpl(logger *zap.SugaredLogger, } func (impl *DeploymentHistoryServiceImpl) GetCdPipelineDeploymentHistory(req *historyBean.CdPipelineDeploymentHistoryListReq) (resp historyBean.DeploymentHistoryResp, err error) { - var wfs []pipelineConfig.CdWorkflowWithArtifact + var wfs []bean.CdWorkflowWithArtifact wfs, err = impl.cdHandler.GetCdBuildHistory(req.AppId, req.EnvId, req.PipelineId, req.Offset, req.Limit) if err != nil { impl.logger.Errorw("service err, List", "err", err, "req", req) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index e21d01a34f..442cbd76bf 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -59,11 +59,11 @@ const ( type CdHandler interface { UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, error) - GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineConfig.CdWorkflowWithArtifact, error) + GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) DownloadCdWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - FetchCdPrePostStageStatus(pipelineId int) ([]pipelineConfig.CdWorkflowWithArtifact, error) + FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) CancelStage(workflowRunnerId int, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) @@ -308,9 +308,9 @@ func (impl *CdHandlerImpl) stateChanged(status string, podStatus string, msg str return savedWorkflow.Status != status || savedWorkflow.PodStatus != podStatus || savedWorkflow.Message != msg || savedWorkflow.FinishedOn != finishedAt } -func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineConfig.CdWorkflowWithArtifact, error) { +func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) { - var cdWorkflowArtifact []pipelineConfig.CdWorkflowWithArtifact + var cdWorkflowArtifact []pipelineBean.CdWorkflowWithArtifact // this map contains artifactId -> array of tags of that artifact imageTagsDataMap, err := impl.imageTaggingService.GetTagsDataMapByAppId(appId) if err != nil { @@ -392,7 +392,7 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel } ciMaterialsArr = append(ciMaterialsArr, res) } - var newCdWorkflowArtifact []pipelineConfig.CdWorkflowWithArtifact + var newCdWorkflowArtifact []pipelineBean.CdWorkflowWithArtifact for _, cdWfA := range cdWorkflowArtifact { gitTriggers := make(map[int]pipelineConfig.GitCommit) @@ -741,8 +741,8 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return file, nil } -func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineConfig.CdWorkflowWithArtifact { - workflow := pipelineConfig.CdWorkflowWithArtifact{} +func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineBean.CdWorkflowWithArtifact { + workflow := pipelineBean.CdWorkflowWithArtifact{} if wfr.Id > 0 { workflow.Name = wfr.Name workflow.Id = wfr.Id @@ -767,9 +767,9 @@ func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pip return workflow } -func (impl *CdHandlerImpl) converterWFRList(wfrList []pipelineConfig.CdWorkflowRunner) []pipelineConfig.CdWorkflowWithArtifact { - var workflowList []pipelineConfig.CdWorkflowWithArtifact - var results []pipelineConfig.CdWorkflowWithArtifact +func (impl *CdHandlerImpl) converterWFRList(wfrList []pipelineConfig.CdWorkflowRunner) []pipelineBean.CdWorkflowWithArtifact { + var workflowList []pipelineBean.CdWorkflowWithArtifact + var results []pipelineBean.CdWorkflowWithArtifact var ids []int32 for _, item := range wfrList { ids = append(ids, item.TriggeredBy) @@ -790,8 +790,8 @@ func (impl *CdHandlerImpl) converterWFRList(wfrList []pipelineConfig.CdWorkflowR return results } -func (impl *CdHandlerImpl) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineConfig.CdWorkflowWithArtifact, error) { - var results []pipelineConfig.CdWorkflowWithArtifact +func (impl *CdHandlerImpl) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) { + var results []pipelineBean.CdWorkflowWithArtifact wfrPre, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean.CD_WORKFLOW_TYPE_PRE) if err != nil && err != pg.ErrNoRows { return results, err @@ -800,7 +800,7 @@ func (impl *CdHandlerImpl) FetchCdPrePostStageStatus(pipelineId int) ([]pipeline workflowPre := impl.converterWFR(wfrPre) results = append(results, workflowPre) } else { - workflowPre := pipelineConfig.CdWorkflowWithArtifact{Status: "Notbuilt", WorkflowType: string(bean.CD_WORKFLOW_TYPE_PRE), PipelineId: pipelineId} + workflowPre := pipelineBean.CdWorkflowWithArtifact{Status: "Notbuilt", WorkflowType: string(bean.CD_WORKFLOW_TYPE_PRE), PipelineId: pipelineId} results = append(results, workflowPre) } @@ -812,7 +812,7 @@ func (impl *CdHandlerImpl) FetchCdPrePostStageStatus(pipelineId int) ([]pipeline workflowPost := impl.converterWFR(wfrPost) results = append(results, workflowPost) } else { - workflowPost := pipelineConfig.CdWorkflowWithArtifact{Status: "Notbuilt", WorkflowType: string(bean.CD_WORKFLOW_TYPE_POST), PipelineId: pipelineId} + workflowPost := pipelineBean.CdWorkflowWithArtifact{Status: "Notbuilt", WorkflowType: string(bean.CD_WORKFLOW_TYPE_POST), PipelineId: pipelineId} results = append(results, workflowPost) } return results, nil diff --git a/pkg/pipeline/bean/CdHandlerBean.go b/pkg/pipeline/bean/CdHandlerBean.go new file mode 100644 index 0000000000..d255c478d8 --- /dev/null +++ b/pkg/pipeline/bean/CdHandlerBean.go @@ -0,0 +1,36 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CdWorkflowWithArtifact struct { + Id int `json:"id"` + CdWorkflowId int `json:"cd_workflow_id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"pod_status"` + Message string `json:"message"` + StartedOn time.Time `json:"started_on"` + FinishedOn time.Time `json:"finished_on"` + PipelineId int `json:"pipeline_id"` + Namespace string `json:"namespace"` + LogFilePath string `json:"log_file_path"` + TriggeredBy int32 `json:"triggered_by"` + EmailId string `json:"email_id"` + Image string `json:"image"` + MaterialInfo string `json:"material_info,omitempty"` + DataSource string `json:"data_source,omitempty"` + CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + WorkflowType string `json:"workflow_type,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` +} From 49d8a8f0132dd46732277756df088832f206ca3b Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 12:31:40 +0530 Subject: [PATCH 06/29] fix: SubscribeCDStageCompleteEvent status update --- pkg/eventProcessor/in/WorkflowEventProcessorService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 9889c5007d..8aee442952 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -168,6 +168,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { wfr.IsArtifactUploaded = cdStageCompleteEvent.IsArtifactUploaded if wfr.Status != string(v1alpha1.NodeSucceeded) { impl.logger.Debugw("event received from ci runner, updating workflow runner status as succeeded", "savedWorkflowRunnerId", wfr.Id, "oldStatus", wfr.Status, "podStatus", wfr.PodStatus) + wfr.Status = string(v1alpha1.NodeSucceeded) err = impl.cdWorkflowRunnerService.UpdateWfr(wfr, 1) if err != nil { impl.logger.Errorw("update cd-wf-runner failed for id ", "cdWfrId", wfr.Id, "err", err) From f141af60d2b0e121c85f6f7bef4d241b223516ad Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 18:45:25 +0530 Subject: [PATCH 07/29] added: unit test file --- pkg/pipeline/util/CiCdUtil_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 pkg/pipeline/util/CiCdUtil_test.go diff --git a/pkg/pipeline/util/CiCdUtil_test.go b/pkg/pipeline/util/CiCdUtil_test.go new file mode 100644 index 0000000000..fd13091f99 --- /dev/null +++ b/pkg/pipeline/util/CiCdUtil_test.go @@ -0,0 +1,27 @@ +package util + +import "testing" + +func TestIsValidUrlSubPath(t *testing.T) { + type args struct { + subPath string + } + tests := []struct { + name string + args args + want bool + }{ + {name: "Test Case 1: Invalid URL", args: args{subPath: "%!d(string=)/5.zip%!(EXTRA int=5)"}, want: false}, + {name: "Test Case 2: Valid URL", args: args{subPath: "5/5.zip"}, want: true}, + {name: "Test Case 3: Valid URL", args: args{subPath: "prefix/1/5.zip"}, want: true}, + {name: "Test Case 5: Valid URL", args: args{subPath: "/prefix/1/5.zip"}, want: true}, + {name: "Test Case 6: Valid URL", args: args{subPath: "//1/5.zip"}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidUrlSubPath(tt.args.subPath); got != tt.want { + t.Errorf("IsValidUrlSubPath() = %v, want %v", got, tt.want) + } + }) + } +} From 30cbc7bd4826a673d62965f0f1d09201019907fc Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 26 Sep 2024 18:57:22 +0530 Subject: [PATCH 08/29] fix: docker build --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index fc5dbe1dce..1db725eaac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ RUN apt install git gcc musl-dev make -y RUN go install github.com/google/wire/cmd/wire@latest WORKDIR /go/src/github.com/devtron-labs/devtron ADD . /go/src/github.com/devtron-labs/devtron/ +# TODO Asutosh: revert after dev test +ADD ./vendor/github.com/Microsoft/ /go/src/github.com/devtron-labs/devtron/vendor/github.com/microsoft/ RUN GOOS=linux make build-all # uncomment this post build arg From 805f628dac94c3231f1d7c5a938cacb8911f8205 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 27 Sep 2024 13:21:37 +0530 Subject: [PATCH 09/29] removed redundant tables ci_workflow_config and cd_workflow_config --- .../configure/BuildPipelineRestHandler.go | 4 +- .../DeploymentPipelineRestHandler.go | 2 +- client/events/EventBuilder.go | 4 +- .../pipelineConfig/CdWorfkflowRepository.go | 28 - .../pipelineConfig/CiWorkflowRepository.go | 90 +-- .../mocks/CdWorkflowRepository.go | 678 +++++++++++++++--- .../mocks/CiWorkflowRepository.go | 249 ++++++- .../devtronApps/PreStageTriggerService.go | 49 +- pkg/pipeline/CdHandler.go | 63 +- pkg/pipeline/CiHandler.go | 97 +-- pkg/pipeline/CiService.go | 108 ++- pkg/workflow/dag/WorkflowDagExecutor.go | 4 +- .../29200000_artifact_uploaded_flag.down.sql | 100 +++ .../29200000_artifact_uploaded_flag.up.sql | 20 +- 14 files changed, 1068 insertions(+), 428 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 1ae6d4f33d..a68b96cfdb 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -1016,7 +1016,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon return } //RBAC - resp, err := handler.ciHandler.GetHistoricBuildLogs(pipelineId, workflowId, nil) + resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1155,7 +1155,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(pipelineId, workflowId) + logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 35bd4355c5..77d2c64a67 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1903,7 +1903,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } //RBAC CHECK - file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(pipelineId, buildId) + file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index bcd8fc5a34..48318b8091 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -37,7 +37,7 @@ import ( type EventFactory interface { Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) Event BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event - BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event + BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event //BuildFinalData(event Event) *Payload } @@ -163,7 +163,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC return event } -func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo, dockerImage string) Event { +func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event { if material == nil { materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId) if err != nil { diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 7ba6cadd74..334d99c2c7 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -49,7 +49,6 @@ type CdWorkflowRepository interface { UpdateWorkFlowRunners(wfr []*CdWorkflowRunner) error FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error) FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error) - FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error) FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) @@ -95,27 +94,6 @@ type CdWorkflow struct { sql.AuditLog } -type CdWorkflowConfig struct { - tableName struct{} `sql:"cd_workflow_config" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - CdTimeout int64 `sql:"cd_timeout"` - MinCpu string `sql:"min_cpu"` - MaxCpu string `sql:"max_cpu"` - MinMem string `sql:"min_mem"` - MaxMem string `sql:"max_mem"` - MinStorage string `sql:"min_storage"` - MaxStorage string `sql:"max_storage"` - MinEphStorage string `sql:"min_eph_storage"` - MaxEphStorage string `sql:"max_eph_storage"` - CdCacheBucket string `sql:"cd_cache_bucket"` - CdCacheRegion string `sql:"cd_cache_region"` - CdImage string `sql:"cd_image"` - Namespace string `sql:"wf_namespace"` - CdPipelineId int `sql:"cd_pipeline_id"` - LogsBucket string `sql:"logs_bucket"` - CdArtifactLocationFormat string `sql:"cd_artifact_location_format"` -} - type CdWorkflowRunnerWithExtraFields struct { CdWorkflowRunner TotalCount int @@ -251,12 +229,6 @@ func (impl *CdWorkflowRepositoryImpl) FindById(wfId int) (*CdWorkflow, error) { return ddWorkflow, err } -func (impl *CdWorkflowRepositoryImpl) FindConfigByPipelineId(pipelineId int) (*CdWorkflowConfig, error) { - cdWorkflowConfig := &CdWorkflowConfig{} - err := impl.dbConnection.Model(cdWorkflowConfig).Where("cd_pipeline_id = ?", pipelineId).Select() - return cdWorkflowConfig, err -} - func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*CdWorkflow, error) { cdWorkflow := &CdWorkflow{} err := impl.dbConnection.Model(cdWorkflow).Where("pipeline_id in (?)", pg.In(pipelineIds)).Order("id DESC").Limit(1).Select() diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 436ff5cd65..3e1f8a37af 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -26,9 +26,6 @@ import ( ) type CiWorkflowRepository interface { - SaveWorkFlowConfig(config *CiWorkflowConfig) error - FindConfigByPipelineId(pipelineId int) (*CiWorkflowConfig, error) - SaveWorkFlow(wf *CiWorkflow) error FindLastTriggeredWorkflow(pipelineId int) (*CiWorkflow, error) UpdateWorkFlow(wf *CiWorkflow) error @@ -94,34 +91,34 @@ func (ciWorkflow *CiWorkflow) IsExternalRunInJobType() bool { } type WorkflowWithArtifact struct { - Id int `json:"id"` - Name string `json:"name"` - PodName string `json:"podName"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - CiPipelineId int `json:"ci_pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - GitTriggers map[int]GitCommit `json:"git_triggers"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - CiArtifactLocation string `json:"ci_artifact_location"` - CiArtifactId int `json:"ci_artifact_d"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - CiBuildType string `json:"ci_build_type"` - IsArtifactUploadedV2 *bool `json:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future - IsArtifactUploaded bool `json:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - RefCiWorkflowId int `json:"referenceCiWorkflowId"` - ParentCiWorkflowId int `json:"parent_ci_workflow_id"` - ExecutorType cdWorkflow.WorkflowExecutorType `json:"executor_type"` //awf, system - ImagePathReservationId int `json:"image_path_reservation_id"` - ImagePathReservationIds []int `json:"image_path_reservation_ids" pg:",array"` + Id int `psql:"id"` + Name string `psql:"name"` + PodName string `psql:"pod_name"` + Status string `psql:"status"` + PodStatus string `psql:"pod_status"` + Message string `psql:"message"` + StartedOn time.Time `psql:"started_on"` + FinishedOn time.Time `psql:"finished_on"` + CiPipelineId int `psql:"ci_pipeline_id"` + Namespace string `psql:"namespace"` + LogFilePath string `psql:"log_file_path"` + GitTriggers map[int]GitCommit `psql:"git_triggers"` + TriggeredBy int32 `psql:"triggered_by"` + EmailId string `psql:"email_id"` + Image string `psql:"image"` + CiArtifactLocation string `psql:"ci_artifact_location"` + CiArtifactId int `psql:"ci_artifact_id"` + BlobStorageEnabled bool `psql:"blob_storage_enabled"` + CiBuildType string `psql:"ci_build_type"` + IsArtifactUploadedV2 *bool `psql:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future + IsArtifactUploaded bool `psql:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table + EnvironmentId int `psql:"environment_id"` + EnvironmentName string `psql:"environment_name"` + RefCiWorkflowId int `psql:"ref_ci_workflow_id"` + ParentCiWorkflowId int `psql:"parent_ci_workflow_id"` + ExecutorType cdWorkflow.WorkflowExecutorType `psql:"executor_type"` //awf, system + ImagePathReservationId int `psql:"image_path_reservation_id"` + ImagePathReservationIds []int `psql:"image_path_reservation_ids" pg:",array"` } type GitCommit struct { @@ -143,27 +140,6 @@ type WebhookData struct { Data map[string]string `json:"data"` } -type CiWorkflowConfig struct { - tableName struct{} `sql:"ci_workflow_config" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - CiTimeout int64 `sql:"ci_timeout"` - MinCpu string `sql:"min_cpu"` - MaxCpu string `sql:"max_cpu"` - MinMem string `sql:"min_mem"` - MaxMem string `sql:"max_mem"` - MinStorage string `sql:"min_storage"` - MaxStorage string `sql:"max_storage"` - MinEphStorage string `sql:"min_eph_storage"` - MaxEphStorage string `sql:"max_eph_storage"` - CiCacheBucket string `sql:"ci_cache_bucket"` - CiCacheRegion string `sql:"ci_cache_region"` - CiImage string `sql:"ci_image"` - Namespace string `sql:"wf_namespace"` - CiPipelineId int `sql:"ci_pipeline_id"` - LogsBucket string `sql:"logs_bucket"` - CiArtifactLocationFormat string `sql:"ci_artifact_location_format"` -} - func NewCiWorkflowRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *CiWorkflowRepositoryImpl { return &CiWorkflowRepositoryImpl{ dbConnection: dbConnection, @@ -251,16 +227,6 @@ func (impl *CiWorkflowRepositoryImpl) FindCiWorkflowGitTriggersByIds(ids []int) return workflows, err } -func (impl *CiWorkflowRepositoryImpl) SaveWorkFlowConfig(config *CiWorkflowConfig) error { - err := impl.dbConnection.Insert(config) - return err -} - -func (impl *CiWorkflowRepositoryImpl) FindConfigByPipelineId(pipelineId int) (*CiWorkflowConfig, error) { - ciWorkflowConfig := &CiWorkflowConfig{} - err := impl.dbConnection.Model(ciWorkflowConfig).Where("ci_pipeline_id = ?", pipelineId).Select() - return ciWorkflowConfig, err -} func (impl *CiWorkflowRepositoryImpl) SaveWorkFlow(wf *CiWorkflow) error { err := impl.dbConnection.Insert(wf) diff --git a/internal/sql/repository/pipelineConfig/mocks/CdWorkflowRepository.go b/internal/sql/repository/pipelineConfig/mocks/CdWorkflowRepository.go index 48cf4109e7..23822b7d20 100644 --- a/internal/sql/repository/pipelineConfig/mocks/CdWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/mocks/CdWorkflowRepository.go @@ -1,10 +1,12 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery v2.42.0. DO NOT EDIT. package mocks import ( - "context" + context "context" + bean "github.com/devtron-labs/devtron/api/bean" + mock "github.com/stretchr/testify/mock" pg "github.com/go-pg/pg" @@ -17,18 +19,53 @@ type CdWorkflowRepository struct { mock.Mock } +// CheckWorkflowRunnerByReferenceId provides a mock function with given fields: referenceId +func (_m *CdWorkflowRepository) CheckWorkflowRunnerByReferenceId(referenceId string) (bool, error) { + ret := _m.Called(referenceId) + + if len(ret) == 0 { + panic("no return value specified for CheckWorkflowRunnerByReferenceId") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { + return rf(referenceId) + } + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(referenceId) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(referenceId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ExistsByStatus provides a mock function with given fields: status func (_m *CdWorkflowRepository) ExistsByStatus(status string) (bool, error) { ret := _m.Called(status) + if len(ret) == 0 { + panic("no return value specified for ExistsByStatus") + } + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { + return rf(status) + } if rf, ok := ret.Get(0).(func(string) bool); ok { r0 = rf(status) } else { r0 = ret.Get(0).(bool) } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(status) } else { @@ -42,7 +79,15 @@ func (_m *CdWorkflowRepository) ExistsByStatus(status string) (bool, error) { func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntity(pipelineIds []int) ([]*pipelineConfig.CdWorkflowStatus, error) { ret := _m.Called(pipelineIds) + if len(ret) == 0 { + panic("no return value specified for FetchAllCdStagesLatestEntity") + } + var r0 []*pipelineConfig.CdWorkflowStatus + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CdWorkflowStatus, error)); ok { + return rf(pipelineIds) + } if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CdWorkflowStatus); ok { r0 = rf(pipelineIds) } else { @@ -51,7 +96,6 @@ func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntity(pipelineIds []int) } } - var r1 error if rf, ok := ret.Get(1).(func([]int) error); ok { r1 = rf(pipelineIds) } else { @@ -65,7 +109,15 @@ func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntity(pipelineIds []int) func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntityStatus(wfrIds []int) ([]*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(wfrIds) + if len(ret) == 0 { + panic("no return value specified for FetchAllCdStagesLatestEntityStatus") + } + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfrIds) + } if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CdWorkflowRunner); ok { r0 = rf(wfrIds) } else { @@ -74,7 +126,6 @@ func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntityStatus(wfrIds []int) } } - var r1 error if rf, ok := ret.Get(1).(func([]int) error); ok { r1 = rf(wfrIds) } else { @@ -84,22 +135,87 @@ func (_m *CdWorkflowRepository) FetchAllCdStagesLatestEntityStatus(wfrIds []int) return r0, r1 } -// FetchArtifactsByCdPipelineId provides a mock function with given fields: pipelineId, runnerType, offset, limit -func (_m *CdWorkflowRepository) FetchArtifactsByCdPipelineId(pipelineId int, runnerType bean.WorkflowType, offset int, limit int) ([]pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(pipelineId, runnerType, offset, limit) +// FetchArtifactsByCdPipelineId provides a mock function with given fields: pipelineId, runnerType, offset, limit, searchString +func (_m *CdWorkflowRepository) FetchArtifactsByCdPipelineId(pipelineId int, runnerType bean.WorkflowType, offset int, limit int, searchString string) ([]pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(pipelineId, runnerType, offset, limit, searchString) + + if len(ret) == 0 { + panic("no return value specified for FetchArtifactsByCdPipelineId") + } var r0 []pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int, int) []pipelineConfig.CdWorkflowRunner); ok { - r0 = rf(pipelineId, runnerType, offset, limit) + var r1 error + if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int, int, string) ([]pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId, runnerType, offset, limit, searchString) + } + if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int, int, string) []pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(pipelineId, runnerType, offset, limit, searchString) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]pipelineConfig.CdWorkflowRunner) } } + if rf, ok := ret.Get(1).(func(int, bean.WorkflowType, int, int, string) error); ok { + r1 = rf(pipelineId, runnerType, offset, limit, searchString) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FetchEnvAllCdStagesLatestEntityStatus provides a mock function with given fields: wfrIds, envID +func (_m *CdWorkflowRepository) FetchEnvAllCdStagesLatestEntityStatus(wfrIds []int, envID int) ([]*pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(wfrIds, envID) + + if len(ret) == 0 { + panic("no return value specified for FetchEnvAllCdStagesLatestEntityStatus") + } + + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func([]int, int) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfrIds, envID) + } + if rf, ok := ret.Get(0).(func([]int, int) []*pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(wfrIds, envID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CdWorkflowRunner) + } + } + + if rf, ok := ret.Get(1).(func([]int, int) error); ok { + r1 = rf(wfrIds, envID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindAllTriggeredWorkflowCountInLast24Hour provides a mock function with given fields: +func (_m *CdWorkflowRepository) FindAllTriggeredWorkflowCountInLast24Hour() (int, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for FindAllTriggeredWorkflowCountInLast24Hour") + } + + var r0 int var r1 error - if rf, ok := ret.Get(1).(func(int, bean.WorkflowType, int, int) error); ok { - r1 = rf(pipelineId, runnerType, offset, limit) + if rf, ok := ret.Get(0).(func() (int, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() } else { r1 = ret.Error(1) } @@ -107,22 +223,59 @@ func (_m *CdWorkflowRepository) FetchArtifactsByCdPipelineId(pipelineId int, run return r0, r1 } -// FindArtifactByPipelineIdAndRunnerType provides a mock function with given fields: pipelineId, runnerType, limit -func (_m *CdWorkflowRepository) FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType bean.WorkflowType, limit int) ([]pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(pipelineId, runnerType, limit) +// FindArtifactByPipelineIdAndRunnerType provides a mock function with given fields: pipelineId, runnerType, limit, runnerStatuses +func (_m *CdWorkflowRepository) FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType bean.WorkflowType, limit int, runnerStatuses []string) ([]pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(pipelineId, runnerType, limit, runnerStatuses) + + if len(ret) == 0 { + panic("no return value specified for FindArtifactByPipelineIdAndRunnerType") + } var r0 []pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int) []pipelineConfig.CdWorkflowRunner); ok { - r0 = rf(pipelineId, runnerType, limit) + var r1 error + if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int, []string) ([]pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId, runnerType, limit, runnerStatuses) + } + if rf, ok := ret.Get(0).(func(int, bean.WorkflowType, int, []string) []pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(pipelineId, runnerType, limit, runnerStatuses) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]pipelineConfig.CdWorkflowRunner) } } + if rf, ok := ret.Get(1).(func(int, bean.WorkflowType, int, []string) error); ok { + r1 = rf(pipelineId, runnerType, limit, runnerStatuses) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindBasicWorkflowRunnerById provides a mock function with given fields: wfrId +func (_m *CdWorkflowRepository) FindBasicWorkflowRunnerById(wfrId int) (*pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(wfrId) + + if len(ret) == 0 { + panic("no return value specified for FindBasicWorkflowRunnerById") + } + + var r0 *pipelineConfig.CdWorkflowRunner var r1 error - if rf, ok := ret.Get(1).(func(int, bean.WorkflowType, int) error); ok { - r1 = rf(pipelineId, runnerType, limit) + if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfrId) + } + if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(wfrId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CdWorkflowRunner) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(wfrId) } else { r1 = ret.Error(1) } @@ -134,7 +287,15 @@ func (_m *CdWorkflowRepository) FindArtifactByPipelineIdAndRunnerType(pipelineId func (_m *CdWorkflowRepository) FindById(wfId int) (*pipelineConfig.CdWorkflow, error) { ret := _m.Called(wfId) + if len(ret) == 0 { + panic("no return value specified for FindById") + } + var r0 *pipelineConfig.CdWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CdWorkflow, error)); ok { + return rf(wfId) + } if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CdWorkflow); ok { r0 = rf(wfId) } else { @@ -143,7 +304,6 @@ func (_m *CdWorkflowRepository) FindById(wfId int) (*pipelineConfig.CdWorkflow, } } - var r1 error if rf, ok := ret.Get(1).(func(int) error); ok { r1 = rf(wfId) } else { @@ -153,20 +313,27 @@ func (_m *CdWorkflowRepository) FindById(wfId int) (*pipelineConfig.CdWorkflow, return r0, r1 } -// FindByWorkflowIdAndRunnerType provides a mock function with given fields: wfId, runnerType +// FindByWorkflowIdAndRunnerType provides a mock function with given fields: ctx, wfId, runnerType func (_m *CdWorkflowRepository) FindByWorkflowIdAndRunnerType(ctx context.Context, wfId int, runnerType bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(wfId, runnerType) + ret := _m.Called(ctx, wfId, runnerType) + + if len(ret) == 0 { + panic("no return value specified for FindByWorkflowIdAndRunnerType") + } var r0 pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, bean.WorkflowType) pipelineConfig.CdWorkflowRunner); ok { - r0 = rf(wfId, runnerType) + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int, bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(ctx, wfId, runnerType) + } + if rf, ok := ret.Get(0).(func(context.Context, int, bean.WorkflowType) pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(ctx, wfId, runnerType) } else { r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) } - var r1 error - if rf, ok := ret.Get(1).(func(int, bean.WorkflowType) error); ok { - r1 = rf(wfId, runnerType) + if rf, ok := ret.Get(1).(func(context.Context, int, bean.WorkflowType) error); ok { + r1 = rf(ctx, wfId, runnerType) } else { r1 = ret.Error(1) } @@ -178,7 +345,15 @@ func (_m *CdWorkflowRepository) FindByWorkflowIdAndRunnerType(ctx context.Contex func (_m *CdWorkflowRepository) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, size int) ([]pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(appId, environmentId, offset, size) + if len(ret) == 0 { + panic("no return value specified for FindCdWorkflowMetaByEnvironmentId") + } + var r0 []pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int, int, int) ([]pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(appId, environmentId, offset, size) + } if rf, ok := ret.Get(0).(func(int, int, int, int) []pipelineConfig.CdWorkflowRunner); ok { r0 = rf(appId, environmentId, offset, size) } else { @@ -187,7 +362,6 @@ func (_m *CdWorkflowRepository) FindCdWorkflowMetaByEnvironmentId(appId int, env } } - var r1 error if rf, ok := ret.Get(1).(func(int, int, int, int) error); ok { r1 = rf(appId, environmentId, offset, size) } else { @@ -201,7 +375,15 @@ func (_m *CdWorkflowRepository) FindCdWorkflowMetaByEnvironmentId(appId int, env func (_m *CdWorkflowRepository) FindCdWorkflowMetaByPipelineId(pipelineId int, offset int, size int) ([]pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(pipelineId, offset, size) + if len(ret) == 0 { + panic("no return value specified for FindCdWorkflowMetaByPipelineId") + } + var r0 []pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int, int) ([]pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId, offset, size) + } if rf, ok := ret.Get(0).(func(int, int, int) []pipelineConfig.CdWorkflowRunner); ok { r0 = rf(pipelineId, offset, size) } else { @@ -210,7 +392,6 @@ func (_m *CdWorkflowRepository) FindCdWorkflowMetaByPipelineId(pipelineId int, o } } - var r1 error if rf, ok := ret.Get(1).(func(int, int, int) error); ok { r1 = rf(pipelineId, offset, size) } else { @@ -220,20 +401,27 @@ func (_m *CdWorkflowRepository) FindCdWorkflowMetaByPipelineId(pipelineId int, o return r0, r1 } -// FindCdWorkflowRunnerByEnvironmentIdAndRunnerType provides a mock function with given fields: appId, environmentId, runnerType -func (_m *CdWorkflowRepository) FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(appId, environmentId, runnerType) +// FindLastPreOrPostTriggeredByEnvironmentId provides a mock function with given fields: appId, environmentId +func (_m *CdWorkflowRepository) FindLastPreOrPostTriggeredByEnvironmentId(appId int, environmentId int) (pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(appId, environmentId) + + if len(ret) == 0 { + panic("no return value specified for FindLastPreOrPostTriggeredByEnvironmentId") + } var r0 pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, int, bean.WorkflowType) pipelineConfig.CdWorkflowRunner); ok { - r0 = rf(appId, environmentId, runnerType) + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(appId, environmentId) + } + if rf, ok := ret.Get(0).(func(int, int) pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(appId, environmentId) } else { r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) } - var r1 error - if rf, ok := ret.Get(1).(func(int, int, bean.WorkflowType) error); ok { - r1 = rf(appId, environmentId, runnerType) + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(appId, environmentId) } else { r1 = ret.Error(1) } @@ -241,20 +429,25 @@ func (_m *CdWorkflowRepository) FindLatestCdWorkflowRunnerByEnvironmentIdAndRunn return r0, r1 } -// FindConfigByPipelineId provides a mock function with given fields: pipelineId -func (_m *CdWorkflowRepository) FindConfigByPipelineId(pipelineId int) (*pipelineConfig.CdWorkflowConfig, error) { +// FindLastPreOrPostTriggeredByPipelineId provides a mock function with given fields: pipelineId +func (_m *CdWorkflowRepository) FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(pipelineId) - var r0 *pipelineConfig.CdWorkflowConfig - if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CdWorkflowConfig); ok { + if len(ret) == 0 { + panic("no return value specified for FindLastPreOrPostTriggeredByPipelineId") + } + + var r0 pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int) (pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId) + } + if rf, ok := ret.Get(0).(func(int) pipelineConfig.CdWorkflowRunner); ok { r0 = rf(pipelineId) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*pipelineConfig.CdWorkflowConfig) - } + r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) } - var r1 error if rf, ok := ret.Get(1).(func(int) error); ok { r1 = rf(pipelineId) } else { @@ -264,18 +457,27 @@ func (_m *CdWorkflowRepository) FindConfigByPipelineId(pipelineId int) (*pipelin return r0, r1 } -// FindLastPreOrPostTriggeredByEnvironmentId provides a mock function with given fields: appId, environmentId -func (_m *CdWorkflowRepository) FindLastPreOrPostTriggeredByEnvironmentId(appId int, environmentId int) (pipelineConfig.CdWorkflowRunner, error) { +// FindLastUnFailedProcessedRunner provides a mock function with given fields: appId, environmentId +func (_m *CdWorkflowRepository) FindLastUnFailedProcessedRunner(appId int, environmentId int) (*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(appId, environmentId) - var r0 pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, int) pipelineConfig.CdWorkflowRunner); ok { + if len(ret) == 0 { + panic("no return value specified for FindLastUnFailedProcessedRunner") + } + + var r0 *pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(appId, environmentId) + } + if rf, ok := ret.Get(0).(func(int, int) *pipelineConfig.CdWorkflowRunner); ok { r0 = rf(appId, environmentId) } else { - r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CdWorkflowRunner) + } } - var r1 error if rf, ok := ret.Get(1).(func(int, int) error); ok { r1 = rf(appId, environmentId) } else { @@ -285,39 +487,25 @@ func (_m *CdWorkflowRepository) FindLastPreOrPostTriggeredByEnvironmentId(appId return r0, r1 } -// FindLastPreOrPostTriggeredByPipelineId provides a mock function with given fields: pipelineId -func (_m *CdWorkflowRepository) FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(pipelineId) +// FindLatestByPipelineIdAndRunnerType provides a mock function with given fields: pipelineId, runnerType +func (_m *CdWorkflowRepository) FindLatestByPipelineIdAndRunnerType(pipelineId int, runnerType bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(pipelineId, runnerType) - var r0 pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int) pipelineConfig.CdWorkflowRunner); ok { - r0 = rf(pipelineId) - } else { - r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) + if len(ret) == 0 { + panic("no return value specified for FindLatestByPipelineIdAndRunnerType") } + var r0 pipelineConfig.CdWorkflowRunner var r1 error - if rf, ok := ret.Get(1).(func(int) error); ok { - r1 = rf(pipelineId) - } else { - r1 = ret.Error(1) + if rf, ok := ret.Get(0).(func(int, bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId, runnerType) } - - return r0, r1 -} - -// FindLastStatusByPipelineIdAndRunnerType provides a mock function with given fields: pipelineId, runnerType -func (_m *CdWorkflowRepository) FindLastStatusByPipelineIdAndRunnerType(pipelineId int, runnerType bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error) { - ret := _m.Called(pipelineId, runnerType) - - var r0 pipelineConfig.CdWorkflowRunner if rf, ok := ret.Get(0).(func(int, bean.WorkflowType) pipelineConfig.CdWorkflowRunner); ok { r0 = rf(pipelineId, runnerType) } else { r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) } - var r1 error if rf, ok := ret.Get(1).(func(int, bean.WorkflowType) error); ok { r1 = rf(pipelineId, runnerType) } else { @@ -331,7 +519,15 @@ func (_m *CdWorkflowRepository) FindLastStatusByPipelineIdAndRunnerType(pipeline func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineId(pipelineIds []int) (*pipelineConfig.CdWorkflow, error) { ret := _m.Called(pipelineIds) + if len(ret) == 0 { + panic("no return value specified for FindLatestCdWorkflowByPipelineId") + } + var r0 *pipelineConfig.CdWorkflow + var r1 error + if rf, ok := ret.Get(0).(func([]int) (*pipelineConfig.CdWorkflow, error)); ok { + return rf(pipelineIds) + } if rf, ok := ret.Get(0).(func([]int) *pipelineConfig.CdWorkflow); ok { r0 = rf(pipelineIds) } else { @@ -340,7 +536,6 @@ func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineId(pipelineIds []i } } - var r1 error if rf, ok := ret.Get(1).(func([]int) error); ok { r1 = rf(pipelineIds) } else { @@ -354,7 +549,15 @@ func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineId(pipelineIds []i func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineIdV2(pipelineIds []int) ([]*pipelineConfig.CdWorkflow, error) { ret := _m.Called(pipelineIds) + if len(ret) == 0 { + panic("no return value specified for FindLatestCdWorkflowByPipelineIdV2") + } + var r0 []*pipelineConfig.CdWorkflow + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CdWorkflow, error)); ok { + return rf(pipelineIds) + } if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CdWorkflow); ok { r0 = rf(pipelineIds) } else { @@ -363,7 +566,6 @@ func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineIdV2(pipelineIds [ } } - var r1 error if rf, ok := ret.Get(1).(func([]int) error); ok { r1 = rf(pipelineIds) } else { @@ -373,32 +575,107 @@ func (_m *CdWorkflowRepository) FindLatestCdWorkflowByPipelineIdV2(pipelineIds [ return r0, r1 } +// FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType provides a mock function with given fields: appId, environmentId, runnerType +func (_m *CdWorkflowRepository) FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(appId, environmentId, runnerType) + + if len(ret) == 0 { + panic("no return value specified for FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType") + } + + var r0 pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int, bean.WorkflowType) (pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(appId, environmentId, runnerType) + } + if rf, ok := ret.Get(0).(func(int, int, bean.WorkflowType) pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(appId, environmentId, runnerType) + } else { + r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) + } + + if rf, ok := ret.Get(1).(func(int, int, bean.WorkflowType) error); ok { + r1 = rf(appId, environmentId, runnerType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindLatestRunnerByPipelineIdsAndRunnerType provides a mock function with given fields: ctx, pipelineIds, runnerType +func (_m *CdWorkflowRepository) FindLatestRunnerByPipelineIdsAndRunnerType(ctx context.Context, pipelineIds []int, runnerType bean.WorkflowType) ([]pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(ctx, pipelineIds, runnerType) + + if len(ret) == 0 { + panic("no return value specified for FindLatestRunnerByPipelineIdsAndRunnerType") + } + + var r0 []pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []int, bean.WorkflowType) ([]pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(ctx, pipelineIds, runnerType) + } + if rf, ok := ret.Get(0).(func(context.Context, []int, bean.WorkflowType) []pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(ctx, pipelineIds, runnerType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]pipelineConfig.CdWorkflowRunner) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []int, bean.WorkflowType) error); ok { + r1 = rf(ctx, pipelineIds, runnerType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindLatestWfrByAppIdAndEnvironmentId provides a mock function with given fields: appId, environmentId func (_m *CdWorkflowRepository) FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(appId, environmentId) - var r0 pipelineConfig.CdWorkflowRunner - if rf, ok := ret.Get(0).(func(int, int) pipelineConfig.CdWorkflowRunner); ok { + if len(ret) == 0 { + panic("no return value specified for FindLatestWfrByAppIdAndEnvironmentId") + } + + var r0 *pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(appId, environmentId) + } + if rf, ok := ret.Get(0).(func(int, int) *pipelineConfig.CdWorkflowRunner); ok { r0 = rf(appId, environmentId) } else { - r0 = ret.Get(0).(pipelineConfig.CdWorkflowRunner) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CdWorkflowRunner) + } } - var r1 error if rf, ok := ret.Get(1).(func(int, int) error); ok { r1 = rf(appId, environmentId) } else { r1 = ret.Error(1) } - return &r0, r1 + return r0, r1 } // FindPreviousCdWfRunnerByStatus provides a mock function with given fields: pipelineId, currentWFRunnerId, status func (_m *CdWorkflowRepository) FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(pipelineId, currentWFRunnerId, status) + if len(ret) == 0 { + panic("no return value specified for FindPreviousCdWfRunnerByStatus") + } + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int, []string) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(pipelineId, currentWFRunnerId, status) + } if rf, ok := ret.Get(0).(func(int, int, []string) []*pipelineConfig.CdWorkflowRunner); ok { r0 = rf(pipelineId, currentWFRunnerId, status) } else { @@ -407,7 +684,6 @@ func (_m *CdWorkflowRepository) FindPreviousCdWfRunnerByStatus(pipelineId int, c } } - var r1 error if rf, ok := ret.Get(1).(func(int, int, []string) error); ok { r1 = rf(pipelineId, currentWFRunnerId, status) } else { @@ -417,11 +693,47 @@ func (_m *CdWorkflowRepository) FindPreviousCdWfRunnerByStatus(pipelineId int, c return r0, r1 } +// FindRetriedWorkflowCountByReferenceId provides a mock function with given fields: wfrId +func (_m *CdWorkflowRepository) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) { + ret := _m.Called(wfrId) + + if len(ret) == 0 { + panic("no return value specified for FindRetriedWorkflowCountByReferenceId") + } + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(int) (int, error)); ok { + return rf(wfrId) + } + if rf, ok := ret.Get(0).(func(int) int); ok { + r0 = rf(wfrId) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(wfrId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindWorkflowRunnerByCdWorkflowId provides a mock function with given fields: wfIds func (_m *CdWorkflowRepository) FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(wfIds) + if len(ret) == 0 { + panic("no return value specified for FindWorkflowRunnerByCdWorkflowId") + } + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfIds) + } if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CdWorkflowRunner); ok { r0 = rf(wfIds) } else { @@ -430,7 +742,6 @@ func (_m *CdWorkflowRepository) FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([ } } - var r1 error if rf, ok := ret.Get(1).(func([]int) error); ok { r1 = rf(wfIds) } else { @@ -444,7 +755,15 @@ func (_m *CdWorkflowRepository) FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([ func (_m *CdWorkflowRepository) FindWorkflowRunnerById(wfrId int) (*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(wfrId) + if len(ret) == 0 { + panic("no return value specified for FindWorkflowRunnerById") + } + var r0 *pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfrId) + } if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CdWorkflowRunner); ok { r0 = rf(wfrId) } else { @@ -453,7 +772,6 @@ func (_m *CdWorkflowRepository) FindWorkflowRunnerById(wfrId int) (*pipelineConf } } - var r1 error if rf, ok := ret.Get(1).(func(int) error); ok { r1 = rf(wfrId) } else { @@ -467,6 +785,10 @@ func (_m *CdWorkflowRepository) FindWorkflowRunnerById(wfrId int) (*pipelineConf func (_m *CdWorkflowRepository) GetConnection() *pg.DB { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for GetConnection") + } + var r0 *pg.DB if rf, ok := ret.Get(0).(func() *pg.DB); ok { r0 = rf() @@ -479,18 +801,113 @@ func (_m *CdWorkflowRepository) GetConnection() *pg.DB { return r0 } +// GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses provides a mock function with given fields: getPipelineDeployedWithinHours +func (_m *CdWorkflowRepository) GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours int) ([]*pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(getPipelineDeployedWithinHours) + + if len(ret) == 0 { + panic("no return value specified for GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses") + } + + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(getPipelineDeployedWithinHours) + } + if rf, ok := ret.Get(0).(func(int) []*pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(getPipelineDeployedWithinHours) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CdWorkflowRunner) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(getPipelineDeployedWithinHours) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetPreviousQueuedRunners provides a mock function with given fields: cdWfrId, pipelineId +func (_m *CdWorkflowRepository) GetPreviousQueuedRunners(cdWfrId int, pipelineId int) ([]*pipelineConfig.CdWorkflowRunner, error) { + ret := _m.Called(cdWfrId, pipelineId) + + if len(ret) == 0 { + panic("no return value specified for GetPreviousQueuedRunners") + } + + var r0 []*pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(int, int) ([]*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(cdWfrId, pipelineId) + } + if rf, ok := ret.Get(0).(func(int, int) []*pipelineConfig.CdWorkflowRunner); ok { + r0 = rf(cdWfrId, pipelineId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CdWorkflowRunner) + } + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(cdWfrId, pipelineId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IsLatestCDWfr provides a mock function with given fields: pipelineId, wfrId +func (_m *CdWorkflowRepository) IsLatestCDWfr(pipelineId int, wfrId int) (bool, error) { + ret := _m.Called(pipelineId, wfrId) + + if len(ret) == 0 { + panic("no return value specified for IsLatestCDWfr") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (bool, error)); ok { + return rf(pipelineId, wfrId) + } + if rf, ok := ret.Get(0).(func(int, int) bool); ok { + r0 = rf(pipelineId, wfrId) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(int, int) error); ok { + r1 = rf(pipelineId, wfrId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IsLatestWf provides a mock function with given fields: pipelineId, wfId func (_m *CdWorkflowRepository) IsLatestWf(pipelineId int, wfId int) (bool, error) { ret := _m.Called(pipelineId, wfId) + if len(ret) == 0 { + panic("no return value specified for IsLatestWf") + } + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(int, int) (bool, error)); ok { + return rf(pipelineId, wfId) + } if rf, ok := ret.Get(0).(func(int, int) bool); ok { r0 = rf(pipelineId, wfId) } else { r0 = ret.Get(0).(bool) } - var r1 error if rf, ok := ret.Get(1).(func(int, int) error); ok { r1 = rf(pipelineId, wfId) } else { @@ -500,13 +917,17 @@ func (_m *CdWorkflowRepository) IsLatestWf(pipelineId int, wfId int) (bool, erro return r0, r1 } -// SaveWorkFlow provides a mock function with given fields: wf +// SaveWorkFlow provides a mock function with given fields: ctx, wf func (_m *CdWorkflowRepository) SaveWorkFlow(ctx context.Context, wf *pipelineConfig.CdWorkflow) error { - ret := _m.Called(wf) + ret := _m.Called(ctx, wf) + + if len(ret) == 0 { + panic("no return value specified for SaveWorkFlow") + } var r0 error - if rf, ok := ret.Get(0).(func(*pipelineConfig.CdWorkflow) error); ok { - r0 = rf(wf) + if rf, ok := ret.Get(0).(func(context.Context, *pipelineConfig.CdWorkflow) error); ok { + r0 = rf(ctx, wf) } else { r0 = ret.Error(0) } @@ -518,7 +939,15 @@ func (_m *CdWorkflowRepository) SaveWorkFlow(ctx context.Context, wf *pipelineCo func (_m *CdWorkflowRepository) SaveWorkFlowRunner(wfr *pipelineConfig.CdWorkflowRunner) (*pipelineConfig.CdWorkflowRunner, error) { ret := _m.Called(wfr) + if len(ret) == 0 { + panic("no return value specified for SaveWorkFlowRunner") + } + var r0 *pipelineConfig.CdWorkflowRunner + var r1 error + if rf, ok := ret.Get(0).(func(*pipelineConfig.CdWorkflowRunner) (*pipelineConfig.CdWorkflowRunner, error)); ok { + return rf(wfr) + } if rf, ok := ret.Get(0).(func(*pipelineConfig.CdWorkflowRunner) *pipelineConfig.CdWorkflowRunner); ok { r0 = rf(wfr) } else { @@ -527,7 +956,6 @@ func (_m *CdWorkflowRepository) SaveWorkFlowRunner(wfr *pipelineConfig.CdWorkflo } } - var r1 error if rf, ok := ret.Get(1).(func(*pipelineConfig.CdWorkflowRunner) error); ok { r1 = rf(wfr) } else { @@ -547,6 +975,10 @@ func (_m *CdWorkflowRepository) SaveWorkFlows(wfs ...*pipelineConfig.CdWorkflow) _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for SaveWorkFlows") + } + var r0 error if rf, ok := ret.Get(0).(func(...*pipelineConfig.CdWorkflow) error); ok { r0 = rf(wfs...) @@ -557,10 +989,57 @@ func (_m *CdWorkflowRepository) SaveWorkFlows(wfs ...*pipelineConfig.CdWorkflow) return r0 } +// UpdateIsArtifactUploaded provides a mock function with given fields: wfrId, isArtifactUploaded +func (_m *CdWorkflowRepository) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error { + ret := _m.Called(wfrId, isArtifactUploaded) + + if len(ret) == 0 { + panic("no return value specified for UpdateIsArtifactUploaded") + } + + var r0 error + if rf, ok := ret.Get(0).(func(int, bool) error); ok { + r0 = rf(wfrId, isArtifactUploaded) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateRunnerStatusToFailedForIds provides a mock function with given fields: errMsg, triggeredBy, cdWfrIds +func (_m *CdWorkflowRepository) UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error { + _va := make([]interface{}, len(cdWfrIds)) + for _i := range cdWfrIds { + _va[_i] = cdWfrIds[_i] + } + var _ca []interface{} + _ca = append(_ca, errMsg, triggeredBy) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for UpdateRunnerStatusToFailedForIds") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, int32, ...int) error); ok { + r0 = rf(errMsg, triggeredBy, cdWfrIds...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UpdateWorkFlow provides a mock function with given fields: wf func (_m *CdWorkflowRepository) UpdateWorkFlow(wf *pipelineConfig.CdWorkflow) error { ret := _m.Called(wf) + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlow") + } + var r0 error if rf, ok := ret.Get(0).(func(*pipelineConfig.CdWorkflow) error); ok { r0 = rf(wf) @@ -575,6 +1054,10 @@ func (_m *CdWorkflowRepository) UpdateWorkFlow(wf *pipelineConfig.CdWorkflow) er func (_m *CdWorkflowRepository) UpdateWorkFlowRunner(wfr *pipelineConfig.CdWorkflowRunner) error { ret := _m.Called(wfr) + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlowRunner") + } + var r0 error if rf, ok := ret.Get(0).(func(*pipelineConfig.CdWorkflowRunner) error); ok { r0 = rf(wfr) @@ -589,6 +1072,10 @@ func (_m *CdWorkflowRepository) UpdateWorkFlowRunner(wfr *pipelineConfig.CdWorkf func (_m *CdWorkflowRepository) UpdateWorkFlowRunners(wfr []*pipelineConfig.CdWorkflowRunner) error { ret := _m.Called(wfr) + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlowRunners") + } + var r0 error if rf, ok := ret.Get(0).(func([]*pipelineConfig.CdWorkflowRunner) error); ok { r0 = rf(wfr) @@ -603,6 +1090,10 @@ func (_m *CdWorkflowRepository) UpdateWorkFlowRunners(wfr []*pipelineConfig.CdWo func (_m *CdWorkflowRepository) UpdateWorkFlowRunnersWithTxn(wfrs []*pipelineConfig.CdWorkflowRunner, tx *pg.Tx) error { ret := _m.Called(wfrs, tx) + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlowRunnersWithTxn") + } + var r0 error if rf, ok := ret.Get(0).(func([]*pipelineConfig.CdWorkflowRunner, *pg.Tx) error); ok { r0 = rf(wfrs, tx) @@ -613,13 +1104,12 @@ func (_m *CdWorkflowRepository) UpdateWorkFlowRunnersWithTxn(wfrs []*pipelineCon return r0 } -type mockConstructorTestingTNewCdWorkflowRepository interface { +// NewCdWorkflowRepository creates a new instance of CdWorkflowRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewCdWorkflowRepository(t interface { mock.TestingT Cleanup(func()) -} - -// NewCdWorkflowRepository creates a new instance of CdWorkflowRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewCdWorkflowRepository(t mockConstructorTestingTNewCdWorkflowRepository) *CdWorkflowRepository { +}) *CdWorkflowRepository { mock := &CdWorkflowRepository{} mock.Mock.Test(t) diff --git a/internal/sql/repository/pipelineConfig/mocks/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/mocks/CiWorkflowRepository.go index d0ebafb3ba..03b7ba8a73 100644 --- a/internal/sql/repository/pipelineConfig/mocks/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/mocks/CiWorkflowRepository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.20.0. DO NOT EDIT. +// Code generated by mockery v2.42.0. DO NOT EDIT. package mocks @@ -16,6 +16,10 @@ type CiWorkflowRepository struct { func (_m *CiWorkflowRepository) ExistsByStatus(status string) (bool, error) { ret := _m.Called(status) + if len(ret) == 0 { + panic("no return value specified for ExistsByStatus") + } + var r0 bool var r1 error if rf, ok := ret.Get(0).(func(string) (bool, error)); ok { @@ -36,10 +40,102 @@ func (_m *CiWorkflowRepository) ExistsByStatus(status string) (bool, error) { return r0, r1 } +// FIndCiWorkflowStatusesByAppId provides a mock function with given fields: appId +func (_m *CiWorkflowRepository) FIndCiWorkflowStatusesByAppId(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { + ret := _m.Called(appId) + + if len(ret) == 0 { + panic("no return value specified for FIndCiWorkflowStatusesByAppId") + } + + var r0 []*pipelineConfig.CiWorkflowStatus + var r1 error + if rf, ok := ret.Get(0).(func(int) ([]*pipelineConfig.CiWorkflowStatus, error)); ok { + return rf(appId) + } + if rf, ok := ret.Get(0).(func(int) []*pipelineConfig.CiWorkflowStatus); ok { + r0 = rf(appId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiWorkflowStatus) + } + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(appId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindAllLastTriggeredWorkflowByArtifactId provides a mock function with given fields: ciArtifactId +func (_m *CiWorkflowRepository) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) ([]*pipelineConfig.CiWorkflow, error) { + ret := _m.Called(ciArtifactId) + + if len(ret) == 0 { + panic("no return value specified for FindAllLastTriggeredWorkflowByArtifactId") + } + + var r0 []*pipelineConfig.CiWorkflow + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CiWorkflow, error)); ok { + return rf(ciArtifactId) + } + if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CiWorkflow); ok { + r0 = rf(ciArtifactId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiWorkflow) + } + } + + if rf, ok := ret.Get(1).(func([]int) error); ok { + r1 = rf(ciArtifactId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindAllTriggeredWorkflowCountInLast24Hour provides a mock function with given fields: +func (_m *CiWorkflowRepository) FindAllTriggeredWorkflowCountInLast24Hour() (int, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for FindAllTriggeredWorkflowCountInLast24Hour") + } + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func() (int, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindBuildTypeAndStatusDataOfLast1Day provides a mock function with given fields: func (_m *CiWorkflowRepository) FindBuildTypeAndStatusDataOfLast1Day() []*pipelineConfig.BuildTypeCount { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for FindBuildTypeAndStatusDataOfLast1Day") + } + var r0 []*pipelineConfig.BuildTypeCount if rf, ok := ret.Get(0).(func() []*pipelineConfig.BuildTypeCount); ok { r0 = rf() @@ -56,6 +152,10 @@ func (_m *CiWorkflowRepository) FindBuildTypeAndStatusDataOfLast1Day() []*pipeli func (_m *CiWorkflowRepository) FindById(id int) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(id) + if len(ret) == 0 { + panic("no return value specified for FindById") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflow, error)); ok { @@ -82,6 +182,10 @@ func (_m *CiWorkflowRepository) FindById(id int) (*pipelineConfig.CiWorkflow, er func (_m *CiWorkflowRepository) FindByName(name string) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for FindByName") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(string) (*pipelineConfig.CiWorkflow, error)); ok { @@ -108,6 +212,10 @@ func (_m *CiWorkflowRepository) FindByName(name string) (*pipelineConfig.CiWorkf func (_m *CiWorkflowRepository) FindByPipelineId(pipelineId int, offset int, size int) ([]pipelineConfig.WorkflowWithArtifact, error) { ret := _m.Called(pipelineId, offset, size) + if len(ret) == 0 { + panic("no return value specified for FindByPipelineId") + } + var r0 []pipelineConfig.WorkflowWithArtifact var r1 error if rf, ok := ret.Get(0).(func(int, int, int) ([]pipelineConfig.WorkflowWithArtifact, error)); ok { @@ -134,6 +242,10 @@ func (_m *CiWorkflowRepository) FindByPipelineId(pipelineId int, offset int, siz func (_m *CiWorkflowRepository) FindByStatusesIn(activeStatuses []string) ([]*pipelineConfig.CiWorkflow, error) { ret := _m.Called(activeStatuses) + if len(ret) == 0 { + panic("no return value specified for FindByStatusesIn") + } + var r0 []*pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func([]string) ([]*pipelineConfig.CiWorkflow, error)); ok { @@ -160,6 +272,10 @@ func (_m *CiWorkflowRepository) FindByStatusesIn(activeStatuses []string) ([]*pi func (_m *CiWorkflowRepository) FindCiWorkflowGitTriggersById(id int) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(id) + if len(ret) == 0 { + panic("no return value specified for FindCiWorkflowGitTriggersById") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflow, error)); ok { @@ -182,25 +298,29 @@ func (_m *CiWorkflowRepository) FindCiWorkflowGitTriggersById(id int) (*pipeline return r0, r1 } -// FindConfigByPipelineId provides a mock function with given fields: pipelineId -func (_m *CiWorkflowRepository) FindConfigByPipelineId(pipelineId int) (*pipelineConfig.CiWorkflowConfig, error) { - ret := _m.Called(pipelineId) +// FindCiWorkflowGitTriggersByIds provides a mock function with given fields: ids +func (_m *CiWorkflowRepository) FindCiWorkflowGitTriggersByIds(ids []int) ([]*pipelineConfig.CiWorkflow, error) { + ret := _m.Called(ids) - var r0 *pipelineConfig.CiWorkflowConfig + if len(ret) == 0 { + panic("no return value specified for FindCiWorkflowGitTriggersByIds") + } + + var r0 []*pipelineConfig.CiWorkflow var r1 error - if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflowConfig, error)); ok { - return rf(pipelineId) + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CiWorkflow, error)); ok { + return rf(ids) } - if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CiWorkflowConfig); ok { - r0 = rf(pipelineId) + if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.CiWorkflow); ok { + r0 = rf(ids) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*pipelineConfig.CiWorkflowConfig) + r0 = ret.Get(0).([]*pipelineConfig.CiWorkflow) } } - if rf, ok := ret.Get(1).(func(int) error); ok { - r1 = rf(pipelineId) + if rf, ok := ret.Get(1).(func([]int) error); ok { + r1 = rf(ids) } else { r1 = ret.Error(1) } @@ -212,6 +332,10 @@ func (_m *CiWorkflowRepository) FindConfigByPipelineId(pipelineId int) (*pipelin func (_m *CiWorkflowRepository) FindLastTriggeredWorkflow(pipelineId int) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(pipelineId) + if len(ret) == 0 { + panic("no return value specified for FindLastTriggeredWorkflow") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflow, error)); ok { @@ -238,6 +362,10 @@ func (_m *CiWorkflowRepository) FindLastTriggeredWorkflow(pipelineId int) (*pipe func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(ciArtifactId) + if len(ret) == 0 { + panic("no return value specified for FindLastTriggeredWorkflowByArtifactId") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflow, error)); ok { @@ -264,6 +392,10 @@ func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowByArtifactId(ciArtifact func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowByCiIds(pipelineId []int) ([]*pipelineConfig.CiWorkflow, error) { ret := _m.Called(pipelineId) + if len(ret) == 0 { + panic("no return value specified for FindLastTriggeredWorkflowByCiIds") + } + var r0 []*pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.CiWorkflow, error)); ok { @@ -290,6 +422,10 @@ func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowByCiIds(pipelineId []in func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowGitTriggersByArtifactId(ciArtifactId int) (*pipelineConfig.CiWorkflow, error) { ret := _m.Called(ciArtifactId) + if len(ret) == 0 { + panic("no return value specified for FindLastTriggeredWorkflowGitTriggersByArtifactId") + } + var r0 *pipelineConfig.CiWorkflow var r1 error if rf, ok := ret.Get(0).(func(int) (*pipelineConfig.CiWorkflow, error)); ok { @@ -312,10 +448,72 @@ func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowGitTriggersByArtifactId return r0, r1 } +// FindLastTriggeredWorkflowGitTriggersByArtifactIds provides a mock function with given fields: ciArtifactIds +func (_m *CiWorkflowRepository) FindLastTriggeredWorkflowGitTriggersByArtifactIds(ciArtifactIds []int) ([]*pipelineConfig.WorkflowWithArtifact, error) { + ret := _m.Called(ciArtifactIds) + + if len(ret) == 0 { + panic("no return value specified for FindLastTriggeredWorkflowGitTriggersByArtifactIds") + } + + var r0 []*pipelineConfig.WorkflowWithArtifact + var r1 error + if rf, ok := ret.Get(0).(func([]int) ([]*pipelineConfig.WorkflowWithArtifact, error)); ok { + return rf(ciArtifactIds) + } + if rf, ok := ret.Get(0).(func([]int) []*pipelineConfig.WorkflowWithArtifact); ok { + r0 = rf(ciArtifactIds) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.WorkflowWithArtifact) + } + } + + if rf, ok := ret.Get(1).(func([]int) error); ok { + r1 = rf(ciArtifactIds) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindRetriedWorkflowCountByReferenceId provides a mock function with given fields: id +func (_m *CiWorkflowRepository) FindRetriedWorkflowCountByReferenceId(id int) (int, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for FindRetriedWorkflowCountByReferenceId") + } + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(int) (int, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(int) int); ok { + r0 = rf(id) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // SaveWorkFlow provides a mock function with given fields: wf func (_m *CiWorkflowRepository) SaveWorkFlow(wf *pipelineConfig.CiWorkflow) error { ret := _m.Called(wf) + if len(ret) == 0 { + panic("no return value specified for SaveWorkFlow") + } + var r0 error if rf, ok := ret.Get(0).(func(*pipelineConfig.CiWorkflow) error); ok { r0 = rf(wf) @@ -326,13 +524,17 @@ func (_m *CiWorkflowRepository) SaveWorkFlow(wf *pipelineConfig.CiWorkflow) erro return r0 } -// SaveWorkFlowConfig provides a mock function with given fields: config -func (_m *CiWorkflowRepository) SaveWorkFlowConfig(config *pipelineConfig.CiWorkflowConfig) error { - ret := _m.Called(config) +// UpdateArtifactUploaded provides a mock function with given fields: id, isUploaded +func (_m *CiWorkflowRepository) UpdateArtifactUploaded(id int, isUploaded bool) error { + ret := _m.Called(id, isUploaded) + + if len(ret) == 0 { + panic("no return value specified for UpdateArtifactUploaded") + } var r0 error - if rf, ok := ret.Get(0).(func(*pipelineConfig.CiWorkflowConfig) error); ok { - r0 = rf(config) + if rf, ok := ret.Get(0).(func(int, bool) error); ok { + r0 = rf(id, isUploaded) } else { r0 = ret.Error(0) } @@ -344,6 +546,10 @@ func (_m *CiWorkflowRepository) SaveWorkFlowConfig(config *pipelineConfig.CiWork func (_m *CiWorkflowRepository) UpdateWorkFlow(wf *pipelineConfig.CiWorkflow) error { ret := _m.Called(wf) + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlow") + } + var r0 error if rf, ok := ret.Get(0).(func(*pipelineConfig.CiWorkflow) error); ok { r0 = rf(wf) @@ -354,13 +560,12 @@ func (_m *CiWorkflowRepository) UpdateWorkFlow(wf *pipelineConfig.CiWorkflow) er return r0 } -type mockConstructorTestingTNewCiWorkflowRepository interface { +// NewCiWorkflowRepository creates a new instance of CiWorkflowRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewCiWorkflowRepository(t interface { mock.TestingT Cleanup(func()) -} - -// NewCiWorkflowRepository creates a new instance of CiWorkflowRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewCiWorkflowRepository(t mockConstructorTestingTNewCiWorkflowRepository) *CiWorkflowRepository { +}) *CiWorkflowRepository { mock := &CiWorkflowRepository{} mock.Mock.Test(t) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 29d310b48a..856cb6c55b 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -364,10 +364,6 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow } cdPipeline.App = *appModel } - cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(cdPipeline.Id) - if err != nil && !util.IsErrNoRows(err) { - return nil, err - } workflowExecutor := runner.ExecutorType @@ -714,9 +710,9 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow if pipelineReleaseCounter > 0 { cdStageWorkflowRequest.DeploymentReleaseCounter = pipelineReleaseCounter } - if cdWorkflowConfig.CdCacheRegion == "" { - cdWorkflowConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() - } + cdWorkflowConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + // For Pre-CD / Post-CD workflow, cache is not uploaded; hence no need to set cache bucket + cdWorkflowConfigCdCacheBucket := "" if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { //populate input variables of steps with extra env variables @@ -730,19 +726,19 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow switch cdStageWorkflowRequest.CloudProvider { case types.BLOB_STORAGE_S3: //No AccessKey is used for uploading artifacts, instead IAM based auth is used - cdStageWorkflowRequest.CdCacheRegion = cdWorkflowConfig.CdCacheRegion - cdStageWorkflowRequest.CdCacheLocation = cdWorkflowConfig.CdCacheBucket - cdStageWorkflowRequest.ArtifactLocation, cdStageWorkflowRequest.CiArtifactBucket, cdStageWorkflowRequest.CiArtifactFileName = impl.buildArtifactLocationForS3(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.CdCacheRegion = cdWorkflowConfigCdCacheRegion + cdStageWorkflowRequest.CdCacheLocation = cdWorkflowConfigCdCacheBucket + cdStageWorkflowRequest.ArtifactLocation, cdStageWorkflowRequest.CiArtifactBucket, cdStageWorkflowRequest.CiArtifactFileName = impl.buildArtifactLocationForS3(cdWf, runner) cdStageWorkflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ AccessKey: impl.config.BlobStorageS3AccessKey, Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: cdWorkflowConfig.CdCacheBucket, - CiCacheRegion: cdWorkflowConfig.CdCacheRegion, + CiCacheBucketName: cdWorkflowConfigCdCacheBucket, + CiCacheRegion: cdWorkflowConfigCdCacheRegion, CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, CiArtifactBucketName: cdStageWorkflowRequest.CiArtifactBucket, - CiArtifactRegion: cdWorkflowConfig.CdCacheRegion, + CiArtifactRegion: cdWorkflowConfigCdCacheRegion, CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), @@ -754,7 +750,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow ArtifactBucketName: impl.config.GetDefaultBuildLogsBucket(), LogBucketName: impl.config.GetDefaultBuildLogsBucket(), } - cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWf, runner) cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation case types.BLOB_STORAGE_AZURE: cdStageWorkflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ @@ -772,7 +768,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow CiLogRegion: "", AccessKey: impl.config.AzureAccountName, } - cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWf, runner) cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation default: if impl.config.BlobStorageEnabled { @@ -935,24 +931,17 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { - cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if len(cdArtifactLocationFormat) == 0 { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } - if cdWorkflowConfig.LogsBucket == "" { - cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfig.LogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) +func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) artifactFileName := fmt.Sprintf(cdArtifactLocationFormat, cdWf.Id, runner.Id) - return ArtifactLocation, cdWorkflowConfig.LogsBucket, artifactFileName + return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName } -func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { - cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if len(cdArtifactLocationFormat) == 0 { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } +func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 442cbd76bf..605f25d43d 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -62,7 +62,7 @@ type CdHandler interface { GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) - DownloadCdWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) + DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) CancelStage(workflowRunnerId int, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) @@ -217,19 +217,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus return 0, "", err } - cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(savedWorkflow.CdWorkflow.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("unable to fetch cdWorkflowConfig", "err", err) - return 0, "", err - } - - cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if len(cdArtifactLocationFormat) == 0 { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } - if cdWorkflowConfig.LogsBucket == "" { - cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if savedWorkflow.Status != executors.WorkflowCancel { @@ -496,18 +484,8 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) - cdConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(pipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", err) - return nil, nil, err - } - - if cdConfig.LogsBucket == "" { - cdConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() // TODO -fixme - } - if cdConfig.CdCacheRegion == "" { - cdConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() - } + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() cdLogRequest := types.BuildLogRequest{ PipelineId: cdWorkflow.CdWorkflow.PipelineId, @@ -526,12 +504,12 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfig.LogsBucket, - Region: cdConfig.CdCacheRegion, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, }, GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfig.LogsBucket, + BucketName: cdConfigLogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } @@ -647,7 +625,7 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, } -func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { +func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil { impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) @@ -658,18 +636,8 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return nil, errors.New("logs-not-stored-in-repository") } - cdConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(pipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("unable to fetch ciCdConfig", "err", err) - return nil, err - } - - if cdConfig.LogsBucket == "" { - cdConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } - if cdConfig.CdCacheRegion == "" { - cdConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() - } + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() item := strconv.Itoa(wfr.Id) awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ @@ -677,8 +645,8 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfig.LogsBucket, - Region: cdConfig.CdCacheRegion, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, } azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ @@ -688,13 +656,10 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i BlobContainerName: impl.config.AzureBlobContainerCiLog, } gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfig.LogsBucket, + BucketName: cdConfigLogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, } - cdArtifactLocationFormat := cdConfig.CdArtifactLocationFormat - if len(cdArtifactLocationFormat) == 0 { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { key = wfr.CdArtifactLocation diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 343d044eac..a9332f79f0 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -71,8 +71,8 @@ type CiHandler interface { //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) CancelBuild(workflowId int, forceAbort bool) (int, error) - GetRunningWorkflowLogs(pipelineId int, workflowId int) (*bufio.Reader, func() error, error) - GetHistoricBuildLogs(pipelineId int, workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) + GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) + GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) //SyncWorkflows() error GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) @@ -772,16 +772,16 @@ func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.Artifacts } return artifactsResponse, nil } -func (impl *CiHandlerImpl) GetRunningWorkflowLogs(pipelineId int, workflowId int) (*bufio.Reader, func() error, error) { +func (impl *CiHandlerImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("err", "err", err) return nil, nil, err } - return impl.getWorkflowLogs(pipelineId, ciWorkflow) + return impl.getWorkflowLogs(ciWorkflow) } -func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { +func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { return bufio.NewReader(strings.NewReader("")), nil, nil } @@ -810,7 +810,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == executors.WorkflowCancel { impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt) + return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) } if err != nil { impl.Logger.Errorw("err on fetch workflow logs", "err", err) @@ -823,21 +823,10 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return logReader, cleanUp, err } -func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { +func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs") - - ciConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(pipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - - if ciConfig.LogsBucket == "" { - ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } - if ciConfig.CiCacheRegion == "" { - ciConfig.CiCacheRegion = impl.config.DefaultCacheBucketRegion - } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty if strings.Contains(ciWorkflow.LogLocation, "main.log") { logsFilePath = ciWorkflow.LogLocation @@ -859,12 +848,12 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfig.LogsBucket, - Region: ciConfig.CiCacheRegion, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, }, GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfig.LogsBucket, + BucketName: ciConfigLogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } @@ -905,20 +894,9 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i return nil, errors.New("invalid request, wf not in pipeline") } - ciConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(pipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("unable to fetch ciCdConfig", "err", err) - return nil, err - } - - if ciConfig.LogsBucket == "" { - ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } - + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() item := strconv.Itoa(ciWorkflow.Id) - if ciConfig.CiCacheRegion == "" { - ciConfig.CiCacheRegion = impl.config.DefaultCacheBucketRegion - } + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, @@ -930,19 +908,16 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfig.LogsBucket, - Region: ciConfig.CiCacheRegion, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, } gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfig.LogsBucket, + BucketName: ciConfigLogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, } - ciArtifactLocationFormat := ciConfig.CiArtifactLocationFormat - if len(ciArtifactLocationFormat) == 0 { - ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { key = ciWorkflow.CiArtifactLocation @@ -994,12 +969,8 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i return file, nil } -func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { - ciConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(pipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("err", "err", err) - return nil, err - } +func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { + var err error if ciWorkflow == nil { ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) if err != nil { @@ -1007,9 +978,8 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, return nil, err } } - if ciConfig.LogsBucket == "" { - ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion ciLogRequest := types.BuildLogRequest{ PipelineId: ciWorkflow.CiPipelineId, WorkflowId: ciWorkflow.Id, @@ -1027,12 +997,12 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfig.LogsBucket, - Region: ciConfig.CiCacheRegion, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, }, GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfig.LogsBucket, + BucketName: ciConfigLogsBucket, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } @@ -1157,16 +1127,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus return 0, err } - ciWorkflowConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(savedWorkflow.CiPipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("unable to fetch ciWorkflowConfig", "err", err) - return 0, err - } - - ciArtifactLocationFormat := ciWorkflowConfig.CiArtifactLocationFormat - if len(ciArtifactLocationFormat) == 0 { - ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { @@ -1199,7 +1160,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) if extractErrorCode(savedWorkflow.Message) != CiStageFailErrorCode { - go impl.WriteCIFailEvent(savedWorkflow, ciWorkflowConfig.CiImage) + go impl.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } @@ -1220,13 +1181,13 @@ func extractErrorCode(msg string) int { return -1 } -func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow, ciImage string) { +func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { event := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) material := &client.MaterialTriggerInfo{} material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material, ciImage) + event = impl.eventFactory.BuildExtraCIData(event, material) event.CiArtifactId = 0 _, evtErr := impl.eventClient.WriteNotificationEvent(event) if evtErr != nil { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 5d3b1f6393..28f8baa2e6 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -254,38 +254,30 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) break } - ciWorkflowConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(trigger.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("could not fetch ci config", "pipeline", trigger.PipelineId) - return 0, err - } - scope := resourceQualifiers.Scope{ AppId: pipeline.App.Id, } - env, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) + ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() + envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) if err != nil { return 0, err } - if isJob && env != nil { - ciWorkflowConfig.Namespace = env.Namespace + if isJob && envModal != nil { + ciWorkflowConfigNamespace = envModal.Namespace // This will be populated for jobs running in selected environment - scope.EnvId = env.Id - scope.ClusterId = env.ClusterId + scope.EnvId = envModal.Id + scope.ClusterId = envModal.ClusterId scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - EnvironmentName: env.Name, - ClusterName: env.Cluster.ClusterName, - Namespace: env.Namespace, + EnvironmentName: envModal.Name, + ClusterName: envModal.Cluster.ClusterName, + Namespace: envModal.Namespace, } } - if ciWorkflowConfig.Namespace == "" { - ciWorkflowConfig.Namespace = impl.config.GetDefaultNamespace() - } if scope.SystemMetadata == nil { scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - Namespace: ciWorkflowConfig.Namespace, + Namespace: ciWorkflowConfigNamespace, AppName: pipeline.App.AppName, } } @@ -304,7 +296,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) if len(preCiSteps) == 0 && isJob { return 0, &util.ApiError{HttpStatusCode: http.StatusNotFound, UserMessage: "No tasks are configured in this job pipeline"} } - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfig, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) if err != nil { impl.Logger.Errorw("could not save new workflow", "err", err) return 0, err @@ -318,7 +310,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) } maps.Copy(trigger.ExtraEnvironmentVariables, gitTriggerEnvVariables) - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfig, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) if err != nil { impl.Logger.Errorw("make workflow req", "err", err) return 0, err @@ -355,7 +347,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) } workflowRequest.AppId = pipeline.AppId workflowRequest.AppLabels = appLabels - workflowRequest.Env = env + workflowRequest.Env = envModal if isJob { workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE } else { @@ -434,7 +426,7 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline * event.UserId = int(trigger.TriggeredBy) event.CiWorkflowRunnerId = workflowRequest.WorkflowId - event = impl.eventFactory.BuildExtraCIData(event, material, workflowRequest.CiImage) + event = impl.eventFactory.BuildExtraCIData(event, material) _, evtErr := impl.eventClient.WriteNotificationEvent(event) if evtErr != nil { impl.Logger.Errorw("error in writing event", "err", evtErr) @@ -449,7 +441,7 @@ func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelin return payload } -func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, wfConfig *pipelineConfig.CiWorkflowConfig, +func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { ciWorkflow := &pipelineConfig.CiWorkflow{ @@ -467,7 +459,7 @@ func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ExecutorType: impl.config.GetWorkflowExecutorType(), } if isJob { - ciWorkflow.Namespace = wfConfig.Namespace + ciWorkflow.Namespace = ciWorkflowConfigNamespace ciWorkflow.EnvironmentId = EnvironmentId } err := impl.ciWorkflowRepository.SaveWorkFlow(ciWorkflow) @@ -488,26 +480,20 @@ func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequ return nil } -func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { - ciArtifactLocationFormat := ciWorkflowConfig.CiArtifactLocationFormat - if ciArtifactLocationFormat == "" { - ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfig.LogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) +func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation, ciWorkflowConfig.LogsBucket, artifactFileName + return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName } -func (impl *CiServiceImpl) buildDefaultArtifactLocation(ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, savedWf *pipelineConfig.CiWorkflow) string { - ciArtifactLocationFormat := ciWorkflowConfig.CiArtifactLocationFormat - if ciArtifactLocationFormat == "" { - ciArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } +func (impl *CiServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) return ArtifactLocation } -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { +func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { var ciProjectDetails []pipelineConfigBean.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { @@ -605,22 +591,14 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) return nil, err } - if ciWorkflowConfig.CiCacheBucket == "" { - ciWorkflowConfig.CiCacheBucket = impl.config.DefaultCacheBucket - } + ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket - if ciWorkflowConfig.CiCacheRegion == "" { - ciWorkflowConfig.CiCacheRegion = impl.config.DefaultCacheBucketRegion - } + ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - if ciWorkflowConfig.CiImage == "" { - ciWorkflowConfig.CiImage = impl.config.GetDefaultImage() - } + ciWorkflowConfigCiImage := impl.config.GetDefaultImage() - if ciWorkflowConfig.CiTimeout == 0 { - // get it from infraConfig - ciWorkflowConfig.CiTimeout = infraConfiguration.GetCiDefaultTimeout() - } + // get it from infraConfig + ciWorkflowConfigCiTimeout := infraConfiguration.GetCiDefaultTimeout() ciTemplate := pipeline.CiTemplate ciLevelArgs := pipeline.DockerArgs @@ -781,10 +759,10 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. PipelineId: pipeline.Id, CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfig.Namespace, + Namespace: ciWorkflowConfigNamespace, BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfig.CiImage, - ActiveDeadlineSeconds: ciWorkflowConfig.CiTimeout, + CiImage: ciWorkflowConfigCiImage, + ActiveDeadlineSeconds: ciWorkflowConfigCiTimeout, WorkflowId: savedWf.Id, TriggeredBy: savedWf.TriggeredBy, CacheLimit: impl.config.CacheLimit, @@ -843,23 +821,21 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. workflowRequest.DockerCert = dockerRegistry.Cert } - if ciWorkflowConfig.LogsBucket == "" { - ciWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } + ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() switch workflowRequest.CloudProvider { case types.BLOB_STORAGE_S3: // No AccessKey is used for uploading artifacts, instead IAM based auth is used - workflowRequest.CiCacheRegion = ciWorkflowConfig.CiCacheRegion - workflowRequest.CiCacheLocation = ciWorkflowConfig.CiCacheBucket - workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfig, savedWf) + workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion + workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket + workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ AccessKey: impl.config.BlobStorageS3AccessKey, Passkey: impl.config.BlobStorageS3SecretKey, EndpointUrl: impl.config.BlobStorageS3Endpoint, IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: ciWorkflowConfig.CiCacheBucket, - CiCacheRegion: ciWorkflowConfig.CiCacheRegion, + CiCacheBucketName: ciWorkflowConfigCiCacheBucket, + CiCacheRegion: ciWorkflowConfigCiCacheRegion, CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, CiArtifactBucketName: workflowRequest.CiArtifactBucket, CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), @@ -871,11 +847,11 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. case types.BLOB_STORAGE_GCP: workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - CacheBucketName: ciWorkflowConfig.CiCacheBucket, - LogBucketName: ciWorkflowConfig.LogsBucket, - ArtifactBucketName: ciWorkflowConfig.LogsBucket, + CacheBucketName: ciWorkflowConfigCiCacheBucket, + LogBucketName: ciWorkflowConfigLogsBucket, + ArtifactBucketName: ciWorkflowConfigLogsBucket, } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(ciWorkflowConfig, savedWf) + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation case types.BLOB_STORAGE_AZURE: workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ @@ -894,7 +870,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, AccessKey: impl.config.AzureAccountName, } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(ciWorkflowConfig, savedWf) + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation default: if impl.config.BlobStorageEnabled { diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 2038be0185..a9d4d31765 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -941,7 +941,7 @@ func (impl *WorkflowDagExecutorImpl) WriteCiSuccessEvent(request *bean2.CiArtifa event.CiWorkflowRunnerId = *artifact.WorkflowId } event.UserId = int(request.UserId) - event = impl.eventFactory.BuildExtraCIData(event, nil, artifact.Image) + event = impl.eventFactory.BuildExtraCIData(event, nil) _, evtErr := impl.eventClient.WriteNotificationEvent(event) if evtErr != nil { impl.logger.Errorw("error in writing event", "err", evtErr) @@ -991,7 +991,7 @@ func (impl *WorkflowDagExecutorImpl) WriteCiStepFailedEvent(pipeline *pipelineCo material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material, request.Image) + event = impl.eventFactory.BuildExtraCIData(event, material) event.CiArtifactId = 0 event.Payload.FailureReason = request.FailureReason _, evtErr := impl.eventClient.WriteNotificationEvent(event) diff --git a/scripts/sql/29200000_artifact_uploaded_flag.down.sql b/scripts/sql/29200000_artifact_uploaded_flag.down.sql index 7018f6d3f4..c35a5c4f82 100644 --- a/scripts/sql/29200000_artifact_uploaded_flag.down.sql +++ b/scripts/sql/29200000_artifact_uploaded_flag.down.sql @@ -1,5 +1,105 @@ +--- +--- Drop is_artifact_uploaded flag from ci_workflow and cd_workflow_runner tables +--- + ALTER TABLE public.ci_workflow DROP COLUMN IF EXISTS is_artifact_uploaded; ALTER TABLE public.cd_workflow_runner DROP COLUMN IF EXISTS is_artifact_uploaded, DROP COLUMN IF EXISTS cd_artifact_location; + +-- +-- Name: ci_workflow_config; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE IF NOT EXISTS public.ci_workflow_config ( + id integer NOT NULL, + ci_timeout bigint, + min_cpu character varying(250), + max_cpu character varying(250), + min_mem character varying(250), + max_mem character varying(250), + min_storage character varying(250), + max_storage character varying(250), + min_eph_storage character varying(250), + max_eph_storage character varying(250), + ci_cache_bucket character varying(250), + ci_cache_region character varying(250), + ci_image character varying(250), + wf_namespace character varying(250), + logs_bucket character varying(250), + ci_pipeline_id integer NOT NULL, + ci_artifact_location_format character varying(256) +); + + +ALTER TABLE public.ci_workflow_config OWNER TO postgres; + +-- +-- Name: ci_workflow_config_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE IF NOT EXISTS public.ci_workflow_config_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.ci_workflow_config_id_seq OWNER TO postgres; + +-- +-- Name: ci_workflow_config_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE public.ci_workflow_config_id_seq OWNED BY public.ci_workflow_config.id; + +-- +-- Name: cd_workflow_config; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE IF NOT EXISTS public.cd_workflow_config ( + id integer NOT NULL, + cd_timeout integer, + min_cpu character varying(256), + max_cpu character varying(256), + min_mem character varying(256), + max_mem character varying(256), + min_storage character varying(256), + max_storage character varying(256), + min_eph_storage character varying(256), + max_eph_storage character varying(256), + cd_cache_bucket character varying(256), + cd_cache_region character varying(256), + cd_image character varying(256), + wf_namespace character varying(256), + cd_pipeline_id integer, + logs_bucket character varying(256), + cd_artifact_location_format character varying(256) +); + + +ALTER TABLE public.cd_workflow_config OWNER TO postgres; + +-- +-- Name: cd_workflow_config_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE IF NOT EXISTS public.cd_workflow_config_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.cd_workflow_config_id_seq OWNER TO postgres; + +-- +-- Name: cd_workflow_config_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE public.cd_workflow_config_id_seq OWNED BY public.cd_workflow_config.id; \ No newline at end of file diff --git a/scripts/sql/29200000_artifact_uploaded_flag.up.sql b/scripts/sql/29200000_artifact_uploaded_flag.up.sql index 174048f72b..8d07f7b582 100644 --- a/scripts/sql/29200000_artifact_uploaded_flag.up.sql +++ b/scripts/sql/29200000_artifact_uploaded_flag.up.sql @@ -1,5 +1,21 @@ -ALTER TABLE public.ci_workflow +--- +--- Add is_artifact_uploaded flag to ci_workflow and cd_workflow_runner tables +--- + +ALTER TABLE "public"."ci_workflow" ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN; -ALTER TABLE public.cd_workflow_runner +ALTER TABLE "public"."cd_workflow_runner" ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN, ADD COLUMN IF NOT EXISTS cd_artifact_location varchar(256); + +--- +--- Drop ci_workflow_config table; redundant table +--- + +DROP TABLE IF EXISTS "public"."ci_workflow_config"; + +--- +--- Drop cd_workflow_config table; redundant table +--- + +DROP TABLE IF EXISTS "public"."cd_workflow_config"; From fa64e2ef1aa55329e3d77cd55e3519dd5b84f224 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 27 Sep 2024 14:00:46 +0530 Subject: [PATCH 10/29] updated ci fail notification logic --- pkg/pipeline/CiHandler.go | 5 ++--- pkg/pipeline/bean/CiPipeline/CiStatus.go | 11 +++++++++++ pkg/workflow/dag/WorkflowDagExecutor.go | 10 +++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 pkg/pipeline/bean/CiPipeline/CiStatus.go diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index a9332f79f0..096cdf5f2e 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "io/ioutil" "net/http" @@ -1071,8 +1072,6 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -const CiStageFailErrorCode = 2 - func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { @@ -1159,7 +1158,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) - if extractErrorCode(savedWorkflow.Message) != CiStageFailErrorCode { + if extractErrorCode(savedWorkflow.Message) != CiPipeline.CiStageFailErrorCode { go impl.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) diff --git a/pkg/pipeline/bean/CiPipeline/CiStatus.go b/pkg/pipeline/bean/CiPipeline/CiStatus.go new file mode 100644 index 0000000000..6956cd6ac8 --- /dev/null +++ b/pkg/pipeline/bean/CiPipeline/CiStatus.go @@ -0,0 +1,11 @@ +package CiPipeline + +type CiFailReason string + +func (r CiFailReason) String() string { + return string(r) +} + +const CiFailed CiFailReason = "CI Failed: exit code 1" + +const CiStageFailErrorCode = 2 diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index a9d4d31765..ddaf05faae 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -978,10 +978,14 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r } } impl.asyncRunnable.Execute(customTagServiceRunnableFunc) - notificationServiceRunnableFunc := func() { - impl.WriteCiStepFailedEvent(pipelineModel, request, savedWorkflow) + if request.FailureReason != CiPipeline.CiFailed.String() { + notificationServiceRunnableFunc := func() { + impl.WriteCiStepFailedEvent(pipelineModel, request, savedWorkflow) + } + impl.asyncRunnable.Execute(notificationServiceRunnableFunc) + } else { + // this case has been handled CiHandlerImpl.UpdateWorkflow function. } - impl.asyncRunnable.Execute(notificationServiceRunnableFunc) return nil } From 2fd54eed49be48ac25f9dedc88d206bb6578c051 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Fri, 27 Sep 2024 16:13:06 +0530 Subject: [PATCH 11/29] fix: FindByPipelineId build histroy list query --- .../pipelineConfig/CiWorkflowRepository.go | 78 ++++++++++++------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 3e1f8a37af..935201e10e 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -91,34 +91,34 @@ func (ciWorkflow *CiWorkflow) IsExternalRunInJobType() bool { } type WorkflowWithArtifact struct { - Id int `psql:"id"` - Name string `psql:"name"` - PodName string `psql:"pod_name"` - Status string `psql:"status"` - PodStatus string `psql:"pod_status"` - Message string `psql:"message"` - StartedOn time.Time `psql:"started_on"` - FinishedOn time.Time `psql:"finished_on"` - CiPipelineId int `psql:"ci_pipeline_id"` - Namespace string `psql:"namespace"` - LogFilePath string `psql:"log_file_path"` - GitTriggers map[int]GitCommit `psql:"git_triggers"` - TriggeredBy int32 `psql:"triggered_by"` - EmailId string `psql:"email_id"` - Image string `psql:"image"` - CiArtifactLocation string `psql:"ci_artifact_location"` - CiArtifactId int `psql:"ci_artifact_id"` - BlobStorageEnabled bool `psql:"blob_storage_enabled"` - CiBuildType string `psql:"ci_build_type"` - IsArtifactUploadedV2 *bool `psql:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future - IsArtifactUploaded bool `psql:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table - EnvironmentId int `psql:"environment_id"` - EnvironmentName string `psql:"environment_name"` - RefCiWorkflowId int `psql:"ref_ci_workflow_id"` - ParentCiWorkflowId int `psql:"parent_ci_workflow_id"` - ExecutorType cdWorkflow.WorkflowExecutorType `psql:"executor_type"` //awf, system - ImagePathReservationId int `psql:"image_path_reservation_id"` - ImagePathReservationIds []int `psql:"image_path_reservation_ids" pg:",array"` + Id int `sql:"id"` + Name string `sql:"name"` + PodName string `sql:"pod_name"` + Status string `sql:"status"` + PodStatus string `sql:"pod_status"` + Message string `sql:"message"` + StartedOn time.Time `sql:"started_on"` + FinishedOn time.Time `sql:"finished_on"` + CiPipelineId int `sql:"ci_pipeline_id"` + Namespace string `sql:"namespace"` + LogFilePath string `sql:"log_file_path"` + GitTriggers map[int]GitCommit `sql:"git_triggers"` + TriggeredBy int32 `sql:"triggered_by"` + EmailId string `sql:"email_id"` + Image string `sql:"image"` + CiArtifactLocation string `sql:"ci_artifact_location"` + CiArtifactId int `sql:"ci_artifact_id"` + BlobStorageEnabled bool `sql:"blob_storage_enabled"` + CiBuildType string `sql:"ci_build_type"` + IsArtifactUploadedV2 *bool `sql:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future + IsArtifactUploaded bool `sql:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table + EnvironmentId int `sql:"environment_id"` + EnvironmentName string `sql:"environment_name"` + RefCiWorkflowId int `sql:"ref_ci_workflow_id"` + ParentCiWorkflowId int `sql:"parent_ci_workflow_id"` + ExecutorType cdWorkflow.WorkflowExecutorType `sql:"executor_type"` //awf, system + ImagePathReservationId int `sql:"image_path_reservation_id"` + ImagePathReservationIds []int `sql:"image_path_reservation_ids" pg:",array"` } type GitCommit struct { @@ -170,8 +170,26 @@ func (impl *CiWorkflowRepositoryImpl) FindByStatusesIn(activeStatuses []string) // FindByPipelineId gets only those workflowWithArtifact whose parent_ci_workflow_id is null, this is done to accommodate multiple ci_artifacts through a single workflow(parent), making child workflows for other ci_artifacts (this has been done due to design understanding and db constraint) single workflow single ci-artifact func (impl *CiWorkflowRepositoryImpl) FindByPipelineId(pipelineId int, offset int, limit int) ([]WorkflowWithArtifact, error) { var wfs []WorkflowWithArtifact - queryTemp := "select cia.id as ci_artifact_id, env.environment_name, cia.image, cia.is_artifact_uploaded as old_is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id left join environment env on env.id = wf.environment_id where wf.ci_pipeline_id = ? and parent_ci_workflow_id is null order by wf.started_on desc offset ? limit ?;" - _, err := impl.dbConnection.Query(&wfs, queryTemp, pipelineId, offset, limit) + err := impl.dbConnection.Model(). + TableExpr("ci_workflow AS wf"). + ColumnExpr("cia.id AS ci_artifact_id"). + ColumnExpr("env.environment_name"). + ColumnExpr("cia.image"). + ColumnExpr("cia.is_artifact_uploaded AS old_is_artifact_uploaded"). + ColumnExpr("wf.*"). + ColumnExpr("u.email_id"). + Join("LEFT JOIN users u"). + JoinOn("u.id = wf.triggered_by"). + Join("LEFT JOIN ci_artifact cia"). + JoinOn("wf.id = cia.ci_workflow_id"). + Join("LEFT JOIN environment env"). + JoinOn("env.id = wf.environment_id"). + Where("wf.ci_pipeline_id = ?", pipelineId). + Where("parent_ci_workflow_id is NULL"). + Order("wf.started_on DESC"). + Offset(offset). + Limit(limit). + Select(&wfs) if err != nil { return nil, err } From 71410bc93f7f67fe348b49592e1669b86ac70066 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 11:26:08 +0530 Subject: [PATCH 12/29] common constants added and removed todos --- Dockerfile | 2 - go.mod | 2 +- go.sum | 4 +- pkg/pipeline/CiHandler.go | 4 +- pkg/pipeline/bean/CiPipeline/CiStatus.go | 11 ----- pkg/pipeline/types/Workflow.go | 4 +- pkg/workflow/dag/WorkflowDagExecutor.go | 3 +- .../common-lib/utils/workFlow/CiCdStage.go | 42 +++++++++++++++++++ vendor/modules.txt | 3 +- 9 files changed, 54 insertions(+), 21 deletions(-) delete mode 100644 pkg/pipeline/bean/CiPipeline/CiStatus.go create mode 100644 vendor/github.com/devtron-labs/common-lib/utils/workFlow/CiCdStage.go diff --git a/Dockerfile b/Dockerfile index 1db725eaac..fc5dbe1dce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,6 @@ RUN apt install git gcc musl-dev make -y RUN go install github.com/google/wire/cmd/wire@latest WORKDIR /go/src/github.com/devtron-labs/devtron ADD . /go/src/github.com/devtron-labs/devtron/ -# TODO Asutosh: revert after dev test -ADD ./vendor/github.com/Microsoft/ /go/src/github.com/devtron-labs/devtron/vendor/github.com/microsoft/ RUN GOOS=linux make build-all # uncomment this post build arg diff --git a/go.mod b/go.mod index 2ab051aa6b..629bfddbce 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 - github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214 + github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 github.com/devtron-labs/go-bitbucket v0.9.60-beta github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 github.com/evanphx/json-patch v5.7.0+incompatible diff --git a/go.sum b/go.sum index c524c2bf66..79fc619233 100644 --- a/go.sum +++ b/go.sum @@ -794,8 +794,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI= -github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214 h1:QBBienyRjnNw/IwVt8O4WxIQvQJUbqvLp51mQqfwcTM= -github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= +github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 h1:FA9Lxn3IF1KSX6Ohru8b4NHlE5/qa1zMOwpH6z0w/kU= +github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys= diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 096cdf5f2e..12969e2aa1 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -21,8 +21,8 @@ import ( "context" "errors" "fmt" + "github.com/devtron-labs/common-lib/utils/workFlow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "io/ioutil" "net/http" @@ -1158,7 +1158,7 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) - if extractErrorCode(savedWorkflow.Message) != CiPipeline.CiStageFailErrorCode { + if extractErrorCode(savedWorkflow.Message) != workFlow.CiStageFailErrorCode { go impl.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) diff --git a/pkg/pipeline/bean/CiPipeline/CiStatus.go b/pkg/pipeline/bean/CiPipeline/CiStatus.go deleted file mode 100644 index 6956cd6ac8..0000000000 --- a/pkg/pipeline/bean/CiPipeline/CiStatus.go +++ /dev/null @@ -1,11 +0,0 @@ -package CiPipeline - -type CiFailReason string - -func (r CiFailReason) String() string { - return string(r) -} - -const CiFailed CiFailReason = "CI Failed: exit code 1" - -const CiStageFailErrorCode = 2 diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 5a74c5624e..0a56f40675 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -22,6 +22,7 @@ import ( "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/common-lib/utils/workFlow" bean3 "github.com/devtron-labs/devtron/api/bean" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" @@ -492,7 +493,8 @@ func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdCon SecurityContext: &v1.SecurityContext{ Privileged: &privileged, }, - Resources: workflowRequest.GetLimitReqCpuMem(config, infraConfigurations), + TerminationMessagePath: workFlow.GetTerminalLogFilePath(), + Resources: workflowRequest.GetLimitReqCpuMem(config, infraConfigurations), } if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { workflowMainContainer.Ports = []v1.ContainerPort{{ diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index ddaf05faae..c98e3f8c89 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" + "github.com/devtron-labs/common-lib/utils/workFlow" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" @@ -978,7 +979,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r } } impl.asyncRunnable.Execute(customTagServiceRunnableFunc) - if request.FailureReason != CiPipeline.CiFailed.String() { + if request.FailureReason != workFlow.CiFailed.String() { notificationServiceRunnableFunc := func() { impl.WriteCiStepFailedEvent(pipelineModel, request, savedWorkflow) } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/workFlow/CiCdStage.go b/vendor/github.com/devtron-labs/common-lib/utils/workFlow/CiCdStage.go new file mode 100644 index 0000000000..e6cc1ede75 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/utils/workFlow/CiCdStage.go @@ -0,0 +1,42 @@ +package workFlow + +import "path" + +type CiFailReason string + +type CdFailReason string + +func (r CiFailReason) String() string { + return string(r) +} + +func (r CdFailReason) String() string { + return string(r) +} + +const ( + PreCiFailed CiFailReason = "Pre-CI task failed: %s" + PostCiFailed CiFailReason = "Post-CI task failed: %s" + BuildFailed CiFailReason = "Docker build failed" + PushFailed CiFailReason = "Docker push failed" + ScanFailed CiFailReason = "Image scan failed" + CiFailed CiFailReason = "CI failed" + + CdStageTaskFailed CdFailReason = "%s task failed: %s" + CdStageFailed CdFailReason = "%s failed. Reason: %s" +) + +const ( + TerminalLogDir = "/dev" + TerminalLogFile = "/termination-log" +) + +func GetTerminalLogFilePath() string { + return path.Join(TerminalLogDir, TerminalLogFile) +} + +const ( + DefaultErrorCode = 1 + AbortErrorCode = 143 + CiStageFailErrorCode = 2 +) diff --git a/vendor/modules.txt b/vendor/modules.txt index a3d4431457..c782c3974f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -344,7 +344,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.16.1-0.20240923091856-41253742c214 +# github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -369,6 +369,7 @@ github.com/devtron-labs/common-lib/utils/k8sObjectsUtil github.com/devtron-labs/common-lib/utils/registry github.com/devtron-labs/common-lib/utils/remoteConnection/bean github.com/devtron-labs/common-lib/utils/runTime +github.com/devtron-labs/common-lib/utils/workFlow github.com/devtron-labs/common-lib/utils/yaml # github.com/devtron-labs/go-bitbucket v0.9.60-beta ## explicit; go 1.14 From 505a38b7bbd0d8de59883ff8a79bf18760b111e0 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 11:56:05 +0530 Subject: [PATCH 13/29] deprecated env flags --- env_gen.md | 2 -- pkg/pipeline/types/CiCdConfig.go | 11 +++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/env_gen.md b/env_gen.md index d6c935a303..b81314405b 100644 --- a/env_gen.md +++ b/env_gen.md @@ -40,7 +40,6 @@ | BUILD_LOG_TTL_VALUE_IN_SECS | 3600 | | | CACHE_LIMIT | 5000000000 | | | CASBIN_DATABASE | casbin | | - | CD_ARTIFACT_LOCATION_FORMAT | %d/%d.zip | | | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | | | | CD_DEFAULT_ADDRESS_POOL_SIZE | | | | CD_HELM_PIPELINE_STATUS_CRON_TIME | */2 * * * * | | @@ -59,7 +58,6 @@ | CD_WORKFLOW_EXECUTOR_TYPE | AWF | | | CD_WORKFLOW_SERVICE_ACCOUNT | cd-runner | | | CExpirationTime | 600 | | - | CI_ARTIFACT_LOCATION_FORMAT | %d/%d.zip | | | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | | | | CI_DEFAULT_ADDRESS_POOL_SIZE | | | | CI_IGNORE_DOCKER_CACHE | | | diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 90ebf555e4..077d96178a 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -58,7 +58,6 @@ type CiCdConfig struct { ExternalCiApiSecret string `env:"EXTERNAL_CI_API_SECRET" envDefault:"devtroncd-secret"` ExternalCiWebhookUrl string `env:"EXTERNAL_CI_WEB_HOOK_URL" envDefault:""` ExternalCiPayload string `env:"EXTERNAL_CI_PAYLOAD" envDefault:"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}"` - CiArtifactLocationFormat string `env:"CI_ARTIFACT_LOCATION_FORMAT" envDefault:"%d/%d.zip"` ImageScannerEndpoint string `env:"IMAGE_SCANNER_ENDPOINT" envDefault:"http://image-scanner-new-demo-devtroncd-service.devtroncd:80"` CiDefaultAddressPoolBaseCidr string `env:"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR"` CiDefaultAddressPoolSize int `env:"CI_DEFAULT_ADDRESS_POOL_SIZE"` @@ -101,7 +100,6 @@ type CiCdConfig struct { CdDefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" ` CdNodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR"` ExternalCdNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR"` - CdArtifactLocationFormat string `env:"CD_ARTIFACT_LOCATION_FORMAT" envDefault:"%d/%d.zip"` CdDefaultNamespace string `env:"DEFAULT_CD_NAMESPACE"` CdDefaultImage string `env:"DEFAULT_CI_IMAGE"` CdDefaultTimeout int64 `env:"DEFAULT_CD_TIMEOUT" envDefault:"3600"` @@ -168,6 +166,11 @@ const ( CdConfigType = "CdConfig" ) +const ( + CiArtifactLocationFormat = "%d/%d.zip" + CdArtifactLocationFormat = "%d/%d.zip" +) + func GetCiConfig() (*CiConfig, error) { ciCdConfig := &CiCdConfig{} err := env.Parse(ciCdConfig) @@ -389,13 +392,13 @@ func (impl *CiCdConfig) GetWorkflowServiceAccount() string { func (impl *CiCdConfig) GetArtifactLocationFormat() string { switch impl.Type { case CiConfigType: - ciArtifactLocationFormat := impl.CiArtifactLocationFormat + ciArtifactLocationFormat := CiArtifactLocationFormat if len(impl.getDefaultArtifactKeyPrefix()) != 0 { ciArtifactLocationFormat = path.Join(impl.getDefaultArtifactKeyPrefix(), ciArtifactLocationFormat) } return ciArtifactLocationFormat case CdConfigType: - cdArtifactLocationFormat := impl.CdArtifactLocationFormat + cdArtifactLocationFormat := CdArtifactLocationFormat if len(impl.getDefaultArtifactKeyPrefix()) != 0 { cdArtifactLocationFormat = path.Join(impl.getDefaultArtifactKeyPrefix(), cdArtifactLocationFormat) } From 3a0077eff2c06e4789e4083bd68bfa3cfa5cf60b Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 12:02:50 +0530 Subject: [PATCH 14/29] fix: panic handling --- pkg/pipeline/CdHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 605f25d43d..152566c904 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -723,7 +723,7 @@ func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pip workflow.Image = wfr.CdWorkflow.CiArtifact.Image workflow.PipelineId = wfr.CdWorkflow.PipelineId workflow.CiArtifactId = wfr.CdWorkflow.CiArtifactId - if wfr.CdWorkflow.CiArtifact != nil { + if wfr.IsArtifactUploaded != nil { workflow.IsArtifactUploaded = *wfr.IsArtifactUploaded } workflow.BlobStorageEnabled = wfr.BlobStorageEnabled From bd2b0a66034148c0e5f7aef2fac3d131d6f72784 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 12:14:03 +0530 Subject: [PATCH 15/29] fix: data updation for CD compelete event --- pkg/eventProcessor/in/WorkflowEventProcessorService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 8aee442952..169eb1d899 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -174,7 +174,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { impl.logger.Errorw("update cd-wf-runner failed for id ", "cdWfrId", wfr.Id, "err", err) return } - } else if cdStageCompleteEvent.IsArtifactUploaded { + } else { err = impl.cdWorkflowRunnerService.UpdateIsArtifactUploaded(wfr.Id, cdStageCompleteEvent.IsArtifactUploaded) if err != nil { impl.logger.Errorw("error in updating isArtifactUploaded", "cdWfrId", wfr.Id, "err", err) From 2d704f56694ad31ad62a5ebc4ad985580f506d80 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 12:18:13 +0530 Subject: [PATCH 16/29] fix: data updation on ci compelete event --- pkg/workflow/dag/WorkflowDagExecutor.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index c98e3f8c89..0f4fd4e263 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -959,11 +959,10 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r impl.logger.Errorw("cannot get saved wf", "wf ID: ", *request.WorkflowId, "err", err) return err } - if request.IsArtifactUploaded { - dbErr := impl.ciWorkflowRepository.UpdateArtifactUploaded(savedWorkflow.Id, request.IsArtifactUploaded) - if dbErr != nil { - impl.logger.Errorw("update workflow status", "ciWorkflowId", savedWorkflow.Id, "err", dbErr) - } + // update IsArtifactUploaded flag in workflow + dbErr := impl.ciWorkflowRepository.UpdateArtifactUploaded(savedWorkflow.Id, request.IsArtifactUploaded) + if dbErr != nil { + impl.logger.Errorw("update workflow status", "ciWorkflowId", savedWorkflow.Id, "err", dbErr) } pipelineModel, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) if err != nil { From b565ad2f6a20755ddcd5c304d3dd110fec0fcaf6 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 12:54:47 +0530 Subject: [PATCH 17/29] populate IsArtifactUploaded flag in triggerInfo api --- pkg/pipeline/CdHandler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 27169a8f8b..cae5bb02b2 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -619,6 +619,7 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, IsVirtualEnvironment: workflowR.CdWorkflow.Pipeline.Environment.IsVirtualEnvironment, PodName: workflowR.PodName, ArtifactId: workflow.CiArtifactId, + IsArtifactUploaded: workflow.IsArtifactUploaded, CiPipelineId: ciWf.CiPipelineId, } return workflowResponse, nil From 9de9d4ea5924d3eaba06a31c2df27659feb1a1a9 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 13:10:53 +0530 Subject: [PATCH 18/29] updated common-lib version --- go.mod | 2 +- go.sum | 4 ++-- .../common-lib/blob-storage/BlobStorageService.go | 3 ++- vendor/modules.txt | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 629bfddbce..162c23e84c 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 - github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 + github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 github.com/devtron-labs/go-bitbucket v0.9.60-beta github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 github.com/evanphx/json-patch v5.7.0+incompatible diff --git a/go.sum b/go.sum index 79fc619233..e09729899a 100644 --- a/go.sum +++ b/go.sum @@ -794,8 +794,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI= -github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 h1:FA9Lxn3IF1KSX6Ohru8b4NHlE5/qa1zMOwpH6z0w/kU= -github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= +github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 h1:ikMVToilPandd3iKEGkISMcyvWva440n/zAlMZT2M3E= +github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys= diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go index 3e59b5aa52..47d0f743e5 100644 --- a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobStorageService.go @@ -23,6 +23,7 @@ import ( "go.uber.org/zap" "log" "os" + "path/filepath" ) type BlobStorageService interface { @@ -70,7 +71,7 @@ func (impl *BlobStorageServiceImpl) Get(request *BlobStorageRequest) (bool, int6 downloadSuccess := false numBytes := int64(0) - file, err := os.Create("/" + request.DestinationKey) + file, err := os.Create(filepath.Clean("/" + request.DestinationKey)) defer file.Close() if err != nil { log.Println(err) diff --git a/vendor/modules.txt b/vendor/modules.txt index c782c3974f..9b60fe1fe5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -344,7 +344,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20240929204207-9278f7840690 +# github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage From f85a7c2a383803e7b6eb5b0d68f35b6582158d6e Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 13:11:49 +0530 Subject: [PATCH 19/29] updated destinationKey in request --- pkg/pipeline/CiHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 12969e2aa1..4843712e1a 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -929,7 +929,7 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i request := &blob_storage.BlobStorageRequest{ StorageType: impl.config.CloudProvider, SourceKey: key, - DestinationKey: baseLogLocationPathConfig + item, + DestinationKey: destinationKey, AzureBlobBaseConfig: azureBlobConfig, AwsS3BaseConfig: awsS3BaseConfig, GcpBlobBaseConfig: gcpBlobBaseConfig, From 24af3091f46ac304a797d13cae1ccfc524b521f9 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 14:32:22 +0530 Subject: [PATCH 20/29] updated logging --- pkg/pipeline/CiHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 4843712e1a..9db376c9ef 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -825,7 +825,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow } func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs") + impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty @@ -956,7 +956,7 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i } _, numBytes, err := blobStorageService.Get(request) if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request) + impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) return nil, errors.New("failed to download resource") } From 26e44fe8a93bf26c3af721bfad9d5a8bc9421169 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 16:34:10 +0530 Subject: [PATCH 21/29] updated db column type --- cmd/external-app/wire_gen.go | 2 +- .../pipelineConfig/CdWorfkflowRepository.go | 30 +++++++++++++--- .../pipelineConfig/CiWorkflowRepository.go | 36 ++++++++++++++++--- .../pipelineConfig/PipelineRepository.go | 2 +- .../bean/workflow/CommonBean.go | 31 ++++++++++++++++ .../cdWorkflow/CdWorkflowBean.go} | 0 pkg/app/AppListingService.go | 2 +- pkg/app/AppService.go | 2 +- pkg/appStore/bean/bean.go | 2 +- pkg/appStore/installedApp/adapter/Adapter.go | 2 +- .../repository/InstalledAppRepository.go | 2 +- .../repository/InstalledAppVersionHistory.go | 2 +- .../service/AppStoreDeploymentDBService.go | 2 +- .../service/AppStoreDeploymentService.go | 2 +- .../deployment/DeploymentStatusService.go | 2 +- .../deployment/FullModeDeploymentService.go | 2 +- .../devtronApps/PostStageTriggerService.go | 2 +- .../devtronApps/PreStageTriggerService.go | 2 +- .../trigger/devtronApps/TriggerService.go | 2 +- .../UserDeploymentRequestRepository.go | 2 +- .../in/AppStoreAppsEventProcessorService.go | 2 +- .../in/WorkflowEventProcessorService.go | 2 +- .../out/WorkflowEventPublishService.go | 2 +- .../AppDeploymentTypeChangeManager.go | 2 +- pkg/pipeline/CdHandler.go | 8 +++-- pkg/pipeline/CiHandler.go | 22 +++++++----- pkg/pipeline/CiService.go | 2 +- .../DeploymentPipelineConfigService.go | 2 +- pkg/pipeline/WorkflowService.go | 2 +- pkg/pipeline/WorkflowServiceIT_test.go | 2 +- pkg/pipeline/types/CiCdConfig.go | 2 +- pkg/pipeline/types/Workflow.go | 2 +- pkg/workflow/cd/CdWorkflowCommonService.go | 2 +- pkg/workflow/cd/CdWorkflowRunnerService.go | 3 +- pkg/workflow/cd/adapter/adapter.go | 5 +-- pkg/workflow/cd/bean/bean.go | 2 +- pkg/workflow/dag/WorkflowDagExecutor.go | 7 ++-- pkg/workflow/dag/bean/bean.go | 1 - pkg/workflow/status/WorkflowStatusService.go | 2 +- .../29200000_artifact_uploaded_flag.up.sql | 4 +-- wire_gen.go | 2 +- 41 files changed, 148 insertions(+), 59 deletions(-) create mode 100644 internal/sql/repository/pipelineConfig/bean/workflow/CommonBean.go rename internal/sql/repository/pipelineConfig/bean/{cdWorkflow/bean.go => workflow/cdWorkflow/CdWorkflowBean.go} (100%) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index bab4ef2b47..ab3bca55cd 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 334d99c2c7..37d6ec21be 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -23,7 +23,8 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" @@ -42,7 +43,7 @@ type CdWorkflowRepository interface { FindArtifactByPipelineIdAndRunnerType(pipelineId int, runnerType apiBean.WorkflowType, limit int, runnerStatuses []string) ([]CdWorkflowRunner, error) SaveWorkFlowRunner(wfr *CdWorkflowRunner) (*CdWorkflowRunner, error) UpdateWorkFlowRunner(wfr *CdWorkflowRunner) error - UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error + UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error GetPreviousQueuedRunners(cdWfrId, pipelineId int) ([]*CdWorkflowRunner, error) UpdateRunnerStatusToFailedForIds(errMsg string, triggeredBy int32, cdWfrIds ...int) error UpdateWorkFlowRunnersWithTxn(wfrs []*CdWorkflowRunner, tx *pg.Tx) error @@ -75,6 +76,8 @@ type CdWorkflowRepository interface { FetchArtifactsByCdPipelineId(pipelineId int, runnerType apiBean.WorkflowType, offset, limit int, searchString string) ([]CdWorkflowRunner, error) GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours int) ([]*CdWorkflowRunner, error) FindLatestRunnerByPipelineIdsAndRunnerType(ctx context.Context, pipelineIds []int, runnerType apiBean.WorkflowType) ([]CdWorkflowRunner, error) + + MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) } type CdWorkflowRepositoryImpl struct { @@ -113,7 +116,7 @@ type CdWorkflowRunner struct { Namespace string `sql:"namespace"` LogLocation string `sql:"log_file_path"` CdArtifactLocation string `sql:"cd_artifact_location"` - IsArtifactUploaded *bool `sql:"is_artifact_uploaded"` + IsArtifactUploaded workflow.ArtifactUploadedType `sql:"is_artifact_uploaded"` TriggeredBy int32 `sql:"triggered_by"` CdWorkflowId int `sql:"cd_workflow_id"` PodName string `sql:"pod_name"` @@ -125,6 +128,15 @@ type CdWorkflowRunner struct { sql.AuditLog } +func (c *CdWorkflowRunner) GetIsArtifactUploaded() (isArtifactUploaded bool, isMigrationRequired bool) { + return workflow.IsArtifactUploaded(c.IsArtifactUploaded) +} + +func (c *CdWorkflowRunner) WithIsArtifactUploaded(isArtifactUploaded bool) *CdWorkflowRunner { + c.IsArtifactUploaded = workflow.GetArtifactUploadedType(isArtifactUploaded) + return c +} + func (c *CdWorkflowRunner) IsExternalRun() bool { var isExtCluster bool if c.WorkflowType == cdWorkflow.WorkflowTypePre { @@ -448,7 +460,7 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunner(wfr *CdWorkflowRunner return err } -func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error { +func (impl *CdWorkflowRepositoryImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded workflow.ArtifactUploadedType) error { _, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)). Set("is_artifact_uploaded = ?", isArtifactUploaded). Where("id = ?", wfrId). @@ -727,3 +739,13 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestRunnerByPipelineIdsAndRunnerType } return latestWfrs, err } + +func (impl *CdWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) { + _, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)). + Set("is_artifact_uploaded = ?", workflow.GetArtifactUploadedType(isArtifactUploaded)). + Where("id = ?", wfrId). + Update() + if err != nil { + impl.logger.Errorw("error in updating is artifact uploaded", "wfrId", wfrId, "err", err) + } +} diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 935201e10e..7e2989988b 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -19,7 +19,8 @@ package pipelineConfig import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/go-pg/pg" "go.uber.org/zap" "time" @@ -29,7 +30,7 @@ type CiWorkflowRepository interface { SaveWorkFlow(wf *CiWorkflow) error FindLastTriggeredWorkflow(pipelineId int) (*CiWorkflow, error) UpdateWorkFlow(wf *CiWorkflow) error - UpdateArtifactUploaded(id int, isUploaded bool) error + UpdateArtifactUploaded(id int, isUploaded workflow.ArtifactUploadedType) error FindByStatusesIn(activeStatuses []string) ([]*CiWorkflow, error) FindByPipelineId(pipelineId int, offset int, size int) ([]WorkflowWithArtifact, error) FindById(id int) (*CiWorkflow, error) @@ -47,6 +48,8 @@ type CiWorkflowRepository interface { ExistsByStatus(status string) (bool, error) FindBuildTypeAndStatusDataOfLast1Day() []*BuildTypeCount FIndCiWorkflowStatusesByAppId(appId int) ([]*CiWorkflowStatus, error) + + MigrateIsArtifactUploaded(wfId int, isArtifactUploaded bool) } type CiWorkflowRepositoryImpl struct { @@ -70,7 +73,7 @@ type CiWorkflow struct { GitTriggers map[int]GitCommit `sql:"git_triggers"` TriggeredBy int32 `sql:"triggered_by"` CiArtifactLocation string `sql:"ci_artifact_location"` - IsArtifactUploaded *bool `sql:"is_artifact_uploaded"` + IsArtifactUploaded workflow.ArtifactUploadedType `sql:"is_artifact_uploaded"` PodName string `sql:"pod_name"` CiBuildType string `sql:"ci_build_type"` EnvironmentId int `sql:"environment_id"` @@ -82,6 +85,15 @@ type CiWorkflow struct { CiPipeline *CiPipeline } +func (ciWorkflow *CiWorkflow) GetIsArtifactUploaded() (isArtifactUploaded bool, isMigrationRequired bool) { + return workflow.IsArtifactUploaded(ciWorkflow.IsArtifactUploaded) +} + +func (ciWorkflow *CiWorkflow) WithIsArtifactUploaded(isArtifactUploaded bool) *CiWorkflow { + ciWorkflow.IsArtifactUploaded = workflow.GetArtifactUploadedType(isArtifactUploaded) + return ciWorkflow +} + func (ciWorkflow *CiWorkflow) InProgress() bool { return ciWorkflow.Status == "Running" || ciWorkflow.Status == "Starting" } @@ -110,7 +122,7 @@ type WorkflowWithArtifact struct { CiArtifactId int `sql:"ci_artifact_id"` BlobStorageEnabled bool `sql:"blob_storage_enabled"` CiBuildType string `sql:"ci_build_type"` - IsArtifactUploadedV2 *bool `sql:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future + IsArtifactUploadedV2 workflow.ArtifactUploadedType `sql:"is_artifact_uploaded"` // IsArtifactUploadedV2 is the new column from ci_workflow table, IsArtifactUploaded is Deprecated and will be removed in future IsArtifactUploaded bool `sql:"old_is_artifact_uploaded"` // Deprecated; Use IsArtifactUploadedV2 instead. IsArtifactUploaded is the column from ci_artifact table EnvironmentId int `sql:"environment_id"` EnvironmentName string `sql:"environment_name"` @@ -121,6 +133,10 @@ type WorkflowWithArtifact struct { ImagePathReservationIds []int `sql:"image_path_reservation_ids" pg:",array"` } +func (w *WorkflowWithArtifact) GetIsArtifactUploaded() (isArtifactUploaded bool, isMigrationRequired bool) { + return workflow.IsArtifactUploaded(w.IsArtifactUploadedV2) +} + type GitCommit struct { Commit string //git hash Author string @@ -256,7 +272,7 @@ func (impl *CiWorkflowRepositoryImpl) UpdateWorkFlow(wf *CiWorkflow) error { return err } -func (impl *CiWorkflowRepositoryImpl) UpdateArtifactUploaded(id int, isUploaded bool) error { +func (impl *CiWorkflowRepositoryImpl) UpdateArtifactUploaded(id int, isUploaded workflow.ArtifactUploadedType) error { _, err := impl.dbConnection.Model(&CiWorkflow{}). Set("is_artifact_uploaded = ?", isUploaded). Where("id = ?", id). @@ -360,3 +376,13 @@ func (impl *CiWorkflowRepositoryImpl) FIndCiWorkflowStatusesByAppId(appId int) ( } return ciworkflowStatuses, err } + +func (impl *CiWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfId int, isArtifactUploaded bool) { + _, err := impl.dbConnection.Model((*CiWorkflow)(nil)). + Set("is_artifact_uploaded = ?", workflow.GetArtifactUploadedType(isArtifactUploaded)). + Where("id = ?", wfId). + Update() + if err != nil { + impl.logger.Errorw("error occurred while updating is_artifact_uploaded", "wfId", wfId, "err", err) + } +} diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index ddddc4f858..ebd6885591 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -22,8 +22,8 @@ import ( "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" "github.com/devtron-labs/devtron/pkg/cluster/repository" diff --git a/internal/sql/repository/pipelineConfig/bean/workflow/CommonBean.go b/internal/sql/repository/pipelineConfig/bean/workflow/CommonBean.go new file mode 100644 index 0000000000..34ab63185b --- /dev/null +++ b/internal/sql/repository/pipelineConfig/bean/workflow/CommonBean.go @@ -0,0 +1,31 @@ +package workflow + +type ArtifactUploadedType string + +func (r ArtifactUploadedType) String() string { + return string(r) +} + +func GetArtifactUploadedType(isUploaded bool) ArtifactUploadedType { + if isUploaded { + return ArtifactUploaded + } + return ArtifactNotUploaded +} + +func IsArtifactUploaded(s ArtifactUploadedType) (isArtifactUploaded bool, isMigrationRequired bool) { + switch s { + case ArtifactUploaded: + return true, false + case ArtifactNotUploaded: + return false, false + default: + return false, true + } +} + +const ( + NullArtifactUploaded ArtifactUploadedType = "NA" + ArtifactUploaded ArtifactUploadedType = "Uploaded" + ArtifactNotUploaded ArtifactUploadedType = "NotUploaded" +) diff --git a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go b/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow/CdWorkflowBean.go similarity index 100% rename from internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go rename to internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow/CdWorkflowBean.go diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 3825252084..0655168154 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -20,7 +20,7 @@ import ( "context" "fmt" argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "net/http" "strconv" diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 7f7c3c0108..a31d15d334 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -25,8 +25,8 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 4fcd57fc0a..85ffa89de5 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -23,7 +23,7 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean/gitOps" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" bean3 "github.com/devtron-labs/devtron/api/helm-app/service/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/util/gitUtil" diff --git a/pkg/appStore/installedApp/adapter/Adapter.go b/pkg/appStore/installedApp/adapter/Adapter.go index 583d5744da..58ad1543eb 100644 --- a/pkg/appStore/installedApp/adapter/Adapter.go +++ b/pkg/appStore/installedApp/adapter/Adapter.go @@ -19,7 +19,7 @@ package adapter import ( "encoding/json" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index b5f8905dbc..aead7617b4 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -20,8 +20,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" util2 "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" diff --git a/pkg/appStore/installedApp/repository/InstalledAppVersionHistory.go b/pkg/appStore/installedApp/repository/InstalledAppVersionHistory.go index 978ae60c8e..1627c37e90 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppVersionHistory.go +++ b/pkg/appStore/installedApp/repository/InstalledAppVersionHistory.go @@ -18,7 +18,7 @@ package repository import ( "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.uber.org/zap" diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go index 30313678ce..a4e3a9c040 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go @@ -22,7 +22,7 @@ import ( "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" appStoreAdapter "github.com/devtron-labs/devtron/pkg/appStore/adapter" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index e0f103a83c..e542f24e46 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -29,8 +29,8 @@ import ( openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/appStore/adapter" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/DeploymentStatusService.go b/pkg/appStore/installedApp/service/FullMode/deployment/DeploymentStatusService.go index d4e06f9c04..db9c54a1d9 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/DeploymentStatusService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/DeploymentStatusService.go @@ -20,8 +20,8 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/sql" diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go index f09e95d7da..777c426db3 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go @@ -23,8 +23,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/adapter" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" "github.com/devtron-labs/devtron/pkg/argoRepositoryCreds" diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 0b58e7ab41..2e43e52781 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -19,7 +19,7 @@ package devtronApps import ( "context" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 856cb6c55b..1a29178a00 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -25,7 +25,7 @@ import ( gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" bean6 "github.com/devtron-labs/devtron/pkg/attributes/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index aebad499af..deb986e076 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -39,8 +39,8 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" diff --git a/pkg/deployment/trigger/devtronApps/userDeploymentRequest/repository/UserDeploymentRequestRepository.go b/pkg/deployment/trigger/devtronApps/userDeploymentRequest/repository/UserDeploymentRequestRepository.go index 689421e25e..4fc9b9d4b9 100644 --- a/pkg/deployment/trigger/devtronApps/userDeploymentRequest/repository/UserDeploymentRequestRepository.go +++ b/pkg/deployment/trigger/devtronApps/userDeploymentRequest/repository/UserDeploymentRequestRepository.go @@ -21,8 +21,8 @@ import ( "context" apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/models" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.opentelemetry.io/otel" diff --git a/pkg/eventProcessor/in/AppStoreAppsEventProcessorService.go b/pkg/eventProcessor/in/AppStoreAppsEventProcessorService.go index 5c4774ae5a..7a04f4ba67 100644 --- a/pkg/eventProcessor/in/AppStoreAppsEventProcessorService.go +++ b/pkg/eventProcessor/in/AppStoreAppsEventProcessorService.go @@ -21,7 +21,7 @@ import ( "errors" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 169eb1d899..57f2015da3 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -30,7 +30,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - cdWorkflowModelBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + cdWorkflowModelBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" diff --git a/pkg/eventProcessor/out/WorkflowEventPublishService.go b/pkg/eventProcessor/out/WorkflowEventPublishService.go index d68fab3a5c..32a3c7d5c3 100644 --- a/pkg/eventProcessor/out/WorkflowEventPublishService.go +++ b/pkg/eventProcessor/out/WorkflowEventPublishService.go @@ -23,7 +23,7 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" internalUtil "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/app/status" diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 02e4bb24b9..1611ddb0a6 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -27,7 +27,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" app2 "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/bean" diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index cae5bb02b2..e7bf55806e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -724,9 +724,13 @@ func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pip workflow.Image = wfr.CdWorkflow.CiArtifact.Image workflow.PipelineId = wfr.CdWorkflow.PipelineId workflow.CiArtifactId = wfr.CdWorkflow.CiArtifactId - if wfr.IsArtifactUploaded != nil { - workflow.IsArtifactUploaded = *wfr.IsArtifactUploaded + isArtifactUploaded, isMigrationRequired := wfr.GetIsArtifactUploaded() + if isMigrationRequired { + // Migrate isArtifactUploaded. For old records, set isArtifactUploaded -> Uploaded + impl.cdWorkflowRepository.MigrateIsArtifactUploaded(wfr.Id, true) + isArtifactUploaded = true } + workflow.IsArtifactUploaded = isArtifactUploaded workflow.BlobStorageEnabled = wfr.BlobStorageEnabled workflow.RefCdWorkflowRunnerId = wfr.RefCdWorkflowRunnerId } diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 9db376c9ef..8317a78c31 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" "github.com/devtron-labs/common-lib/utils/workFlow" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" "io/ioutil" "net/http" @@ -530,9 +530,11 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int var ciWorkLowResponses []types.WorkflowResponse for _, w := range workFlows { - isArtifactUploaded := w.IsArtifactUploaded - if w.IsArtifactUploadedV2 != nil { - isArtifactUploaded = *w.IsArtifactUploadedV2 + isArtifactUploaded, isMigrationRequired := w.GetIsArtifactUploaded() + if isMigrationRequired { + // Migrate isArtifactUploaded. For old records, set isArtifactUploaded -> w.IsArtifactUploaded + impl.ciWorkflowRepository.MigrateIsArtifactUploaded(w.Id, w.IsArtifactUploaded) + isArtifactUploaded = w.IsArtifactUploaded } wfResponse := types.WorkflowResponse{ Id: w.Id, @@ -724,16 +726,18 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build } environmentName := "" if workflow.EnvironmentId != 0 { - env, err := impl.envRepository.FindById(workflow.EnvironmentId) + envModel, err := impl.envRepository.FindById(workflow.EnvironmentId) if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in fetching environment details ", "err", err) return types.WorkflowResponse{}, err } - environmentName = env.Name + environmentName = envModel.Name } - isArtifactUploaded := ciArtifact.IsArtifactUploaded - if workflow.IsArtifactUploaded != nil { - isArtifactUploaded = *workflow.IsArtifactUploaded + isArtifactUploaded, isMigrationRequired := workflow.GetIsArtifactUploaded() + if isMigrationRequired { + // Migrate isArtifactUploaded. For old records, set isArtifactUploaded -> ciArtifact.IsArtifactUploaded + impl.ciWorkflowRepository.MigrateIsArtifactUploaded(workflow.Id, ciArtifact.IsArtifactUploaded) + isArtifactUploaded = ciArtifact.IsArtifactUploaded } workflowResponse := types.WorkflowResponse{ Id: workflow.Id, diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 28f8baa2e6..8e3fa78500 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -23,7 +23,7 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils" bean3 "github.com/devtron-labs/common-lib/utils/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/infraConfig" diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 9c004660e3..92b3e61fc3 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -36,7 +36,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/bean" diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 5c7b42a5e5..406949617f 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -24,7 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/workflow/util" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/infraConfig" diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/pipeline/WorkflowServiceIT_test.go index 34523ceb55..ece775c66d 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/pipeline/WorkflowServiceIT_test.go @@ -24,7 +24,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 077d96178a..91a48f7d95 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -24,7 +24,7 @@ import ( "github.com/caarlos0/env" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/pipeline/bean" v12 "k8s.io/api/core/v1" "k8s.io/client-go/rest" diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 0a56f40675..23fa62023d 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -27,7 +27,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/infraConfig" diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go index 8733e5db57..f84b891125 100644 --- a/pkg/workflow/cd/CdWorkflowCommonService.go +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -25,8 +25,8 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/status" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go index 2a5028142b..410a00f022 100644 --- a/pkg/workflow/cd/CdWorkflowRunnerService.go +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -18,6 +18,7 @@ package cd import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" "github.com/go-pg/pg" @@ -75,7 +76,7 @@ func (impl *CdWorkflowRunnerServiceImpl) UpdateWfr(dto *bean.CdWorkflowRunnerDto } func (impl *CdWorkflowRunnerServiceImpl) UpdateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) error { - err := impl.cdWorkflowRepository.UpdateIsArtifactUploaded(wfrId, isArtifactUploaded) + err := impl.cdWorkflowRepository.UpdateIsArtifactUploaded(wfrId, workflow.GetArtifactUploadedType(isArtifactUploaded)) if err != nil { impl.logger.Errorw("error in updating isArtifactUploaded in db", "wfrId", wfrId, "err", err) return err diff --git a/pkg/workflow/cd/adapter/adapter.go b/pkg/workflow/cd/adapter/adapter.go index af6c00adbb..419e58499c 100644 --- a/pkg/workflow/cd/adapter/adapter.go +++ b/pkg/workflow/cd/adapter/adapter.go @@ -64,7 +64,7 @@ func ConvertCdWorkflowDtoToDbObj(dto *bean.CdWorkflowDto) *pipelineConfig.CdWork } func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineConfig.CdWorkflowRunner { - return &pipelineConfig.CdWorkflowRunner{ + model := &pipelineConfig.CdWorkflowRunner{ Id: dto.Id, Name: dto.Name, WorkflowType: dto.WorkflowType, @@ -83,7 +83,6 @@ func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineC RefCdWorkflowRunnerId: dto.RefCdWorkflowRunnerId, ImagePathReservationIds: dto.ImagePathReservationIds, ReferenceId: dto.ReferenceId, - IsArtifactUploaded: &dto.IsArtifactUploaded, AuditLog: sql.AuditLog{ CreatedOn: dto.StartedOn, CreatedBy: dto.TriggeredBy, @@ -91,4 +90,6 @@ func ConvertCdWorkflowRunnerDtoToDbObj(dto *bean.CdWorkflowRunnerDto) *pipelineC UpdatedBy: dto.TriggeredBy, }, } + model.WithIsArtifactUploaded(dto.IsArtifactUploaded) + return model } diff --git a/pkg/workflow/cd/bean/bean.go b/pkg/workflow/cd/bean/bean.go index 314a87107e..8ea11094af 100644 --- a/pkg/workflow/cd/bean/bean.go +++ b/pkg/workflow/cd/bean/bean.go @@ -18,7 +18,7 @@ package bean import ( "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "time" ) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 0f4fd4e263..3834d1ccfc 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -30,7 +30,8 @@ import ( argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/build/artifacts" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" @@ -727,7 +728,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger return 0, err } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) - savedWorkflow.IsArtifactUploaded = &request.IsArtifactUploaded + savedWorkflow.IsArtifactUploaded = workflow.GetArtifactUploadedType(request.IsArtifactUploaded) impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) if err != nil { @@ -960,7 +961,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r return err } // update IsArtifactUploaded flag in workflow - dbErr := impl.ciWorkflowRepository.UpdateArtifactUploaded(savedWorkflow.Id, request.IsArtifactUploaded) + dbErr := impl.ciWorkflowRepository.UpdateArtifactUploaded(savedWorkflow.Id, workflow.GetArtifactUploadedType(request.IsArtifactUploaded)) if dbErr != nil { impl.logger.Errorw("update workflow status", "ciWorkflowId", savedWorkflow.Id, "err", dbErr) } diff --git a/pkg/workflow/dag/bean/bean.go b/pkg/workflow/dag/bean/bean.go index 62a70fa611..10470c42e1 100644 --- a/pkg/workflow/dag/bean/bean.go +++ b/pkg/workflow/dag/bean/bean.go @@ -31,7 +31,6 @@ type CiArtifactWebhookRequest struct { UserId int32 `json:"userId"` IsArtifactUploaded bool `json:"isArtifactUploaded"` FailureReason string `json:"failureReason"` // FailureReason is used for notifying the failure reason to the user. Should be short and user-friendly - InternalError string `json:"internalError,omitempty"` // InternalError is used as workflow failure message. Should be more detailed and technical PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") } diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go index f34a71b295..f176bf5ab0 100644 --- a/pkg/workflow/status/WorkflowStatusService.go +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -28,8 +28,8 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/app/status" app_status "github.com/devtron-labs/devtron/pkg/appStatus" diff --git a/scripts/sql/29200000_artifact_uploaded_flag.up.sql b/scripts/sql/29200000_artifact_uploaded_flag.up.sql index 8d07f7b582..2af533c40b 100644 --- a/scripts/sql/29200000_artifact_uploaded_flag.up.sql +++ b/scripts/sql/29200000_artifact_uploaded_flag.up.sql @@ -3,9 +3,9 @@ --- ALTER TABLE "public"."ci_workflow" - ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN; + ADD COLUMN IF NOT EXISTS is_artifact_uploaded varchar(50); ALTER TABLE "public"."cd_workflow_runner" - ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN, + ADD COLUMN IF NOT EXISTS is_artifact_uploaded varchar(50), ADD COLUMN IF NOT EXISTS cd_artifact_location varchar(256); --- diff --git a/wire_gen.go b/wire_gen.go index ab237b4562..10bb0d2362 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 8635dc21a478fe07ec656f015b8085592ccc7b25 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 16:38:37 +0530 Subject: [PATCH 22/29] removed space --- pkg/deployment/trigger/devtronApps/PreStageTriggerService.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 1a29178a00..13258f6475 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -934,7 +934,6 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) artifactFileName := fmt.Sprintf(cdArtifactLocationFormat, cdWf.Id, runner.Id) return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName From b1d5934485ec64073930297262e32ea4975c6a7a Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 16:47:31 +0530 Subject: [PATCH 23/29] updated backward compatibility --- pkg/workflow/dag/WorkflowDagExecutor.go | 39 +++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 3834d1ccfc..3da18ea29b 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -768,15 +768,16 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger createdOn = imagePushedAt } buildArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: pipeline.Id, - WorkflowId: request.WorkflowId, - ScanEnabled: pipeline.ScanEnabled, - Scanned: false, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: pipeline.Id, + WorkflowId: request.WorkflowId, + ScanEnabled: pipeline.ScanEnabled, + IsArtifactUploaded: request.IsArtifactUploaded, // for backward compatibility + Scanned: false, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, } plugin, err := impl.globalPluginRepository.GetPluginByName(bean3.VULNERABILITY_SCANNING_PLUGIN) if err != nil || len(plugin) == 0 { @@ -816,6 +817,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger ParentCiArtifact: buildArtifact.Id, Scanned: buildArtifact.Scanned, ScanEnabled: buildArtifact.ScanEnabled, + IsArtifactUploaded: request.IsArtifactUploaded, // for backward compatibility } pluginArtifacts = append(pluginArtifacts, pluginArtifact) } @@ -837,15 +839,16 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var ciArtifactArr []*repository.CiArtifact for _, ci := range childrenCi { ciArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: ci.Id, - ParentCiArtifact: buildArtifact.Id, - ScanEnabled: ci.ScanEnabled, - Scanned: false, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: ci.Id, + ParentCiArtifact: buildArtifact.Id, + IsArtifactUploaded: request.IsArtifactUploaded, // for backward compatibility + ScanEnabled: ci.ScanEnabled, + Scanned: false, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if ci.ScanEnabled { ciArtifact.Scanned = true From 37410f60a16e355944b3ac191eaadfd41002357b Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 30 Sep 2024 23:03:33 +0530 Subject: [PATCH 24/29] fix: cdArtifactLocation in UpdateWorkflow method --- pkg/pipeline/CdHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index e7bf55806e..cbb68a761a 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -218,7 +218,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) + cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.CdWorkflowId, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { if savedWorkflow.Status != executors.WorkflowCancel { savedWorkflow.Status = status From 84974115bac1a6bd8d80c05445615c71f88edd22 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 1 Oct 2024 00:24:12 +0530 Subject: [PATCH 25/29] updated migration for artifact location --- .../pipelineConfig/CdWorfkflowRepository.go | 11 +++++++++++ .../repository/pipelineConfig/CiWorkflowRepository.go | 11 +++++++++++ pkg/pipeline/CdHandler.go | 2 ++ pkg/pipeline/CiHandler.go | 2 ++ 4 files changed, 26 insertions(+) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 37d6ec21be..cb81f4ca1b 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -78,6 +78,7 @@ type CdWorkflowRepository interface { FindLatestRunnerByPipelineIdsAndRunnerType(ctx context.Context, pipelineIds []int, runnerType apiBean.WorkflowType) ([]CdWorkflowRunner, error) MigrateIsArtifactUploaded(wfrId int, isArtifactUploaded bool) + MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string) } type CdWorkflowRepositoryImpl struct { @@ -749,3 +750,13 @@ func (impl *CdWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfrId int, isArt impl.logger.Errorw("error in updating is artifact uploaded", "wfrId", wfrId, "err", err) } } + +func (impl *CdWorkflowRepositoryImpl) MigrateCdArtifactLocation(wfrId int, cdArtifactLocation string) { + _, err := impl.dbConnection.Model((*CdWorkflowRunner)(nil)). + Set("cd_artifact_location = ?", cdArtifactLocation). + Where("id = ?", wfrId). + Update() + if err != nil { + impl.logger.Errorw("error in updating cd artifact location", "wfrId", wfrId, "err", err) + } +} diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 7e2989988b..48f66755a7 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -50,6 +50,7 @@ type CiWorkflowRepository interface { FIndCiWorkflowStatusesByAppId(appId int) ([]*CiWorkflowStatus, error) MigrateIsArtifactUploaded(wfId int, isArtifactUploaded bool) + MigrateCiArtifactLocation(wfId int, artifactLocation string) } type CiWorkflowRepositoryImpl struct { @@ -386,3 +387,13 @@ func (impl *CiWorkflowRepositoryImpl) MigrateIsArtifactUploaded(wfId int, isArti impl.logger.Errorw("error occurred while updating is_artifact_uploaded", "wfId", wfId, "err", err) } } + +func (impl *CiWorkflowRepositoryImpl) MigrateCiArtifactLocation(wfId int, artifactLocation string) { + _, err := impl.dbConnection.Model((*CiWorkflow)(nil)). + Set("ci_artifact_location = ?", artifactLocation). + Where("id = ?", wfId). + Update() + if err != nil { + impl.logger.Errorw("error occurred while updating ci_artifact_location", "wfId", wfId, "err", err) + } +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index cbb68a761a..6890014614 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -664,6 +664,8 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, e key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { key = wfr.CdArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) } baseLogLocationPathConfig := impl.config.BaseLogLocationPath blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 8317a78c31..65237aa2e2 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -926,6 +926,8 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { key = ciWorkflow.CiArtifactLocation + } else if util3.IsValidUrlSubPath(key) { + impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) } baseLogLocationPathConfig := impl.config.BaseLogLocationPath blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) From cb2bef4e01a0002ef1472d6d73ec43af53db2a11 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 1 Oct 2024 01:05:39 +0530 Subject: [PATCH 26/29] updated HandleCiSuccessEvent func --- .../in/WorkflowEventProcessorService.go | 15 +++- pkg/workflow/dag/WorkflowDagExecutor.go | 85 ++++++++++--------- 2 files changed, 57 insertions(+), 43 deletions(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 57f2015da3..552843bb80 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -546,7 +546,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error { impl.logger.Error("Error while creating request for pipelineID", "pipelineId", ciCompleteEvent.PipelineId, "err", err) return } - resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, request, detail.LastUpdatedOn) + resp, err := impl.validateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, request, detail.LastUpdatedOn) if err != nil { return } @@ -555,7 +555,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error { } } else { globalUtil.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.globalEnvVariables.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName) - resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, req, time.Time{}) + resp, err := impl.validateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, req, time.Time{}) if err != nil { return } @@ -582,12 +582,21 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error { return nil } -func (impl *WorkflowEventProcessorImpl) ValidateAndHandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *wrokflowDagBean.CiArtifactWebhookRequest, imagePushedAt time.Time) (int, error) { +func (impl *WorkflowEventProcessorImpl) validateAndHandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *wrokflowDagBean.CiArtifactWebhookRequest, imagePushedAt time.Time) (int, error) { + if request.WorkflowId != nil { + err := impl.workflowDagExecutor.UpdateCiWorkflowForCiSuccess(request) + if err != nil { + return 0, err + } + } + // Validate request, must be performed after workflowDagExecutor.UpdateCiWorkflow func + // As it is required to update IsArtifactUploaded field in UpdateCiWorkflow table, irrespective of CiArtifact creation validationErr := impl.validator.Struct(request) if validationErr != nil { impl.logger.Errorw("validation err, HandleCiSuccessEvent", "err", validationErr, "payload", request) return 0, validationErr } + // Create CiArtifact and Trigger CI Success event buildArtifactId, err := impl.workflowDagExecutor.HandleCiSuccessEvent(triggerContext, ciPipelineId, request, imagePushedAt) if err != nil { impl.logger.Error("Error while sending event for CI success for pipelineID", diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 3da18ea29b..38b89bbd43 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -51,6 +51,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/workflow/dag/bean" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" + "gopkg.in/go-playground/validator.v9" "strings" "sync" "time" @@ -76,6 +77,7 @@ import ( ) type WorkflowDagExecutor interface { + UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error HandleDeploymentSuccessEvent(triggerContext triggerBean.TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error @@ -122,6 +124,7 @@ type WorkflowDagExecutorImpl struct { commonArtifactService artifacts.CommonArtifactService deploymentConfigService common2.DeploymentConfigService asyncRunnable *async.Runnable + validator *validator.Validate } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -145,7 +148,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, deploymentConfigService common2.DeploymentConfigService, - asyncRunnable *async.Runnable) *WorkflowDagExecutorImpl { + asyncRunnable *async.Runnable, validator *validator.Validate) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -169,6 +172,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi commonArtifactService: commonArtifactService, deploymentConfigService: deploymentConfigService, asyncRunnable: asyncRunnable, + validator: validator, } config, err := types.GetCdConfig() if err != nil { @@ -715,42 +719,43 @@ func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext return nil } -func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) { - impl.logger.Infow("webhook for artifact save", "req", request) - if request.WorkflowId != nil { - savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) - if err != nil { - impl.logger.Errorw("cannot get saved wf", "err", err) - return 0, err - } - // if workflow already cancelled then return, this state arises when user force aborts a ci - if savedWorkflow.Status == executors.WorkflowCancel { - return 0, err - } - savedWorkflow.Status = string(v1alpha1.NodeSucceeded) - savedWorkflow.IsArtifactUploaded = workflow.GetArtifactUploadedType(request.IsArtifactUploaded) - impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) - err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) - if err != nil { - impl.logger.Errorw("update wf failed for id ", "err", err) - return 0, err - } - - err = impl.deactivateUnusedPaths(savedWorkflow.ImagePathReservationIds, request.PluginRegistryArtifactDetails) - if err != nil { - impl.logger.Errorw("error in deactivation images", "err", err) - return 0, err - } +func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) { + savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) + if err != nil { + impl.logger.Errorw("cannot get saved wf", "err", err) + return err + } + // if workflow already cancelled then return, this state arises when user force aborts a ci + if savedWorkflow.Status == executors.WorkflowCancel { + return err + } + savedWorkflow.Status = string(v1alpha1.NodeSucceeded) + savedWorkflow.IsArtifactUploaded = workflow.GetArtifactUploadedType(request.IsArtifactUploaded) + impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) + err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) + if err != nil { + impl.logger.Errorw("update wf failed for id ", "err", err) + return err } - pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) - if request.PipelineName == "" { - request.PipelineName = pipeline.Name + err = impl.deactivateUnusedPaths(savedWorkflow.ImagePathReservationIds, request.PluginRegistryArtifactDetails) + if err != nil { + impl.logger.Errorw("error in deactivation images", "err", err) + return err } + return nil +} + +func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) { + impl.logger.Infow("webhook for artifact save", "req", request) + pipelineModal, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) if err != nil { - impl.logger.Errorw("unable to find pipeline", "name", request.PipelineName, "err", err) + impl.logger.Errorw("unable to find pipelineModal", "ciPipelineId", ciPipelineId, "err", err) return 0, err } + if request.PipelineName == "" { + request.PipelineName = pipelineModal.Name + } materialJson, err := request.MaterialInfo.MarshalJSON() if err != nil { impl.logger.Errorw("unable to marshal material metadata", "err", err) @@ -772,9 +777,9 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger ImageDigest: request.ImageDigest, MaterialInfo: string(materialJson), DataSource: request.DataSource, - PipelineId: pipeline.Id, + PipelineId: pipelineModal.Id, WorkflowId: request.WorkflowId, - ScanEnabled: pipeline.ScanEnabled, + ScanEnabled: pipelineModal.ScanEnabled, IsArtifactUploaded: request.IsArtifactUploaded, // for backward compatibility Scanned: false, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, @@ -784,12 +789,12 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger impl.logger.Errorw("error in getting image scanning plugin", "err", err) return 0, err } - isScanPluginConfigured, err := impl.pipelineStageRepository.CheckPluginExistsInCiPipeline(pipeline.Id, string(repository4.PIPELINE_STAGE_TYPE_POST_CI), plugin[0].Id) + isScanPluginConfigured, err := impl.pipelineStageRepository.CheckPluginExistsInCiPipeline(pipelineModal.Id, string(repository4.PIPELINE_STAGE_TYPE_POST_CI), plugin[0].Id) if err != nil { - impl.logger.Errorw("error in getting ci pipeline plugin", "err", err, "pipelineId", pipeline.Id, "pluginId", plugin[0].Id) + impl.logger.Errorw("error in getting ci pipelineModal plugin", "err", err, "pipelineId", pipelineModal.Id, "pluginId", plugin[0].Id) return 0, err } - if pipeline.ScanEnabled || isScanPluginConfigured { + if pipelineModal.ScanEnabled || isScanPluginConfigured { buildArtifact.Scanned = true buildArtifact.ScanEnabled = true } @@ -801,7 +806,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipeline.PipelineType == string(CiPipeline.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(CiPipeline.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ @@ -809,8 +814,8 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger ImageDigest: request.ImageDigest, MaterialInfo: string(materialJson), DataSource: request.PluginArtifactStage, - ComponentId: pipeline.Id, - PipelineId: pipeline.Id, + ComponentId: pipelineModal.Id, + PipelineId: pipelineModal.Id, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, CredentialSourceValue: registry, @@ -869,7 +874,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger } else { ciArtifactArr = append(ciArtifactArr, pluginArtifacts[0]) } - go impl.WriteCiSuccessEvent(request, pipeline, buildArtifact) + go impl.WriteCiSuccessEvent(request, pipelineModal, buildArtifact) async := false // execute auto trigger in batch on CI success event From 4d33c1eea81cf07e72c09720202015d017f53a17 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 1 Oct 2024 01:07:00 +0530 Subject: [PATCH 27/29] removed unused wire injection --- pkg/workflow/dag/WorkflowDagExecutor.go | 5 +---- wire_gen.go | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 38b89bbd43..f2545e81be 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -51,7 +51,6 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/workflow/dag/bean" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" - "gopkg.in/go-playground/validator.v9" "strings" "sync" "time" @@ -124,7 +123,6 @@ type WorkflowDagExecutorImpl struct { commonArtifactService artifacts.CommonArtifactService deploymentConfigService common2.DeploymentConfigService asyncRunnable *async.Runnable - validator *validator.Validate } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -148,7 +146,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, deploymentConfigService common2.DeploymentConfigService, - asyncRunnable *async.Runnable, validator *validator.Validate) *WorkflowDagExecutorImpl { + asyncRunnable *async.Runnable) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -172,7 +170,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi commonArtifactService: commonArtifactService, deploymentConfigService: deploymentConfigService, asyncRunnable: asyncRunnable, - validator: validator, } config, err := types.GetCdConfig() if err != nil { diff --git a/wire_gen.go b/wire_gen.go index 10bb0d2362..2cf0e49f94 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -665,7 +665,7 @@ func InitializeApp() (*App, error) { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, validate) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From 3734135eac5140771744479d0549edd880245543 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 1 Oct 2024 01:07:40 +0530 Subject: [PATCH 28/29] updated wire_gen file --- wire_gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wire_gen.go b/wire_gen.go index 2cf0e49f94..10bb0d2362 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -665,7 +665,7 @@ func InitializeApp() (*App, error) { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, validate) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From c21168444419e58cc7369616bf8a498ecdc2d1b2 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 3 Oct 2024 12:51:22 +0530 Subject: [PATCH 29/29] updated common-lib version --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 162c23e84c..28b83cf7db 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 - github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 + github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f github.com/devtron-labs/go-bitbucket v0.9.60-beta github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 github.com/evanphx/json-patch v5.7.0+incompatible diff --git a/go.sum b/go.sum index e09729899a..9f48ec4b43 100644 --- a/go.sum +++ b/go.sum @@ -794,8 +794,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI= -github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 h1:ikMVToilPandd3iKEGkISMcyvWva440n/zAlMZT2M3E= -github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= +github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f h1:5PDhi7bAc9r67OpwXZDG7y1n9lhHX26Ke4Y9VE9+kd8= +github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys= diff --git a/vendor/modules.txt b/vendor/modules.txt index 9b60fe1fe5..1c716e9c1b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -344,7 +344,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20240930074010-674233f8aaa6 +# github.com/devtron-labs/common-lib v0.18.1-0.20241003071930-acfb6a1b1e7f ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage