From 88538e872159c4e7b6d8805d909cf6db3fd3a000 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Fri, 30 Sep 2022 17:17:34 +0530 Subject: [PATCH 01/50] Ci build config introduced --- api/appbean/AppDetail.go | 21 +++++++-- api/restHandler/CoreAppRestHandler.go | 7 ++- .../pipelineConfig/CiBuildConfigRepository.go | 32 +++++++++++++ .../pipelineConfig/CiTemplateRepository.go | 2 + pkg/bean/app.go | 47 ++++++++++++------- pkg/pipeline/CiBuildConfigService.go | 26 ++++++++++ pkg/pipeline/PipelineBuilder.go | 2 +- pkg/pipeline/bean/CiBuildConfig.go | 33 +++++++++++++ 8 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go create mode 100644 pkg/pipeline/CiBuildConfigService.go create mode 100644 pkg/pipeline/bean/CiBuildConfig.go diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index 81f049d89c..2b5fa32712 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -35,9 +35,24 @@ type GitMaterial struct { } type DockerConfig struct { - DockerRegistry string `json:"dockerRegistry" validate:"required"` - DockerRepository string `json:"dockerRepository" validate:"required"` - BuildConfig *DockerBuildConfig `json:"dockerBuildConfig"` + DockerRegistry string `json:"dockerRegistry" validate:"required"` + DockerRepository string `json:"dockerRepository" validate:"required"` + CiBuildConfig *CiBuildConfig `json:"ciBuildConfig" validate:"required"` +} + +type CiBuildConfig struct { + GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` + CiBuildType string `json:"ciBuildType"` + DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` + BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +} + +type BuildPackConfig struct { + BuilderId string `json:"builderId"` + Language string `json:"language"` + LanguageVersion string `json:"languageVersion"` + BuildPacks []string `json:"buildPacks"` + Args map[string]string `json:"args"` } type DockerBuildConfig struct { diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index c0c19cd4ca..ab5af4a885 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -487,15 +487,18 @@ func (handler CoreAppRestHandlerImpl) buildDockerConfig(appId int) (*appBean.Doc } //getting gitMaterialUrl by id - gitMaterial, err := handler.materialRepository.FindById(ciConfig.DockerBuildConfig.GitMaterialId) + gitMaterial, err := handler.materialRepository.FindById(ciConfig.CiBuildConfig.GitMaterialId) if err != nil { - handler.logger.Errorw("error in fetching materialUrl by ID in GetAppAllDetail", "err", err, "gitMaterialId", ciConfig.DockerBuildConfig.GitMaterialId) + handler.logger.Errorw("error in fetching materialUrl by ID in GetAppAllDetail", "err", err, "gitMaterialId", ciConfig.CiBuildConfig.GitMaterialId) return nil, err, http.StatusInternalServerError } dockerConfig := &appBean.DockerConfig{ DockerRegistry: ciConfig.DockerRegistry, DockerRepository: ciConfig.DockerRepository, + CiBuildConfig: &appBean.CiBuildConfig{ + //BuildPackConfig: ciConfig.CiBuildConfig.BuildPackConfig, + }, BuildConfig: &appBean.DockerBuildConfig{ Args: ciConfig.DockerBuildConfig.Args, DockerfileRelativePath: ciConfig.DockerBuildConfig.DockerfilePath, diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go new file mode 100644 index 0000000000..ce5153fd2f --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go @@ -0,0 +1,32 @@ +package pipelineConfig + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type CiBuildConfig struct { + tableName struct{} `sql:"ci_build_config" pg:",discard_unknown_columns"` + Id int `sql:"id"` + CiTemplateId int `sql:"ci_template_id"` + CiPipelineId int `sql:"ci_pipeline_id"` + Type string `sql:"type"` + BuildMetadata string `sql:"build_metadata"` + sql.AuditLog +} + +type CiBuildConfigRepository interface { +} + +type CiBuildConfigRepositoryImpl struct { + dbConnection *pg.DB + logger *zap.SugaredLogger +} + +func NewCiBuildConfigRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *CiBuildConfigRepositoryImpl { + return &CiBuildConfigRepositoryImpl{ + dbConnection: dbConnection, + logger: logger, + } +} diff --git a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go index b6be184df9..0f56a9bd94 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go @@ -41,10 +41,12 @@ type CiTemplate struct { Version string `sql:"version"` //gocd etage Active bool `sql:"active,notnull"` GitMaterialId int `sql:"git_material_id"` + CiBuildConfigId int `sql:"ci_build_config_id"` sql.AuditLog App *app.App DockerRegistry *repository.DockerArtifactStore GitMaterial *GitMaterial + CiBuildConfig *CiBuildConfig } type CiTemplateRepository interface { diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 29e7461f75..1f4122a5d2 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -249,22 +249,22 @@ type Material struct { } type CiConfigRequest struct { - Id int `json:"id,omitempty" validate:"number"` //ciTemplateId - AppId int `json:"appId,omitempty" validate:"required,number"` - DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry - DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` - CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial - AppName string `json:"appName,omitempty"` - Version string `json:"version,omitempty"` //gocd etag used for edit purpose - DockerRegistryUrl string `json:"-"` - CiTemplateName string `json:"-"` - UserId int32 `json:"-"` - Materials []Material `json:"materials"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Id int `json:"id,omitempty" validate:"number"` //ciTemplateId + AppId int `json:"appId,omitempty" validate:"required,number"` + DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry + DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr + CiBuildConfig *CiBuildConfig `json:"ciBuildConfig"` + CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial + AppName string `json:"appName,omitempty"` + Version string `json:"version,omitempty"` //gocd etag used for edit purpose + DockerRegistryUrl string `json:"-"` + CiTemplateName string `json:"-"` + UserId int32 `json:"-"` + Materials []Material `json:"materials"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type TestExecutorImageProperties struct { @@ -273,6 +273,13 @@ type TestExecutorImageProperties struct { ReportDir string `json:"reportDir,omitempty"` } +type CiBuildConfig struct { + GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` + CiBuildType string `json:"ciBuildType"` + DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` + BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +} + type DockerBuildConfig struct { GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` @@ -281,6 +288,14 @@ type DockerBuildConfig struct { //Name Tag DockerfilePath RepoUrl } +type BuildPackConfig struct { + BuilderId string `json:"builderId"` + Language string `json:"language"` + LanguageVersion string `json:"languageVersion"` + BuildPacks []string `json:"buildPacks"` + Args map[string]string `json:"args"` +} + type PipelineCreateResponse struct { AppName string `json:"appName,omitempty"` AppId int `json:"appId,omitempty"` diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go new file mode 100644 index 0000000000..d6d7ee1582 --- /dev/null +++ b/pkg/pipeline/CiBuildConfigService.go @@ -0,0 +1,26 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "go.uber.org/zap" +) + +type CiBuildConfigService interface { + Get(ciBuildConfigId int) (*bean.CiBuildConfig, error) + Save(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) + Update(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) + Delete(ciBuildConfigId int) error +} + +type CiBuildConfigImpl struct { + Logger *zap.SugaredLogger + CiBuildConfigRepository *pipelineConfig.CiBuildConfigRepository +} + +func NewCiBuildConfigImpl(logger *zap.SugaredLogger, ciBuildConfigRepository *pipelineConfig.CiBuildConfigRepository) *CiBuildConfigImpl { + return &CiBuildConfigImpl{ + Logger: logger, + CiBuildConfigRepository: ciBuildConfigRepository, + } +} diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index f86bdd0c23..7e71ba5335 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -724,7 +724,7 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq ciTemplate := &pipelineConfig.CiTemplate{ DockerRegistryId: createRequest.DockerRegistry, DockerRepository: createRequest.DockerRepository, - GitMaterialId: createRequest.DockerBuildConfig.GitMaterialId, + GitMaterialId: createRequest.CiBuildConfig.GitMaterialId, DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, Args: string(argByte), TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go new file mode 100644 index 0000000000..2227f33f63 --- /dev/null +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -0,0 +1,33 @@ +package bean + +type CiBuildType string + +const ( + SELF_DOCKERFILE_BUILD_TYPE CiBuildType = "self-dockerfile-build" + MANAGED_DOCKERFILE_BUILD_TYPE CiBuildType = "managed-dockerfile-build" + SKIP_BUILD_BUILD_TYPE CiBuildType = "skip-build" + BUILDPACK_BUILD_TYPE CiBuildType = "buildpack-build" +) + +type CiBuildConfig struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` + CiBuildType CiBuildType `json:"ciBuildType"` + DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` + BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +} + +type DockerBuildConfig struct { + DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` + DockerfileContent string `json:"DockerfileContent"` + Args map[string]string `json:"args,omitempty"` + TargetPlatform string `json:"targetPlatform,omitempty"` +} + +type BuildPackConfig struct { + BuilderId string `json:"builderId"` + Language string `json:"language"` + LanguageVersion string `json:"languageVersion"` + BuildPacks []string `json:"buildPacks"` + Args map[string]string `json:"args"` +} From 1635a80154a80021568176d0f84639b33251b7a0 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Fri, 30 Sep 2022 19:35:45 +0530 Subject: [PATCH 02/50] ci build service commit --- pkg/bean/app.go | 83 ++++++++++++++-------------- pkg/pipeline/CiBuildConfigService.go | 20 +++++++ pkg/pipeline/PipelineBuilder.go | 22 +++++--- 3 files changed, 77 insertions(+), 48 deletions(-) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 1f4122a5d2..abe6341f53 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -110,9 +110,10 @@ type CiPipeline struct { } type DockerConfigOverride struct { - DockerRegistry string `json:"dockerRegistry,omitempty"` - DockerRepository string `json:"dockerRepository,omitempty"` - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` + DockerRegistry string `json:"dockerRegistry,omitempty"` + DockerRepository string `json:"dockerRepository,omitempty"` + CiBuildConfig *bean.CiBuildConfig `json:"ciBuildConfig,omitEmpty"` + //DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` } type CiPipelineMin struct { @@ -249,22 +250,22 @@ type Material struct { } type CiConfigRequest struct { - Id int `json:"id,omitempty" validate:"number"` //ciTemplateId - AppId int `json:"appId,omitempty" validate:"required,number"` - DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry - DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr - CiBuildConfig *CiBuildConfig `json:"ciBuildConfig"` - CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial - AppName string `json:"appName,omitempty"` - Version string `json:"version,omitempty"` //gocd etag used for edit purpose - DockerRegistryUrl string `json:"-"` - CiTemplateName string `json:"-"` - UserId int32 `json:"-"` - Materials []Material `json:"materials"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Id int `json:"id,omitempty" validate:"number"` //ciTemplateId + AppId int `json:"appId,omitempty" validate:"required,number"` + DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry + DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr + CiBuildConfig *bean.CiBuildConfig `json:"ciBuildConfig"` + CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial + AppName string `json:"appName,omitempty"` + Version string `json:"version,omitempty"` //gocd etag used for edit purpose + DockerRegistryUrl string `json:"-"` + CiTemplateName string `json:"-"` + UserId int32 `json:"-"` + Materials []Material `json:"materials"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type TestExecutorImageProperties struct { @@ -273,28 +274,28 @@ type TestExecutorImageProperties struct { ReportDir string `json:"reportDir,omitempty"` } -type CiBuildConfig struct { - GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` - CiBuildType string `json:"ciBuildType"` - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` - BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` -} - -type DockerBuildConfig struct { - GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` - DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` - Args map[string]string `json:"args,omitempty"` - TargetPlatform string `json:"targetPlatform,omitempty"` - //Name Tag DockerfilePath RepoUrl -} - -type BuildPackConfig struct { - BuilderId string `json:"builderId"` - Language string `json:"language"` - LanguageVersion string `json:"languageVersion"` - BuildPacks []string `json:"buildPacks"` - Args map[string]string `json:"args"` -} +//type CiBuildConfig struct { +// GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` +// CiBuildType string `json:"ciBuildType"` +// DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` +// BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +//} +// +//type DockerBuildConfig struct { +// GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` +// DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` +// Args map[string]string `json:"args,omitempty"` +// TargetPlatform string `json:"targetPlatform,omitempty"` +// //Name Tag DockerfilePath RepoUrl +//} +// +//type BuildPackConfig struct { +// BuilderId string `json:"builderId"` +// Language string `json:"language"` +// LanguageVersion string `json:"languageVersion"` +// BuildPacks []string `json:"buildPacks"` +// Args map[string]string `json:"args"` +//} type PipelineCreateResponse struct { AppName string `json:"appName,omitempty"` diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index d6d7ee1582..0eac6f7e04 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -24,3 +24,23 @@ func NewCiBuildConfigImpl(logger *zap.SugaredLogger, ciBuildConfigRepository *pi CiBuildConfigRepository: ciBuildConfigRepository, } } + +func (impl *CiBuildConfigImpl) Get(ciBuildConfigId int) (*bean.CiBuildConfig, error) { + //TODO implement me + panic("implement me") +} + +func (impl *CiBuildConfigImpl) Save(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) { + //TODO implement me + panic("implement me") +} + +func (impl *CiBuildConfigImpl) Update(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) { + //TODO implement me + panic("implement me") +} + +func (impl *CiBuildConfigImpl) Delete(ciBuildConfigId int) error { + //TODO implement me + panic("implement me") +} diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 7e71ba5335..6c9dc73ce6 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -148,6 +148,7 @@ type PipelineBuilderImpl struct { deploymentGroupRepository repository.DeploymentGroupRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + ciBuildConfigService CiBuildConfigService } func NewPipelineBuilderImpl(logger *zap.SugaredLogger, @@ -185,7 +186,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, deploymentGroupRepository repository.DeploymentGroupRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *PipelineBuilderImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciBuildConfigService CiBuildConfigService) *PipelineBuilderImpl { return &PipelineBuilderImpl{ logger: logger, dbPipelineOrchestrator: dbPipelineOrchestrator, @@ -227,6 +228,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, deploymentGroupRepository: deploymentGroupRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciBuildConfigService: ciBuildConfigService, } } @@ -721,13 +723,14 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq return nil, err } + buildConfig := createRequest.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ - DockerRegistryId: createRequest.DockerRegistry, - DockerRepository: createRequest.DockerRepository, - GitMaterialId: createRequest.CiBuildConfig.GitMaterialId, - DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, - Args: string(argByte), - TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, + DockerRegistryId: createRequest.DockerRegistry, + DockerRepository: createRequest.DockerRepository, + GitMaterialId: buildConfig.GitMaterialId, + //DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, + //Args: string(argByte), + //TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, Active: true, TemplateName: createRequest.CiTemplateName, Version: createRequest.Version, @@ -743,6 +746,11 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq //TODO delete template from gocd otherwise dangling+ no create in future return nil, err } + _, err = impl.ciBuildConfigService.Save(buildConfig) + if err != nil { + impl.logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) + return nil, err + } //-- template config end createRequest.Id = ciTemplate.Id createRequest.CiTemplateName = ciTemplate.TemplateName From 160943597fda1f7e70bbd1b2135ffccafc30b504 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Sun, 2 Oct 2022 20:07:55 +0530 Subject: [PATCH 03/50] migrated to ci template service & ciBuildConfig created --- Wire.go | 9 + api/appbean/AppDetail.go | 53 +++--- api/restHandler/CoreAppRestHandler.go | 45 ++--- .../pipelineConfig/CiBuildConfigRepository.go | 40 ++++- .../CiTemplateOverrideRepository.go | 2 + pkg/appClone/AppCloneService.go | 76 +++++---- pkg/bean/app.go | 40 ++--- pkg/pipeline/CiBuildConfigService.go | 58 ++++--- pkg/pipeline/CiService.go | 12 +- pkg/pipeline/CiTemplateService.go | 161 ++++++++++++++++++ pkg/pipeline/DbPipelineOrchestrator.go | 45 +++-- pkg/pipeline/PipelineBuilder.go | 160 +++++++++-------- pkg/pipeline/bean/CiBuildConfig.go | 69 +++++++- pkg/pipeline/bean/CiTemplateBean.go | 9 + wire_gen.go | 16 +- 15 files changed, 563 insertions(+), 232 deletions(-) create mode 100644 pkg/pipeline/CiTemplateService.go create mode 100644 pkg/pipeline/bean/CiTemplateBean.go diff --git a/Wire.go b/Wire.go index 7e3445e68f..5482cdeb56 100644 --- a/Wire.go +++ b/Wire.go @@ -742,6 +742,15 @@ func InitializeApp() (*App, error) { pipelineConfig.NewCiTemplateOverrideRepositoryImpl, wire.Bind(new(pipelineConfig.CiTemplateOverrideRepository), new(*pipelineConfig.CiTemplateOverrideRepositoryImpl)), + + pipelineConfig.NewCiBuildConfigRepositoryImpl, + wire.Bind(new(pipelineConfig.CiBuildConfigRepository), new(*pipelineConfig.CiBuildConfigRepositoryImpl)), + + pipeline.NewCiBuildConfigServiceImpl, + wire.Bind(new(pipeline.CiBuildConfigService), new(*pipeline.CiBuildConfigServiceImpl)), + + pipeline.NewCiTemplateServiceImpl, + wire.Bind(new(pipeline.CiTemplateService), new(*pipeline.CiTemplateServiceImpl)), ) return &App{}, nil } diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index 2b5fa32712..afff943523 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -35,32 +35,33 @@ type GitMaterial struct { } type DockerConfig struct { - DockerRegistry string `json:"dockerRegistry" validate:"required"` - DockerRepository string `json:"dockerRepository" validate:"required"` - CiBuildConfig *CiBuildConfig `json:"ciBuildConfig" validate:"required"` -} - -type CiBuildConfig struct { - GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` - CiBuildType string `json:"ciBuildType"` - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` - BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` -} - -type BuildPackConfig struct { - BuilderId string `json:"builderId"` - Language string `json:"language"` - LanguageVersion string `json:"languageVersion"` - BuildPacks []string `json:"buildPacks"` - Args map[string]string `json:"args"` -} - -type DockerBuildConfig struct { - GitCheckoutPath string `json:"gitCheckoutPath,omitempty" validate:"required"` - DockerfileRelativePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` - Args map[string]string `json:"args,omitempty"` - TargetPlatform string `json:"targetPlatform"` -} + DockerRegistry string `json:"dockerRegistry" validate:"required"` + DockerRepository string `json:"dockerRepository" validate:"required"` + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig" validate:"required"` + CheckoutPath string `json:"checkoutPath"` +} + +//type CiBuildConfig struct { +// GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` +// CiBuildType string `json:"ciBuildType"` +// DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` +// BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +//} +// +//type BuildPackConfig struct { +// BuilderId string `json:"builderId"` +// Language string `json:"language"` +// LanguageVersion string `json:"languageVersion"` +// BuildPacks []string `json:"buildPacks"` +// Args map[string]string `json:"args"` +//} +// +//type DockerBuildConfig struct { +// GitCheckoutPath string `json:"gitCheckoutPath,omitempty" validate:"required"` +// DockerfileRelativePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` +// Args map[string]string `json:"args,omitempty"` +// TargetPlatform string `json:"targetPlatform"` +//} type DeploymentTemplate struct { ChartRefId int `json:"chartRefId,notnull" validate:"required"` diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index ab5af4a885..b3eedafe6f 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -496,15 +496,14 @@ func (handler CoreAppRestHandlerImpl) buildDockerConfig(appId int) (*appBean.Doc dockerConfig := &appBean.DockerConfig{ DockerRegistry: ciConfig.DockerRegistry, DockerRepository: ciConfig.DockerRepository, - CiBuildConfig: &appBean.CiBuildConfig{ - //BuildPackConfig: ciConfig.CiBuildConfig.BuildPackConfig, - }, - BuildConfig: &appBean.DockerBuildConfig{ - Args: ciConfig.DockerBuildConfig.Args, - DockerfileRelativePath: ciConfig.DockerBuildConfig.DockerfilePath, - TargetPlatform: ciConfig.DockerBuildConfig.TargetPlatform, - GitCheckoutPath: gitMaterial.CheckoutPath, - }, + CiBuildConfig: ciConfig.CiBuildConfig, + CheckoutPath: gitMaterial.CheckoutPath, + //BuildConfig: &appBean.DockerBuildConfig{ + // Args: ciConfig.DockerBuildConfig.Args, + // DockerfileRelativePath: ciConfig.DockerBuildConfig.DockerfilePath, + // TargetPlatform: ciConfig.DockerBuildConfig.TargetPlatform, + // GitCheckoutPath: gitMaterial.CheckoutPath, + //}, } return dockerConfig, nil, http.StatusOK @@ -1256,24 +1255,26 @@ func (handler CoreAppRestHandlerImpl) createDockerConfig(appId int, dockerConfig } //finding gitMaterial by appId and checkoutPath - gitMaterial, err := handler.materialRepository.FindByAppIdAndCheckoutPath(appId, dockerConfig.BuildConfig.GitCheckoutPath) + gitMaterial, err := handler.materialRepository.FindByAppIdAndCheckoutPath(appId, dockerConfig.CheckoutPath) if err != nil { handler.logger.Errorw("service err, FindByAppIdAndCheckoutPath in CreateDockerConfig", "err", err, "appId", appId) return err, http.StatusInternalServerError } - dockerBuildArgs := make(map[string]string) - if dockerConfig.BuildConfig.Args != nil { - dockerBuildArgs = dockerConfig.BuildConfig.Args - } - - dockerBuildConfigRequest := &bean.DockerBuildConfig{ - GitMaterialId: gitMaterial.Id, - DockerfilePath: dockerConfig.BuildConfig.DockerfileRelativePath, - Args: dockerBuildArgs, - TargetPlatform: dockerConfig.BuildConfig.TargetPlatform, - } - createDockerConfigRequest.DockerBuildConfig = dockerBuildConfigRequest + //dockerBuildArgs := make(map[string]string) + //if dockerConfig.BuildConfig.Args != nil { + // dockerBuildArgs = dockerConfig.BuildConfig.Args + //} + + //dockerBuildConfigRequest := &bean.DockerBuildConfig{ + // GitMaterialId: gitMaterial.Id, + // DockerfilePath: dockerConfig.BuildConfig.DockerfileRelativePath, + // Args: dockerBuildArgs, + // TargetPlatform: dockerConfig.BuildConfig.TargetPlatform, + //} + ciBuildConfig := dockerConfig.CiBuildConfig + ciBuildConfig.GitMaterialId = gitMaterial.Id + createDockerConfigRequest.CiBuildConfig = ciBuildConfig _, err = handler.pipelineBuilder.CreateCiPipeline(createDockerConfigRequest) if err != nil { diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go index ce5153fd2f..da7c5f3858 100644 --- a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go @@ -7,16 +7,19 @@ import ( ) type CiBuildConfig struct { - tableName struct{} `sql:"ci_build_config" pg:",discard_unknown_columns"` - Id int `sql:"id"` - CiTemplateId int `sql:"ci_template_id"` - CiPipelineId int `sql:"ci_pipeline_id"` - Type string `sql:"type"` - BuildMetadata string `sql:"build_metadata"` + tableName struct{} `sql:"ci_build_config" pg:",discard_unknown_columns"` + Id int `sql:"id"` + Type string `sql:"type"` + CiTemplateId int `sql:"ci_template_id"` + CiTemplateOverrideId int `sql:"ci_template_override_id"` + BuildMetadata string `sql:"build_metadata"` sql.AuditLog } type CiBuildConfigRepository interface { + Save(ciBuildConfig *CiBuildConfig) error + Update(ciBuildConfig *CiBuildConfig) error + Delete(ciBuildConfigId int) error } type CiBuildConfigRepositoryImpl struct { @@ -30,3 +33,28 @@ func NewCiBuildConfigRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogg logger: logger, } } + +func (impl CiBuildConfigRepositoryImpl) Save(ciBuildConfig *CiBuildConfig) error { + + err := impl.dbConnection.Insert(ciBuildConfig) + if err != nil { + impl.logger.Errorw("error occurred while saving ciBuildConfig", "ciBuildConfig", ciBuildConfig, "err", err) + } + return err +} + +func (impl CiBuildConfigRepositoryImpl) Update(ciBuildConfig *CiBuildConfig) error { + err := impl.dbConnection.Update(ciBuildConfig) + if err != nil { + impl.logger.Errorw("error occurred while updating ciBuildConfig", "err", err) + } + return err +} + +func (impl CiBuildConfigRepositoryImpl) Delete(ciBuildConfigId int) error { + err := impl.dbConnection.Delete(ciBuildConfigId) + if err != nil { + impl.logger.Errorw("error occurred while deleting ciBuildConfig", "ciBuildConfigId", ciBuildConfigId, "err", err) + } + return err +} diff --git a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go index 9358248cf8..44772751ae 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go @@ -16,9 +16,11 @@ type CiTemplateOverride struct { DockerfilePath string `sql:"dockerfile_path"` GitMaterialId int `sql:"git_material_id"` Active bool `sql:"active,notnull"` + CiBuildConfigId int `sql:"ci_build_config_id"` sql.AuditLog GitMaterial *GitMaterial DockerRegistry *repository.DockerArtifactStore + CiBuildConfig *CiBuildConfig } type CiTemplateOverrideRepository interface { diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 3516d19cfe..dbdbbd0d9c 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -37,16 +37,17 @@ type AppCloneService interface { CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { - logger *zap.SugaredLogger - pipelineBuilder pipeline.PipelineBuilder - materialRepository pipelineConfig.MaterialRepository - chartService chart.ChartService - configMapService pipeline.ConfigMapService - appWorkflowService appWorkflow.AppWorkflowService - appListingService app.AppListingService - propertiesConfigService pipeline.PropertiesConfigService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository - pipelineStageService pipeline.PipelineStageService + logger *zap.SugaredLogger + pipelineBuilder pipeline.PipelineBuilder + materialRepository pipelineConfig.MaterialRepository + chartService chart.ChartService + configMapService pipeline.ConfigMapService + appWorkflowService appWorkflow.AppWorkflowService + appListingService app.AppListingService + propertiesConfigService pipeline.PropertiesConfigService + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + pipelineStageService pipeline.PipelineStageService + ciTemplateService pipeline.CiTemplateService } func NewAppCloneServiceImpl(logger *zap.SugaredLogger, @@ -58,18 +59,19 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, appListingService app.AppListingService, propertiesConfigService pipeline.PropertiesConfigService, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, - pipelineStageService pipeline.PipelineStageService) *AppCloneServiceImpl { + pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService) *AppCloneServiceImpl { return &AppCloneServiceImpl{ - logger: logger, - pipelineBuilder: pipelineBuilder, - materialRepository: materialRepository, - chartService: chartService, - configMapService: configMapService, - appWorkflowService: appWorkflowService, - appListingService: appListingService, - propertiesConfigService: propertiesConfigService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, - pipelineStageService: pipelineStageService, + logger: logger, + pipelineBuilder: pipelineBuilder, + materialRepository: materialRepository, + chartService: chartService, + configMapService: configMapService, + appWorkflowService: appWorkflowService, + appListingService: appListingService, + propertiesConfigService: propertiesConfigService, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + pipelineStageService: pipelineStageService, + ciTemplateService: ciTemplateService, } } @@ -244,9 +246,9 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId if len(gitMaterials) == 1 { dockerfileGitMaterial = gitMaterials[0].Id } else { - refGitmaterial, err := impl.materialRepository.FindById(refCiConf.DockerBuildConfig.GitMaterialId) + refGitmaterial, err := impl.materialRepository.FindById(refCiConf.CiBuildConfig.GitMaterialId) if err != nil { - impl.logger.Errorw("error in fetching ref git material", "id", refCiConf.DockerBuildConfig.GitMaterialId, "err", err) + impl.logger.Errorw("error in fetching ref git material", "id", refCiConf.CiBuildConfig.GitMaterialId, "err", err) return nil, err } //first repo with same checkout path @@ -278,12 +280,13 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId AppId: newAppId, DockerRegistry: refCiConf.DockerRegistry, DockerRepository: refCiConf.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: dockerfileGitMaterial, - DockerfilePath: refCiConf.DockerBuildConfig.DockerfilePath, - Args: refCiConf.DockerBuildConfig.Args, - TargetPlatform: refCiConf.DockerBuildConfig.TargetPlatform, - }, + CiBuildConfig: refCiConf.CiBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{ + // GitMaterialId: dockerfileGitMaterial, + // DockerfilePath: refCiConf.DockerBuildConfig.DockerfilePath, + // Args: refCiConf.DockerBuildConfig.Args, + // TargetPlatform: refCiConf.DockerBuildConfig.TargetPlatform, + //}, DockerRegistryUrl: refCiConf.DockerRegistry, CiTemplateName: refCiConf.CiTemplateName, UserId: userId, @@ -746,24 +749,27 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( } if !refCiPipeline.IsExternal && refCiPipeline.IsDockerConfigOverridden { //get template override - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(refCiPipeline.Id) + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(refCiPipeline.Id) if err != nil { - impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", refCiPipeline.Id) return nil, err } + templateOverride := templateOverrideBean.CiTemplateOverride + ciBuildConfig := templateOverrideBean.CiBuildConfig //getting new git material for this app gitMaterial, err := impl.materialRepository.FindByAppIdAndCheckoutPath(req.appId, templateOverride.GitMaterial.CheckoutPath) if err != nil { impl.logger.Errorw("error in getting git material by appId and checkoutPath", "err", err, "appid", req.refAppId, "checkoutPath", templateOverride.GitMaterial.CheckoutPath) return nil, err } + ciBuildConfig.GitMaterialId = gitMaterial.Id ciPatchReq.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - DockerfilePath: templateOverride.DockerfilePath, - GitMaterialId: gitMaterial.Id, - }, + CiBuildConfig: ciBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{ + // DockerfilePath: templateOverride.DockerfilePath, + // GitMaterialId: gitMaterial.Id, + //}, } } diff --git a/pkg/bean/app.go b/pkg/bean/app.go index abe6341f53..47c3408d98 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -110,9 +110,9 @@ type CiPipeline struct { } type DockerConfigOverride struct { - DockerRegistry string `json:"dockerRegistry,omitempty"` - DockerRepository string `json:"dockerRepository,omitempty"` - CiBuildConfig *bean.CiBuildConfig `json:"ciBuildConfig,omitEmpty"` + DockerRegistry string `json:"dockerRegistry,omitempty"` + DockerRepository string `json:"dockerRepository,omitempty"` + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig,omitEmpty"` //DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` } @@ -250,22 +250,22 @@ type Material struct { } type CiConfigRequest struct { - Id int `json:"id,omitempty" validate:"number"` //ciTemplateId - AppId int `json:"appId,omitempty" validate:"required,number"` - DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry - DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr - CiBuildConfig *bean.CiBuildConfig `json:"ciBuildConfig"` - CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial - AppName string `json:"appName,omitempty"` - Version string `json:"version,omitempty"` //gocd etag used for edit purpose - DockerRegistryUrl string `json:"-"` - CiTemplateName string `json:"-"` - UserId int32 `json:"-"` - Materials []Material `json:"materials"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Id int `json:"id,omitempty" validate:"number"` //ciTemplateId + AppId int `json:"appId,omitempty" validate:"required,number"` + DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry + DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig"` + CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial + AppName string `json:"appName,omitempty"` + Version string `json:"version,omitempty"` //gocd etag used for edit purpose + DockerRegistryUrl string `json:"-"` + CiTemplateName string `json:"-"` + UserId int32 `json:"-"` + Materials []Material `json:"materials"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type TestExecutorImageProperties struct { @@ -274,7 +274,7 @@ type TestExecutorImageProperties struct { ReportDir string `json:"reportDir,omitempty"` } -//type CiBuildConfig struct { +//type CiBuildConfigBean struct { // GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` // CiBuildType string `json:"ciBuildType"` // DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index 0eac6f7e04..3d53c3a7a4 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -1,46 +1,62 @@ package pipeline import ( + "errors" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" ) type CiBuildConfigService interface { - Get(ciBuildConfigId int) (*bean.CiBuildConfig, error) - Save(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) - Update(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) + Save(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) error + Update(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) Delete(ciBuildConfigId int) error } -type CiBuildConfigImpl struct { +type CiBuildConfigServiceImpl struct { Logger *zap.SugaredLogger - CiBuildConfigRepository *pipelineConfig.CiBuildConfigRepository + CiBuildConfigRepository pipelineConfig.CiBuildConfigRepository } -func NewCiBuildConfigImpl(logger *zap.SugaredLogger, ciBuildConfigRepository *pipelineConfig.CiBuildConfigRepository) *CiBuildConfigImpl { - return &CiBuildConfigImpl{ +func NewCiBuildConfigServiceImpl(logger *zap.SugaredLogger, ciBuildConfigRepository pipelineConfig.CiBuildConfigRepository) *CiBuildConfigServiceImpl { + return &CiBuildConfigServiceImpl{ Logger: logger, CiBuildConfigRepository: ciBuildConfigRepository, } } -func (impl *CiBuildConfigImpl) Get(ciBuildConfigId int) (*bean.CiBuildConfig, error) { - //TODO implement me - panic("implement me") -} - -func (impl *CiBuildConfigImpl) Save(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) { - //TODO implement me - panic("implement me") +func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean) error { + ciBuildConfig, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, + "overrideTemplateId", overrideTemplateId, "ciBuildConfigBean", ciBuildConfigBean, "err", err) + return errors.New("error while saving build config") + } + err = impl.CiBuildConfigRepository.Save(ciBuildConfig) + if err != nil { + return errors.New("error while saving build config") + } + return nil } -func (impl *CiBuildConfigImpl) Update(ciBuildConfig *bean.CiBuildConfig) (*bean.CiBuildConfig, error) { - //TODO implement me - panic("implement me") +func (impl *CiBuildConfigServiceImpl) Update(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) { + if ciBuildConfig == nil || ciBuildConfig.Id == 0 { + impl.Logger.Warnw("not updating build config as object is empty", "ciBuildConfig", ciBuildConfig) + return nil, nil + } + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, + "overrideTemplateId", overrideTemplateId, "ciBuildConfig", ciBuildConfig, "err", err) + return nil, errors.New("error while saving build config") + } + err = impl.CiBuildConfigRepository.Update(ciBuildConfigEntity) + if err != nil { + return nil, errors.New("error while updating build config") + } + return ciBuildConfig, nil } -func (impl *CiBuildConfigImpl) Delete(ciBuildConfigId int) error { - //TODO implement me - panic("implement me") +func (impl *CiBuildConfigServiceImpl) Delete(ciBuildConfigId int) error { + return impl.CiBuildConfigRepository.Delete(ciBuildConfigId) } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index ba635a971f..e76163d14e 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -59,7 +59,8 @@ type CiServiceImpl struct { prePostCiScriptHistoryService history.PrePostCiScriptHistoryService pipelineStageService PipelineStageService userService user.UserService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + ciTemplateService CiTemplateService } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -69,7 +70,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService prePostCiScriptHistoryService history.PrePostCiScriptHistoryService, pipelineStageService PipelineStageService, userService user.UserService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *CiServiceImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciTemplateService CiTemplateService) *CiServiceImpl { return &CiServiceImpl{ Logger: Logger, workflowService: workflowService, @@ -83,7 +84,8 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService prePostCiScriptHistoryService: prePostCiScriptHistoryService, pipelineStageService: pipelineStageService, userService: userService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciTemplateService: ciTemplateService, } } @@ -390,11 +392,11 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var checkoutPath string dockerRegistry := &repository3.DockerArtifactStore{} if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(pipeline.Id) + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) if err != nil { - impl.Logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", pipeline.Id) return nil, err } + templateOverride := templateOverrideBean.CiTemplateOverride dockerfilePath = filepath.Join(templateOverride.GitMaterial.CheckoutPath, templateOverride.DockerfilePath) dockerRepository = templateOverride.DockerRepository dockerRegistry = templateOverride.DockerRegistry diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go new file mode 100644 index 0000000000..d842b32e45 --- /dev/null +++ b/pkg/pipeline/CiTemplateService.go @@ -0,0 +1,161 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type CiTemplateService interface { + Save(ciTemplateBean *bean.CiTemplateBean) error + FindByAppId(appId int) (ciTemplateBean *bean.CiTemplateBean, err error) + FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) + FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) + Update(ciTemplateBean *bean.CiTemplateBean) error + FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) + FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) +} +type CiTemplateServiceImpl struct { + Logger *zap.SugaredLogger + CiBuildConfigService CiBuildConfigService + CiTemplateRepository pipelineConfig.CiTemplateRepository + CiTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository +} + +func NewCiTemplateServiceImpl(logger *zap.SugaredLogger, ciBuildConfigService CiBuildConfigService, + ciTemplateRepository pipelineConfig.CiTemplateRepository, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *CiTemplateServiceImpl { + return &CiTemplateServiceImpl{ + Logger: logger, + CiBuildConfigService: ciBuildConfigService, + CiTemplateRepository: ciTemplateRepository, + CiTemplateOverrideRepository: ciTemplateOverrideRepository, + } +} + +func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) error { + ciTemplate := ciTemplateBean.CiTemplate + ciTemplateOverride := ciTemplateBean.CiTemplateOverride + ciTemplateId := 0 + ciTemplateOverrideId := 0 + if ciTemplateOverride == nil { + err := impl.CiTemplateRepository.Save(ciTemplate) + if err != nil { + impl.Logger.Errorw("error in saving ci template in db ", "template", ciTemplate, "err", err) + //TODO delete template from gocd otherwise dangling+ no create in future + return err + } + ciTemplateId = ciTemplate.Id + } else { + _, err := impl.CiTemplateOverrideRepository.Save(ciTemplateOverride) + if err != nil { + impl.Logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", ciTemplateOverride) + return err + } + ciTemplateOverrideId = ciTemplateOverride.Id + } + buildConfig := ciTemplateBean.CiBuildConfig + err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) + } + return err +} + +func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.CiTemplateBean, err error) { + ciTemplate, err := impl.CiTemplateRepository.FindByAppId(appId) + if err != nil { + return nil, err + } + //dockerArgs := map[string]string{} + //if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { + // impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) + // return nil, err + //} + ciBuildConfig := ciTemplate.CiBuildConfig + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", + ciBuildConfig, "error", err) + } + return &bean.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: ciBuildConfigBean, + }, err +} + +func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) { + templateOverrides, err := impl.CiTemplateOverrideRepository.FindByAppId(appId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting ciTemplateOverrides by appId", "err", err, "appId", appId) + return nil, err + } + var templateBeanOverrides []*bean.CiTemplateBean + for _, templateOverride := range templateOverrides { + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", + templateOverride.CiBuildConfig, "error", err) + return nil, err + } + overrideBean := &bean.CiTemplateBean{ + CiTemplateOverride: templateOverride, + CiBuildConfig: ciBuildConfigBean, + } + templateBeanOverrides = append(templateBeanOverrides, overrideBean) + } + return templateBeanOverrides, nil +} + +func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) { + templateOverride, err := impl.CiTemplateOverrideRepository.FindByCiPipelineId(ciPipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) + return nil, err + } + ciBuildConfig := templateOverride.CiBuildConfig + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", + ciBuildConfig, "error", err) + } + return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err +} + +func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) error { + ciTemplate := ciTemplateBean.CiTemplate + ciTemplateOverride := ciTemplateBean.CiTemplateOverride + ciTemplateId := 0 + ciTemplateOverrideId := 0 + if ciTemplateOverride == nil { + err := impl.CiTemplateRepository.Update(ciTemplate) + if err != nil { + impl.Logger.Errorw("error in updating ci template in db", "template", ciTemplate, "err", err) + return err + } + ciTemplateId = ciTemplate.Id + } else { + _, err := impl.CiTemplateOverrideRepository.Update(ciTemplateOverride) + if err != nil { + impl.Logger.Errorw("error in updating template override", "err", err, "templateOverrideConfig", ciTemplateOverride) + return err + } + ciTemplateOverrideId = ciTemplateOverride.Id + } + ciBuildConfig := ciTemplateBean.CiBuildConfig + _, err := impl.CiBuildConfigService.Update(ciTemplateId, ciTemplateOverrideId, ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error in updating ci build config in db", "ciBuildConfig", ciBuildConfig, "err", err) + } + return err +} + +func (impl CiTemplateServiceImpl) FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) { + //TODO implement me + panic("implement me") +} + +func (impl CiTemplateServiceImpl) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { + //TODO implement me + panic("implement me") +} diff --git a/pkg/pipeline/DbPipelineOrchestrator.go b/pkg/pipeline/DbPipelineOrchestrator.go index aefd8aa995..be33d1f8f3 100644 --- a/pkg/pipeline/DbPipelineOrchestrator.go +++ b/pkg/pipeline/DbPipelineOrchestrator.go @@ -27,6 +27,7 @@ import ( "fmt" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" @@ -90,7 +91,8 @@ type DbPipelineOrchestratorImpl struct { prePostCdScriptHistoryService history3.PrePostCdScriptHistoryService prePostCiScriptHistoryService history3.PrePostCiScriptHistoryService pipelineStageService PipelineStageService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + ciTemplateService CiTemplateService } func NewDbPipelineOrchestrator( @@ -110,7 +112,7 @@ func NewDbPipelineOrchestrator( prePostCdScriptHistoryService history3.PrePostCdScriptHistoryService, prePostCiScriptHistoryService history3.PrePostCiScriptHistoryService, pipelineStageService PipelineStageService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *DbPipelineOrchestratorImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciTemplateService CiTemplateService) *DbPipelineOrchestratorImpl { return &DbPipelineOrchestratorImpl{ appRepository: pipelineGroupRepository, logger: logger, @@ -129,7 +131,8 @@ func NewDbPipelineOrchestrator( prePostCdScriptHistoryService: prePostCdScriptHistoryService, prePostCiScriptHistoryService: prePostCiScriptHistoryService, pipelineStageService: pipelineStageService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciTemplateService: ciTemplateService, } } @@ -290,7 +293,7 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci } if !createRequest.IsExternal && createRequest.IsDockerConfigOverridden { //get override - savedTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(createRequest.Id) + savedTemplateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(createRequest.Id) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting templateOverride by ciPipelineId", "err", err, "ciPipelineId", createRequest.Id) return nil, err @@ -299,9 +302,9 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci CiPipelineId: createRequest.Id, DockerRegistryId: createRequest.DockerConfigOverride.DockerRegistry, DockerRepository: createRequest.DockerConfigOverride.DockerRepository, - DockerfilePath: createRequest.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - GitMaterialId: createRequest.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + //DockerfilePath: createRequest.DockerConfigOverride.DockerBuildConfig.DockerfilePath, + //GitMaterialId: createRequest.DockerConfigOverride.DockerBuildConfig.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), CreatedBy: userId, @@ -309,19 +312,26 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci UpdatedBy: userId, }, } + savedTemplateOverride := savedTemplateOverrideBean.CiTemplateOverride if savedTemplateOverride != nil && savedTemplateOverride.Id > 0 { templateOverrideReq.Id = savedTemplateOverride.Id templateOverrideReq.CreatedOn = savedTemplateOverride.CreatedOn templateOverrideReq.CreatedBy = savedTemplateOverride.CreatedBy - _, err = impl.ciTemplateOverrideRepository.Update(templateOverrideReq) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverrideReq, + CiBuildConfig: createRequest.DockerConfigOverride.CiBuildConfig, + } + err = impl.ciTemplateService.Update(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in updating template override", "err", err, "templateOverrideConfig", templateOverrideReq) return nil, err } } else { - _, err = impl.ciTemplateOverrideRepository.Save(templateOverrideReq) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverrideReq, + CiBuildConfig: createRequest.DockerConfigOverride.CiBuildConfig, + } + err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", templateOverrideReq) return nil, err } } @@ -527,9 +537,9 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig CiPipelineId: ciPipeline.Id, DockerRegistryId: ciPipeline.DockerConfigOverride.DockerRegistry, DockerRepository: ciPipeline.DockerConfigOverride.DockerRepository, - DockerfilePath: ciPipeline.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - GitMaterialId: ciPipeline.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + //DockerfilePath: ciPipeline.DockerConfigOverride.DockerBuildConfig.DockerfilePath, + //GitMaterialId: ciPipeline.DockerConfigOverride.DockerBuildConfig.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedBy: createRequest.UserId, CreatedOn: time.Now(), @@ -537,9 +547,12 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig UpdatedOn: time.Now(), }, } - _, err = impl.ciTemplateOverrideRepository.Save(templateOverride) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverride, + CiBuildConfig: ciPipeline.DockerConfigOverride.CiBuildConfig, + } + err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", templateOverride) return nil, err } } diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 6c9dc73ce6..1c52538303 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -108,14 +109,14 @@ type PipelineBuilder interface { } type PipelineBuilderImpl struct { - logger *zap.SugaredLogger - dbPipelineOrchestrator DbPipelineOrchestrator - dockerArtifactStoreRepository repository.DockerArtifactStoreRepository - materialRepo pipelineConfig.MaterialRepository - appRepo app2.AppRepository - pipelineRepository pipelineConfig.PipelineRepository - propertiesConfigService PropertiesConfigService - ciTemplateRepository pipelineConfig.CiTemplateRepository + logger *zap.SugaredLogger + dbPipelineOrchestrator DbPipelineOrchestrator + dockerArtifactStoreRepository repository.DockerArtifactStoreRepository + materialRepo pipelineConfig.MaterialRepository + appRepo app2.AppRepository + pipelineRepository pipelineConfig.PipelineRepository + propertiesConfigService PropertiesConfigService + //ciTemplateRepository pipelineConfig.CiTemplateRepository ciPipelineRepository pipelineConfig.CiPipelineRepository application application.ServiceClient chartRepository chartRepoRepository.ChartRepository @@ -147,8 +148,9 @@ type PipelineBuilderImpl struct { helmAppService client.HelmAppService deploymentGroupRepository repository.DeploymentGroupRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository - ciBuildConfigService CiBuildConfigService + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciBuildConfigService CiBuildConfigService + ciTemplateService CiTemplateService } func NewPipelineBuilderImpl(logger *zap.SugaredLogger, @@ -188,15 +190,15 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciBuildConfigService CiBuildConfigService) *PipelineBuilderImpl { return &PipelineBuilderImpl{ - logger: logger, - dbPipelineOrchestrator: dbPipelineOrchestrator, - dockerArtifactStoreRepository: dockerArtifactStoreRepository, - materialRepo: materialRepo, - appService: appService, - appRepo: pipelineGroupRepo, - pipelineRepository: pipelineRepository, - propertiesConfigService: propertiesConfigService, - ciTemplateRepository: ciTemplateRepository, + logger: logger, + dbPipelineOrchestrator: dbPipelineOrchestrator, + dockerArtifactStoreRepository: dockerArtifactStoreRepository, + materialRepo: materialRepo, + appService: appService, + appRepo: pipelineGroupRepo, + pipelineRepository: pipelineRepository, + propertiesConfigService: propertiesConfigService, + //ciTemplateRepository: ciTemplateRepository, ciPipelineRepository: ciPipelineRepository, application: application, chartRepository: chartRepository, @@ -227,8 +229,8 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, helmAppService: helmAppService, deploymentGroupRepository: deploymentGroupRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, - ciBuildConfigService: ciBuildConfigService, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + //ciBuildConfigService: ciBuildConfigService, } } @@ -280,13 +282,16 @@ func (impl PipelineBuilderImpl) DeleteMaterial(request *bean.UpdateMaterialDTO) if len(pipelines) > 0 { //pipelines are present, in this case we will check if this material is used in docker config //if it is used, then we won't delete - ciTemplate, err := impl.ciTemplateRepository.FindByAppId(request.AppId) + ciTemplateBean, err := impl.ciTemplateService.FindByAppId(request.AppId) if err != nil && err == errors.NotFoundf(err.Error()) { impl.logger.Errorw("err in getting docker registry", "appId", request.AppId, "err", err) return err } - if ciTemplate != nil && ciTemplate.GitMaterialId == request.Material.Id { - return fmt.Errorf("cannot delete git material, is being used in docker config") + if ciTemplateBean != nil { + ciTemplate := ciTemplateBean.CiTemplate + if ciTemplate != nil && ciTemplate.GitMaterialId == request.Material.Id { + return fmt.Errorf("cannot delete git material, is being used in docker config") + } } } existingMaterial, err := impl.materialRepo.FindById(request.Material.Id) @@ -359,11 +364,13 @@ func (impl PipelineBuilderImpl) getDefaultArtifactStore(id string) (store *repos } func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bean.CiConfigRequest, err error) { - template, err := impl.ciTemplateRepository.FindByAppId(appId) + //template, err := impl.ciTemplateRepository.FindByAppId(appId) + ciTemplateBean, err := impl.ciTemplateService.FindByAppId(appId) if err != nil && !errors.IsNotFound(err) { impl.logger.Errorw("error in fetching ci pipeline", "appId", appId, "err", err) return nil, err } + template := ciTemplateBean.CiTemplate if errors.IsNotFound(err) { impl.logger.Debugw("no ci pipeline exists", "appId", appId, "err", err) err = &util.ApiError{Code: "404", HttpStatusCode: 200, UserMessage: "no ci pipeline exists"} @@ -390,11 +397,11 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea materials = append(materials, m) } - dockerArgs := map[string]string{} - if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { - impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) - return nil, err - } + //dockerArgs := map[string]string{} + //if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { + // impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) + // return nil, err + //} regHost, err := template.DockerRegistry.GetRegistryLocation() if err != nil { impl.logger.Errorw("invalid reg url", "err", err) @@ -407,13 +414,15 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea DockerRepository: template.DockerRepository, DockerRegistry: template.DockerRegistry.Id, DockerRegistryUrl: regHost, - DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform}, - Version: template.Version, - CiTemplateName: template.TemplateName, - Materials: materials, + CiBuildConfig: ciTemplateBean.CiBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform}, + Version: template.Version, + CiTemplateName: template.TemplateName, + Materials: materials, } return ciConfig, err } + func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfigRequest, err error) { ciConfig, err = impl.getCiTemplateVariables(appId) if err != nil { @@ -439,14 +448,15 @@ func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfi } } //map of ciPipelineId and their templateOverrideConfig - templateOverrideMap := make(map[int]*pipelineConfig.CiTemplateOverride) - templateOverrides, err := impl.ciTemplateOverrideRepository.FindByAppId(appId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting ciTemplateOverrides by appId", "err", err, "appId", appId) + ciOverrideTemplateMap := make(map[int]*bean3.CiTemplateBean) + ciTemplateBeanOverrides, err := impl.ciTemplateService.FindTemplateOverrideByAppId(appId) + if err != nil { return nil, err } - for _, templateOverride := range templateOverrides { - templateOverrideMap[templateOverride.CiPipelineId] = templateOverride + + for _, templateBeanOverride := range ciTemplateBeanOverrides { + ciTemplateOverride := templateBeanOverride.CiTemplateOverride + ciOverrideTemplateMap[ciTemplateOverride.CiPipelineId] = templateBeanOverride } var ciPipelineResp []*bean.CiPipeline for _, pipeline := range pipelines { @@ -513,14 +523,12 @@ func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfi ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, } - if templateOverride, ok := templateOverrideMap[pipeline.Id]; ok { + if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { + templateOverride := ciTemplateBean.CiTemplateOverride ciPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: templateOverride.GitMaterialId, - DockerfilePath: templateOverride.DockerfilePath, - }, + CiBuildConfig: ciTemplateBean.CiBuildConfig, } } for _, material := range pipeline.CiPipelineMaterials { @@ -628,15 +636,15 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq originalCiConf.AfterDockerBuild = updateRequest.AfterDockerBuild originalCiConf.BeforeDockerBuild = updateRequest.BeforeDockerBuild - originalCiConf.DockerBuildConfig = updateRequest.DockerBuildConfig + //originalCiConf.CiBuildConfigBean = updateRequest.CiBuildConfigBean originalCiConf.DockerRegistry = updateRequest.DockerRegistry originalCiConf.DockerRepository = updateRequest.DockerRepository originalCiConf.DockerRegistryUrl = regHost - argByte, err := json.Marshal(originalCiConf.DockerBuildConfig.Args) - if err != nil { - return nil, err - } + //argByte, err := json.Marshal(originalCiConf.DockerBuildConfig.Args) + //if err != nil { + // return nil, err + //} afterByte, err := json.Marshal(originalCiConf.AfterDockerBuild) if err != nil { return nil, err @@ -646,10 +654,10 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq return nil, err } ciTemplate := &pipelineConfig.CiTemplate{ - DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, - GitMaterialId: originalCiConf.DockerBuildConfig.GitMaterialId, - Args: string(argByte), - TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, + //DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, + GitMaterialId: originalCiConf.CiBuildConfig.GitMaterialId, + //Args: string(argByte), + //TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, BeforeDockerBuild: string(beforeByte), AfterDockerBuild: string(afterByte), Version: originalCiConf.Version, @@ -659,11 +667,15 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq Active: true, } - err = impl.ciTemplateRepository.Update(ciTemplate) + ciTemplateBean := &bean3.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: updateRequest.CiBuildConfig, + } + err = impl.ciTemplateService.Update(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in updating ci template in db", "template", ciTemplate, "err", err) return nil, err } + originalCiConf.CiBuildConfig = updateRequest.CiBuildConfig return originalCiConf, nil } @@ -710,10 +722,10 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq //--ecr config end //-- template config start - argByte, err := json.Marshal(createRequest.DockerBuildConfig.Args) - if err != nil { - return nil, err - } + //argByte, err := json.Marshal(createRequest.DockerBuildConfig.Args) + //if err != nil { + // return nil, err + //} afterByte, err := json.Marshal(createRequest.AfterDockerBuild) if err != nil { return nil, err @@ -740,17 +752,15 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq AuditLog: sql.AuditLog{CreatedOn: time.Now(), UpdatedOn: time.Now(), CreatedBy: createRequest.UserId, UpdatedBy: createRequest.UserId}, } - err = impl.ciTemplateRepository.Save(ciTemplate) - if err != nil { - impl.logger.Errorw("error in saving ci template in db ", "template", ciTemplate, "err", err) - //TODO delete template from gocd otherwise dangling+ no create in future - return nil, err + ciTemplateBean := &bean3.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: createRequest.CiBuildConfig, } - _, err = impl.ciBuildConfigService.Save(buildConfig) + err = impl.ciTemplateService.Save(ciTemplateBean) if err != nil { - impl.logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) return nil, err } + //-- template config end createRequest.Id = ciTemplate.Id createRequest.CiTemplateName = ciTemplate.TemplateName @@ -2383,18 +2393,20 @@ func (impl PipelineBuilderImpl) GetCiPipelineById(pipelineId int) (ciPipeline *b IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, } if !ciPipeline.IsExternal && ciPipeline.IsDockerConfigOverridden { - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipeline.Id) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipeline.IsDockerConfigOverridden) + ciTemplateBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(ciPipeline.Id) + if err != nil { return nil, err } + templateOverride := ciTemplateBean.CiTemplateOverride + ciBuildConfig := ciTemplateBean.CiBuildConfig ciPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: templateOverride.GitMaterialId, - DockerfilePath: templateOverride.DockerfilePath, - }, + CiBuildConfig: ciBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{ + // GitMaterialId: templateOverride.GitMaterialId, + // DockerfilePath: templateOverride.DockerfilePath, + //}, } } for _, material := range pipeline.CiPipelineMaterials { diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 2227f33f63..41d7c230dc 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -1,5 +1,10 @@ package bean +import ( + "encoding/json" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" +) + type CiBuildType string const ( @@ -9,7 +14,7 @@ const ( BUILDPACK_BUILD_TYPE CiBuildType = "buildpack-build" ) -type CiBuildConfig struct { +type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` CiBuildType CiBuildType `json:"ciBuildType"` @@ -31,3 +36,65 @@ type BuildPackConfig struct { BuildPacks []string `json:"buildPacks"` Args map[string]string `json:"args"` } + +func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean) (*pipelineConfig.CiBuildConfig, error) { + buildMetadata := "" + ciBuildType := ciBuildConfigBean.CiBuildType + if ciBuildType == BUILDPACK_BUILD_TYPE { + buildPackConfigMetadataBytes, err := json.Marshal(ciBuildConfigBean.BuildPackConfig) + if err != nil { + return nil, err + } + buildMetadata = string(buildPackConfigMetadataBytes) + } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildMetadataBytes, err := json.Marshal(ciBuildConfigBean.DockerBuildConfig) + if err != nil { + return nil, err + } + buildMetadata = string(dockerBuildMetadataBytes) + } + ciBuildConfig := &pipelineConfig.CiBuildConfig{ + Type: string(ciBuildType), + CiTemplateId: templateId, + CiTemplateOverrideId: overrideTemplateId, + BuildMetadata: buildMetadata, + } + return ciBuildConfig, nil +} + +func ConvertDbBuildConfigToBean(dbConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { + var buildPackConfig *BuildPackConfig + var dockerBuildConfig *DockerBuildConfig + var err error + ciBuildType := CiBuildType(dbConfig.Type) + if ciBuildType == BUILDPACK_BUILD_TYPE { + buildPackConfig, err = convertMetadataToBuildPackConfig(dbConfig.BuildMetadata) + if err != nil { + return nil, err + } + } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbConfig.BuildMetadata) + if err != nil { + return nil, err + } + } + ciBuildConfigBean := &CiBuildConfigBean{ + Id: dbConfig.Id, + CiBuildType: ciBuildType, + BuildPackConfig: buildPackConfig, + DockerBuildConfig: dockerBuildConfig, + } + return ciBuildConfigBean, nil +} + +func convertMetadataToBuildPackConfig(buildConfMetadata string) (*BuildPackConfig, error) { + buildPackConfig := &BuildPackConfig{} + err := json.Unmarshal([]byte(buildConfMetadata), buildPackConfig) + return buildPackConfig, err +} + +func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuildConfig, error) { + dockerBuildConfig := &DockerBuildConfig{} + err := json.Unmarshal([]byte(dockerBuildMetadata), dockerBuildConfig) + return dockerBuildConfig, err +} diff --git a/pkg/pipeline/bean/CiTemplateBean.go b/pkg/pipeline/bean/CiTemplateBean.go new file mode 100644 index 0000000000..dc03675c49 --- /dev/null +++ b/pkg/pipeline/bean/CiTemplateBean.go @@ -0,0 +1,9 @@ +package bean + +import "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + +type CiTemplateBean struct { + CiTemplate *pipelineConfig.CiTemplate + CiTemplateOverride *pipelineConfig.CiTemplateOverride + CiBuildConfig *CiBuildConfigBean +} diff --git a/wire_gen.go b/wire_gen.go index 6c56c5031c..5dd9c0673a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire -//+build !wireinject +//go:build !wireinject +// +build !wireinject package main @@ -348,17 +349,20 @@ func InitializeApp() (*App, error) { globalPluginRepositoryImpl := repository8.NewGlobalPluginRepository(sugaredLogger, db) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) - dbPipelineOrchestratorImpl := pipeline.NewDbPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl) - propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, dbPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + dbPipelineOrchestratorImpl := pipeline.NewDbPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) + propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, dbPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciTemplateOverrideRepositoryImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciTemplateOverrideRepositoryImpl, ciBuildConfigServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) ciLogServiceImpl := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, ciConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciConfig, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, gitSensorClientImpl) @@ -370,7 +374,7 @@ func InitializeApp() (*App, error) { cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, cdConfig, userServiceImpl, cdWorkflowRepositoryImpl, cdWorkflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentFailureHandlerImpl) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, dbPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) - appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl) + appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) From cef3406c51614090441cff07e6fc9bb982dead21 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 3 Oct 2022 00:19:24 +0530 Subject: [PATCH 04/50] old data handling + workflow handling --- pkg/pipeline/CiService.go | 54 ++++++++++++++++++++---------- pkg/pipeline/CiTemplateService.go | 23 ++++++++++--- pkg/pipeline/WorkflowService.go | 5 ++- pkg/pipeline/bean/CiBuildConfig.go | 27 +++++++++++++++ 4 files changed, 83 insertions(+), 26 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index e76163d14e..9a92b15dfd 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -18,6 +18,7 @@ package pipeline import ( + "errors" "fmt" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -370,7 +371,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciWorkflowConfig.CiTimeout = impl.ciConfig.DefaultTimeout } - args := pipeline.CiTemplate.Args + ciTemplate := pipeline.CiTemplate + args := ciTemplate.Args ciLevelArgs := pipeline.DockerArgs if ciLevelArgs == "" { @@ -390,38 +392,53 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var dockerfilePath string var dockerRepository string var checkoutPath string + var ciBuildConfigBean *bean2.CiBuildConfigBean dockerRegistry := &repository3.DockerArtifactStore{} if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) if err != nil { return nil, err } + ciBuildConfigBean = templateOverrideBean.CiBuildConfig templateOverride := templateOverrideBean.CiTemplateOverride - dockerfilePath = filepath.Join(templateOverride.GitMaterial.CheckoutPath, templateOverride.DockerfilePath) + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = filepath.Join(checkoutPath, templateOverride.DockerfilePath) dockerRepository = templateOverride.DockerRepository dockerRegistry = templateOverride.DockerRegistry - checkoutPath = templateOverride.GitMaterial.CheckoutPath } else { - dockerfilePath = filepath.Join(pipeline.CiTemplate.GitMaterial.CheckoutPath, pipeline.CiTemplate.DockerfilePath) - dockerRegistry = pipeline.CiTemplate.DockerRegistry - dockerRepository = pipeline.CiTemplate.DockerRepository - checkoutPath = pipeline.CiTemplate.GitMaterial.CheckoutPath + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = filepath.Join(checkoutPath, ciTemplate.DockerfilePath) + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigEntity := ciTemplate.CiBuildConfig + ciBuildConfigBean, err = bean2.ConvertDbBuildConfigToBean(ciBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } } if checkoutPath == "" { checkoutPath = "./" } + mergedArgs := string(merged) + ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, mergedArgs, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "args", mergedArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } workflowRequest := &WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - DockerRegistryId: dockerRegistry.Id, - DockerRegistryType: string(dockerRegistry.RegistryType), - DockerImageTag: dockerImageTag, - DockerRegistryURL: dockerRegistry.RegistryURL, - DockerRepository: dockerRepository, - DockerBuildArgs: string(merged), - DockerBuildTargetPlatform: pipeline.CiTemplate.TargetPlatform, - DockerFileLocation: dockerfilePath, + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + DockerRegistryId: dockerRegistry.Id, + DockerRegistryType: string(dockerRegistry.RegistryType), + DockerImageTag: dockerImageTag, + DockerRegistryURL: dockerRegistry.RegistryURL, + DockerRepository: dockerRepository, + //DockerBuildArgs: string(merged), + //DockerBuildTargetPlatform: ciTemplate.TargetPlatform, + //DockerFileLocation: dockerfilePath, + CheckoutPath: checkoutPath, DockerUsername: dockerRegistry.Username, DockerPassword: dockerRegistry.Password, AwsRegion: dockerRegistry.AWSRegion, @@ -448,6 +465,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. RefPlugins: refPluginsData, AppName: pipeline.App.AppName, TriggerByAuthor: user.EmailId, + CiBuildConfig: ciBuildConfigBean, } if ciWorkflowConfig.LogsBucket == "" { diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index d842b32e45..157ad8d2f4 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -67,17 +67,18 @@ func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.C if err != nil { return nil, err } - //dockerArgs := map[string]string{} - //if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { - // impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) - // return nil, err - //} ciBuildConfig := ciTemplate.CiBuildConfig ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) if err != nil { impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", ciBuildConfig, "error", err) } + if ciBuildConfigBean == nil { + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, ciTemplate.TargetPlatform, nil) + if err != nil { + impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) + } + } return &bean.CiTemplateBean{ CiTemplate: ciTemplate, CiBuildConfig: ciBuildConfigBean, @@ -98,6 +99,12 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp templateOverride.CiBuildConfig, "error", err) return nil, err } + if ciBuildConfigBean == nil { + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", nil) + if err != nil { + impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) + } + } overrideBean := &bean.CiTemplateBean{ CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean, @@ -119,6 +126,12 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", ciBuildConfig, "error", err) } + if ciBuildConfigBean == nil { + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", nil) + if err != nil { + impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) + } + } return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err } diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 3825ca2472..44dc03a69c 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -71,10 +71,8 @@ type WorkflowRequest struct { DockerRegistryURL string `json:"dockerRegistryURL"` DockerConnection string `json:"dockerConnection"` DockerCert string `json:"dockerCert"` - DockerBuildArgs string `json:"dockerBuildArgs"` - DockerBuildTargetPlatform string `json:"dockerBuildTargetPlatform"` DockerRepository string `json:"dockerRepository"` - DockerFileLocation string `json:"dockerfileLocation"` + CheckoutPath string `json:"checkoutPath"` DockerUsername string `json:"dockerUsername"` DockerPassword string `json:"dockerPassword"` AwsRegion string `json:"awsRegion"` @@ -111,6 +109,7 @@ type WorkflowRequest struct { RefPlugins []*bean2.RefPluginObject `json:"refPlugins"` AppName string `json:"appName"` TriggerByAuthor string `json:"triggerByAuthor"` + CiBuildConfig *bean2.CiBuildConfigBean `json:"ciBuildConfig"` } const BLOB_STORAGE_AZURE = "AZURE" diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 41d7c230dc..2100881b3a 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -66,6 +66,9 @@ func ConvertDbBuildConfigToBean(dbConfig *pipelineConfig.CiBuildConfig) (*CiBuil var buildPackConfig *BuildPackConfig var dockerBuildConfig *DockerBuildConfig var err error + if dbConfig == nil { + return nil, nil + } ciBuildType := CiBuildType(dbConfig.Type) if ciBuildType == BUILDPACK_BUILD_TYPE { buildPackConfig, err = convertMetadataToBuildPackConfig(dbConfig.BuildMetadata) @@ -98,3 +101,27 @@ func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuil err := json.Unmarshal([]byte(dockerBuildMetadata), dockerBuildConfig) return dockerBuildConfig, err } + +func OverrideCiBuildConfig(dockerfilePath string, args string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { + dockerArgs := map[string]string{} + if args != "" { + if err := json.Unmarshal([]byte(args), &dockerArgs); err != nil { + return nil, err + } + } + if ciBuildConfigBean == nil { + ciBuildConfigBean = &CiBuildConfigBean{ + CiBuildType: SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &DockerBuildConfig{ + DockerfilePath: dockerfilePath, + Args: dockerArgs, + TargetPlatform: targetPlatform, + }, + } + } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerBuildConfig.DockerfilePath = dockerfilePath + dockerBuildConfig.Args = dockerArgs + } + return ciBuildConfigBean, nil +} From 2d2261f8363bec2d2d554c3262790b23a96e7321 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 3 Oct 2022 01:00:57 +0530 Subject: [PATCH 05/50] gitMaterialId handling --- pkg/pipeline/CiTemplateService.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 157ad8d2f4..e9de175ad4 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -79,6 +79,7 @@ func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.C impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } } + ciBuildConfigBean.GitMaterialId = ciTemplate.GitMaterialId return &bean.CiTemplateBean{ CiTemplate: ciTemplate, CiBuildConfig: ciBuildConfigBean, @@ -105,6 +106,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } } + ciBuildConfigBean.GitMaterialId = templateOverride.GitMaterialId overrideBean := &bean.CiTemplateBean{ CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean, @@ -132,6 +134,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } } + ciBuildConfigBean.GitMaterialId = templateOverride.GitMaterialId return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err } From 47f9142227c8a950bf37d37483ae229323f40467 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 3 Oct 2022 13:30:49 +0530 Subject: [PATCH 06/50] reqd. check removed --- pkg/pipeline/bean/CiBuildConfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 2100881b3a..7d9998fa57 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -23,7 +23,7 @@ type CiBuildConfigBean struct { } type DockerBuildConfig struct { - DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` + DockerfilePath string `json:"dockerfileRelativePath,omitempty"` DockerfileContent string `json:"DockerfileContent"` Args map[string]string `json:"args,omitempty"` TargetPlatform string `json:"targetPlatform,omitempty"` From 6386f6fbb362c5f06cc05d5cb1a75b0b800e0114 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 3 Oct 2022 15:01:49 +0530 Subject: [PATCH 07/50] ci template service incorporated --- pkg/pipeline/PipelineBuilder.go | 3 ++- scripts/sql/84_ci_build_config.up.sql | 26 ++++++++++++++++++++++++++ wire_gen.go | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 scripts/sql/84_ci_build_config.up.sql diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 1c52538303..15f74b5506 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -188,7 +188,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, deploymentGroupRepository repository.DeploymentGroupRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciBuildConfigService CiBuildConfigService) *PipelineBuilderImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciTemplateService CiTemplateService) *PipelineBuilderImpl { return &PipelineBuilderImpl{ logger: logger, dbPipelineOrchestrator: dbPipelineOrchestrator, @@ -229,6 +229,7 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, helmAppService: helmAppService, deploymentGroupRepository: deploymentGroupRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, + ciTemplateService: ciTemplateService, //ciTemplateOverrideRepository: ciTemplateOverrideRepository, //ciBuildConfigService: ciBuildConfigService, } diff --git a/scripts/sql/84_ci_build_config.up.sql b/scripts/sql/84_ci_build_config.up.sql new file mode 100644 index 0000000000..cb7525ad20 --- /dev/null +++ b/scripts/sql/84_ci_build_config.up.sql @@ -0,0 +1,26 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_ci_build_config; + +-- Table Definition +CREATE TABLE "public"."ci_build_config" ( + "id" integer NOT NULL DEFAULT nextval('id_seq_ci_build_config'::regclass), + "type" varchar(100), + "ci_template_id" integer, + "ci_template_override_id" integer, + "build_metadata" text, + "created_on" timestamptz, + "created_by" integer, + "updated_on" timestamptz, + "updated_by" integer, + PRIMARY KEY ("id") +); + +ALTER TABLE ci_template ADD COLUMN ci_build_config_id integer; + +ALTER TABLE ONLY public.ci_template + ADD CONSTRAINT ci_template_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + + +ALTER TABLE ci_template_override ADD COLUMN ci_build_config_id integer; + +ALTER TABLE ONLY public.ci_template_override + ADD CONSTRAINT ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); diff --git a/wire_gen.go b/wire_gen.go index 5dd9c0673a..b69d8d3a0b 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -359,7 +359,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciTemplateOverrideRepositoryImpl, ciBuildConfigServiceImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) From 0a65e074a2c22f835dd409e035420b05d5fe8a28 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Tue, 4 Oct 2022 09:18:32 +0530 Subject: [PATCH 08/50] id update fix & test case --- .../CiTemplateOverrideRepository.go | 2 +- .../pipelineConfig/CiTemplateRepository.go | 2 +- pkg/pipeline/CiBuildConfigService.go | 19 +++++--- pkg/pipeline/CiTemplateService.go | 31 ++++++++----- pkg/pipeline/CiTemplateService_test.go | 44 +++++++++++++++++++ pkg/pipeline/bean/CiBuildConfig.go | 5 ++- 6 files changed, 81 insertions(+), 22 deletions(-) create mode 100644 pkg/pipeline/CiTemplateService_test.go diff --git a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go index 44772751ae..d223eee797 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go @@ -80,7 +80,7 @@ func (repo *CiTemplateOverrideRepositoryImpl) FindByAppId(appId int) ([]*CiTempl func (repo *CiTemplateOverrideRepositoryImpl) FindByCiPipelineId(ciPipelineId int) (*CiTemplateOverride, error) { ciTemplateOverride := &CiTemplateOverride{} err := repo.dbConnection.Model(ciTemplateOverride). - Column("ci_template_override.*", "GitMaterial", "DockerRegistry"). + Column("ci_template_override.*", "GitMaterial", "DockerRegistry", "CiBuildConfig"). Where("ci_pipeline_id = ?", ciPipelineId). Where("ci_template_override.active = ?", true). Select() diff --git a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go index 0f56a9bd94..82ca73b6db 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go @@ -82,7 +82,7 @@ func (impl CiTemplateRepositoryImpl) FindByAppId(appId int) (ciTemplate *CiTempl template := &CiTemplate{} err = impl.dbConnection.Model(template). Where("app_id =? ", appId). - Column("ci_template.*", "App", "DockerRegistry"). + Column("ci_template.*", "App", "DockerRegistry", "CiBuildConfig"). Select() if pg.ErrNoRows == err { return nil, errors.NotFoundf(err.Error()) diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index 3d53c3a7a4..924abb26f4 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -9,7 +9,7 @@ import ( type CiBuildConfigService interface { Save(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) error - Update(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) + UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) Delete(ciBuildConfigId int) error } @@ -26,21 +26,22 @@ func NewCiBuildConfigServiceImpl(logger *zap.SugaredLogger, ciBuildConfigReposit } func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean) error { - ciBuildConfig, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean) + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean) if err != nil { impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, "overrideTemplateId", overrideTemplateId, "ciBuildConfigBean", ciBuildConfigBean, "err", err) return errors.New("error while saving build config") } - err = impl.CiBuildConfigRepository.Save(ciBuildConfig) + err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) + ciBuildConfigBean.Id = ciBuildConfigEntity.Id if err != nil { return errors.New("error while saving build config") } return nil } -func (impl *CiBuildConfigServiceImpl) Update(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) { - if ciBuildConfig == nil || ciBuildConfig.Id == 0 { +func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) { + if ciBuildConfig == nil { impl.Logger.Warnw("not updating build config as object is empty", "ciBuildConfig", ciBuildConfig) return nil, nil } @@ -50,8 +51,14 @@ func (impl *CiBuildConfigServiceImpl) Update(templateId int, overrideTemplateId "overrideTemplateId", overrideTemplateId, "ciBuildConfig", ciBuildConfig, "err", err) return nil, errors.New("error while saving build config") } - err = impl.CiBuildConfigRepository.Update(ciBuildConfigEntity) + if ciBuildConfig.Id == 0 { + err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) + ciBuildConfig.Id = ciBuildConfigEntity.Id + } else { + err = impl.CiBuildConfigRepository.Update(ciBuildConfigEntity) + } if err != nil { + impl.Logger.Errorw("error occurred while updating/saving ciBuildConfig", "entity", ciBuildConfigEntity, "err", err) return nil, errors.New("error while updating build config") } return ciBuildConfig, nil diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index e9de175ad4..91955399ac 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -38,6 +38,12 @@ func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) erro ciTemplateOverride := ciTemplateBean.CiTemplateOverride ciTemplateId := 0 ciTemplateOverrideId := 0 + + buildConfig := ciTemplateBean.CiBuildConfig + err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) + } if ciTemplateOverride == nil { err := impl.CiTemplateRepository.Save(ciTemplate) if err != nil { @@ -54,11 +60,7 @@ func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) erro } ciTemplateOverrideId = ciTemplateOverride.Id } - buildConfig := ciTemplateBean.CiBuildConfig - err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig) - if err != nil { - impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) - } + return err } @@ -143,25 +145,30 @@ func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) er ciTemplateOverride := ciTemplateBean.CiTemplateOverride ciTemplateId := 0 ciTemplateOverrideId := 0 + ciBuildConfig := ciTemplateBean.CiBuildConfig if ciTemplateOverride == nil { + ciTemplateId = ciTemplate.Id + } else { + ciTemplateOverrideId = ciTemplateOverride.Id + } + _, err := impl.CiBuildConfigService.UpdateOrSave(ciTemplateId, ciTemplateOverrideId, ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error in updating ci build config in db", "ciBuildConfig", ciBuildConfig, "err", err) + } + if ciTemplateOverride == nil { + ciTemplate.CiBuildConfigId = ciBuildConfig.Id err := impl.CiTemplateRepository.Update(ciTemplate) if err != nil { impl.Logger.Errorw("error in updating ci template in db", "template", ciTemplate, "err", err) return err } - ciTemplateId = ciTemplate.Id } else { + ciTemplateOverride.CiBuildConfigId = ciBuildConfig.Id _, err := impl.CiTemplateOverrideRepository.Update(ciTemplateOverride) if err != nil { impl.Logger.Errorw("error in updating template override", "err", err, "templateOverrideConfig", ciTemplateOverride) return err } - ciTemplateOverrideId = ciTemplateOverride.Id - } - ciBuildConfig := ciTemplateBean.CiBuildConfig - _, err := impl.CiBuildConfigService.Update(ciTemplateId, ciTemplateOverrideId, ciBuildConfig) - if err != nil { - impl.Logger.Errorw("error in updating ci build config in db", "ciBuildConfig", ciBuildConfig, "err", err) } return err } diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go new file mode 100644 index 0000000000..35be0fbbd7 --- /dev/null +++ b/pkg/pipeline/CiTemplateService_test.go @@ -0,0 +1,44 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCiTemplateService(t *testing.T) { + + t.SkipNow() + t.Run("getCiTemplate", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + appId := 1 + ciTemplateBean, err := ciTemplateServiceImpl.FindByAppId(appId) + assert.True(t, err == nil, err) + assert.True(t, ciTemplateBean != nil, ciTemplateBean) + assert.True(t, ciTemplateBean.CiTemplate != nil) + assert.True(t, ciTemplateBean.CiTemplate.AppId == appId) + assert.True(t, ciTemplateBean.CiBuildConfig != nil) + + buildPackConfig := &bean.BuildPackConfig{ + BuilderId: "gcr.io/buildpacks/builder:v1", + } + ciBuildConfig := ciTemplateBean.CiBuildConfig + ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE + ciBuildConfig.BuildPackConfig = buildPackConfig + err = ciTemplateServiceImpl.Update(ciTemplateBean) + assert.True(t, err == nil, err) + }) + +} diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 7d9998fa57..f51aed523e 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -53,13 +53,14 @@ func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ci } buildMetadata = string(dockerBuildMetadataBytes) } - ciBuildConfig := &pipelineConfig.CiBuildConfig{ + ciBuildConfigEntity := &pipelineConfig.CiBuildConfig{ + Id: ciBuildConfigBean.Id, Type: string(ciBuildType), CiTemplateId: templateId, CiTemplateOverrideId: overrideTemplateId, BuildMetadata: buildMetadata, } - return ciBuildConfig, nil + return ciBuildConfigEntity, nil } func ConvertDbBuildConfigToBean(dbConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { From c7a76d3e08d38415367e3a53738c29d6ec2da1c9 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Tue, 4 Oct 2022 09:35:53 +0530 Subject: [PATCH 09/50] ciBuildConfig fetched from db --- .../sql/repository/pipelineConfig/CiPipelineMaterial.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go b/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go index 6e7142faa3..cf25319ce1 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go @@ -79,7 +79,7 @@ func (impl CiPipelineMaterialRepositoryImpl) GetById(id int) (*CiPipelineMateria func (impl CiPipelineMaterialRepositoryImpl) GetByPipelineId(id int) ([]*CiPipelineMaterial, error) { var ciPipelineMaterials []*CiPipelineMaterial err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.ci_pipeline_id = ?", id). Where("ci_pipeline_material.active = ?", true). Where("ci_pipeline_material.type != ?", SOURCE_TYPE_BRANCH_REGEX). @@ -127,7 +127,7 @@ func (impl CiPipelineMaterialRepositoryImpl) Update(tx *pg.Tx, materials ...*CiP func (impl CiPipelineMaterialRepositoryImpl) GetRegexByPipelineId(id int) ([]*CiPipelineMaterial, error) { var ciPipelineMaterials []*CiPipelineMaterial err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.ci_pipeline_id = ?", id). Where("ci_pipeline_material.active = ?", true). Where("ci_pipeline_material.type = ?", SOURCE_TYPE_BRANCH_REGEX). @@ -138,7 +138,7 @@ func (impl CiPipelineMaterialRepositoryImpl) GetRegexByPipelineId(id int) ([]*Ci func (impl CiPipelineMaterialRepositoryImpl) CheckRegexExistsForMaterial(id int) bool { var ciPipelineMaterials []*CiPipelineMaterial exists, err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.id = ?", id). Where("ci_pipeline_material.regex != ?", ""). Exists() From 6bf64dfc0df5a21a1f4e65fd6fc8c5f80007df57 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Tue, 4 Oct 2022 11:34:35 +0530 Subject: [PATCH 10/50] heroku id added --- pkg/pipeline/CiTemplateService_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 35be0fbbd7..cd3f3565c3 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -34,6 +34,7 @@ func TestCiTemplateService(t *testing.T) { buildPackConfig := &bean.BuildPackConfig{ BuilderId: "gcr.io/buildpacks/builder:v1", } + //buildPackConfig.BuilderId = "heroku/buildpacks:20" ciBuildConfig := ciTemplateBean.CiBuildConfig ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE ciBuildConfig.BuildPackConfig = buildPackConfig From b179846489d73fa27c3915bb41fd1ed1548dda44 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Tue, 4 Oct 2022 13:39:32 +0530 Subject: [PATCH 11/50] args and ciLevel args merging handling --- pkg/pipeline/CiService.go | 18 ++++++++--------- pkg/pipeline/CiTemplateService.go | 6 +++--- pkg/pipeline/CiTemplateService_test.go | 26 ++++++++++++++++++------- pkg/pipeline/bean/CiBuildConfig.go | 27 ++++++++++++++++++++++---- 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 9a92b15dfd..25ce77f6d7 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -372,18 +372,17 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } ciTemplate := pipeline.CiTemplate - args := ciTemplate.Args ciLevelArgs := pipeline.DockerArgs if ciLevelArgs == "" { ciLevelArgs = "{}" } - merged, err := impl.mergeUtil.JsonPatch([]byte(args), []byte(ciLevelArgs)) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } + //merged, err := impl.mergeUtil.JsonPatch([]byte(args), []byte(ciLevelArgs)) + //if err != nil { + // impl.Logger.Errorw("err", "err", err) + // return nil, err + //} user, err := impl.userService.GetById(trigger.TriggeredBy) if err != nil { impl.Logger.Errorw("unable to find user by id", "err", err, "id", trigger.TriggeredBy) @@ -420,10 +419,11 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if checkoutPath == "" { checkoutPath = "./" } - mergedArgs := string(merged) - ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, mergedArgs, ciTemplate.TargetPlatform, ciBuildConfigBean) + //mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.TargetPlatform, ciBuildConfigBean) if err != nil { - impl.Logger.Errorw("error occurred while overriding ci build config", "args", mergedArgs, "error", err) + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) return nil, errors.New("error while parsing ci build config") } workflowRequest := &WorkflowRequest{ diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 91955399ac..5363615f4f 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -76,7 +76,7 @@ func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.C ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, ciTemplate.TargetPlatform, nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.TargetPlatform, nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -103,7 +103,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp return nil, err } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -131,7 +131,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index cd3f3565c3..b26825bffa 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -23,7 +23,7 @@ func TestCiTemplateService(t *testing.T) { ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) - appId := 1 + appId := 3 ciTemplateBean, err := ciTemplateServiceImpl.FindByAppId(appId) assert.True(t, err == nil, err) assert.True(t, ciTemplateBean != nil, ciTemplateBean) @@ -31,13 +31,25 @@ func TestCiTemplateService(t *testing.T) { assert.True(t, ciTemplateBean.CiTemplate.AppId == appId) assert.True(t, ciTemplateBean.CiBuildConfig != nil) - buildPackConfig := &bean.BuildPackConfig{ - BuilderId: "gcr.io/buildpacks/builder:v1", - } - //buildPackConfig.BuilderId = "heroku/buildpacks:20" ciBuildConfig := ciTemplateBean.CiBuildConfig - ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE - ciBuildConfig.BuildPackConfig = buildPackConfig + + //buildPackConfig := &bean.BuildPackConfig{ + // BuilderId: "gcr.io/buildpacks/builder:v1", + //} + //buildPackConfig.BuilderId = "heroku/buildpacks:20" + //ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE + //ciBuildConfig.BuildPackConfig = buildPackConfig + + args := make(map[string]string) + args["hello"] = "world" + dockerBuildConfig := &bean.DockerBuildConfig{ + DockerfilePath: "Dockerfile", + Args: args, + TargetPlatform: "linux/amd64", + } + ciBuildConfig.CiBuildType = bean.SELF_DOCKERFILE_BUILD_TYPE + ciBuildConfig.DockerBuildConfig = dockerBuildConfig + err = ciTemplateServiceImpl.Update(ciTemplateBean) assert.True(t, err == nil, err) }) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index f51aed523e..3cd361ae99 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -103,14 +103,21 @@ func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuil return dockerBuildConfig, err } -func OverrideCiBuildConfig(dockerfilePath string, args string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { - dockerArgs := map[string]string{} - if args != "" { - if err := json.Unmarshal([]byte(args), &dockerArgs); err != nil { +func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { + oldDockerArgs := map[string]string{} + ciLevelDockerArgs := map[string]string{} + if oldArgs != "" { + if err := json.Unmarshal([]byte(oldArgs), &oldDockerArgs); err != nil { + return nil, err + } + } + if ciLevelArgs != "" { + if err := json.Unmarshal([]byte(oldArgs), &ciLevelDockerArgs); err != nil { return nil, err } } if ciBuildConfigBean == nil { + dockerArgs := mergeMap(oldDockerArgs, ciLevelDockerArgs) ciBuildConfigBean = &CiBuildConfigBean{ CiBuildType: SELF_DOCKERFILE_BUILD_TYPE, DockerBuildConfig: &DockerBuildConfig{ @@ -121,8 +128,20 @@ func OverrideCiBuildConfig(dockerfilePath string, args string, targetPlatform st } } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE { dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) dockerBuildConfig.DockerfilePath = dockerfilePath dockerBuildConfig.Args = dockerArgs } return ciBuildConfigBean, nil } + +func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]string) map[string]string { + dockerArgs := make(map[string]string) + for key, value := range oldDockerArgs { + dockerArgs[key] = value + } + for key, value := range ciLevelDockerArgs { + dockerArgs[key] = value + } + return dockerArgs +} From 3faffaac0a086fca3dd674e3ad4b2a75000c10c4 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Thu, 6 Oct 2022 08:49:53 +0530 Subject: [PATCH 12/50] IT test case commit --- pkg/pipeline/CiTemplateService_test.go | 35 +++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index b26825bffa..5a76c06b8b 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -1,11 +1,14 @@ package pipeline import ( + "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/sql" "github.com/stretchr/testify/assert" + "log" + "os" "testing" ) @@ -23,7 +26,7 @@ func TestCiTemplateService(t *testing.T) { ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) - appId := 3 + appId := 1 ciTemplateBean, err := ciTemplateServiceImpl.FindByAppId(appId) assert.True(t, err == nil, err) assert.True(t, ciTemplateBean != nil, ciTemplateBean) @@ -41,17 +44,37 @@ func TestCiTemplateService(t *testing.T) { //ciBuildConfig.BuildPackConfig = buildPackConfig args := make(map[string]string) - args["hello"] = "world" + //args["hello"] = "world" + input := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" dockerBuildConfig := &bean.DockerBuildConfig{ - DockerfilePath: "Dockerfile", - Args: args, - TargetPlatform: "linux/amd64", + DockerfilePath: "Dockerfile", + DockerfileContent: input, + Args: args, + //TargetPlatform: "linux/amd64", } - ciBuildConfig.CiBuildType = bean.SELF_DOCKERFILE_BUILD_TYPE + ciBuildConfig.CiBuildType = bean.MANAGED_DOCKERFILE_BUILD_TYPE ciBuildConfig.DockerBuildConfig = dockerBuildConfig err = ciTemplateServiceImpl.Update(ciTemplateBean) assert.True(t, err == nil, err) }) + t.Run("escaping char test", func(t *testing.T) { + input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" + //fmt.Println(input) + output := fmt.Sprint(input) + fmt.Println(output) + f, err := os.Create("/tmp/devtroncd/data.txt") + if err != nil { + log.Fatal(err) + } + defer f.Close() + _, err2 := f.WriteString(input) + + if err2 != nil { + log.Fatal(err2) + } + fmt.Println("done") + }) + } From fa86aa206fcde4d03aeb66f4bdef116a987da297 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Thu, 6 Oct 2022 13:22:06 +0530 Subject: [PATCH 13/50] nil bug fix --- .../repository/pipelineConfig/CiTemplateOverrideRepository.go | 2 +- pkg/pipeline/CiTemplateService_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go index d223eee797..4510795e79 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go @@ -86,7 +86,7 @@ func (repo *CiTemplateOverrideRepositoryImpl) FindByCiPipelineId(ciPipelineId in Select() if err != nil { repo.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) - return nil, err + return ciTemplateOverride, err } return ciTemplateOverride, nil } diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 5a76c06b8b..c8d73e4367 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -14,7 +14,7 @@ import ( func TestCiTemplateService(t *testing.T) { - t.SkipNow() + //t.SkipNow() t.Run("getCiTemplate", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() assert.True(t, err == nil, err) @@ -26,6 +26,7 @@ func TestCiTemplateService(t *testing.T) { ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + _, err = ciTemplateServiceImpl.FindTemplateOverrideByCiPipelineId(1) appId := 1 ciTemplateBean, err := ciTemplateServiceImpl.FindByAppId(appId) assert.True(t, err == nil, err) From 7719f7d053f191b301ccb062f49952e30622455e Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Thu, 6 Oct 2022 16:45:08 +0530 Subject: [PATCH 14/50] DockerBuildOptions in buildconfig --- pkg/pipeline/bean/CiBuildConfig.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 3cd361ae99..5c86bf8b6d 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -23,10 +23,11 @@ type CiBuildConfigBean struct { } type DockerBuildConfig struct { - DockerfilePath string `json:"dockerfileRelativePath,omitempty"` - DockerfileContent string `json:"DockerfileContent"` - Args map[string]string `json:"args,omitempty"` - TargetPlatform string `json:"targetPlatform,omitempty"` + DockerfilePath string `json:"dockerfileRelativePath,omitempty"` + DockerfileContent string `json:"dockerfileContent"` + Args map[string]string `json:"args,omitempty"` + TargetPlatform string `json:"targetPlatform,omitempty"` + DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` } type BuildPackConfig struct { From 0d01be4c0ca53839f98c700dd72c59e25680db64 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Thu, 6 Oct 2022 17:27:42 +0530 Subject: [PATCH 15/50] dockerBuildOptions handling --- pkg/pipeline/CiService.go | 8 ++++---- pkg/pipeline/CiTemplateService.go | 6 +++--- pkg/pipeline/bean/CiBuildConfig.go | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index a05f0c2d61..9f2b0c5a74 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -383,9 +383,9 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. // impl.Logger.Errorw("err", "err", err) // return nil, err //} - //if pipeline.CiTemplate.DockerBuildOptions == "" { - // pipeline.CiTemplate.DockerBuildOptions = "{}" - //} + if pipeline.CiTemplate.DockerBuildOptions == "" { + pipeline.CiTemplate.DockerBuildOptions = "{}" + } user, err := impl.userService.GetById(trigger.TriggeredBy) if err != nil { impl.Logger.Errorw("unable to find user by id", "err", err, "id", trigger.TriggeredBy) @@ -424,7 +424,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } //mergedArgs := string(merged) oldArgs := ciTemplate.Args - ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.TargetPlatform, ciBuildConfigBean) + ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) if err != nil { impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) return nil, errors.New("error while parsing ci build config") diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 5363615f4f..64d1a0c07f 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -76,7 +76,7 @@ func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.C ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.TargetPlatform, nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -103,7 +103,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp return nil, err } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -131,7 +131,7 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", nil) + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 5c86bf8b6d..929fe632b8 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -104,9 +104,10 @@ func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuil return dockerBuildConfig, err } -func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { +func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, dockerBuildOptions string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { oldDockerArgs := map[string]string{} ciLevelDockerArgs := map[string]string{} + dockerBuildOptionsMap := map[string]string{} if oldArgs != "" { if err := json.Unmarshal([]byte(oldArgs), &oldDockerArgs); err != nil { return nil, err @@ -117,17 +118,23 @@ func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs st return nil, err } } + if dockerBuildOptions != "" { + if err := json.Unmarshal([]byte(dockerBuildOptions), &dockerBuildOptionsMap); err != nil { + return nil, err + } + } if ciBuildConfigBean == nil { dockerArgs := mergeMap(oldDockerArgs, ciLevelDockerArgs) ciBuildConfigBean = &CiBuildConfigBean{ CiBuildType: SELF_DOCKERFILE_BUILD_TYPE, DockerBuildConfig: &DockerBuildConfig{ - DockerfilePath: dockerfilePath, - Args: dockerArgs, - TargetPlatform: targetPlatform, + DockerfilePath: dockerfilePath, + Args: dockerArgs, + TargetPlatform: targetPlatform, + DockerBuildOptions: dockerBuildOptionsMap, }, } - } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE { + } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) dockerBuildConfig.DockerfilePath = dockerfilePath From 17c58c5cc47b54773de9ef21c54ae53e11c9c208 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Thu, 6 Oct 2022 23:37:14 +0530 Subject: [PATCH 16/50] buildconfig audit log and id for update handling --- pkg/pipeline/CiBuildConfigService.go | 17 +++++++++++------ pkg/pipeline/CiTemplateService.go | 4 ++-- pkg/pipeline/DbPipelineOrchestrator.go | 13 +++++++++---- pkg/pipeline/PipelineBuilder.go | 10 +++++++--- pkg/pipeline/bean/CiBuildConfig.go | 5 ++++- pkg/pipeline/bean/CiTemplateBean.go | 1 + 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index 924abb26f4..104357ade5 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -5,11 +5,12 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" + "time" ) type CiBuildConfigService interface { - Save(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) error - UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) + Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error + UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) Delete(ciBuildConfigId int) error } @@ -25,13 +26,15 @@ func NewCiBuildConfigServiceImpl(logger *zap.SugaredLogger, ciBuildConfigReposit } } -func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean) error { - ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean) +func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean, userId) if err != nil { impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, "overrideTemplateId", overrideTemplateId, "ciBuildConfigBean", ciBuildConfigBean, "err", err) return errors.New("error while saving build config") } + ciBuildConfigEntity.CreatedOn = time.Now() + ciBuildConfigEntity.CreatedBy = userId err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) ciBuildConfigBean.Id = ciBuildConfigEntity.Id if err != nil { @@ -40,18 +43,20 @@ func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId in return nil } -func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean) (*bean.CiBuildConfigBean, error) { +func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) { if ciBuildConfig == nil { impl.Logger.Warnw("not updating build config as object is empty", "ciBuildConfig", ciBuildConfig) return nil, nil } - ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig) + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig, userId) if err != nil { impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, "overrideTemplateId", overrideTemplateId, "ciBuildConfig", ciBuildConfig, "err", err) return nil, errors.New("error while saving build config") } if ciBuildConfig.Id == 0 { + ciBuildConfigEntity.CreatedOn = time.Now() + ciBuildConfigEntity.CreatedBy = userId err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) ciBuildConfig.Id = ciBuildConfigEntity.Id } else { diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 64d1a0c07f..36a235d840 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -40,7 +40,7 @@ func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) erro ciTemplateOverrideId := 0 buildConfig := ciTemplateBean.CiBuildConfig - err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig) + err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig, ciTemplateBean.UserId) if err != nil { impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) } @@ -151,7 +151,7 @@ func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) er } else { ciTemplateOverrideId = ciTemplateOverride.Id } - _, err := impl.CiBuildConfigService.UpdateOrSave(ciTemplateId, ciTemplateOverrideId, ciBuildConfig) + _, err := impl.CiBuildConfigService.UpdateOrSave(ciTemplateId, ciTemplateOverrideId, ciBuildConfig, ciTemplateBean.UserId) if err != nil { impl.Logger.Errorw("error in updating ci build config in db", "ciBuildConfig", ciBuildConfig, "err", err) } diff --git a/pkg/pipeline/DbPipelineOrchestrator.go b/pkg/pipeline/DbPipelineOrchestrator.go index be33d1f8f3..5c13f31c7a 100644 --- a/pkg/pipeline/DbPipelineOrchestrator.go +++ b/pkg/pipeline/DbPipelineOrchestrator.go @@ -298,13 +298,14 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci impl.logger.Errorw("error in getting templateOverride by ciPipelineId", "err", err, "ciPipelineId", createRequest.Id) return nil, err } + ciBuildConfigBean := createRequest.DockerConfigOverride.CiBuildConfig templateOverrideReq := &pipelineConfig.CiTemplateOverride{ CiPipelineId: createRequest.Id, DockerRegistryId: createRequest.DockerConfigOverride.DockerRegistry, DockerRepository: createRequest.DockerConfigOverride.DockerRepository, //DockerfilePath: createRequest.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - //GitMaterialId: createRequest.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + GitMaterialId: ciBuildConfigBean.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), CreatedBy: userId, @@ -314,12 +315,14 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci } savedTemplateOverride := savedTemplateOverrideBean.CiTemplateOverride if savedTemplateOverride != nil && savedTemplateOverride.Id > 0 { + ciBuildConfigBean.Id = savedTemplateOverride.CiBuildConfigId templateOverrideReq.Id = savedTemplateOverride.Id templateOverrideReq.CreatedOn = savedTemplateOverride.CreatedOn templateOverrideReq.CreatedBy = savedTemplateOverride.CreatedBy ciTemplateBean := &bean2.CiTemplateBean{ CiTemplateOverride: templateOverrideReq, - CiBuildConfig: createRequest.DockerConfigOverride.CiBuildConfig, + CiBuildConfig: ciBuildConfigBean, + UserId: userId, } err = impl.ciTemplateService.Update(ciTemplateBean) if err != nil { @@ -328,7 +331,8 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci } else { ciTemplateBean := &bean2.CiTemplateBean{ CiTemplateOverride: templateOverrideReq, - CiBuildConfig: createRequest.DockerConfigOverride.CiBuildConfig, + CiBuildConfig: ciBuildConfigBean, + UserId: userId, } err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { @@ -550,6 +554,7 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig ciTemplateBean := &bean2.CiTemplateBean{ CiTemplateOverride: templateOverride, CiBuildConfig: ciPipeline.DockerConfigOverride.CiBuildConfig, + UserId: createRequest.UserId, } err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index a206737456..900e114e94 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -659,9 +659,10 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq // impl.logger.Errorw("error in marshaling dockerBuildOptions", "err", err) // return nil, err //} + originalCiBuildConfig := originalCiConf.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ //DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, - GitMaterialId: originalCiConf.CiBuildConfig.GitMaterialId, + GitMaterialId: originalCiBuildConfig.GitMaterialId, //Args: string(argByte), //TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, BeforeDockerBuild: string(beforeByte), @@ -673,15 +674,18 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq Active: true, } + ciBuildConfig := updateRequest.CiBuildConfig + ciBuildConfig.Id = originalCiBuildConfig.Id ciTemplateBean := &bean3.CiTemplateBean{ CiTemplate: ciTemplate, - CiBuildConfig: updateRequest.CiBuildConfig, + CiBuildConfig: ciBuildConfig, + UserId: updateRequest.UserId, } err = impl.ciTemplateService.Update(ciTemplateBean) if err != nil { return nil, err } - originalCiConf.CiBuildConfig = updateRequest.CiBuildConfig + originalCiConf.CiBuildConfig = ciBuildConfig return originalCiConf, nil } diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 929fe632b8..0d89d10bc3 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -3,6 +3,8 @@ package bean import ( "encoding/json" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/sql" + "time" ) type CiBuildType string @@ -38,7 +40,7 @@ type BuildPackConfig struct { Args map[string]string `json:"args"` } -func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean) (*pipelineConfig.CiBuildConfig, error) { +func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean, userId int32) (*pipelineConfig.CiBuildConfig, error) { buildMetadata := "" ciBuildType := ciBuildConfigBean.CiBuildType if ciBuildType == BUILDPACK_BUILD_TYPE { @@ -60,6 +62,7 @@ func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ci CiTemplateId: templateId, CiTemplateOverrideId: overrideTemplateId, BuildMetadata: buildMetadata, + AuditLog: sql.AuditLog{UpdatedOn: time.Now(), UpdatedBy: userId}, } return ciBuildConfigEntity, nil } diff --git a/pkg/pipeline/bean/CiTemplateBean.go b/pkg/pipeline/bean/CiTemplateBean.go index dc03675c49..d49e4b2634 100644 --- a/pkg/pipeline/bean/CiTemplateBean.go +++ b/pkg/pipeline/bean/CiTemplateBean.go @@ -6,4 +6,5 @@ type CiTemplateBean struct { CiTemplate *pipelineConfig.CiTemplate CiTemplateOverride *pipelineConfig.CiTemplateOverride CiBuildConfig *CiBuildConfigBean + UserId int32 } From e7b01a0e9e98d9de6d087e37c5ed89150ff171b0 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Sat, 8 Oct 2022 09:51:45 +0530 Subject: [PATCH 17/50] unit tese cases --- cmd/external-app/wire_gen.go | 5 +- .../mocks/CiTemplateOverrideRepository.go | 120 +++++++++++++++ .../repository/mocks/CiTemplateRepository.go | 123 +++++++++++++++ pkg/pipeline/CiTemplateRepository_mock.go | 43 ++++++ pkg/pipeline/CiTemplateService.go | 12 -- pkg/pipeline/CiTemplateService_test.go | 142 +++++++++++++++--- pkg/pipeline/bean/CiBuildConfig.go | 12 +- 7 files changed, 420 insertions(+), 37 deletions(-) create mode 100644 internal/sql/repository/mocks/CiTemplateOverrideRepository.go create mode 100644 internal/sql/repository/mocks/CiTemplateRepository.go create mode 100644 pkg/pipeline/CiTemplateRepository_mock.go diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index fc2e114a47..b38b138231 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire -//+build !wireinject +//go:build !wireinject +// +build !wireinject package main @@ -184,7 +185,7 @@ func InitializeApp() (*App, error) { serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl) + helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) diff --git a/internal/sql/repository/mocks/CiTemplateOverrideRepository.go b/internal/sql/repository/mocks/CiTemplateOverrideRepository.go new file mode 100644 index 0000000000..94175b3255 --- /dev/null +++ b/internal/sql/repository/mocks/CiTemplateOverrideRepository.go @@ -0,0 +1,120 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + pipelineConfig "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + mock "github.com/stretchr/testify/mock" +) + +// CiTemplateOverrideRepository is an autogenerated mock type for the CiTemplateOverrideRepository type +type CiTemplateOverrideRepository struct { + mock.Mock +} + +// FindByAppId provides a mock function with given fields: appId +func (_m *CiTemplateOverrideRepository) FindByAppId(appId int) ([]*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(appId) + + var r0 []*pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(int) []*pipelineConfig.CiTemplateOverride); ok { + r0 = rf(appId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(appId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByCiPipelineId provides a mock function with given fields: ciPipelineId +func (_m *CiTemplateOverrideRepository) FindByCiPipelineId(ciPipelineId int) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(ciPipelineId) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(ciPipelineId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(ciPipelineId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: templateOverrideConfig +func (_m *CiTemplateOverrideRepository) Save(templateOverrideConfig *pipelineConfig.CiTemplateOverride) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(templateOverrideConfig) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(templateOverrideConfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*pipelineConfig.CiTemplateOverride) error); ok { + r1 = rf(templateOverrideConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: templateOverrideConfig +func (_m *CiTemplateOverrideRepository) Update(templateOverrideConfig *pipelineConfig.CiTemplateOverride) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(templateOverrideConfig) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(templateOverrideConfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*pipelineConfig.CiTemplateOverride) error); ok { + r1 = rf(templateOverrideConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewCiTemplateOverrideRepository interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiTemplateOverrideRepository creates a new instance of CiTemplateOverrideRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiTemplateOverrideRepository(t mockConstructorTestingTNewCiTemplateOverrideRepository) *CiTemplateOverrideRepository { + mock := &CiTemplateOverrideRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/sql/repository/mocks/CiTemplateRepository.go b/internal/sql/repository/mocks/CiTemplateRepository.go new file mode 100644 index 0000000000..c8c2124914 --- /dev/null +++ b/internal/sql/repository/mocks/CiTemplateRepository.go @@ -0,0 +1,123 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + pipelineConfig "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + mock "github.com/stretchr/testify/mock" +) + +// CiTemplateRepository is an autogenerated mock type for the CiTemplateRepository type +type CiTemplateRepository struct { + mock.Mock +} + +// FindByAppId provides a mock function with given fields: appId +func (_m *CiTemplateRepository) FindByAppId(appId int) (*pipelineConfig.CiTemplate, error) { + ret := _m.Called(appId) + + var r0 *pipelineConfig.CiTemplate + if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CiTemplate); ok { + r0 = rf(appId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplate) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(appId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByDockerRegistryId provides a mock function with given fields: dockerRegistryId +func (_m *CiTemplateRepository) FindByDockerRegistryId(dockerRegistryId string) ([]*pipelineConfig.CiTemplate, error) { + ret := _m.Called(dockerRegistryId) + + var r0 []*pipelineConfig.CiTemplate + if rf, ok := ret.Get(0).(func(string) []*pipelineConfig.CiTemplate); ok { + r0 = rf(dockerRegistryId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiTemplate) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(dockerRegistryId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindNumberOfAppsWithDockerConfigured provides a mock function with given fields: appIds +func (_m *CiTemplateRepository) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { + ret := _m.Called(appIds) + + var r0 int + if rf, ok := ret.Get(0).(func([]int) int); ok { + r0 = rf(appIds) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func([]int) error); ok { + r1 = rf(appIds) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: material +func (_m *CiTemplateRepository) Save(material *pipelineConfig.CiTemplate) error { + ret := _m.Called(material) + + var r0 error + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplate) error); ok { + r0 = rf(material) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Update provides a mock function with given fields: material +func (_m *CiTemplateRepository) Update(material *pipelineConfig.CiTemplate) error { + ret := _m.Called(material) + + var r0 error + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplate) error); ok { + r0 = rf(material) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewCiTemplateRepository interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiTemplateRepository creates a new instance of CiTemplateRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiTemplateRepository(t mockConstructorTestingTNewCiTemplateRepository) *CiTemplateRepository { + mock := &CiTemplateRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/pipeline/CiTemplateRepository_mock.go b/pkg/pipeline/CiTemplateRepository_mock.go new file mode 100644 index 0000000000..134bcbea66 --- /dev/null +++ b/pkg/pipeline/CiTemplateRepository_mock.go @@ -0,0 +1,43 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/stretchr/testify/mock" +) + +type CiTemplateRepositoryMock struct { + mock.Mock +} + +func (impl CiTemplateRepositoryMock) Save(material *pipelineConfig.CiTemplate) error { + //TODO implement me + panic("implement me") +} + +func (impl CiTemplateRepositoryMock) Update(material *pipelineConfig.CiTemplate) error { + //TODO implement me + panic("implement me") +} + +func (impl CiTemplateRepositoryMock) FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) { + //TODO implement me + panic("implement me") +} + +func (impl CiTemplateRepositoryMock) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { + //TODO implement me + panic("implement me") +} + +func (impl *CiTemplateRepositoryMock) FindByAppId(appId int) (ciTemplate *pipelineConfig.CiTemplate, err error) { + called := impl.Called(appId) + ciTemplateInterface := called.Get(0) + if ciTemplateInterface != nil { + ciTemplate = ciTemplateInterface.(*pipelineConfig.CiTemplate) + } + errInterface := called.Get(1) + if errInterface != nil { + err = errInterface.(error) + } + return ciTemplate, err +} diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 36a235d840..da26d34f24 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -13,8 +13,6 @@ type CiTemplateService interface { FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) Update(ciTemplateBean *bean.CiTemplateBean) error - FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) - FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) } type CiTemplateServiceImpl struct { Logger *zap.SugaredLogger @@ -172,13 +170,3 @@ func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) er } return err } - -func (impl CiTemplateServiceImpl) FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) { - //TODO implement me - panic("implement me") -} - -func (impl CiTemplateServiceImpl) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { - //TODO implement me - panic("implement me") -} diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index c8d73e4367..55af031231 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -1,7 +1,9 @@ package pipeline import ( + "encoding/json" "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/mocks" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -15,7 +17,112 @@ import ( func TestCiTemplateService(t *testing.T) { //t.SkipNow() + t.Run("TemplateWithBuildpackConfiguration", func(t *testing.T) { + + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + builderId := "sample-builder" + //mockCiTemplateRepository := &CiTemplateRepositoryMock{} + ciTemplateRepositoryMocked := mocks.NewCiTemplateRepository(t) + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId + "\"}", + }, + } + ciTemplateRepositoryMocked.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, ciTemplateRepositoryMocked, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + template := templateBean.CiTemplate + assert.True(t, templateDbEntity.Id == template.Id) + assert.True(t, templateBean.CiBuildConfig != nil) + assert.True(t, templateBean.CiBuildConfig.CiBuildType == bean.BUILDPACK_BUILD_TYPE) + assert.True(t, templateBean.CiBuildConfig.BuildPackConfig.BuilderId == builderId) + }) + + t.Run("TemplateWithOldDockerData", func(t *testing.T) { + + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockCiTemplateRepository := &CiTemplateRepositoryMock{} + argsKey := "hello" + argsValue := "world" + buildOptionsKey := "volume" + buildOptionsValue := "hello1" + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + TargetPlatform: "linux/amd64", + DockerBuildOptions: "{\"" + buildOptionsKey + "\":\"" + buildOptionsValue + "\"}", + Args: "{\"" + argsKey + "\":\"" + argsValue + "\"}", + } + mockCiTemplateRepository.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, mockCiTemplateRepository, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + template := templateBean.CiTemplate + assert.True(t, templateDbEntity.Id == template.Id) + assert.True(t, templateDbEntity.TargetPlatform == template.TargetPlatform) + assert.True(t, template.CiBuildConfig == nil) + ciBuildConfig := templateBean.CiBuildConfig + assert.True(t, ciBuildConfig.CiBuildType == bean.SELF_DOCKERFILE_BUILD_TYPE) + assert.True(t, ciBuildConfig.DockerBuildConfig != nil) + assert.True(t, ciBuildConfig.DockerBuildConfig.TargetPlatform == templateDbEntity.TargetPlatform) + args := ciBuildConfig.DockerBuildConfig.Args + assert.True(t, args != nil) + assert.True(t, args[argsKey] == argsValue) + dockerBuildOptions := ciBuildConfig.DockerBuildConfig.DockerBuildOptions + assert.True(t, dockerBuildOptions != nil) + assert.True(t, dockerBuildOptions[buildOptionsKey] == buildOptionsValue, dockerBuildOptions) + }) + + t.Run("TemplateWithManagedDockerData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockCiTemplateRepository := &CiTemplateRepositoryMock{} + targetPlatform := "linux/amd64" + dockerfileContent := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + notPlatform := "linux/arm64" + gitMaterialId := 2 + ciBuildConfigId := 3 + managedDockerfileBuildType := bean.MANAGED_DOCKERFILE_BUILD_TYPE + buildConfigMetadata := &bean.DockerBuildConfig{ + DockerfileContent: dockerfileContent, + TargetPlatform: targetPlatform, + } + buildMetadata, err := json.Marshal(buildConfigMetadata) + assert.True(t, err == nil, err) + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + GitMaterialId: gitMaterialId, + TargetPlatform: notPlatform, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Id: ciBuildConfigId, + Type: string(managedDockerfileBuildType), + BuildMetadata: string(buildMetadata), + }, + } + mockCiTemplateRepository.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, mockCiTemplateRepository, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + ciTemplateEntity := templateBean.CiTemplate + ciBuildConfig := templateBean.CiBuildConfig + assert.True(t, ciTemplateEntity.Id == templateDbEntity.Id) + assert.True(t, ciBuildConfig.Id == ciBuildConfigId) + assert.True(t, ciBuildConfig.CiBuildType == managedDockerfileBuildType) + assert.True(t, ciBuildConfig.GitMaterialId == gitMaterialId) + assert.True(t, ciBuildConfig.BuildPackConfig == nil) + dockerBuildConfig := ciBuildConfig.DockerBuildConfig + assert.True(t, dockerBuildConfig != nil) + assert.True(t, dockerBuildConfig.DockerfileContent == dockerfileContent) + assert.True(t, dockerBuildConfig.TargetPlatform == targetPlatform) + }) + + t.Run("ciTemplateOverride", func(t *testing.T) { + + }) + t.Run("getCiTemplate", func(t *testing.T) { + t.SkipNow() sugaredLogger, err := util.NewSugardLogger() assert.True(t, err == nil, err) config, err := sql.GetConfig() @@ -37,30 +144,31 @@ func TestCiTemplateService(t *testing.T) { ciBuildConfig := ciTemplateBean.CiBuildConfig - //buildPackConfig := &bean.BuildPackConfig{ - // BuilderId: "gcr.io/buildpacks/builder:v1", - //} - //buildPackConfig.BuilderId = "heroku/buildpacks:20" - //ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE - //ciBuildConfig.BuildPackConfig = buildPackConfig - - args := make(map[string]string) - //args["hello"] = "world" - input := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" - dockerBuildConfig := &bean.DockerBuildConfig{ - DockerfilePath: "Dockerfile", - DockerfileContent: input, - Args: args, - //TargetPlatform: "linux/amd64", + buildPackConfig := &bean.BuildPackConfig{ + BuilderId: "gcr.io/buildpacks/builder:v1", } - ciBuildConfig.CiBuildType = bean.MANAGED_DOCKERFILE_BUILD_TYPE - ciBuildConfig.DockerBuildConfig = dockerBuildConfig + //buildPackConfig.BuilderId = "heroku/buildpacks:20" + ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE + ciBuildConfig.BuildPackConfig = buildPackConfig + + //args := make(map[string]string) + ////args["hello"] = "world" + //input := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + //dockerBuildConfig := &bean.DockerBuildConfig{ + // DockerfilePath: "Dockerfile", + // DockerfileContent: input, + // Args: args, + // //TargetPlatform: "linux/amd64", + //} + //ciBuildConfig.CiBuildType = bean.MANAGED_DOCKERFILE_BUILD_TYPE + //ciBuildConfig.DockerBuildConfig = dockerBuildConfig err = ciTemplateServiceImpl.Update(ciTemplateBean) assert.True(t, err == nil, err) }) t.Run("escaping char test", func(t *testing.T) { + t.SkipNow() input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" //fmt.Println(input) output := fmt.Sprint(input) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 0d89d10bc3..da2301dd37 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -67,27 +67,27 @@ func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ci return ciBuildConfigEntity, nil } -func ConvertDbBuildConfigToBean(dbConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { +func ConvertDbBuildConfigToBean(dbBuildConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { var buildPackConfig *BuildPackConfig var dockerBuildConfig *DockerBuildConfig var err error - if dbConfig == nil { + if dbBuildConfig == nil { return nil, nil } - ciBuildType := CiBuildType(dbConfig.Type) + ciBuildType := CiBuildType(dbBuildConfig.Type) if ciBuildType == BUILDPACK_BUILD_TYPE { - buildPackConfig, err = convertMetadataToBuildPackConfig(dbConfig.BuildMetadata) + buildPackConfig, err = convertMetadataToBuildPackConfig(dbBuildConfig.BuildMetadata) if err != nil { return nil, err } } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { - dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbConfig.BuildMetadata) + dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbBuildConfig.BuildMetadata) if err != nil { return nil, err } } ciBuildConfigBean := &CiBuildConfigBean{ - Id: dbConfig.Id, + Id: dbBuildConfig.Id, CiBuildType: ciBuildType, BuildPackConfig: buildPackConfig, DockerBuildConfig: dockerBuildConfig, From 600edf410a0b2fc02dd723f75d6135c510f031ab Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Sat, 8 Oct 2022 17:09:15 +0530 Subject: [PATCH 18/50] template override cases --- pkg/pipeline/CiTemplateService_test.go | 136 +++++++++++++++++++------ 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 55af031231..34e46e4f95 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -16,11 +16,10 @@ import ( func TestCiTemplateService(t *testing.T) { - //t.SkipNow() t.Run("TemplateWithBuildpackConfiguration", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() - assert.True(t, err == nil, err) + assert.Nil(t, err) builderId := "sample-builder" //mockCiTemplateRepository := &CiTemplateRepositoryMock{} ciTemplateRepositoryMocked := mocks.NewCiTemplateRepository(t) @@ -35,17 +34,17 @@ func TestCiTemplateService(t *testing.T) { ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, ciTemplateRepositoryMocked, nil) templateBean, err := ciTemplateServiceImpl.FindByAppId(1) template := templateBean.CiTemplate - assert.True(t, templateDbEntity.Id == template.Id) - assert.True(t, templateBean.CiBuildConfig != nil) - assert.True(t, templateBean.CiBuildConfig.CiBuildType == bean.BUILDPACK_BUILD_TYPE) - assert.True(t, templateBean.CiBuildConfig.BuildPackConfig.BuilderId == builderId) + assert.Equal(t, template.Id, templateDbEntity.Id) + assert.NotNil(t, templateBean.CiBuildConfig) + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, templateBean.CiBuildConfig.CiBuildType) + assert.Equal(t, builderId, templateBean.CiBuildConfig.BuildPackConfig.BuilderId) }) t.Run("TemplateWithOldDockerData", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() - assert.True(t, err == nil, err) - mockCiTemplateRepository := &CiTemplateRepositoryMock{} + assert.Nil(t, err) + mockCiTemplateRepository := mocks.NewCiTemplateRepository(t) argsKey := "hello" argsValue := "world" buildOptionsKey := "volume" @@ -60,25 +59,25 @@ func TestCiTemplateService(t *testing.T) { ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, mockCiTemplateRepository, nil) templateBean, err := ciTemplateServiceImpl.FindByAppId(1) template := templateBean.CiTemplate - assert.True(t, templateDbEntity.Id == template.Id) - assert.True(t, templateDbEntity.TargetPlatform == template.TargetPlatform) - assert.True(t, template.CiBuildConfig == nil) + assert.Equal(t, template.Id, templateDbEntity.Id) + assert.Equal(t, template.TargetPlatform, templateDbEntity.TargetPlatform) + assert.Nil(t, template.CiBuildConfig) ciBuildConfig := templateBean.CiBuildConfig - assert.True(t, ciBuildConfig.CiBuildType == bean.SELF_DOCKERFILE_BUILD_TYPE) - assert.True(t, ciBuildConfig.DockerBuildConfig != nil) - assert.True(t, ciBuildConfig.DockerBuildConfig.TargetPlatform == templateDbEntity.TargetPlatform) + assert.Equal(t, bean.SELF_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.NotNil(t, ciBuildConfig.DockerBuildConfig) + assert.Equal(t, templateDbEntity.TargetPlatform, ciBuildConfig.DockerBuildConfig.TargetPlatform) args := ciBuildConfig.DockerBuildConfig.Args - assert.True(t, args != nil) - assert.True(t, args[argsKey] == argsValue) + assert.NotEmpty(t, args) + assert.Equal(t, argsValue, args[argsKey]) dockerBuildOptions := ciBuildConfig.DockerBuildConfig.DockerBuildOptions - assert.True(t, dockerBuildOptions != nil) - assert.True(t, dockerBuildOptions[buildOptionsKey] == buildOptionsValue, dockerBuildOptions) + assert.NotEmpty(t, dockerBuildOptions) + assert.Equal(t, buildOptionsValue, dockerBuildOptions[buildOptionsKey]) }) t.Run("TemplateWithManagedDockerData", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() - assert.True(t, err == nil, err) - mockCiTemplateRepository := &CiTemplateRepositoryMock{} + assert.Nil(t, err) + mockCiTemplateRepository := mocks.NewCiTemplateRepository(t) targetPlatform := "linux/amd64" dockerfileContent := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" notPlatform := "linux/arm64" @@ -90,7 +89,7 @@ func TestCiTemplateService(t *testing.T) { TargetPlatform: targetPlatform, } buildMetadata, err := json.Marshal(buildConfigMetadata) - assert.True(t, err == nil, err) + assert.Nil(t, err) templateDbEntity := &pipelineConfig.CiTemplate{ Id: 1, GitMaterialId: gitMaterialId, @@ -106,19 +105,95 @@ func TestCiTemplateService(t *testing.T) { templateBean, err := ciTemplateServiceImpl.FindByAppId(1) ciTemplateEntity := templateBean.CiTemplate ciBuildConfig := templateBean.CiBuildConfig - assert.True(t, ciTemplateEntity.Id == templateDbEntity.Id) - assert.True(t, ciBuildConfig.Id == ciBuildConfigId) - assert.True(t, ciBuildConfig.CiBuildType == managedDockerfileBuildType) - assert.True(t, ciBuildConfig.GitMaterialId == gitMaterialId) - assert.True(t, ciBuildConfig.BuildPackConfig == nil) + assert.Equal(t, templateDbEntity.Id, ciTemplateEntity.Id) + assert.Equal(t, ciBuildConfigId, ciBuildConfig.Id) + assert.Equal(t, managedDockerfileBuildType, ciBuildConfig.CiBuildType) + assert.Equal(t, gitMaterialId, ciBuildConfig.GitMaterialId) + assert.Nil(t, ciBuildConfig.BuildPackConfig) dockerBuildConfig := ciBuildConfig.DockerBuildConfig - assert.True(t, dockerBuildConfig != nil) - assert.True(t, dockerBuildConfig.DockerfileContent == dockerfileContent) - assert.True(t, dockerBuildConfig.TargetPlatform == targetPlatform) + assert.NotNil(t, dockerBuildConfig) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, dockerfileContent, dockerBuildConfig.DockerfileContent) + assert.Equal(t, targetPlatform, dockerBuildConfig.TargetPlatform) }) - t.Run("ciTemplateOverride", func(t *testing.T) { + t.Run("ciTemplateOverrideWithOldDockerData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.Nil(t, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + DockerfilePath: "Dockerfile", + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + DockerfilePath: "Dockerfile_ea", + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + assert.Equal(t, bean.SELF_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.Empty(t, ciBuildConfig.DockerBuildConfig.DockerBuildOptions, "docker build options not supported in pipeline override") + assert.Equal(t, mockedTemplateOverride.DockerfilePath, ciBuildConfig.DockerBuildConfig.DockerfilePath) + } + }) + t.Run("ciTemplateOverrideWithBuildPackData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + builderId1 := "sample-builder-1" + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId1 + "\"}", + }, + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId1 + "\"}", + }, + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.DockerBuildConfig) + assert.NotNil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, builderId1, ciBuildConfig.BuildPackConfig.BuilderId) + } }) t.Run("getCiTemplate", func(t *testing.T) { @@ -169,6 +244,7 @@ func TestCiTemplateService(t *testing.T) { t.Run("escaping char test", func(t *testing.T) { t.SkipNow() + input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" //fmt.Println(input) output := fmt.Sprint(input) From 246bec2482d9e7ce6542cb88b54a72e62fc39829 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 10 Oct 2022 01:27:51 +0530 Subject: [PATCH 19/50] ci template update service --- pkg/pipeline/CiTemplateService.go | 36 +++-- pkg/pipeline/CiTemplateService_test.go | 149 +++++++++++++++++++++ pkg/pipeline/mocks/CiBuildConfigService.go | 79 +++++++++++ 3 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 pkg/pipeline/mocks/CiBuildConfigService.go diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index da26d34f24..d42553f0a7 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -94,19 +94,10 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp } var templateBeanOverrides []*bean.CiTemplateBean for _, templateOverride := range templateOverrides { - ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) + ciBuildConfigBean, err := impl.extractBuildConfigBean(templateOverride) if err != nil { - impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", - templateOverride.CiBuildConfig, "error", err) - return nil, err + return templateBeanOverrides, err } - if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) - if err != nil { - impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) - } - } - ciBuildConfigBean.GitMaterialId = templateOverride.GitMaterialId overrideBean := &bean.CiTemplateBean{ CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean, @@ -116,17 +107,12 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemp return templateBeanOverrides, nil } -func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) { - templateOverride, err := impl.CiTemplateOverrideRepository.FindByCiPipelineId(ciPipelineId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) - return nil, err - } - ciBuildConfig := templateOverride.CiBuildConfig - ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) +func (impl CiTemplateServiceImpl) extractBuildConfigBean(templateOverride *pipelineConfig.CiTemplateOverride) (*bean.CiBuildConfigBean, error) { + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) if err != nil { impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", - ciBuildConfig, "error", err) + templateOverride.CiBuildConfig, "error", err) + return nil, err } if ciBuildConfigBean == nil { ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) @@ -135,6 +121,16 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI } } ciBuildConfigBean.GitMaterialId = templateOverride.GitMaterialId + return ciBuildConfigBean, nil +} + +func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) { + templateOverride, err := impl.CiTemplateOverrideRepository.FindByCiPipelineId(ciPipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) + return nil, err + } + ciBuildConfigBean, err := impl.extractBuildConfigBean(templateOverride) return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err } diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 34e46e4f95..7c30610130 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -7,8 +7,10 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineMocks "github.com/devtron-labs/devtron/pkg/pipeline/mocks" "github.com/devtron-labs/devtron/pkg/sql" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "log" "os" "testing" @@ -196,6 +198,153 @@ func TestCiTemplateService(t *testing.T) { } }) + t.Run("templateOverrideWithManagedDockerfileAndBuildpack", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + dockerfileContent := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + targetPlatform := "linux/amd64" + builderId := "sample-builder" + buildConfigMetadata := &bean.DockerBuildConfig{ + DockerfileContent: dockerfileContent, + TargetPlatform: targetPlatform, + } + buildMetadata, err := json.Marshal(buildConfigMetadata) + assert.Nil(t, err) + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.MANAGED_DOCKERFILE_BUILD_TYPE), + BuildMetadata: string(buildMetadata), + }, + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId + "\"}", + }, + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + if ciBuildConfig.CiBuildType == bean.MANAGED_DOCKERFILE_BUILD_TYPE { + assert.Equal(t, bean.MANAGED_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.NotNil(t, ciBuildConfig.DockerBuildConfig) + assert.Equal(t, dockerfileContent, ciBuildConfig.DockerBuildConfig.DockerfileContent) + } else if ciBuildConfig.CiBuildType == bean.BUILDPACK_BUILD_TYPE { + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.DockerBuildConfig) + assert.NotNil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, builderId, ciBuildConfig.BuildPackConfig.BuilderId) + } + } + }) + + t.Run("UpdateTemplateOverrideWithBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, nil, mockedCiTemplateOverrideRepository) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateOverrideId := 1 + mockedCiBuildConfigId := 5 + mockedTemplateOverride := &pipelineConfig.CiTemplateOverride{Id: mockedTemplateOverrideId, CiPipelineId: 2, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplateOverride = mockedTemplateOverride + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: mockedCiBuildConfigId, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedCiTemplateOverrideRepository.On("Update", mock.AnythingOfType("*pipelineConfig.CiTemplateOverride")). + Return(func(templateOverride *pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride { + assert.Equal(t, mockedCiBuildConfigId, templateOverride.CiBuildConfigId) + return nil + }, nil) + mockedBuildConfigService.On("UpdateOrSave", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) *bean.CiBuildConfigBean { + assert.Equal(t, 0, templateId) + assert.Equal(t, mockedTemplateOverrideId, overrideTemplateId) + assert.Equal(t, mockedUserId, userId) + mockedBuildConfigBean := mockedCiTemplateBean.CiBuildConfig + assert.Equal(t, mockedBuildConfigBean, ciBuildConfig) + return ciBuildConfig + }, + nil, + ) + err = ciTemplateServiceImpl.Update(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedCiBuildConfigId, mockedTemplateOverride.CiBuildConfigId) + }) + + t.Run("UpdateTemplateWithBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateRepository := mocks.NewCiTemplateRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, mockedCiTemplateRepository, nil) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateId := 1 + mockedCiBuildConfigId := 5 + mockedTemplate := &pipelineConfig.CiTemplate{Id: mockedTemplateId, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplate = mockedTemplate + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: mockedCiBuildConfigId, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedCiTemplateRepository.On("Update", mock.AnythingOfType("*pipelineConfig.CiTemplate")). + Return(func(template *pipelineConfig.CiTemplate) error { + assert.Equal(t, mockedCiBuildConfigId, template.CiBuildConfigId) + return nil + }) + mockedBuildConfigService.On("UpdateOrSave", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) *bean.CiBuildConfigBean { + assert.Equal(t, 0, overrideTemplateId) + assert.Equal(t, mockedTemplateId, templateId) + assert.Equal(t, mockedUserId, userId) + assert.Equal(t, mockedCiTemplateBean.CiBuildConfig, ciBuildConfig) + return ciBuildConfig + }, + nil, + ) + err = ciTemplateServiceImpl.Update(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedCiBuildConfigId, mockedTemplate.CiBuildConfigId) + }) + t.Run("getCiTemplate", func(t *testing.T) { t.SkipNow() sugaredLogger, err := util.NewSugardLogger() diff --git a/pkg/pipeline/mocks/CiBuildConfigService.go b/pkg/pipeline/mocks/CiBuildConfigService.go new file mode 100644 index 0000000000..d9d9e86268 --- /dev/null +++ b/pkg/pipeline/mocks/CiBuildConfigService.go @@ -0,0 +1,79 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + bean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + mock "github.com/stretchr/testify/mock" +) + +// CiBuildConfigService is an autogenerated mock type for the CiBuildConfigService type +type CiBuildConfigService struct { + mock.Mock +} + +// Delete provides a mock function with given fields: ciBuildConfigId +func (_m *CiBuildConfigService) Delete(ciBuildConfigId int) error { + ret := _m.Called(ciBuildConfigId) + + var r0 error + if rf, ok := ret.Get(0).(func(int) error); ok { + r0 = rf(ciBuildConfigId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Save provides a mock function with given fields: templateId, overrideTemplateId, ciBuildConfigBean, userId +func (_m *CiBuildConfigService) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { + ret := _m.Called(templateId, overrideTemplateId, ciBuildConfigBean, userId) + + var r0 error + if rf, ok := ret.Get(0).(func(int, int, *bean.CiBuildConfigBean, int32) error); ok { + r0 = rf(templateId, overrideTemplateId, ciBuildConfigBean, userId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateOrSave provides a mock function with given fields: templateId, overrideTemplateId, ciBuildConfig, userId +func (_m *CiBuildConfigService) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) { + ret := _m.Called(templateId, overrideTemplateId, ciBuildConfig, userId) + + var r0 *bean.CiBuildConfigBean + if rf, ok := ret.Get(0).(func(int, int, *bean.CiBuildConfigBean, int32) *bean.CiBuildConfigBean); ok { + r0 = rf(templateId, overrideTemplateId, ciBuildConfig, userId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bean.CiBuildConfigBean) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int, int, *bean.CiBuildConfigBean, int32) error); ok { + r1 = rf(templateId, overrideTemplateId, ciBuildConfig, userId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewCiBuildConfigService interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiBuildConfigService creates a new instance of CiBuildConfigService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiBuildConfigService(t mockConstructorTestingTNewCiBuildConfigService) *CiBuildConfigService { + mock := &CiBuildConfigService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From cb56b650f03455a0de87d47ab4ae5e3bb53d7576 Mon Sep 17 00:00:00 2001 From: kripanshdevtron Date: Mon, 10 Oct 2022 02:11:19 +0530 Subject: [PATCH 20/50] save template and buildconfig --- pkg/pipeline/CiTemplateService.go | 2 ++ pkg/pipeline/CiTemplateService_test.go | 45 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index d42553f0a7..b4c4d54d72 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -43,6 +43,7 @@ func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) erro impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) } if ciTemplateOverride == nil { + ciTemplate.CiBuildConfigId = buildConfig.Id err := impl.CiTemplateRepository.Save(ciTemplate) if err != nil { impl.Logger.Errorw("error in saving ci template in db ", "template", ciTemplate, "err", err) @@ -51,6 +52,7 @@ func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) erro } ciTemplateId = ciTemplate.Id } else { + ciTemplateOverride.CiBuildConfigId = buildConfig.Id _, err := impl.CiTemplateOverrideRepository.Save(ciTemplateOverride) if err != nil { impl.Logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", ciTemplateOverride) diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 7c30610130..0294570076 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -345,6 +345,51 @@ func TestCiTemplateService(t *testing.T) { assert.Equal(t, mockedCiBuildConfigId, mockedTemplate.CiBuildConfigId) }) + t.Run("SaveTemplateAndBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateRepository := mocks.NewCiTemplateRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, mockedCiTemplateRepository, nil) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateId := 7 + mockedCiBuildConfigId := 6 + mockedTemplate := &pipelineConfig.CiTemplate{Id: 0, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplate = mockedTemplate + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: 0, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedBuildConfigService.On("Save", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) error { + assert.Equal(t, 0, overrideTemplateId) + assert.Equal(t, mockedTemplate.Id, templateId) + assert.Equal(t, mockedUserId, userId) + assert.Equal(t, mockedCiTemplateBean.CiBuildConfig, ciBuildConfig) + ciBuildConfig.Id = mockedCiBuildConfigId + return nil + }, + ) + mockedCiTemplateRepository.On("Save", mock.AnythingOfType("*pipelineConfig.CiTemplate")). + Return(func(template *pipelineConfig.CiTemplate) error { + assert.Equal(t, mockedCiBuildConfigId, template.CiBuildConfigId) + template.Id = mockedTemplateId + return nil + }) + err = ciTemplateServiceImpl.Save(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedTemplateId, mockedTemplate.Id) + }) + t.Run("getCiTemplate", func(t *testing.T) { t.SkipNow() sugaredLogger, err := util.NewSugardLogger() From 254a8912aedd53df2ff4456376825d6caa7b21d9 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 10 Oct 2022 19:34:26 +0530 Subject: [PATCH 21/50] refactoring --- pkg/pipeline/CiTemplateRepository_mock.go | 43 ----------------------- pkg/pipeline/CiTemplateService_test.go | 8 ++--- 2 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 pkg/pipeline/CiTemplateRepository_mock.go diff --git a/pkg/pipeline/CiTemplateRepository_mock.go b/pkg/pipeline/CiTemplateRepository_mock.go deleted file mode 100644 index 134bcbea66..0000000000 --- a/pkg/pipeline/CiTemplateRepository_mock.go +++ /dev/null @@ -1,43 +0,0 @@ -package pipeline - -import ( - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/stretchr/testify/mock" -) - -type CiTemplateRepositoryMock struct { - mock.Mock -} - -func (impl CiTemplateRepositoryMock) Save(material *pipelineConfig.CiTemplate) error { - //TODO implement me - panic("implement me") -} - -func (impl CiTemplateRepositoryMock) Update(material *pipelineConfig.CiTemplate) error { - //TODO implement me - panic("implement me") -} - -func (impl CiTemplateRepositoryMock) FindByDockerRegistryId(dockerRegistryId string) (ciTemplates []*pipelineConfig.CiTemplate, err error) { - //TODO implement me - panic("implement me") -} - -func (impl CiTemplateRepositoryMock) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { - //TODO implement me - panic("implement me") -} - -func (impl *CiTemplateRepositoryMock) FindByAppId(appId int) (ciTemplate *pipelineConfig.CiTemplate, err error) { - called := impl.Called(appId) - ciTemplateInterface := called.Get(0) - if ciTemplateInterface != nil { - ciTemplate = ciTemplateInterface.(*pipelineConfig.CiTemplate) - } - errInterface := called.Get(1) - if errInterface != nil { - err = errInterface.(error) - } - return ciTemplate, err -} diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index 0294570076..c0785ac9de 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -18,12 +18,11 @@ import ( func TestCiTemplateService(t *testing.T) { - t.Run("TemplateWithBuildpackConfiguration", func(t *testing.T) { + t.Run("FetchTemplateWithBuildpackConfiguration", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() assert.Nil(t, err) builderId := "sample-builder" - //mockCiTemplateRepository := &CiTemplateRepositoryMock{} ciTemplateRepositoryMocked := mocks.NewCiTemplateRepository(t) templateDbEntity := &pipelineConfig.CiTemplate{ Id: 1, @@ -42,7 +41,7 @@ func TestCiTemplateService(t *testing.T) { assert.Equal(t, builderId, templateBean.CiBuildConfig.BuildPackConfig.BuilderId) }) - t.Run("TemplateWithOldDockerData", func(t *testing.T) { + t.Run("FetchTemplateWithOldDockerData", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() assert.Nil(t, err) @@ -76,7 +75,7 @@ func TestCiTemplateService(t *testing.T) { assert.Equal(t, buildOptionsValue, dockerBuildOptions[buildOptionsKey]) }) - t.Run("TemplateWithManagedDockerData", func(t *testing.T) { + t.Run("FetchTemplateWithManagedDockerData", func(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() assert.Nil(t, err) mockCiTemplateRepository := mocks.NewCiTemplateRepository(t) @@ -438,7 +437,6 @@ func TestCiTemplateService(t *testing.T) { t.Run("escaping char test", func(t *testing.T) { t.SkipNow() - input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" //fmt.Println(input) output := fmt.Sprint(input) From d5872605218e00e2094ee6b733242158a6656cb5 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 11 Oct 2022 13:20:59 +0530 Subject: [PATCH 22/50] empty template nil handling --- pkg/pipeline/PipelineBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 900e114e94..27d922ac39 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -371,12 +371,12 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea impl.logger.Errorw("error in fetching ci pipeline", "appId", appId, "err", err) return nil, err } - template := ciTemplateBean.CiTemplate if errors.IsNotFound(err) { impl.logger.Debugw("no ci pipeline exists", "appId", appId, "err", err) err = &util.ApiError{Code: "404", HttpStatusCode: 200, UserMessage: "no ci pipeline exists"} return nil, err } + template := ciTemplateBean.CiTemplate gitMaterials, err := impl.materialRepo.FindByAppId(appId) if err != nil && err != pg.ErrNoRows { From 1adbf1a066c5ff3aa8440a742f1137ff6d88b8f1 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 11 Oct 2022 23:57:48 +0530 Subject: [PATCH 23/50] ci build config data from db --- .../CiTemplateOverrideRepository.go | 2 ++ pkg/pipeline/CiTemplateService_test.go | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go index 4510795e79..129469a013 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go @@ -64,7 +64,9 @@ func (repo *CiTemplateOverrideRepositoryImpl) Update(templateOverrideConfig *CiT func (repo *CiTemplateOverrideRepositoryImpl) FindByAppId(appId int) ([]*CiTemplateOverride, error) { var ciTemplateOverrides []*CiTemplateOverride err := repo.dbConnection.Model(&ciTemplateOverrides). + Column("ci_template_override.*", "CiBuildConfig"). Join("INNER JOIN ci_pipeline cp on cp.id=ci_template_override.ci_pipeline_id"). + Join("INNER JOIN ci_build_config cbc on cbc.id=ci_template_override.ci_build_config_id"). Where("app_id = ?", appId). Where("is_docker_config_overridden = ?", true). Where("ci_template_override.active = ?", true). diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go index c0785ac9de..5632c07715 100644 --- a/pkg/pipeline/CiTemplateService_test.go +++ b/pkg/pipeline/CiTemplateService_test.go @@ -435,6 +435,28 @@ func TestCiTemplateService(t *testing.T) { assert.True(t, err == nil, err) }) + t.Run("fetch ci pipeline details", func(t *testing.T) { + t.SkipNow() + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + appId := 7 + templateBeans, _ := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + for _, templateBean := range templateBeans { + buildConfig := templateBean.CiBuildConfig + fmt.Println(buildConfig) + } + ciTemplateBean, _ := ciTemplateServiceImpl.FindByAppId(appId) + fmt.Println(ciTemplateBean.CiBuildConfig) + }) + t.Run("escaping char test", func(t *testing.T) { t.SkipNow() input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" From 58023a36289e37f5d8d22a9e560386e0aacd12b5 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 17 Oct 2022 15:57:18 +0530 Subject: [PATCH 24/50] validate param removed --- pkg/pipeline/bean/CiBuildConfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index da2301dd37..adc44adb07 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -20,7 +20,7 @@ type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` CiBuildType CiBuildType `json:"ciBuildType"` - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` + DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` } From 6313d92d350b4bc3bf1dd618b7d72c2be2536d76 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 17 Oct 2022 16:49:23 +0530 Subject: [PATCH 25/50] bug fix --- pkg/pipeline/bean/CiBuildConfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index adc44adb07..5537bc8923 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -117,7 +117,7 @@ func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs st } } if ciLevelArgs != "" { - if err := json.Unmarshal([]byte(oldArgs), &ciLevelDockerArgs); err != nil { + if err := json.Unmarshal([]byte(ciLevelArgs), &ciLevelDockerArgs); err != nil { return nil, err } } From 50287537a6bc0235fb9cac2000638230e3bc338e Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 17 Oct 2022 17:45:31 +0530 Subject: [PATCH 26/50] extract dockerfile path from templates --- pkg/pipeline/CiService.go | 4 ++++ pkg/pipeline/bean/CiBuildConfig.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 9f2b0c5a74..77a1e1aa29 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -429,6 +429,10 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) return nil, errors.New("error while parsing ci build config") } + if ciBuildConfigBean.CiBuildType == bean2.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean2.MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + } workflowRequest := &WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, PipelineName: pipeline.Name, diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index 5537bc8923..aa98043b0c 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -140,7 +140,7 @@ func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs st } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) - dockerBuildConfig.DockerfilePath = dockerfilePath + //dockerBuildConfig.DockerfilePath = dockerfilePath dockerBuildConfig.Args = dockerArgs } return ciBuildConfigBean, nil From 1694db000e59d2c34d0fae8ae080184baff89288 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 17 Oct 2022 17:59:56 +0530 Subject: [PATCH 27/50] append checkout path --- pkg/pipeline/CiService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 77a1e1aa29..1535374132 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -432,6 +432,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if ciBuildConfigBean.CiBuildType == bean2.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean2.MANAGED_DOCKERFILE_BUILD_TYPE { dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath } workflowRequest := &WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, From 8949bac68a9c0e37793cf052a46075b42994b1fd Mon Sep 17 00:00:00 2001 From: Kripansh Date: Fri, 21 Oct 2022 11:06:23 +0530 Subject: [PATCH 28/50] managed template language and language fw support --- pkg/pipeline/bean/CiBuildConfig.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index aa98043b0c..b836491412 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -29,6 +29,8 @@ type DockerBuildConfig struct { DockerfileContent string `json:"dockerfileContent"` Args map[string]string `json:"args,omitempty"` TargetPlatform string `json:"targetPlatform,omitempty"` + Language string `json:"language,omitempty"` + LanguageFramework string `json:"languageFramework,omitempty"` DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` } From ad303272525750d155045e7fdc44e729bf1295b3 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Fri, 21 Oct 2022 17:35:16 +0530 Subject: [PATCH 29/50] project path support added --- pkg/pipeline/bean/CiBuildConfig.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index b836491412..bd3c7b0b84 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -40,6 +40,7 @@ type BuildPackConfig struct { LanguageVersion string `json:"languageVersion"` BuildPacks []string `json:"buildPacks"` Args map[string]string `json:"args"` + ProjectPath string `json:"projectPath,omitempty"` } func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean, userId int32) (*pipelineConfig.CiBuildConfig, error) { From fd6d50e816e22ba063ba4aa5426f1ea7273a8f52 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 25 Oct 2022 12:27:15 +0530 Subject: [PATCH 30/50] migration version update --- .../sql/{84_ci_build_config.up.sql => 87_ci_build_config.up.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{84_ci_build_config.up.sql => 87_ci_build_config.up.sql} (100%) diff --git a/scripts/sql/84_ci_build_config.up.sql b/scripts/sql/87_ci_build_config.up.sql similarity index 100% rename from scripts/sql/84_ci_build_config.up.sql rename to scripts/sql/87_ci_build_config.up.sql From c7040f3a86d9e786c62b8ff8e1ad30c311c262db Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 25 Oct 2022 13:29:40 +0530 Subject: [PATCH 31/50] 87 down file commit --- scripts/sql/87_ci_build_config.down.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/sql/87_ci_build_config.down.sql diff --git a/scripts/sql/87_ci_build_config.down.sql b/scripts/sql/87_ci_build_config.down.sql new file mode 100644 index 0000000000..5a08563c96 --- /dev/null +++ b/scripts/sql/87_ci_build_config.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE ci_template DROP COLUMN ci_build_config_id; +ALTER TABLE ci_template_override DROP COLUMN ci_build_config_id; + +DROP TABLE IF EXISTS "public"."ci_build_config"; \ No newline at end of file From 1b8cc55bc5b8003091a5387d0e599f2eed92a5fd Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 25 Oct 2022 13:39:07 +0530 Subject: [PATCH 32/50] rename migration version --- .../{87_ci_build_config.down.sql => 89_ci_build_config.down.sql} | 0 .../sql/{87_ci_build_config.up.sql => 89_ci_build_config.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{87_ci_build_config.down.sql => 89_ci_build_config.down.sql} (100%) rename scripts/sql/{87_ci_build_config.up.sql => 89_ci_build_config.up.sql} (100%) diff --git a/scripts/sql/87_ci_build_config.down.sql b/scripts/sql/89_ci_build_config.down.sql similarity index 100% rename from scripts/sql/87_ci_build_config.down.sql rename to scripts/sql/89_ci_build_config.down.sql diff --git a/scripts/sql/87_ci_build_config.up.sql b/scripts/sql/89_ci_build_config.up.sql similarity index 100% rename from scripts/sql/87_ci_build_config.up.sql rename to scripts/sql/89_ci_build_config.up.sql From b551bcc566591dc14ca9829efa6a1d7226602e6a Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 25 Oct 2022 14:01:27 +0530 Subject: [PATCH 33/50] indentation fix --- scripts/sql/89_ci_build_config.up.sql | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/sql/89_ci_build_config.up.sql b/scripts/sql/89_ci_build_config.up.sql index cb7525ad20..7cb1486fd1 100644 --- a/scripts/sql/89_ci_build_config.up.sql +++ b/scripts/sql/89_ci_build_config.up.sql @@ -1,26 +1,31 @@ CREATE SEQUENCE IF NOT EXISTS id_seq_ci_build_config; -- Table Definition -CREATE TABLE "public"."ci_build_config" ( - "id" integer NOT NULL DEFAULT nextval('id_seq_ci_build_config'::regclass), - "type" varchar(100), - "ci_template_id" integer, - "ci_template_override_id" integer, - "build_metadata" text, - "created_on" timestamptz, - "created_by" integer, - "updated_on" timestamptz, - "updated_by" integer, - PRIMARY KEY ("id") +CREATE TABLE public.ci_build_config +( + "id" integer NOT NULL DEFAULT nextval('id_seq_ci_build_config'::regclass), + "type" varchar(100), + "ci_template_id" integer, + "ci_template_override_id" integer, + "build_metadata" text, + "created_on" timestamptz, + "created_by" integer, + "updated_on" timestamptz, + "updated_by" integer, + PRIMARY KEY ("id") ); -ALTER TABLE ci_template ADD COLUMN ci_build_config_id integer; + + +ALTER TABLE ci_template + ADD COLUMN ci_build_config_id integer; ALTER TABLE ONLY public.ci_template ADD CONSTRAINT ci_template_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); -ALTER TABLE ci_template_override ADD COLUMN ci_build_config_id integer; +ALTER TABLE ci_template_override + ADD COLUMN ci_build_config_id integer; ALTER TABLE ONLY public.ci_template_override ADD CONSTRAINT ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); From 23a293f4b090232bbbf7985e41205a30b3ed5acc Mon Sep 17 00:00:00 2001 From: Kripansh Date: Wed, 26 Oct 2022 01:38:52 +0530 Subject: [PATCH 34/50] dockerfile template doc update --- sample-docker-templates/django/Dockerfile | 4 ++++ sample-docker-templates/flask/Dockerfile | 2 ++ sample-docker-templates/flask/start.sh | 1 + sample-docker-templates/node/Dockerfile | 1 + 4 files changed, 8 insertions(+) diff --git a/sample-docker-templates/django/Dockerfile b/sample-docker-templates/django/Dockerfile index db241a3995..b84c90ebe6 100644 --- a/sample-docker-templates/django/Dockerfile +++ b/sample-docker-templates/django/Dockerfile @@ -31,7 +31,10 @@ RUN pip install -r requirements.txt # install nginx RUN apt-get update && apt-get install nginx vim -y --no-install-recommends + +#Refer https://github.com/devtron-labs/devtron/blob/main/sample-docker-templates/django/nginx.default for sample nginx.default file COPY nginx.default /etc/nginx/sites-available/default + RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log @@ -41,4 +44,5 @@ EXPOSE 8000 STOPSIGNAL SIGTERM +# Refer https://github.com/devtron-labs/devtron/blob/main/sample-docker-templates/django/start-server.sh for sample start-server.sh file CMD ["/app/start-server.sh"] \ No newline at end of file diff --git a/sample-docker-templates/flask/Dockerfile b/sample-docker-templates/flask/Dockerfile index a2cb7c3085..ad20d787cc 100644 --- a/sample-docker-templates/flask/Dockerfile +++ b/sample-docker-templates/flask/Dockerfile @@ -29,8 +29,10 @@ ADD . /app/ #Installing dependencies RUN pip3 install -r requirements.txt +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/nginx.default for sample nginx.default file COPY nginx.default /etc/nginx/sites-available/default +#Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/start.sh for sample start.sh file #Making start.sh executable RUN chmod +x ./start.sh diff --git a/sample-docker-templates/flask/start.sh b/sample-docker-templates/flask/start.sh index a1e9b76869..0909a01c2c 100644 --- a/sample-docker-templates/flask/start.sh +++ b/sample-docker-templates/flask/start.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash service nginx start +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/uwsgi.ini for sample uwsgi.ini file uwsgi --ini uwsgi.ini diff --git a/sample-docker-templates/node/Dockerfile b/sample-docker-templates/node/Dockerfile index 8d05a864a7..123a903771 100644 --- a/sample-docker-templates/node/Dockerfile +++ b/sample-docker-templates/node/Dockerfile @@ -24,6 +24,7 @@ WORKDIR /app # Adding complete files and dirs in app dir in container ADD . /app/ +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/node/nginx.default for sample nginx.default COPY nginx.default /etc/nginx/sites-available/default # Installing dependencies From cf09e3af1c726c98e2c13e57f70f92efe527c554 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Wed, 26 Oct 2022 21:42:40 +0530 Subject: [PATCH 35/50] ci build config id handling for app clone --- pkg/appClone/AppCloneService.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 75b3679a29..20cc444c7d 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -274,12 +274,14 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId } } + ciBuildConfig := refCiConf.CiBuildConfig + ciBuildConfig.Id = 0 ciConfRequest := &bean.CiConfigRequest{ Id: 0, AppId: newAppId, DockerRegistry: refCiConf.DockerRegistry, DockerRepository: refCiConf.DockerRepository, - CiBuildConfig: refCiConf.CiBuildConfig, + CiBuildConfig: ciBuildConfig, //DockerBuildConfig: &bean.DockerBuildConfig{ // GitMaterialId: dockerfileGitMaterial, // DockerfilePath: refCiConf.DockerBuildConfig.DockerfilePath, From 446f8beb09eb111c7f55d0093c62d916f10faf17 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Wed, 26 Oct 2022 22:17:54 +0530 Subject: [PATCH 36/50] ciBuildconfig id handling for new entry --- pkg/appClone/AppCloneService.go | 1 - pkg/pipeline/CiBuildConfigService.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 20cc444c7d..bd0326a35e 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -275,7 +275,6 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId } ciBuildConfig := refCiConf.CiBuildConfig - ciBuildConfig.Id = 0 ciConfRequest := &bean.CiConfigRequest{ Id: 0, AppId: newAppId, diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index 104357ade5..c4b1d1fef6 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -35,6 +35,7 @@ func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId in } ciBuildConfigEntity.CreatedOn = time.Now() ciBuildConfigEntity.CreatedBy = userId + ciBuildConfigEntity.Id = 0 err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) ciBuildConfigBean.Id = ciBuildConfigEntity.Id if err != nil { From 7bbb262b4e88ccdc35c2caf7150fab8b0cd27167 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 00:00:24 +0530 Subject: [PATCH 37/50] build type count metadata in telemetry --- .../telemetry/TelemetryEventClientExtended.go | 17 +++++++++++++ .../pipelineConfig/CiBuildConfigRepository.go | 23 +++++++++++++++++ pkg/pipeline/CiBuildConfigService.go | 13 ++++++++++ pkg/pipeline/CiBuildConfigService_test.go | 25 +++++++++++++++++++ pkg/pipeline/mocks/CiBuildConfigService.go | 16 ++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 pkg/pipeline/CiBuildConfigService_test.go diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index e8a6c135ba..537adb1922 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -9,6 +9,8 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/cluster" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" "github.com/devtron-labs/devtron/pkg/sso" "github.com/devtron-labs/devtron/pkg/user" @@ -37,6 +39,7 @@ type TelemetryEventClientImplExtended struct { materialRepository pipelineConfig.MaterialRepository ciTemplateRepository pipelineConfig.CiTemplateRepository chartRepository chartRepoRepository.ChartRepository + ciBuildConfigService pipeline.CiBuildConfigService *TelemetryEventClientImpl } @@ -138,6 +141,9 @@ type TelemetryEventDto struct { InstallingIntegrations []string `json:"installingIntegrations,omitempty"` DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + SelfDockerfileCount int `json:"selfDockerfileCount"` + ManagedDockerfileCount int `json:"managedDockerfileCount"` + BuildPackCount int `json:"buildPackCount"` } func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { @@ -259,6 +265,8 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) return err } + selfDockerfileCount, managedDockerfileCount, buildpackCount := impl.getCiBuildTypeData() + devtronVersion := util.GetDevtronVersion() payload.ProdAppCount = prodApps payload.NonProdAppCount = nonProdApps @@ -268,6 +276,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.EnvironmentCount = len(environments) payload.ClusterCount = len(clusters) payload.CiCountPerDay = len(ciPipeline) + payload.CdCountPerDay = len(cdPipeline) payload.GitAccountsCount = len(gitAccounts) payload.GitOpsCount = len(gitOps) @@ -290,6 +299,9 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) } payload.LastLoginTime = loginTime } + payload.SelfDockerfileCount = selfDockerfileCount + payload.ManagedDockerfileCount = managedDockerfileCount + payload.BuildPackCount = buildpackCount reqBody, err := json.Marshal(payload) if err != nil { @@ -310,3 +322,8 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) } return nil } + +func (impl *TelemetryEventClientImplExtended) getCiBuildTypeData() (int, int, int) { + countByBuildType := impl.ciBuildConfigService.GetCountByBuildType() + return countByBuildType[bean.SELF_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.MANAGED_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.BUILDPACK_BUILD_TYPE] +} diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go index da7c5f3858..f35cc0bfc4 100644 --- a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go @@ -16,10 +16,16 @@ type CiBuildConfig struct { sql.AuditLog } +type BuildTypeCount struct { + Type string `json:"type"` + Count int `json:"count"` +} + type CiBuildConfigRepository interface { Save(ciBuildConfig *CiBuildConfig) error Update(ciBuildConfig *CiBuildConfig) error Delete(ciBuildConfigId int) error + GetCountByBuildType() (map[string]int, error) } type CiBuildConfigRepositoryImpl struct { @@ -58,3 +64,20 @@ func (impl CiBuildConfigRepositoryImpl) Delete(ciBuildConfigId int) error { } return err } + +func (impl CiBuildConfigRepositoryImpl) GetCountByBuildType() (map[string]int, error) { + + var buildTypeCounts []*BuildTypeCount + result := make(map[string]int) + query := "SELECT type, count(*) as count from ci_build_config group by type" + _, err := impl.dbConnection.Query(&buildTypeCounts, query) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error occurred while fetching config type vs count", "err", err) + } else if err == pg.ErrNoRows { + return result, nil + } + for _, elem := range buildTypeCounts { + result[elem.Type] = elem.Count + } + return result, err +} diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index c4b1d1fef6..4d7862d69e 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -12,6 +12,7 @@ type CiBuildConfigService interface { Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) Delete(ciBuildConfigId int) error + GetCountByBuildType() map[bean.CiBuildType]int } type CiBuildConfigServiceImpl struct { @@ -73,3 +74,15 @@ func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTempl func (impl *CiBuildConfigServiceImpl) Delete(ciBuildConfigId int) error { return impl.CiBuildConfigRepository.Delete(ciBuildConfigId) } + +func (impl *CiBuildConfigServiceImpl) GetCountByBuildType() map[bean.CiBuildType]int { + result := make(map[bean.CiBuildType]int) + buildTypeVsCount, err := impl.CiBuildConfigRepository.GetCountByBuildType() + if err != nil { + return result + } + for buildType, count := range buildTypeVsCount { + result[bean.CiBuildType(buildType)] = count + } + return result +} diff --git a/pkg/pipeline/CiBuildConfigService_test.go b/pkg/pipeline/CiBuildConfigService_test.go new file mode 100644 index 0000000000..e27a6ba09d --- /dev/null +++ b/pkg/pipeline/CiBuildConfigService_test.go @@ -0,0 +1,25 @@ +package pipeline + +import ( + "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCiBuildConfigService(t *testing.T) { + t.SkipNow() + t.Run("buildTypeVsCount", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + countByBuildType := ciBuildConfigServiceImpl.GetCountByBuildType() + fmt.Println(countByBuildType) + }) +} diff --git a/pkg/pipeline/mocks/CiBuildConfigService.go b/pkg/pipeline/mocks/CiBuildConfigService.go index d9d9e86268..4064426ce4 100644 --- a/pkg/pipeline/mocks/CiBuildConfigService.go +++ b/pkg/pipeline/mocks/CiBuildConfigService.go @@ -26,6 +26,22 @@ func (_m *CiBuildConfigService) Delete(ciBuildConfigId int) error { return r0 } +// GetCountByBuildType provides a mock function with given fields: +func (_m *CiBuildConfigService) GetCountByBuildType() map[bean.CiBuildType]int { + ret := _m.Called() + + var r0 map[bean.CiBuildType]int + if rf, ok := ret.Get(0).(func() map[bean.CiBuildType]int); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[bean.CiBuildType]int) + } + } + + return r0 +} + // Save provides a mock function with given fields: templateId, overrideTemplateId, ciBuildConfigBean, userId func (_m *CiBuildConfigService) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { ret := _m.Called(templateId, overrideTemplateId, ciBuildConfigBean, userId) From 2027b01f4f9a226e079f1c2c413a58d33dd518cb Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 17:26:43 +0530 Subject: [PATCH 38/50] docker mtu value commit --- pkg/pipeline/CiConfig.go | 1 + pkg/pipeline/WorkflowService.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/pipeline/CiConfig.go b/pkg/pipeline/CiConfig.go index 371468c768..0d4e7f166c 100644 --- a/pkg/pipeline/CiConfig.go +++ b/pkg/pipeline/CiConfig.go @@ -76,6 +76,7 @@ type CiConfig struct { BlobStorageGcpCredentialJson string `env:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` BuildLogTTLValue int `json:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600"` AzureAccountKey string `env:"AZURE_ACCOUNT_KEY"` + CiRunnerDockerMTUValue int `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1"` ClusterConfig *rest.Config NodeLabel map[string]string } diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 444bcb02d5..ad3201c9ac 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -110,6 +110,7 @@ type WorkflowRequest struct { AppName string `json:"appName"` TriggerByAuthor string `json:"triggerByAuthor"` CiBuildConfig *bean2.CiBuildConfigBean `json:"ciBuildConfig"` + CiRunnerDockerMtuValue int `json:"ciRunnerDockerMtuValue"` } const ( From 0830976b2e1925f3c1867d521354fe663c60078c Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 17:48:27 +0530 Subject: [PATCH 39/50] git material handling in app clone --- pkg/appClone/AppCloneService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index bd0326a35e..122c5efce4 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -275,6 +275,7 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId } ciBuildConfig := refCiConf.CiBuildConfig + ciBuildConfig.GitMaterialId = dockerfileGitMaterial ciConfRequest := &bean.CiConfigRequest{ Id: 0, AppId: newAppId, From b225b483919d847d4833c39ba68e3b6b3b9d923f Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 19:10:07 +0530 Subject: [PATCH 40/50] git material id set in case of create --- pkg/appClone/AppCloneService.go | 2 ++ pkg/pipeline/DbPipelineOrchestrator.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 122c5efce4..e6d03c82c3 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -763,6 +763,8 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( return nil, err } ciBuildConfig.GitMaterialId = gitMaterial.Id + templateOverride.GitMaterialId = gitMaterial.Id + ciBuildConfig.Id = 0 ciPatchReq.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, diff --git a/pkg/pipeline/DbPipelineOrchestrator.go b/pkg/pipeline/DbPipelineOrchestrator.go index 5c13f31c7a..2b5c2f831a 100644 --- a/pkg/pipeline/DbPipelineOrchestrator.go +++ b/pkg/pipeline/DbPipelineOrchestrator.go @@ -542,8 +542,8 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig DockerRegistryId: ciPipeline.DockerConfigOverride.DockerRegistry, DockerRepository: ciPipeline.DockerConfigOverride.DockerRepository, //DockerfilePath: ciPipeline.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - //GitMaterialId: ciPipeline.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + GitMaterialId: ciPipeline.DockerConfigOverride.CiBuildConfig.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedBy: createRequest.UserId, CreatedOn: time.Now(), From fc9cf4d7292940d2c6cf2fc05688a1ddbc231fe3 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 20:29:24 +0530 Subject: [PATCH 41/50] ci build docker mtu value --- pkg/pipeline/WorkflowService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index ad3201c9ac..49126cf1d0 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -110,7 +110,7 @@ type WorkflowRequest struct { AppName string `json:"appName"` TriggerByAuthor string `json:"triggerByAuthor"` CiBuildConfig *bean2.CiBuildConfigBean `json:"ciBuildConfig"` - CiRunnerDockerMtuValue int `json:"ciRunnerDockerMtuValue"` + CiBuildDockerMtuValue int `json:"ciBuildDockerMtuValue"` } const ( From d5656edcf8a285d0146f6fef15b4e0a86718d070 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 31 Oct 2022 23:03:09 +0530 Subject: [PATCH 42/50] mtu value passed --- pkg/pipeline/CiService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index e8d915f58d..1806a9bda6 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -474,6 +474,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. AppName: pipeline.App.AppName, TriggerByAuthor: user.EmailId, CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.ciConfig.CiRunnerDockerMTUValue, //DockerBuildOptions: pipeline.CiTemplate.DockerBuildOptions, } From fffd7a6301a86b10fa1cfd06d1630015f917e97f Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 1 Nov 2022 11:12:40 +0530 Subject: [PATCH 43/50] refactored sql patch version --- scripts/sql/89_ci_build_config.down.sql | 4 ---- scripts/sql/90_ci_build_config.down.sql | 4 ++++ ..._build_config.up.sql => 90_ci_build_config.up.sql} | 11 +++++------ 3 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 scripts/sql/89_ci_build_config.down.sql create mode 100644 scripts/sql/90_ci_build_config.down.sql rename scripts/sql/{89_ci_build_config.up.sql => 90_ci_build_config.up.sql} (59%) diff --git a/scripts/sql/89_ci_build_config.down.sql b/scripts/sql/89_ci_build_config.down.sql deleted file mode 100644 index 5a08563c96..0000000000 --- a/scripts/sql/89_ci_build_config.down.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE ci_template DROP COLUMN ci_build_config_id; -ALTER TABLE ci_template_override DROP COLUMN ci_build_config_id; - -DROP TABLE IF EXISTS "public"."ci_build_config"; \ No newline at end of file diff --git a/scripts/sql/90_ci_build_config.down.sql b/scripts/sql/90_ci_build_config.down.sql new file mode 100644 index 0000000000..642a3b8f34 --- /dev/null +++ b/scripts/sql/90_ci_build_config.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE ci_template DROP COLUMN IF EXISTS ci_build_config_id; +ALTER TABLE ci_template_override DROP COLUMN IF EXISTS ci_build_config_id; + +DROP TABLE IF EXISTS "public"."ci_build_config"; \ No newline at end of file diff --git a/scripts/sql/89_ci_build_config.up.sql b/scripts/sql/90_ci_build_config.up.sql similarity index 59% rename from scripts/sql/89_ci_build_config.up.sql rename to scripts/sql/90_ci_build_config.up.sql index 7cb1486fd1..c153564ab2 100644 --- a/scripts/sql/89_ci_build_config.up.sql +++ b/scripts/sql/90_ci_build_config.up.sql @@ -1,7 +1,7 @@ CREATE SEQUENCE IF NOT EXISTS id_seq_ci_build_config; -- Table Definition -CREATE TABLE public.ci_build_config +CREATE TABLE IF NOT EXISTS public.ci_build_config ( "id" integer NOT NULL DEFAULT nextval('id_seq_ci_build_config'::regclass), "type" varchar(100), @@ -16,16 +16,15 @@ CREATE TABLE public.ci_build_config ); - ALTER TABLE ci_template - ADD COLUMN ci_build_config_id integer; + ADD COLUMN IF NOT EXISTS ci_build_config_id integer; ALTER TABLE ONLY public.ci_template - ADD CONSTRAINT ci_template_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + ADD CONSTRAINT IF NOT EXISTS ci_template_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); ALTER TABLE ci_template_override - ADD COLUMN ci_build_config_id integer; + ADD COLUMN IF NOT EXISTS ci_build_config_id integer; ALTER TABLE ONLY public.ci_template_override - ADD CONSTRAINT ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + ADD CONSTRAINT IF NOT EXISTS ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); From 7a7b588aa3dc03ec0c13a8669c5cb38753b32ce3 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 1 Nov 2022 11:52:21 +0530 Subject: [PATCH 44/50] refactoring done --- api/restHandler/CoreAppRestHandler.go | 24 ------------------------ pkg/appClone/AppCloneService.go | 20 +++++--------------- pkg/bean/app.go | 24 ------------------------ pkg/pipeline/CiService.go | 5 ----- 4 files changed, 5 insertions(+), 68 deletions(-) diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index e2bf8fb810..9f91e497d3 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -498,13 +498,6 @@ func (handler CoreAppRestHandlerImpl) buildDockerConfig(appId int) (*appBean.Doc DockerRepository: ciConfig.DockerRepository, CiBuildConfig: ciConfig.CiBuildConfig, CheckoutPath: gitMaterial.CheckoutPath, - //BuildConfig: &appBean.DockerBuildConfig{ - // Args: ciConfig.DockerBuildConfig.Args, - // DockerfileRelativePath: ciConfig.DockerBuildConfig.DockerfilePath, - // TargetPlatform: ciConfig.DockerBuildConfig.TargetPlatform, - // DockerBuildOptions: ciConfig.DockerBuildConfig.DockerBuildOptions, - // GitCheckoutPath: gitMaterial.CheckoutPath, - //}, } return dockerConfig, nil, http.StatusOK @@ -1262,23 +1255,6 @@ func (handler CoreAppRestHandlerImpl) createDockerConfig(appId int, dockerConfig return err, http.StatusInternalServerError } - //dockerBuildArgs := make(map[string]string) - //if dockerConfig.BuildConfig.Args != nil { - // dockerBuildArgs = dockerConfig.BuildConfig.Args - //} - - //dockerBuildOptions := make(map[string]string) - //if dockerConfig.BuildConfig.DockerBuildOptions != nil { - // dockerBuildOptions = dockerConfig.BuildConfig.DockerBuildOptions - //} - - //dockerBuildConfigRequest := &bean.DockerBuildConfig{ - // GitMaterialId: gitMaterial.Id, - // DockerfilePath: dockerConfig.BuildConfig.DockerfileRelativePath, - // Args: dockerBuildArgs, - // DockerBuildOptions: dockerBuildOptions, - // TargetPlatform: dockerConfig.BuildConfig.TargetPlatform, - //} ciBuildConfig := dockerConfig.CiBuildConfig ciBuildConfig.GitMaterialId = gitMaterial.Id createDockerConfigRequest.CiBuildConfig = ciBuildConfig diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index fbc1d6fb59..ca739a317d 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -275,17 +275,11 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId ciBuildConfig := refCiConf.CiBuildConfig ciBuildConfig.GitMaterialId = dockerfileGitMaterial ciConfRequest := &bean.CiConfigRequest{ - Id: 0, - AppId: newAppId, - DockerRegistry: refCiConf.DockerRegistry, - DockerRepository: refCiConf.DockerRepository, - CiBuildConfig: ciBuildConfig, - //DockerBuildConfig: &bean.DockerBuildConfig{ - // GitMaterialId: dockerfileGitMaterial, - // DockerfilePath: refCiConf.DockerBuildConfig.DockerfilePath, - // Args: refCiConf.DockerBuildConfig.Args, - // TargetPlatform: refCiConf.DockerBuildConfig.TargetPlatform, - //}, + Id: 0, + AppId: newAppId, + DockerRegistry: refCiConf.DockerRegistry, + DockerRepository: refCiConf.DockerRepository, + CiBuildConfig: ciBuildConfig, DockerRegistryUrl: refCiConf.DockerRegistry, CiTemplateName: refCiConf.CiTemplateName, UserId: userId, @@ -773,10 +767,6 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, CiBuildConfig: ciBuildConfig, - //DockerBuildConfig: &bean.DockerBuildConfig{ - // DockerfilePath: templateOverride.DockerfilePath, - // GitMaterialId: gitMaterial.Id, - //}, } } diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 26af1acf0c..7d982e02df 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -274,30 +274,6 @@ type TestExecutorImageProperties struct { ReportDir string `json:"reportDir,omitempty"` } -//type CiBuildConfigBean struct { -// GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` -// CiBuildType string `json:"ciBuildType"` -// DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` -// BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` -//} -// -//type DockerBuildConfig struct { -// GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` -// DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` -// Args map[string]string `json:"args,omitempty"` -// TargetPlatform string `json:"targetPlatform,omitempty"` -// DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` -// //Name Tag DockerfilePath RepoUrl -//} -// -//type BuildPackConfig struct { -// BuilderId string `json:"builderId"` -// Language string `json:"language"` -// LanguageVersion string `json:"languageVersion"` -// BuildPacks []string `json:"buildPacks"` -// Args map[string]string `json:"args"` -//} - type PipelineCreateResponse struct { AppName string `json:"appName,omitempty"` AppId int `json:"appId,omitempty"` diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 65e97bc603..bc21ed023a 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -386,11 +386,6 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciLevelArgs = "{}" } - //mergedArgs, err := impl.mergeUtil.JsonPatch([]byte(args), []byte(ciLevelArgs)) - //if err != nil { - // impl.Logger.Errorw("err", "err", err) - // return nil, err - //} if pipeline.CiTemplate.DockerBuildOptions == "" { pipeline.CiTemplate.DockerBuildOptions = "{}" } From 4d72c4bdb519ffb4518b82e2b90b55809ab1c5d5 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Tue, 1 Nov 2022 15:42:52 +0530 Subject: [PATCH 45/50] checkout path handling for dockerfile and buildpack --- pkg/pipeline/CiService.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index bc21ed023a..9838f4d25c 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -436,6 +436,10 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == bean2.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } workflowRequest := &WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, From b64baf597f4b7f2c93447d8192814878b6cc58c7 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Thu, 3 Nov 2022 09:26:20 +0530 Subject: [PATCH 46/50] ci build type dumped in ci workflow --- .../repository/pipelineConfig/CiWorkflowRepository.go | 1 + pkg/pipeline/CiService.go | 9 +++++++++ scripts/sql/90_ci_build_config.down.sql | 5 ++++- scripts/sql/90_ci_build_config.up.sql | 3 +++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index d1c4fe3916..c4d304e6d5 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -62,6 +62,7 @@ type CiWorkflow struct { TriggeredBy int32 `sql:"triggered_by"` CiArtifactLocation string `sql:"ci_artifact_location"` PodName string `sql:"pod_name"` + CiBuildType string `sql:"ci_build_type"` CiPipeline *CiPipeline } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 9838f4d25c..1f758bd96b 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -148,6 +148,8 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { return 0, err } + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) if err != nil { return 0, err @@ -627,6 +629,13 @@ func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]bean.GitCommit, id return dockerImageTag } +func (impl *CiServiceImpl) updateCiWorkflow(request *WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciWorkflowRepository.UpdateWorkFlow(savedWf) +} + func _getTruncatedImageTag(imageTag string) string { _length := len(imageTag) if _length == 0 { diff --git a/scripts/sql/90_ci_build_config.down.sql b/scripts/sql/90_ci_build_config.down.sql index 642a3b8f34..6cfec2dff0 100644 --- a/scripts/sql/90_ci_build_config.down.sql +++ b/scripts/sql/90_ci_build_config.down.sql @@ -1,4 +1,7 @@ ALTER TABLE ci_template DROP COLUMN IF EXISTS ci_build_config_id; ALTER TABLE ci_template_override DROP COLUMN IF EXISTS ci_build_config_id; -DROP TABLE IF EXISTS "public"."ci_build_config"; \ No newline at end of file +DROP TABLE IF EXISTS "public"."ci_build_config"; + +ALTER TABLE ci_workflow + DROP COLUMN IF EXISTS ci_build_type; \ No newline at end of file diff --git a/scripts/sql/90_ci_build_config.up.sql b/scripts/sql/90_ci_build_config.up.sql index c153564ab2..43db192323 100644 --- a/scripts/sql/90_ci_build_config.up.sql +++ b/scripts/sql/90_ci_build_config.up.sql @@ -28,3 +28,6 @@ ALTER TABLE ci_template_override ALTER TABLE ONLY public.ci_template_override ADD CONSTRAINT IF NOT EXISTS ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + +ALTER TABLE ci_workflow + ADD COLUMN IF NOT EXISTS ci_build_type varchar(100); From 3945821b490769ec97893d4075207fc3c80ed9b4 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Thu, 3 Nov 2022 10:29:44 +0530 Subject: [PATCH 47/50] ci build type added in model --- internal/sql/repository/pipelineConfig/CiWorkflowRepository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index c4d304e6d5..67a9665411 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -85,6 +85,7 @@ type WorkflowWithArtifact struct { CiArtifactLocation string `json:"ci_artifact_location"` CiArtifactId int `json:"ci_artifact_d"` BlobStorageEnabled bool `json:"blobStorageEnabled"` + CiBuildType string `json:"ci_build_type"` } type GitCommit struct { From 50a6f8197617f00476c52f6d5264eca9e55de30e Mon Sep 17 00:00:00 2001 From: Kripansh Date: Thu, 3 Nov 2022 11:50:39 +0530 Subject: [PATCH 48/50] telemetry event data published --- .../telemetry/TelemetryEventClientExtended.go | 37 +++++++++++++++++++ .../pipelineConfig/CiBuildConfigRepository.go | 5 ++- .../CiBuildConfigRepository_test.go | 25 +++++++++++++ .../pipelineConfig/CiWorkflowRepository.go | 11 ++++++ .../CiWorkflowRepository_test.go | 24 ++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go create mode 100644 internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 537adb1922..fa8289a8cf 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -144,6 +144,12 @@ type TelemetryEventDto struct { SelfDockerfileCount int `json:"selfDockerfileCount"` ManagedDockerfileCount int `json:"managedDockerfileCount"` BuildPackCount int `json:"buildPackCount"` + SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` + SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` + ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` + ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` + BuildPackSuccessCount int `json:"buildPackSuccessCount"` + BuildPackFailureCount int `json:"buildPackFailureCount"` } func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { @@ -267,6 +273,8 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) selfDockerfileCount, managedDockerfileCount, buildpackCount := impl.getCiBuildTypeData() + successCount, failureCount := impl.getCiBuildTypeVsStatusVsCount() + devtronVersion := util.GetDevtronVersion() payload.ProdAppCount = prodApps payload.NonProdAppCount = nonProdApps @@ -299,9 +307,18 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) } payload.LastLoginTime = loginTime } + payload.SelfDockerfileCount = selfDockerfileCount + payload.SelfDockerfileSuccessCount = successCount[bean.SELF_DOCKERFILE_BUILD_TYPE] + payload.SelfDockerfileFailureCount = failureCount[bean.SELF_DOCKERFILE_BUILD_TYPE] + payload.ManagedDockerfileCount = managedDockerfileCount + payload.ManagedDockerfileSuccessCount = successCount[bean.MANAGED_DOCKERFILE_BUILD_TYPE] + payload.ManagedDockerfileFailureCount = failureCount[bean.MANAGED_DOCKERFILE_BUILD_TYPE] + payload.BuildPackCount = buildpackCount + payload.BuildPackSuccessCount = successCount[bean.BUILDPACK_BUILD_TYPE] + payload.BuildPackFailureCount = failureCount[bean.BUILDPACK_BUILD_TYPE] reqBody, err := json.Marshal(payload) if err != nil { @@ -327,3 +344,23 @@ func (impl *TelemetryEventClientImplExtended) getCiBuildTypeData() (int, int, in countByBuildType := impl.ciBuildConfigService.GetCountByBuildType() return countByBuildType[bean.SELF_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.MANAGED_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.BUILDPACK_BUILD_TYPE] } + +func (impl *TelemetryEventClientImplExtended) getCiBuildTypeVsStatusVsCount() (successCount map[bean.CiBuildType]int, failureCount map[bean.CiBuildType]int) { + successCount = make(map[bean.CiBuildType]int) + failureCount = make(map[bean.CiBuildType]int) + buildTypeAndStatusVsCount := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day() + for _, buildTypeCount := range buildTypeAndStatusVsCount { + if buildTypeCount == nil { + continue + } + if buildTypeCount.Type == "" { + buildTypeCount.Type = string(bean.SELF_DOCKERFILE_BUILD_TYPE) + } + if buildTypeCount.Status == "Succeeded" { + successCount[bean.CiBuildType(buildTypeCount.Type)] = buildTypeCount.Count + } else { + failureCount[bean.CiBuildType(buildTypeCount.Type)] = buildTypeCount.Count + } + } + return successCount, failureCount +} diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go index f35cc0bfc4..386196b806 100644 --- a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go @@ -17,8 +17,9 @@ type CiBuildConfig struct { } type BuildTypeCount struct { - Type string `json:"type"` - Count int `json:"count"` + Status string `json:"status"` + Type string `json:"type"` + Count int `json:"count"` } type CiBuildConfigRepository interface { diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go new file mode 100644 index 0000000000..b9820ad5b4 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go @@ -0,0 +1,25 @@ +package pipelineConfig + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewCiBuildConfigRepository(t *testing.T) { + t.SkipNow() + t.Run("GetCountByBuildType", func(t *testing.T) { + cfg, _ := sql.GetConfig() + logger, err := utils.NewSugardLogger() + con, _ := sql.NewDbConnection(cfg, logger) + assert.Nil(t, err) + configRepositoryImpl := NewCiBuildConfigRepositoryImpl(con, logger) + countByBuildType, err := configRepositoryImpl.GetCountByBuildType() + assert.Nil(t, err) + for buildType, count := range countByBuildType { + fmt.Println("type:", buildType, ", count:", count) + } + }) +} diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 67a9665411..a6a3df2177 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -38,6 +38,7 @@ type CiWorkflowRepository interface { FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) ExistsByStatus(status string) (bool, error) + FindBuildTypeAndStatusDataOfLast1Day() []*BuildTypeCount } type CiWorkflowRepositoryImpl struct { @@ -230,3 +231,13 @@ func (impl *CiWorkflowRepositoryImpl) ExistsByStatus(status string) (bool, error Exists() return exists, err } + +func (impl *CiWorkflowRepositoryImpl) FindBuildTypeAndStatusDataOfLast1Day() []*BuildTypeCount { + var buildTypeCounts []*BuildTypeCount + query := "select status,ci_build_type as type, count(*) from ci_workflow where status in ('Succeeded','Failed') and started_on > ? group by (ci_build_type, status)" + _, err := impl.dbConnection.Query(&buildTypeCounts, query, time.Now().AddDate(0, 0, -1)) + if err != nil { + impl.logger.Errorw("error occurred while fetching build type vs status vs count data", "err", err) + } + return buildTypeCounts +} diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go new file mode 100644 index 0000000000..f2c26e03c3 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go @@ -0,0 +1,24 @@ +package pipelineConfig + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCiWorkflowRepository(t *testing.T) { + t.SkipNow() + t.Run("FindBuildTypeAndStatusDataOfLast1Day", func(t *testing.T) { + cfg, _ := sql.GetConfig() + logger, err := utils.NewSugardLogger() + con, _ := sql.NewDbConnection(cfg, logger) + assert.Nil(t, err) + workflowRepositoryImpl := NewCiWorkflowRepositoryImpl(con, logger) + statusData := workflowRepositoryImpl.FindBuildTypeAndStatusDataOfLast1Day() + for _, statusDatum := range statusData { + fmt.Println(statusDatum.Count) + } + }) +} From 6df6f9c11d75248b53a4e8a99dd55d35bf5dc92e Mon Sep 17 00:00:00 2001 From: Kripansh Date: Thu, 3 Nov 2022 14:21:38 +0530 Subject: [PATCH 49/50] testcases run --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5cd6b07412..96fa06ffbc 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ SERVER_MODE_EA_ONLY=EA_ONLY include $(ENV_FILE) export -build: clean wire +build: clean wire test $(ENVVAR) GOOS=$(GOOS) go build -o devtron \ -ldflags="-X 'github.com/devtron-labs/devtron/util.GitCommit=${GIT_COMMIT}' \ -X 'github.com/devtron-labs/devtron/util.BuildTime=${BUILD_TIME}' \ @@ -30,6 +30,9 @@ wire: clean: rm -f devtron +test: + go test ./pkg/pipeline + run: build ./devtron From 2793904878ebcc734ac8ae92ea157d91fc764ed5 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Fri, 4 Nov 2022 16:08:22 +0530 Subject: [PATCH 50/50] git material id fix --- pkg/pipeline/PipelineBuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 543272dc5b..274b04e94b 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -667,10 +667,11 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq // impl.logger.Errorw("error in marshaling dockerBuildOptions", "err", err) // return nil, err //} + ciBuildConfig := updateRequest.CiBuildConfig originalCiBuildConfig := originalCiConf.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ //DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, - GitMaterialId: originalCiBuildConfig.GitMaterialId, + GitMaterialId: ciBuildConfig.GitMaterialId, //Args: string(argByte), //TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, BeforeDockerBuild: string(beforeByte), @@ -682,7 +683,6 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq Active: true, } - ciBuildConfig := updateRequest.CiBuildConfig ciBuildConfig.Id = originalCiBuildConfig.Id ciTemplateBean := &bean3.CiTemplateBean{ CiTemplate: ciTemplate,