diff --git a/Makefile b/Makefile index 5cd6b07412..96fa06ffbc 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ SERVER_MODE_EA_ONLY=EA_ONLY include $(ENV_FILE) export -build: clean wire +build: clean wire test $(ENVVAR) GOOS=$(GOOS) go build -o devtron \ -ldflags="-X 'github.com/devtron-labs/devtron/util.GitCommit=${GIT_COMMIT}' \ -X 'github.com/devtron-labs/devtron/util.BuildTime=${BUILD_TIME}' \ @@ -30,6 +30,9 @@ wire: clean: rm -f devtron +test: + go test ./pkg/pipeline + run: build ./devtron diff --git a/Wire.go b/Wire.go index 1c3f833a6d..71eab3df26 100644 --- a/Wire.go +++ b/Wire.go @@ -740,13 +740,16 @@ func InitializeApp() (*App, error) { wire.Bind(new(repository.UserAttributesRepository), new(*repository.UserAttributesRepositoryImpl)), pipelineConfig.NewPipelineStatusTimelineRepositoryImpl, wire.Bind(new(pipelineConfig.PipelineStatusTimelineRepository), new(*pipelineConfig.PipelineStatusTimelineRepositoryImpl)), - wire.Bind(new(pipeline.DeploymentConfigService), new(*pipeline.DeploymentConfigServiceImpl)), pipeline.NewDeploymentConfigServiceImpl, - pipelineConfig.NewCiTemplateOverrideRepositoryImpl, wire.Bind(new(pipelineConfig.CiTemplateOverrideRepository), new(*pipelineConfig.CiTemplateOverrideRepositoryImpl)), - + pipelineConfig.NewCiBuildConfigRepositoryImpl, + wire.Bind(new(pipelineConfig.CiBuildConfigRepository), new(*pipelineConfig.CiBuildConfigRepositoryImpl)), + pipeline.NewCiBuildConfigServiceImpl, + wire.Bind(new(pipeline.CiBuildConfigService), new(*pipeline.CiBuildConfigServiceImpl)), + pipeline.NewCiTemplateServiceImpl, + wire.Bind(new(pipeline.CiTemplateService), new(*pipeline.CiTemplateServiceImpl)), router.NewGlobalCMCSRouterImpl, wire.Bind(new(router.GlobalCMCSRouter), new(*router.GlobalCMCSRouterImpl)), restHandler.NewGlobalCMCSRestHandlerImpl, diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index daaabc256b..f6b197e4e0 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -42,17 +42,10 @@ type GitMaterial struct { } type DockerConfig struct { - DockerRegistry string `json:"dockerRegistry" validate:"required"` - DockerRepository string `json:"dockerRepository" validate:"required"` - BuildConfig *DockerBuildConfig `json:"dockerBuildConfig"` -} - -type DockerBuildConfig struct { - GitCheckoutPath string `json:"gitCheckoutPath,omitempty" validate:"required"` - DockerfileRelativePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` - Args map[string]string `json:"args,omitempty"` - TargetPlatform string `json:"targetPlatform"` - DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` + DockerRegistry string `json:"dockerRegistry" validate:"required"` + DockerRepository string `json:"dockerRepository" validate:"required"` + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig" validate:"required"` + CheckoutPath string `json:"checkoutPath"` } type DeploymentTemplate struct { diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 959f667acd..a5ef04dfa1 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -491,22 +491,17 @@ func (handler CoreAppRestHandlerImpl) buildDockerConfig(appId int) (*appBean.Doc } //getting gitMaterialUrl by id - gitMaterial, err := handler.materialRepository.FindById(ciConfig.DockerBuildConfig.GitMaterialId) + gitMaterial, err := handler.materialRepository.FindById(ciConfig.CiBuildConfig.GitMaterialId) if err != nil { - handler.logger.Errorw("error in fetching materialUrl by ID in GetAppAllDetail", "err", err, "gitMaterialId", ciConfig.DockerBuildConfig.GitMaterialId) + handler.logger.Errorw("error in fetching materialUrl by ID in GetAppAllDetail", "err", err, "gitMaterialId", ciConfig.CiBuildConfig.GitMaterialId) return nil, err, http.StatusInternalServerError } dockerConfig := &appBean.DockerConfig{ DockerRegistry: ciConfig.DockerRegistry, DockerRepository: ciConfig.DockerRepository, - BuildConfig: &appBean.DockerBuildConfig{ - Args: ciConfig.DockerBuildConfig.Args, - DockerfileRelativePath: ciConfig.DockerBuildConfig.DockerfilePath, - TargetPlatform: ciConfig.DockerBuildConfig.TargetPlatform, - GitCheckoutPath: gitMaterial.CheckoutPath, - DockerBuildOptions: ciConfig.DockerBuildConfig.DockerBuildOptions, - }, + CiBuildConfig: ciConfig.CiBuildConfig, + CheckoutPath: gitMaterial.CheckoutPath, } return dockerConfig, nil, http.StatusOK @@ -1258,28 +1253,15 @@ func (handler CoreAppRestHandlerImpl) createDockerConfig(appId int, dockerConfig } //finding gitMaterial by appId and checkoutPath - gitMaterial, err := handler.materialRepository.FindByAppIdAndCheckoutPath(appId, dockerConfig.BuildConfig.GitCheckoutPath) + gitMaterial, err := handler.materialRepository.FindByAppIdAndCheckoutPath(appId, dockerConfig.CheckoutPath) if err != nil { handler.logger.Errorw("service err, FindByAppIdAndCheckoutPath in CreateDockerConfig", "err", err, "appId", appId) return err, http.StatusInternalServerError } - dockerBuildArgs := make(map[string]string) - if dockerConfig.BuildConfig.Args != nil { - dockerBuildArgs = dockerConfig.BuildConfig.Args - } - dockerBuildOptions := make(map[string]string) - if dockerConfig.BuildConfig.DockerBuildOptions != nil { - dockerBuildOptions = dockerConfig.BuildConfig.DockerBuildOptions - } - dockerBuildConfigRequest := &bean.DockerBuildConfig{ - GitMaterialId: gitMaterial.Id, - DockerfilePath: dockerConfig.BuildConfig.DockerfileRelativePath, - Args: dockerBuildArgs, - DockerBuildOptions: dockerBuildOptions, - TargetPlatform: dockerConfig.BuildConfig.TargetPlatform, - } - createDockerConfigRequest.DockerBuildConfig = dockerBuildConfigRequest + ciBuildConfig := dockerConfig.CiBuildConfig + ciBuildConfig.GitMaterialId = gitMaterial.Id + createDockerConfigRequest.CiBuildConfig = ciBuildConfig _, err = handler.pipelineBuilder.CreateCiPipeline(createDockerConfigRequest) if err != nil { diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index e8a6c135ba..fa8289a8cf 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -9,6 +9,8 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/cluster" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" "github.com/devtron-labs/devtron/pkg/sso" "github.com/devtron-labs/devtron/pkg/user" @@ -37,6 +39,7 @@ type TelemetryEventClientImplExtended struct { materialRepository pipelineConfig.MaterialRepository ciTemplateRepository pipelineConfig.CiTemplateRepository chartRepository chartRepoRepository.ChartRepository + ciBuildConfigService pipeline.CiBuildConfigService *TelemetryEventClientImpl } @@ -138,6 +141,15 @@ type TelemetryEventDto struct { InstallingIntegrations []string `json:"installingIntegrations,omitempty"` DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + SelfDockerfileCount int `json:"selfDockerfileCount"` + ManagedDockerfileCount int `json:"managedDockerfileCount"` + BuildPackCount int `json:"buildPackCount"` + SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` + SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` + ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` + ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` + BuildPackSuccessCount int `json:"buildPackSuccessCount"` + BuildPackFailureCount int `json:"buildPackFailureCount"` } func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { @@ -259,6 +271,10 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) return err } + selfDockerfileCount, managedDockerfileCount, buildpackCount := impl.getCiBuildTypeData() + + successCount, failureCount := impl.getCiBuildTypeVsStatusVsCount() + devtronVersion := util.GetDevtronVersion() payload.ProdAppCount = prodApps payload.NonProdAppCount = nonProdApps @@ -268,6 +284,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.EnvironmentCount = len(environments) payload.ClusterCount = len(clusters) payload.CiCountPerDay = len(ciPipeline) + payload.CdCountPerDay = len(cdPipeline) payload.GitAccountsCount = len(gitAccounts) payload.GitOpsCount = len(gitOps) @@ -291,6 +308,18 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.LastLoginTime = loginTime } + payload.SelfDockerfileCount = selfDockerfileCount + payload.SelfDockerfileSuccessCount = successCount[bean.SELF_DOCKERFILE_BUILD_TYPE] + payload.SelfDockerfileFailureCount = failureCount[bean.SELF_DOCKERFILE_BUILD_TYPE] + + payload.ManagedDockerfileCount = managedDockerfileCount + payload.ManagedDockerfileSuccessCount = successCount[bean.MANAGED_DOCKERFILE_BUILD_TYPE] + payload.ManagedDockerfileFailureCount = failureCount[bean.MANAGED_DOCKERFILE_BUILD_TYPE] + + payload.BuildPackCount = buildpackCount + payload.BuildPackSuccessCount = successCount[bean.BUILDPACK_BUILD_TYPE] + payload.BuildPackFailureCount = failureCount[bean.BUILDPACK_BUILD_TYPE] + reqBody, err := json.Marshal(payload) if err != nil { impl.logger.Errorw("SummaryEventForTelemetry, payload marshal error", "error", err) @@ -310,3 +339,28 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) } return nil } + +func (impl *TelemetryEventClientImplExtended) getCiBuildTypeData() (int, int, int) { + countByBuildType := impl.ciBuildConfigService.GetCountByBuildType() + return countByBuildType[bean.SELF_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.MANAGED_DOCKERFILE_BUILD_TYPE], countByBuildType[bean.BUILDPACK_BUILD_TYPE] +} + +func (impl *TelemetryEventClientImplExtended) getCiBuildTypeVsStatusVsCount() (successCount map[bean.CiBuildType]int, failureCount map[bean.CiBuildType]int) { + successCount = make(map[bean.CiBuildType]int) + failureCount = make(map[bean.CiBuildType]int) + buildTypeAndStatusVsCount := impl.ciWorkflowRepository.FindBuildTypeAndStatusDataOfLast1Day() + for _, buildTypeCount := range buildTypeAndStatusVsCount { + if buildTypeCount == nil { + continue + } + if buildTypeCount.Type == "" { + buildTypeCount.Type = string(bean.SELF_DOCKERFILE_BUILD_TYPE) + } + if buildTypeCount.Status == "Succeeded" { + successCount[bean.CiBuildType(buildTypeCount.Type)] = buildTypeCount.Count + } else { + failureCount[bean.CiBuildType(buildTypeCount.Type)] = buildTypeCount.Count + } + } + return successCount, failureCount +} diff --git a/internal/sql/repository/mocks/CiTemplateOverrideRepository.go b/internal/sql/repository/mocks/CiTemplateOverrideRepository.go new file mode 100644 index 0000000000..94175b3255 --- /dev/null +++ b/internal/sql/repository/mocks/CiTemplateOverrideRepository.go @@ -0,0 +1,120 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + pipelineConfig "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + mock "github.com/stretchr/testify/mock" +) + +// CiTemplateOverrideRepository is an autogenerated mock type for the CiTemplateOverrideRepository type +type CiTemplateOverrideRepository struct { + mock.Mock +} + +// FindByAppId provides a mock function with given fields: appId +func (_m *CiTemplateOverrideRepository) FindByAppId(appId int) ([]*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(appId) + + var r0 []*pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(int) []*pipelineConfig.CiTemplateOverride); ok { + r0 = rf(appId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(appId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByCiPipelineId provides a mock function with given fields: ciPipelineId +func (_m *CiTemplateOverrideRepository) FindByCiPipelineId(ciPipelineId int) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(ciPipelineId) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(ciPipelineId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(ciPipelineId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: templateOverrideConfig +func (_m *CiTemplateOverrideRepository) Save(templateOverrideConfig *pipelineConfig.CiTemplateOverride) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(templateOverrideConfig) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(templateOverrideConfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*pipelineConfig.CiTemplateOverride) error); ok { + r1 = rf(templateOverrideConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Update provides a mock function with given fields: templateOverrideConfig +func (_m *CiTemplateOverrideRepository) Update(templateOverrideConfig *pipelineConfig.CiTemplateOverride) (*pipelineConfig.CiTemplateOverride, error) { + ret := _m.Called(templateOverrideConfig) + + var r0 *pipelineConfig.CiTemplateOverride + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride); ok { + r0 = rf(templateOverrideConfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplateOverride) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*pipelineConfig.CiTemplateOverride) error); ok { + r1 = rf(templateOverrideConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewCiTemplateOverrideRepository interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiTemplateOverrideRepository creates a new instance of CiTemplateOverrideRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiTemplateOverrideRepository(t mockConstructorTestingTNewCiTemplateOverrideRepository) *CiTemplateOverrideRepository { + mock := &CiTemplateOverrideRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/sql/repository/mocks/CiTemplateRepository.go b/internal/sql/repository/mocks/CiTemplateRepository.go new file mode 100644 index 0000000000..c8c2124914 --- /dev/null +++ b/internal/sql/repository/mocks/CiTemplateRepository.go @@ -0,0 +1,123 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + pipelineConfig "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + mock "github.com/stretchr/testify/mock" +) + +// CiTemplateRepository is an autogenerated mock type for the CiTemplateRepository type +type CiTemplateRepository struct { + mock.Mock +} + +// FindByAppId provides a mock function with given fields: appId +func (_m *CiTemplateRepository) FindByAppId(appId int) (*pipelineConfig.CiTemplate, error) { + ret := _m.Called(appId) + + var r0 *pipelineConfig.CiTemplate + if rf, ok := ret.Get(0).(func(int) *pipelineConfig.CiTemplate); ok { + r0 = rf(appId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pipelineConfig.CiTemplate) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(appId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByDockerRegistryId provides a mock function with given fields: dockerRegistryId +func (_m *CiTemplateRepository) FindByDockerRegistryId(dockerRegistryId string) ([]*pipelineConfig.CiTemplate, error) { + ret := _m.Called(dockerRegistryId) + + var r0 []*pipelineConfig.CiTemplate + if rf, ok := ret.Get(0).(func(string) []*pipelineConfig.CiTemplate); ok { + r0 = rf(dockerRegistryId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*pipelineConfig.CiTemplate) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(dockerRegistryId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindNumberOfAppsWithDockerConfigured provides a mock function with given fields: appIds +func (_m *CiTemplateRepository) FindNumberOfAppsWithDockerConfigured(appIds []int) (int, error) { + ret := _m.Called(appIds) + + var r0 int + if rf, ok := ret.Get(0).(func([]int) int); ok { + r0 = rf(appIds) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func([]int) error); ok { + r1 = rf(appIds) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: material +func (_m *CiTemplateRepository) Save(material *pipelineConfig.CiTemplate) error { + ret := _m.Called(material) + + var r0 error + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplate) error); ok { + r0 = rf(material) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Update provides a mock function with given fields: material +func (_m *CiTemplateRepository) Update(material *pipelineConfig.CiTemplate) error { + ret := _m.Called(material) + + var r0 error + if rf, ok := ret.Get(0).(func(*pipelineConfig.CiTemplate) error); ok { + r0 = rf(material) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewCiTemplateRepository interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiTemplateRepository creates a new instance of CiTemplateRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiTemplateRepository(t mockConstructorTestingTNewCiTemplateRepository) *CiTemplateRepository { + mock := &CiTemplateRepository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go new file mode 100644 index 0000000000..386196b806 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository.go @@ -0,0 +1,84 @@ +package pipelineConfig + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type CiBuildConfig struct { + tableName struct{} `sql:"ci_build_config" pg:",discard_unknown_columns"` + Id int `sql:"id"` + Type string `sql:"type"` + CiTemplateId int `sql:"ci_template_id"` + CiTemplateOverrideId int `sql:"ci_template_override_id"` + BuildMetadata string `sql:"build_metadata"` + sql.AuditLog +} + +type BuildTypeCount struct { + Status string `json:"status"` + Type string `json:"type"` + Count int `json:"count"` +} + +type CiBuildConfigRepository interface { + Save(ciBuildConfig *CiBuildConfig) error + Update(ciBuildConfig *CiBuildConfig) error + Delete(ciBuildConfigId int) error + GetCountByBuildType() (map[string]int, error) +} + +type CiBuildConfigRepositoryImpl struct { + dbConnection *pg.DB + logger *zap.SugaredLogger +} + +func NewCiBuildConfigRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *CiBuildConfigRepositoryImpl { + return &CiBuildConfigRepositoryImpl{ + dbConnection: dbConnection, + logger: logger, + } +} + +func (impl CiBuildConfigRepositoryImpl) Save(ciBuildConfig *CiBuildConfig) error { + + err := impl.dbConnection.Insert(ciBuildConfig) + if err != nil { + impl.logger.Errorw("error occurred while saving ciBuildConfig", "ciBuildConfig", ciBuildConfig, "err", err) + } + return err +} + +func (impl CiBuildConfigRepositoryImpl) Update(ciBuildConfig *CiBuildConfig) error { + err := impl.dbConnection.Update(ciBuildConfig) + if err != nil { + impl.logger.Errorw("error occurred while updating ciBuildConfig", "err", err) + } + return err +} + +func (impl CiBuildConfigRepositoryImpl) Delete(ciBuildConfigId int) error { + err := impl.dbConnection.Delete(ciBuildConfigId) + if err != nil { + impl.logger.Errorw("error occurred while deleting ciBuildConfig", "ciBuildConfigId", ciBuildConfigId, "err", err) + } + return err +} + +func (impl CiBuildConfigRepositoryImpl) GetCountByBuildType() (map[string]int, error) { + + var buildTypeCounts []*BuildTypeCount + result := make(map[string]int) + query := "SELECT type, count(*) as count from ci_build_config group by type" + _, err := impl.dbConnection.Query(&buildTypeCounts, query) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error occurred while fetching config type vs count", "err", err) + } else if err == pg.ErrNoRows { + return result, nil + } + for _, elem := range buildTypeCounts { + result[elem.Type] = elem.Count + } + return result, err +} diff --git a/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go new file mode 100644 index 0000000000..b9820ad5b4 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiBuildConfigRepository_test.go @@ -0,0 +1,25 @@ +package pipelineConfig + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewCiBuildConfigRepository(t *testing.T) { + t.SkipNow() + t.Run("GetCountByBuildType", func(t *testing.T) { + cfg, _ := sql.GetConfig() + logger, err := utils.NewSugardLogger() + con, _ := sql.NewDbConnection(cfg, logger) + assert.Nil(t, err) + configRepositoryImpl := NewCiBuildConfigRepositoryImpl(con, logger) + countByBuildType, err := configRepositoryImpl.GetCountByBuildType() + assert.Nil(t, err) + for buildType, count := range countByBuildType { + fmt.Println("type:", buildType, ", count:", count) + } + }) +} diff --git a/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go b/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go index 6e7142faa3..cf25319ce1 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineMaterial.go @@ -79,7 +79,7 @@ func (impl CiPipelineMaterialRepositoryImpl) GetById(id int) (*CiPipelineMateria func (impl CiPipelineMaterialRepositoryImpl) GetByPipelineId(id int) ([]*CiPipelineMaterial, error) { var ciPipelineMaterials []*CiPipelineMaterial err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.ci_pipeline_id = ?", id). Where("ci_pipeline_material.active = ?", true). Where("ci_pipeline_material.type != ?", SOURCE_TYPE_BRANCH_REGEX). @@ -127,7 +127,7 @@ func (impl CiPipelineMaterialRepositoryImpl) Update(tx *pg.Tx, materials ...*CiP func (impl CiPipelineMaterialRepositoryImpl) GetRegexByPipelineId(id int) ([]*CiPipelineMaterial, error) { var ciPipelineMaterials []*CiPipelineMaterial err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.ci_pipeline_id = ?", id). Where("ci_pipeline_material.active = ?", true). Where("ci_pipeline_material.type = ?", SOURCE_TYPE_BRANCH_REGEX). @@ -138,7 +138,7 @@ func (impl CiPipelineMaterialRepositoryImpl) GetRegexByPipelineId(id int) ([]*Ci func (impl CiPipelineMaterialRepositoryImpl) CheckRegexExistsForMaterial(id int) bool { var ciPipelineMaterials []*CiPipelineMaterial exists, err := impl.dbConnection.Model(&ciPipelineMaterials). - Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "GitMaterial", "GitMaterial.GitProvider"). + Column("ci_pipeline_material.*", "CiPipeline", "CiPipeline.CiTemplate", "CiPipeline.CiTemplate.GitMaterial", "CiPipeline.App", "CiPipeline.CiTemplate.DockerRegistry", "CiPipeline.CiTemplate.CiBuildConfig", "GitMaterial", "GitMaterial.GitProvider"). Where("ci_pipeline_material.id = ?", id). Where("ci_pipeline_material.regex != ?", ""). Exists() diff --git a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go index 9358248cf8..129469a013 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateOverrideRepository.go @@ -16,9 +16,11 @@ type CiTemplateOverride struct { DockerfilePath string `sql:"dockerfile_path"` GitMaterialId int `sql:"git_material_id"` Active bool `sql:"active,notnull"` + CiBuildConfigId int `sql:"ci_build_config_id"` sql.AuditLog GitMaterial *GitMaterial DockerRegistry *repository.DockerArtifactStore + CiBuildConfig *CiBuildConfig } type CiTemplateOverrideRepository interface { @@ -62,7 +64,9 @@ func (repo *CiTemplateOverrideRepositoryImpl) Update(templateOverrideConfig *CiT func (repo *CiTemplateOverrideRepositoryImpl) FindByAppId(appId int) ([]*CiTemplateOverride, error) { var ciTemplateOverrides []*CiTemplateOverride err := repo.dbConnection.Model(&ciTemplateOverrides). + Column("ci_template_override.*", "CiBuildConfig"). Join("INNER JOIN ci_pipeline cp on cp.id=ci_template_override.ci_pipeline_id"). + Join("INNER JOIN ci_build_config cbc on cbc.id=ci_template_override.ci_build_config_id"). Where("app_id = ?", appId). Where("is_docker_config_overridden = ?", true). Where("ci_template_override.active = ?", true). @@ -78,13 +82,13 @@ func (repo *CiTemplateOverrideRepositoryImpl) FindByAppId(appId int) ([]*CiTempl func (repo *CiTemplateOverrideRepositoryImpl) FindByCiPipelineId(ciPipelineId int) (*CiTemplateOverride, error) { ciTemplateOverride := &CiTemplateOverride{} err := repo.dbConnection.Model(ciTemplateOverride). - Column("ci_template_override.*", "GitMaterial", "DockerRegistry"). + Column("ci_template_override.*", "GitMaterial", "DockerRegistry", "CiBuildConfig"). Where("ci_pipeline_id = ?", ciPipelineId). Where("ci_template_override.active = ?", true). Select() if err != nil { repo.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) - return nil, err + return ciTemplateOverride, err } return ciTemplateOverride, nil } diff --git a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go index ea5bccc701..a9fb8f922f 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go @@ -42,10 +42,12 @@ type CiTemplate struct { Active bool `sql:"active,notnull"` GitMaterialId int `sql:"git_material_id"` DockerBuildOptions string `sql:"docker_build_options"` //json string format of map[string]string + CiBuildConfigId int `sql:"ci_build_config_id"` sql.AuditLog App *app.App DockerRegistry *repository.DockerArtifactStore GitMaterial *GitMaterial + CiBuildConfig *CiBuildConfig } type CiTemplateRepository interface { @@ -81,7 +83,7 @@ func (impl CiTemplateRepositoryImpl) FindByAppId(appId int) (ciTemplate *CiTempl template := &CiTemplate{} err = impl.dbConnection.Model(template). Where("app_id =? ", appId). - Column("ci_template.*", "App", "DockerRegistry"). + Column("ci_template.*", "App", "DockerRegistry", "CiBuildConfig"). Select() if pg.ErrNoRows == err { return nil, errors.NotFoundf(err.Error()) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index d1c4fe3916..a6a3df2177 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -38,6 +38,7 @@ type CiWorkflowRepository interface { FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) ExistsByStatus(status string) (bool, error) + FindBuildTypeAndStatusDataOfLast1Day() []*BuildTypeCount } type CiWorkflowRepositoryImpl struct { @@ -62,6 +63,7 @@ type CiWorkflow struct { TriggeredBy int32 `sql:"triggered_by"` CiArtifactLocation string `sql:"ci_artifact_location"` PodName string `sql:"pod_name"` + CiBuildType string `sql:"ci_build_type"` CiPipeline *CiPipeline } @@ -84,6 +86,7 @@ type WorkflowWithArtifact struct { CiArtifactLocation string `json:"ci_artifact_location"` CiArtifactId int `json:"ci_artifact_d"` BlobStorageEnabled bool `json:"blobStorageEnabled"` + CiBuildType string `json:"ci_build_type"` } type GitCommit struct { @@ -228,3 +231,13 @@ func (impl *CiWorkflowRepositoryImpl) ExistsByStatus(status string) (bool, error Exists() return exists, err } + +func (impl *CiWorkflowRepositoryImpl) FindBuildTypeAndStatusDataOfLast1Day() []*BuildTypeCount { + var buildTypeCounts []*BuildTypeCount + query := "select status,ci_build_type as type, count(*) from ci_workflow where status in ('Succeeded','Failed') and started_on > ? group by (ci_build_type, status)" + _, err := impl.dbConnection.Query(&buildTypeCounts, query, time.Now().AddDate(0, 0, -1)) + if err != nil { + impl.logger.Errorw("error occurred while fetching build type vs status vs count data", "err", err) + } + return buildTypeCounts +} diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go new file mode 100644 index 0000000000..f2c26e03c3 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository_test.go @@ -0,0 +1,24 @@ +package pipelineConfig + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCiWorkflowRepository(t *testing.T) { + t.SkipNow() + t.Run("FindBuildTypeAndStatusDataOfLast1Day", func(t *testing.T) { + cfg, _ := sql.GetConfig() + logger, err := utils.NewSugardLogger() + con, _ := sql.NewDbConnection(cfg, logger) + assert.Nil(t, err) + workflowRepositoryImpl := NewCiWorkflowRepositoryImpl(con, logger) + statusData := workflowRepositoryImpl.FindBuildTypeAndStatusDataOfLast1Day() + for _, statusDatum := range statusData { + fmt.Println(statusDatum.Count) + } + }) +} diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index a45c11b4dc..ca739a317d 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -37,16 +37,16 @@ type AppCloneService interface { CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { - logger *zap.SugaredLogger - pipelineBuilder pipeline.PipelineBuilder - materialRepository pipelineConfig.MaterialRepository - chartService chart.ChartService - configMapService pipeline.ConfigMapService - appWorkflowService appWorkflow.AppWorkflowService - appListingService app.AppListingService - propertiesConfigService pipeline.PropertiesConfigService - pipelineStageService pipeline.PipelineStageService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + logger *zap.SugaredLogger + pipelineBuilder pipeline.PipelineBuilder + materialRepository pipelineConfig.MaterialRepository + chartService chart.ChartService + configMapService pipeline.ConfigMapService + appWorkflowService appWorkflow.AppWorkflowService + appListingService app.AppListingService + propertiesConfigService pipeline.PropertiesConfigService + pipelineStageService pipeline.PipelineStageService + ciTemplateService pipeline.CiTemplateService } func NewAppCloneServiceImpl(logger *zap.SugaredLogger, @@ -58,20 +58,19 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, appListingService app.AppListingService, propertiesConfigService pipeline.PropertiesConfigService, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, - pipelineStageService pipeline.PipelineStageService) *AppCloneServiceImpl { + pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService) *AppCloneServiceImpl { return &AppCloneServiceImpl{ - logger: logger, - pipelineBuilder: pipelineBuilder, - materialRepository: materialRepository, - chartService: chartService, - configMapService: configMapService, - appWorkflowService: appWorkflowService, - appListingService: appListingService, - propertiesConfigService: propertiesConfigService, - pipelineStageService: pipelineStageService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + logger: logger, + pipelineBuilder: pipelineBuilder, + materialRepository: materialRepository, + chartService: chartService, + configMapService: configMapService, + appWorkflowService: appWorkflowService, + appListingService: appListingService, + propertiesConfigService: propertiesConfigService, + pipelineStageService: pipelineStageService, + ciTemplateService: ciTemplateService, } - } type CloneRequest struct { @@ -244,9 +243,9 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId if len(gitMaterials) == 1 { dockerfileGitMaterial = gitMaterials[0].Id } else { - refGitmaterial, err := impl.materialRepository.FindById(refCiConf.DockerBuildConfig.GitMaterialId) + refGitmaterial, err := impl.materialRepository.FindById(refCiConf.CiBuildConfig.GitMaterialId) if err != nil { - impl.logger.Errorw("error in fetching ref git material", "id", refCiConf.DockerBuildConfig.GitMaterialId, "err", err) + impl.logger.Errorw("error in fetching ref git material", "id", refCiConf.CiBuildConfig.GitMaterialId, "err", err) return nil, err } //first repo with same checkout path @@ -273,17 +272,14 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId } } + ciBuildConfig := refCiConf.CiBuildConfig + ciBuildConfig.GitMaterialId = dockerfileGitMaterial ciConfRequest := &bean.CiConfigRequest{ - Id: 0, - AppId: newAppId, - DockerRegistry: refCiConf.DockerRegistry, - DockerRepository: refCiConf.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: dockerfileGitMaterial, - DockerfilePath: refCiConf.DockerBuildConfig.DockerfilePath, - Args: refCiConf.DockerBuildConfig.Args, - TargetPlatform: refCiConf.DockerBuildConfig.TargetPlatform, - }, + Id: 0, + AppId: newAppId, + DockerRegistry: refCiConf.DockerRegistry, + DockerRepository: refCiConf.DockerRepository, + CiBuildConfig: ciBuildConfig, DockerRegistryUrl: refCiConf.DockerRegistry, CiTemplateName: refCiConf.CiTemplateName, UserId: userId, @@ -752,24 +748,25 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( } if !refCiPipeline.IsExternal && refCiPipeline.IsDockerConfigOverridden { //get template override - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(refCiPipeline.Id) + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(refCiPipeline.Id) if err != nil { - impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", refCiPipeline.Id) return nil, err } + templateOverride := templateOverrideBean.CiTemplateOverride + ciBuildConfig := templateOverrideBean.CiBuildConfig //getting new git material for this app gitMaterial, err := impl.materialRepository.FindByAppIdAndCheckoutPath(req.appId, templateOverride.GitMaterial.CheckoutPath) if err != nil { impl.logger.Errorw("error in getting git material by appId and checkoutPath", "err", err, "appid", req.refAppId, "checkoutPath", templateOverride.GitMaterial.CheckoutPath) return nil, err } + ciBuildConfig.GitMaterialId = gitMaterial.Id + templateOverride.GitMaterialId = gitMaterial.Id + ciBuildConfig.Id = 0 ciPatchReq.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - DockerfilePath: templateOverride.DockerfilePath, - GitMaterialId: gitMaterial.Id, - }, + CiBuildConfig: ciBuildConfig, } } diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 5526f0137b..f2f524464a 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -111,9 +111,10 @@ type CiPipeline struct { } type DockerConfigOverride struct { - DockerRegistry string `json:"dockerRegistry,omitempty"` - DockerRepository string `json:"dockerRepository,omitempty"` - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` + DockerRegistry string `json:"dockerRegistry,omitempty"` + DockerRepository string `json:"dockerRepository,omitempty"` + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig,omitEmpty"` + //DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` } type CiPipelineMin struct { @@ -250,22 +251,22 @@ type Material struct { } type CiConfigRequest struct { - Id int `json:"id,omitempty" validate:"number"` //ciTemplateId - AppId int `json:"appId,omitempty" validate:"required,number"` - DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry - DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr - DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty" validate:"required,dive"` - CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial - AppName string `json:"appName,omitempty"` - Version string `json:"version,omitempty"` //gocd etag used for edit purpose - DockerRegistryUrl string `json:"-"` - CiTemplateName string `json:"-"` - UserId int32 `json:"-"` - Materials []Material `json:"materials"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Id int `json:"id,omitempty" validate:"number"` //ciTemplateId + AppId int `json:"appId,omitempty" validate:"required,number"` + DockerRegistry string `json:"dockerRegistry,omitempty" ` //repo id example ecr mapped one-one with gocd registry entry + DockerRepository string `json:"dockerRepository,omitempty"` // example test-app-1 which is inside ecr + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig"` + CiPipelines []*CiPipeline `json:"ciPipelines,omitempty" validate:"dive"` //a pipeline will be built for each ciMaterial + AppName string `json:"appName,omitempty"` + Version string `json:"version,omitempty"` //gocd etag used for edit purpose + DockerRegistryUrl string `json:"-"` + CiTemplateName string `json:"-"` + UserId int32 `json:"-"` + Materials []Material `json:"materials"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type TestExecutorImageProperties struct { @@ -274,15 +275,6 @@ type TestExecutorImageProperties struct { ReportDir string `json:"reportDir,omitempty"` } -type DockerBuildConfig struct { - GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` - DockerfilePath string `json:"dockerfileRelativePath,omitempty" validate:"required"` - Args map[string]string `json:"args,omitempty"` - TargetPlatform string `json:"targetPlatform"` - DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` - //Name Tag DockerfilePath RepoUrl -} - type PipelineCreateResponse struct { AppName string `json:"appName,omitempty"` AppId int `json:"appId,omitempty"` diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go new file mode 100644 index 0000000000..4d7862d69e --- /dev/null +++ b/pkg/pipeline/CiBuildConfigService.go @@ -0,0 +1,88 @@ +package pipeline + +import ( + "errors" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "go.uber.org/zap" + "time" +) + +type CiBuildConfigService interface { + Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error + UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) + Delete(ciBuildConfigId int) error + GetCountByBuildType() map[bean.CiBuildType]int +} + +type CiBuildConfigServiceImpl struct { + Logger *zap.SugaredLogger + CiBuildConfigRepository pipelineConfig.CiBuildConfigRepository +} + +func NewCiBuildConfigServiceImpl(logger *zap.SugaredLogger, ciBuildConfigRepository pipelineConfig.CiBuildConfigRepository) *CiBuildConfigServiceImpl { + return &CiBuildConfigServiceImpl{ + Logger: logger, + CiBuildConfigRepository: ciBuildConfigRepository, + } +} + +func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean, userId) + if err != nil { + impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, + "overrideTemplateId", overrideTemplateId, "ciBuildConfigBean", ciBuildConfigBean, "err", err) + return errors.New("error while saving build config") + } + ciBuildConfigEntity.CreatedOn = time.Now() + ciBuildConfigEntity.CreatedBy = userId + ciBuildConfigEntity.Id = 0 + err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) + ciBuildConfigBean.Id = ciBuildConfigEntity.Id + if err != nil { + return errors.New("error while saving build config") + } + return nil +} + +func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) { + if ciBuildConfig == nil { + impl.Logger.Warnw("not updating build config as object is empty", "ciBuildConfig", ciBuildConfig) + return nil, nil + } + ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig, userId) + if err != nil { + impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, + "overrideTemplateId", overrideTemplateId, "ciBuildConfig", ciBuildConfig, "err", err) + return nil, errors.New("error while saving build config") + } + if ciBuildConfig.Id == 0 { + ciBuildConfigEntity.CreatedOn = time.Now() + ciBuildConfigEntity.CreatedBy = userId + err = impl.CiBuildConfigRepository.Save(ciBuildConfigEntity) + ciBuildConfig.Id = ciBuildConfigEntity.Id + } else { + err = impl.CiBuildConfigRepository.Update(ciBuildConfigEntity) + } + if err != nil { + impl.Logger.Errorw("error occurred while updating/saving ciBuildConfig", "entity", ciBuildConfigEntity, "err", err) + return nil, errors.New("error while updating build config") + } + return ciBuildConfig, nil +} + +func (impl *CiBuildConfigServiceImpl) Delete(ciBuildConfigId int) error { + return impl.CiBuildConfigRepository.Delete(ciBuildConfigId) +} + +func (impl *CiBuildConfigServiceImpl) GetCountByBuildType() map[bean.CiBuildType]int { + result := make(map[bean.CiBuildType]int) + buildTypeVsCount, err := impl.CiBuildConfigRepository.GetCountByBuildType() + if err != nil { + return result + } + for buildType, count := range buildTypeVsCount { + result[bean.CiBuildType(buildType)] = count + } + return result +} diff --git a/pkg/pipeline/CiBuildConfigService_test.go b/pkg/pipeline/CiBuildConfigService_test.go new file mode 100644 index 0000000000..e27a6ba09d --- /dev/null +++ b/pkg/pipeline/CiBuildConfigService_test.go @@ -0,0 +1,25 @@ +package pipeline + +import ( + "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCiBuildConfigService(t *testing.T) { + t.SkipNow() + t.Run("buildTypeVsCount", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + countByBuildType := ciBuildConfigServiceImpl.GetCountByBuildType() + fmt.Println(countByBuildType) + }) +} diff --git a/pkg/pipeline/CiConfig.go b/pkg/pipeline/CiConfig.go index a355d0456f..11247d1c26 100644 --- a/pkg/pipeline/CiConfig.go +++ b/pkg/pipeline/CiConfig.go @@ -76,6 +76,7 @@ type CiConfig struct { BlobStorageGcpCredentialJson string `env:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` BuildLogTTLValue int `json:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600"` AzureAccountKey string `env:"AZURE_ACCOUNT_KEY"` + CiRunnerDockerMTUValue int `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1"` IgnoreDockerCacheForCI bool `env:"CI_IGNORE_DOCKER_CACHE"` VolumeMountsForCiJson string `env:"CI_VOLUME_MOUNTS_JSON"` ClusterConfig *rest.Config diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 6f285522dc..1f758bd96b 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -18,6 +18,7 @@ package pipeline import ( + "errors" "fmt" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/pkg/app" @@ -60,7 +61,7 @@ type CiServiceImpl struct { prePostCiScriptHistoryService history.PrePostCiScriptHistoryService pipelineStageService PipelineStageService userService user.UserService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + ciTemplateService CiTemplateService appCrudOperationService app.AppCrudOperationService } @@ -71,7 +72,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService prePostCiScriptHistoryService history.PrePostCiScriptHistoryService, pipelineStageService PipelineStageService, userService user.UserService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, appCrudOperationService app.AppCrudOperationService) *CiServiceImpl { + ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService) *CiServiceImpl { return &CiServiceImpl{ Logger: Logger, workflowService: workflowService, @@ -85,7 +86,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService prePostCiScriptHistoryService: prePostCiScriptHistoryService, pipelineStageService: pipelineStageService, userService: userService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciTemplateService: ciTemplateService, appCrudOperationService: appCrudOperationService, } } @@ -147,6 +148,8 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { return 0, err } + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) if err != nil { return 0, err @@ -378,18 +381,13 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciWorkflowConfig.CiTimeout = impl.ciConfig.DefaultTimeout } - args := pipeline.CiTemplate.Args + ciTemplate := pipeline.CiTemplate ciLevelArgs := pipeline.DockerArgs if ciLevelArgs == "" { ciLevelArgs = "{}" } - mergedArgs, err := impl.mergeUtil.JsonPatch([]byte(args), []byte(ciLevelArgs)) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } if pipeline.CiTemplate.DockerBuildOptions == "" { pipeline.CiTemplate.DockerBuildOptions = "{}" } @@ -401,26 +399,50 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var dockerfilePath string var dockerRepository string var checkoutPath string + var ciBuildConfigBean *bean2.CiBuildConfigBean dockerRegistry := &repository3.DockerArtifactStore{} if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(pipeline.Id) + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) if err != nil { - impl.Logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "err", err, "ciPipelineId", pipeline.Id) return nil, err } - dockerfilePath = filepath.Join(templateOverride.GitMaterial.CheckoutPath, templateOverride.DockerfilePath) + ciBuildConfigBean = templateOverrideBean.CiBuildConfig + templateOverride := templateOverrideBean.CiTemplateOverride + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = filepath.Join(checkoutPath, templateOverride.DockerfilePath) dockerRepository = templateOverride.DockerRepository dockerRegistry = templateOverride.DockerRegistry - checkoutPath = templateOverride.GitMaterial.CheckoutPath } else { - dockerfilePath = filepath.Join(pipeline.CiTemplate.GitMaterial.CheckoutPath, pipeline.CiTemplate.DockerfilePath) - dockerRegistry = pipeline.CiTemplate.DockerRegistry - dockerRepository = pipeline.CiTemplate.DockerRepository - checkoutPath = pipeline.CiTemplate.GitMaterial.CheckoutPath + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = filepath.Join(checkoutPath, ciTemplate.DockerfilePath) + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigEntity := ciTemplate.CiBuildConfig + ciBuildConfigBean, err = bean2.ConvertDbBuildConfigToBean(ciBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } } if checkoutPath == "" { checkoutPath = "./" } + //mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } + if ciBuildConfigBean.CiBuildType == bean2.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean2.MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == bean2.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) + } workflowRequest := &WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, PipelineName: pipeline.Name, @@ -430,9 +452,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. DockerImageTag: dockerImageTag, DockerRegistryURL: dockerRegistry.RegistryURL, DockerRepository: dockerRepository, - DockerBuildArgs: string(mergedArgs), - DockerBuildTargetPlatform: pipeline.CiTemplate.TargetPlatform, - DockerFileLocation: dockerfilePath, + CheckoutPath: checkoutPath, DockerUsername: dockerRegistry.Username, DockerPassword: dockerRegistry.Password, AwsRegion: dockerRegistry.AWSRegion, @@ -458,7 +478,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. RefPlugins: refPluginsData, AppName: pipeline.App.AppName, TriggerByAuthor: user.EmailId, - DockerBuildOptions: pipeline.CiTemplate.DockerBuildOptions, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.ciConfig.CiRunnerDockerMTUValue, IgnoreDockerCachePush: impl.ciConfig.IgnoreDockerCacheForCI, IgnoreDockerCachePull: impl.ciConfig.IgnoreDockerCacheForCI || trigger.InvalidateCache, } @@ -608,6 +629,13 @@ func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]bean.GitCommit, id return dockerImageTag } +func (impl *CiServiceImpl) updateCiWorkflow(request *WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciWorkflowRepository.UpdateWorkFlow(savedWf) +} + func _getTruncatedImageTag(imageTag string) string { _length := len(imageTag) if _length == 0 { diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go new file mode 100644 index 0000000000..b4c4d54d72 --- /dev/null +++ b/pkg/pipeline/CiTemplateService.go @@ -0,0 +1,170 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type CiTemplateService interface { + Save(ciTemplateBean *bean.CiTemplateBean) error + FindByAppId(appId int) (ciTemplateBean *bean.CiTemplateBean, err error) + FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) + FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) + Update(ciTemplateBean *bean.CiTemplateBean) error +} +type CiTemplateServiceImpl struct { + Logger *zap.SugaredLogger + CiBuildConfigService CiBuildConfigService + CiTemplateRepository pipelineConfig.CiTemplateRepository + CiTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository +} + +func NewCiTemplateServiceImpl(logger *zap.SugaredLogger, ciBuildConfigService CiBuildConfigService, + ciTemplateRepository pipelineConfig.CiTemplateRepository, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *CiTemplateServiceImpl { + return &CiTemplateServiceImpl{ + Logger: logger, + CiBuildConfigService: ciBuildConfigService, + CiTemplateRepository: ciTemplateRepository, + CiTemplateOverrideRepository: ciTemplateOverrideRepository, + } +} + +func (impl CiTemplateServiceImpl) Save(ciTemplateBean *bean.CiTemplateBean) error { + ciTemplate := ciTemplateBean.CiTemplate + ciTemplateOverride := ciTemplateBean.CiTemplateOverride + ciTemplateId := 0 + ciTemplateOverrideId := 0 + + buildConfig := ciTemplateBean.CiBuildConfig + err := impl.CiBuildConfigService.Save(ciTemplateId, ciTemplateOverrideId, buildConfig, ciTemplateBean.UserId) + if err != nil { + impl.Logger.Errorw("error occurred while saving ci build config", "config", buildConfig, "err", err) + } + if ciTemplateOverride == nil { + ciTemplate.CiBuildConfigId = buildConfig.Id + err := impl.CiTemplateRepository.Save(ciTemplate) + if err != nil { + impl.Logger.Errorw("error in saving ci template in db ", "template", ciTemplate, "err", err) + //TODO delete template from gocd otherwise dangling+ no create in future + return err + } + ciTemplateId = ciTemplate.Id + } else { + ciTemplateOverride.CiBuildConfigId = buildConfig.Id + _, err := impl.CiTemplateOverrideRepository.Save(ciTemplateOverride) + if err != nil { + impl.Logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", ciTemplateOverride) + return err + } + ciTemplateOverrideId = ciTemplateOverride.Id + } + + return err +} + +func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.CiTemplateBean, err error) { + ciTemplate, err := impl.CiTemplateRepository.FindByAppId(appId) + if err != nil { + return nil, err + } + ciBuildConfig := ciTemplate.CiBuildConfig + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", + ciBuildConfig, "error", err) + } + if ciBuildConfigBean == nil { + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) + if err != nil { + impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) + } + } + ciBuildConfigBean.GitMaterialId = ciTemplate.GitMaterialId + return &bean.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: ciBuildConfigBean, + }, err +} + +func (impl CiTemplateServiceImpl) FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) { + templateOverrides, err := impl.CiTemplateOverrideRepository.FindByAppId(appId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting ciTemplateOverrides by appId", "err", err, "appId", appId) + return nil, err + } + var templateBeanOverrides []*bean.CiTemplateBean + for _, templateOverride := range templateOverrides { + ciBuildConfigBean, err := impl.extractBuildConfigBean(templateOverride) + if err != nil { + return templateBeanOverrides, err + } + overrideBean := &bean.CiTemplateBean{ + CiTemplateOverride: templateOverride, + CiBuildConfig: ciBuildConfigBean, + } + templateBeanOverrides = append(templateBeanOverrides, overrideBean) + } + return templateBeanOverrides, nil +} + +func (impl CiTemplateServiceImpl) extractBuildConfigBean(templateOverride *pipelineConfig.CiTemplateOverride) (*bean.CiBuildConfigBean, error) { + ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) + if err != nil { + impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", + templateOverride.CiBuildConfig, "error", err) + return nil, err + } + if ciBuildConfigBean == nil { + ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) + if err != nil { + impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) + } + } + ciBuildConfigBean.GitMaterialId = templateOverride.GitMaterialId + return ciBuildConfigBean, nil +} + +func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) { + templateOverride, err := impl.CiTemplateOverrideRepository.FindByCiPipelineId(ciPipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) + return nil, err + } + ciBuildConfigBean, err := impl.extractBuildConfigBean(templateOverride) + return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err +} + +func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) error { + ciTemplate := ciTemplateBean.CiTemplate + ciTemplateOverride := ciTemplateBean.CiTemplateOverride + ciTemplateId := 0 + ciTemplateOverrideId := 0 + ciBuildConfig := ciTemplateBean.CiBuildConfig + if ciTemplateOverride == nil { + ciTemplateId = ciTemplate.Id + } else { + ciTemplateOverrideId = ciTemplateOverride.Id + } + _, err := impl.CiBuildConfigService.UpdateOrSave(ciTemplateId, ciTemplateOverrideId, ciBuildConfig, ciTemplateBean.UserId) + if err != nil { + impl.Logger.Errorw("error in updating ci build config in db", "ciBuildConfig", ciBuildConfig, "err", err) + } + if ciTemplateOverride == nil { + ciTemplate.CiBuildConfigId = ciBuildConfig.Id + err := impl.CiTemplateRepository.Update(ciTemplate) + if err != nil { + impl.Logger.Errorw("error in updating ci template in db", "template", ciTemplate, "err", err) + return err + } + } else { + ciTemplateOverride.CiBuildConfigId = ciBuildConfig.Id + _, err := impl.CiTemplateOverrideRepository.Update(ciTemplateOverride) + if err != nil { + impl.Logger.Errorw("error in updating template override", "err", err, "templateOverrideConfig", ciTemplateOverride) + return err + } + } + return err +} diff --git a/pkg/pipeline/CiTemplateService_test.go b/pkg/pipeline/CiTemplateService_test.go new file mode 100644 index 0000000000..5632c07715 --- /dev/null +++ b/pkg/pipeline/CiTemplateService_test.go @@ -0,0 +1,479 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/mocks" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineMocks "github.com/devtron-labs/devtron/pkg/pipeline/mocks" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "log" + "os" + "testing" +) + +func TestCiTemplateService(t *testing.T) { + + t.Run("FetchTemplateWithBuildpackConfiguration", func(t *testing.T) { + + sugaredLogger, err := util.NewSugardLogger() + assert.Nil(t, err) + builderId := "sample-builder" + ciTemplateRepositoryMocked := mocks.NewCiTemplateRepository(t) + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId + "\"}", + }, + } + ciTemplateRepositoryMocked.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, ciTemplateRepositoryMocked, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + template := templateBean.CiTemplate + assert.Equal(t, template.Id, templateDbEntity.Id) + assert.NotNil(t, templateBean.CiBuildConfig) + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, templateBean.CiBuildConfig.CiBuildType) + assert.Equal(t, builderId, templateBean.CiBuildConfig.BuildPackConfig.BuilderId) + }) + + t.Run("FetchTemplateWithOldDockerData", func(t *testing.T) { + + sugaredLogger, err := util.NewSugardLogger() + assert.Nil(t, err) + mockCiTemplateRepository := mocks.NewCiTemplateRepository(t) + argsKey := "hello" + argsValue := "world" + buildOptionsKey := "volume" + buildOptionsValue := "hello1" + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + TargetPlatform: "linux/amd64", + DockerBuildOptions: "{\"" + buildOptionsKey + "\":\"" + buildOptionsValue + "\"}", + Args: "{\"" + argsKey + "\":\"" + argsValue + "\"}", + } + mockCiTemplateRepository.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, mockCiTemplateRepository, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + template := templateBean.CiTemplate + assert.Equal(t, template.Id, templateDbEntity.Id) + assert.Equal(t, template.TargetPlatform, templateDbEntity.TargetPlatform) + assert.Nil(t, template.CiBuildConfig) + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, bean.SELF_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.NotNil(t, ciBuildConfig.DockerBuildConfig) + assert.Equal(t, templateDbEntity.TargetPlatform, ciBuildConfig.DockerBuildConfig.TargetPlatform) + args := ciBuildConfig.DockerBuildConfig.Args + assert.NotEmpty(t, args) + assert.Equal(t, argsValue, args[argsKey]) + dockerBuildOptions := ciBuildConfig.DockerBuildConfig.DockerBuildOptions + assert.NotEmpty(t, dockerBuildOptions) + assert.Equal(t, buildOptionsValue, dockerBuildOptions[buildOptionsKey]) + }) + + t.Run("FetchTemplateWithManagedDockerData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.Nil(t, err) + mockCiTemplateRepository := mocks.NewCiTemplateRepository(t) + targetPlatform := "linux/amd64" + dockerfileContent := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + notPlatform := "linux/arm64" + gitMaterialId := 2 + ciBuildConfigId := 3 + managedDockerfileBuildType := bean.MANAGED_DOCKERFILE_BUILD_TYPE + buildConfigMetadata := &bean.DockerBuildConfig{ + DockerfileContent: dockerfileContent, + TargetPlatform: targetPlatform, + } + buildMetadata, err := json.Marshal(buildConfigMetadata) + assert.Nil(t, err) + templateDbEntity := &pipelineConfig.CiTemplate{ + Id: 1, + GitMaterialId: gitMaterialId, + TargetPlatform: notPlatform, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Id: ciBuildConfigId, + Type: string(managedDockerfileBuildType), + BuildMetadata: string(buildMetadata), + }, + } + mockCiTemplateRepository.On("FindByAppId", 1).Return(templateDbEntity, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, mockCiTemplateRepository, nil) + templateBean, err := ciTemplateServiceImpl.FindByAppId(1) + ciTemplateEntity := templateBean.CiTemplate + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, templateDbEntity.Id, ciTemplateEntity.Id) + assert.Equal(t, ciBuildConfigId, ciBuildConfig.Id) + assert.Equal(t, managedDockerfileBuildType, ciBuildConfig.CiBuildType) + assert.Equal(t, gitMaterialId, ciBuildConfig.GitMaterialId) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + dockerBuildConfig := ciBuildConfig.DockerBuildConfig + assert.NotNil(t, dockerBuildConfig) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, dockerfileContent, dockerBuildConfig.DockerfileContent) + assert.Equal(t, targetPlatform, dockerBuildConfig.TargetPlatform) + }) + + t.Run("ciTemplateOverrideWithOldDockerData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.Nil(t, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + DockerfilePath: "Dockerfile", + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + DockerfilePath: "Dockerfile_ea", + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + assert.Equal(t, bean.SELF_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.Empty(t, ciBuildConfig.DockerBuildConfig.DockerBuildOptions, "docker build options not supported in pipeline override") + assert.Equal(t, mockedTemplateOverride.DockerfilePath, ciBuildConfig.DockerBuildConfig.DockerfilePath) + } + }) + + t.Run("ciTemplateOverrideWithBuildPackData", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + builderId1 := "sample-builder-1" + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId1 + "\"}", + }, + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId1 + "\"}", + }, + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.DockerBuildConfig) + assert.NotNil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, builderId1, ciBuildConfig.BuildPackConfig.BuilderId) + } + }) + + t.Run("templateOverrideWithManagedDockerfileAndBuildpack", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + appId := 1 + dockerfileContent := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + targetPlatform := "linux/amd64" + builderId := "sample-builder" + buildConfigMetadata := &bean.DockerBuildConfig{ + DockerfileContent: dockerfileContent, + TargetPlatform: targetPlatform, + } + buildMetadata, err := json.Marshal(buildConfigMetadata) + assert.Nil(t, err) + mockedTemplateOverrides := []*pipelineConfig.CiTemplateOverride{{ + Id: 1, + CiPipelineId: 2, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.MANAGED_DOCKERFILE_BUILD_TYPE), + BuildMetadata: string(buildMetadata), + }, + }, { + Id: 2, + CiPipelineId: 3, + GitMaterialId: 3, + CiBuildConfig: &pipelineConfig.CiBuildConfig{ + Type: string(bean.BUILDPACK_BUILD_TYPE), + BuildMetadata: "{\"BuilderId\":\"" + builderId + "\"}", + }, + }} + mockedCiTemplateOverrideRepository.On("FindByAppId", appId).Return(mockedTemplateOverrides, nil) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, nil, nil, mockedCiTemplateOverrideRepository) + templateBeans, err := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + assert.Nil(t, err) + assert.Equal(t, len(mockedTemplateOverrides), len(templateBeans)) + for index, _ := range templateBeans { + templateBean := templateBeans[index] + mockedTemplateOverride := mockedTemplateOverrides[index] + templateOverride := templateBean.CiTemplateOverride + ciBuildConfig := templateBean.CiBuildConfig + assert.Equal(t, mockedTemplateOverride.Id, templateOverride.Id) + assert.Equal(t, mockedTemplateOverride.GitMaterialId, templateOverride.GitMaterialId) + assert.Equal(t, mockedTemplateOverride.CiPipelineId, templateOverride.CiPipelineId) + if ciBuildConfig.CiBuildType == bean.MANAGED_DOCKERFILE_BUILD_TYPE { + assert.Equal(t, bean.MANAGED_DOCKERFILE_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.BuildPackConfig) + assert.NotNil(t, ciBuildConfig.DockerBuildConfig) + assert.Equal(t, dockerfileContent, ciBuildConfig.DockerBuildConfig.DockerfileContent) + } else if ciBuildConfig.CiBuildType == bean.BUILDPACK_BUILD_TYPE { + assert.Equal(t, bean.BUILDPACK_BUILD_TYPE, ciBuildConfig.CiBuildType) + assert.Nil(t, ciBuildConfig.DockerBuildConfig) + assert.NotNil(t, ciBuildConfig.BuildPackConfig) + assert.Equal(t, builderId, ciBuildConfig.BuildPackConfig.BuilderId) + } + } + }) + + t.Run("UpdateTemplateOverrideWithBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateOverrideRepository := mocks.NewCiTemplateOverrideRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, nil, mockedCiTemplateOverrideRepository) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateOverrideId := 1 + mockedCiBuildConfigId := 5 + mockedTemplateOverride := &pipelineConfig.CiTemplateOverride{Id: mockedTemplateOverrideId, CiPipelineId: 2, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplateOverride = mockedTemplateOverride + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: mockedCiBuildConfigId, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedCiTemplateOverrideRepository.On("Update", mock.AnythingOfType("*pipelineConfig.CiTemplateOverride")). + Return(func(templateOverride *pipelineConfig.CiTemplateOverride) *pipelineConfig.CiTemplateOverride { + assert.Equal(t, mockedCiBuildConfigId, templateOverride.CiBuildConfigId) + return nil + }, nil) + mockedBuildConfigService.On("UpdateOrSave", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) *bean.CiBuildConfigBean { + assert.Equal(t, 0, templateId) + assert.Equal(t, mockedTemplateOverrideId, overrideTemplateId) + assert.Equal(t, mockedUserId, userId) + mockedBuildConfigBean := mockedCiTemplateBean.CiBuildConfig + assert.Equal(t, mockedBuildConfigBean, ciBuildConfig) + return ciBuildConfig + }, + nil, + ) + err = ciTemplateServiceImpl.Update(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedCiBuildConfigId, mockedTemplateOverride.CiBuildConfigId) + }) + + t.Run("UpdateTemplateWithBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateRepository := mocks.NewCiTemplateRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, mockedCiTemplateRepository, nil) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateId := 1 + mockedCiBuildConfigId := 5 + mockedTemplate := &pipelineConfig.CiTemplate{Id: mockedTemplateId, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplate = mockedTemplate + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: mockedCiBuildConfigId, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedCiTemplateRepository.On("Update", mock.AnythingOfType("*pipelineConfig.CiTemplate")). + Return(func(template *pipelineConfig.CiTemplate) error { + assert.Equal(t, mockedCiBuildConfigId, template.CiBuildConfigId) + return nil + }) + mockedBuildConfigService.On("UpdateOrSave", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) *bean.CiBuildConfigBean { + assert.Equal(t, 0, overrideTemplateId) + assert.Equal(t, mockedTemplateId, templateId) + assert.Equal(t, mockedUserId, userId) + assert.Equal(t, mockedCiTemplateBean.CiBuildConfig, ciBuildConfig) + return ciBuildConfig + }, + nil, + ) + err = ciTemplateServiceImpl.Update(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedCiBuildConfigId, mockedTemplate.CiBuildConfigId) + }) + + t.Run("SaveTemplateAndBuildConfig", func(t *testing.T) { + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + mockedCiTemplateRepository := mocks.NewCiTemplateRepository(t) + mockedBuildConfigService := pipelineMocks.NewCiBuildConfigService(t) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, mockedBuildConfigService, mockedCiTemplateRepository, nil) + mockedCiTemplateBean := &bean.CiTemplateBean{} + materialId := 3 + mockedTemplateId := 7 + mockedCiBuildConfigId := 6 + mockedTemplate := &pipelineConfig.CiTemplate{Id: 0, GitMaterialId: materialId} + mockedCiTemplateBean.CiTemplate = mockedTemplate + dockerBuildOptions := map[string]string{} + dockerBuildOptions["volume"] = "abcd:defg" + mockedCiTemplateBean.CiBuildConfig = &bean.CiBuildConfigBean{ + Id: 0, + GitMaterialId: materialId, + CiBuildType: bean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: "Dockerfile", TargetPlatform: "linux/amd64", DockerBuildOptions: dockerBuildOptions}, + } + mockedUserId := int32(4) + mockedCiTemplateBean.UserId = mockedUserId + mockedBuildConfigService.On("Save", mock.AnythingOfType("int"), mock.AnythingOfType("int"), + mock.AnythingOfType("*bean.CiBuildConfigBean"), mock.AnythingOfType("int32")). + Return( + func(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) error { + assert.Equal(t, 0, overrideTemplateId) + assert.Equal(t, mockedTemplate.Id, templateId) + assert.Equal(t, mockedUserId, userId) + assert.Equal(t, mockedCiTemplateBean.CiBuildConfig, ciBuildConfig) + ciBuildConfig.Id = mockedCiBuildConfigId + return nil + }, + ) + mockedCiTemplateRepository.On("Save", mock.AnythingOfType("*pipelineConfig.CiTemplate")). + Return(func(template *pipelineConfig.CiTemplate) error { + assert.Equal(t, mockedCiBuildConfigId, template.CiBuildConfigId) + template.Id = mockedTemplateId + return nil + }) + err = ciTemplateServiceImpl.Save(mockedCiTemplateBean) + assert.Nil(t, err) + assert.Equal(t, mockedTemplateId, mockedTemplate.Id) + }) + + t.Run("getCiTemplate", func(t *testing.T) { + t.SkipNow() + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + _, err = ciTemplateServiceImpl.FindTemplateOverrideByCiPipelineId(1) + appId := 1 + ciTemplateBean, err := ciTemplateServiceImpl.FindByAppId(appId) + assert.True(t, err == nil, err) + assert.True(t, ciTemplateBean != nil, ciTemplateBean) + assert.True(t, ciTemplateBean.CiTemplate != nil) + assert.True(t, ciTemplateBean.CiTemplate.AppId == appId) + assert.True(t, ciTemplateBean.CiBuildConfig != nil) + + ciBuildConfig := ciTemplateBean.CiBuildConfig + + buildPackConfig := &bean.BuildPackConfig{ + BuilderId: "gcr.io/buildpacks/builder:v1", + } + //buildPackConfig.BuilderId = "heroku/buildpacks:20" + ciBuildConfig.CiBuildType = bean.BUILDPACK_BUILD_TYPE + ciBuildConfig.BuildPackConfig = buildPackConfig + + //args := make(map[string]string) + ////args["hello"] = "world" + //input := "FROM node:9\r\n\r\nWORKDIR /app\r\n\r\nRUN npm install -g contentful-cli\r\n\r\nCOPY package.json .\r\nRUN npm install\r\n\r\nCOPY . .\r\n\r\nUSER node\r\nEXPOSE 3000\r\n\r\nCMD [\"npm\", \"run\", \"start:dev\"]" + //dockerBuildConfig := &bean.DockerBuildConfig{ + // DockerfilePath: "Dockerfile", + // DockerfileContent: input, + // Args: args, + // //TargetPlatform: "linux/amd64", + //} + //ciBuildConfig.CiBuildType = bean.MANAGED_DOCKERFILE_BUILD_TYPE + //ciBuildConfig.DockerBuildConfig = dockerBuildConfig + + err = ciTemplateServiceImpl.Update(ciTemplateBean) + assert.True(t, err == nil, err) + }) + + t.Run("fetch ci pipeline details", func(t *testing.T) { + t.SkipNow() + sugaredLogger, err := util.NewSugardLogger() + assert.True(t, err == nil, err) + config, err := sql.GetConfig() + assert.True(t, err == nil, err) + db, err := sql.NewDbConnection(config, sugaredLogger) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) + ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + appId := 7 + templateBeans, _ := ciTemplateServiceImpl.FindTemplateOverrideByAppId(appId) + for _, templateBean := range templateBeans { + buildConfig := templateBean.CiBuildConfig + fmt.Println(buildConfig) + } + ciTemplateBean, _ := ciTemplateServiceImpl.FindByAppId(appId) + fmt.Println(ciTemplateBean.CiBuildConfig) + }) + + t.Run("escaping char test", func(t *testing.T) { + t.SkipNow() + input := "FROM debian\r\nRUN export node_version=\"0.10\" \\\r\n&& apt-get update && apt-get -y install nodejs=\"$node_verion\"\r\nCOPY package.json usr/src/app\r\nRUN cd /usr/src/app \\\r\n&& npm install node-static\r\n\r\nEXPOSE 80000\r\nCMD [\"npm\", \"start\"]" + //fmt.Println(input) + output := fmt.Sprint(input) + fmt.Println(output) + f, err := os.Create("/tmp/devtroncd/data.txt") + if err != nil { + log.Fatal(err) + } + defer f.Close() + _, err2 := f.WriteString(input) + + if err2 != nil { + log.Fatal(err2) + } + fmt.Println("done") + }) + +} diff --git a/pkg/pipeline/DbPipelineOrchestrator.go b/pkg/pipeline/DbPipelineOrchestrator.go index aefd8aa995..2b5c2f831a 100644 --- a/pkg/pipeline/DbPipelineOrchestrator.go +++ b/pkg/pipeline/DbPipelineOrchestrator.go @@ -27,6 +27,7 @@ import ( "fmt" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" @@ -90,7 +91,8 @@ type DbPipelineOrchestratorImpl struct { prePostCdScriptHistoryService history3.PrePostCdScriptHistoryService prePostCiScriptHistoryService history3.PrePostCiScriptHistoryService pipelineStageService PipelineStageService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + ciTemplateService CiTemplateService } func NewDbPipelineOrchestrator( @@ -110,7 +112,7 @@ func NewDbPipelineOrchestrator( prePostCdScriptHistoryService history3.PrePostCdScriptHistoryService, prePostCiScriptHistoryService history3.PrePostCiScriptHistoryService, pipelineStageService PipelineStageService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *DbPipelineOrchestratorImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciTemplateService CiTemplateService) *DbPipelineOrchestratorImpl { return &DbPipelineOrchestratorImpl{ appRepository: pipelineGroupRepository, logger: logger, @@ -129,7 +131,8 @@ func NewDbPipelineOrchestrator( prePostCdScriptHistoryService: prePostCdScriptHistoryService, prePostCiScriptHistoryService: prePostCiScriptHistoryService, pipelineStageService: pipelineStageService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciTemplateService: ciTemplateService, } } @@ -290,18 +293,19 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci } if !createRequest.IsExternal && createRequest.IsDockerConfigOverridden { //get override - savedTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(createRequest.Id) + savedTemplateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(createRequest.Id) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting templateOverride by ciPipelineId", "err", err, "ciPipelineId", createRequest.Id) return nil, err } + ciBuildConfigBean := createRequest.DockerConfigOverride.CiBuildConfig templateOverrideReq := &pipelineConfig.CiTemplateOverride{ CiPipelineId: createRequest.Id, DockerRegistryId: createRequest.DockerConfigOverride.DockerRegistry, DockerRepository: createRequest.DockerConfigOverride.DockerRepository, - DockerfilePath: createRequest.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - GitMaterialId: createRequest.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + //DockerfilePath: createRequest.DockerConfigOverride.DockerBuildConfig.DockerfilePath, + GitMaterialId: ciBuildConfigBean.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), CreatedBy: userId, @@ -309,19 +313,29 @@ func (impl DbPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean.Ci UpdatedBy: userId, }, } + savedTemplateOverride := savedTemplateOverrideBean.CiTemplateOverride if savedTemplateOverride != nil && savedTemplateOverride.Id > 0 { + ciBuildConfigBean.Id = savedTemplateOverride.CiBuildConfigId templateOverrideReq.Id = savedTemplateOverride.Id templateOverrideReq.CreatedOn = savedTemplateOverride.CreatedOn templateOverrideReq.CreatedBy = savedTemplateOverride.CreatedBy - _, err = impl.ciTemplateOverrideRepository.Update(templateOverrideReq) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverrideReq, + CiBuildConfig: ciBuildConfigBean, + UserId: userId, + } + err = impl.ciTemplateService.Update(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in updating template override", "err", err, "templateOverrideConfig", templateOverrideReq) return nil, err } } else { - _, err = impl.ciTemplateOverrideRepository.Save(templateOverrideReq) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverrideReq, + CiBuildConfig: ciBuildConfigBean, + UserId: userId, + } + err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", templateOverrideReq) return nil, err } } @@ -527,9 +541,9 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig CiPipelineId: ciPipeline.Id, DockerRegistryId: ciPipeline.DockerConfigOverride.DockerRegistry, DockerRepository: ciPipeline.DockerConfigOverride.DockerRepository, - DockerfilePath: ciPipeline.DockerConfigOverride.DockerBuildConfig.DockerfilePath, - GitMaterialId: ciPipeline.DockerConfigOverride.DockerBuildConfig.GitMaterialId, - Active: true, + //DockerfilePath: ciPipeline.DockerConfigOverride.DockerBuildConfig.DockerfilePath, + GitMaterialId: ciPipeline.DockerConfigOverride.CiBuildConfig.GitMaterialId, + Active: true, AuditLog: sql.AuditLog{ CreatedBy: createRequest.UserId, CreatedOn: time.Now(), @@ -537,9 +551,13 @@ func (impl DbPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfig UpdatedOn: time.Now(), }, } - _, err = impl.ciTemplateOverrideRepository.Save(templateOverride) + ciTemplateBean := &bean2.CiTemplateBean{ + CiTemplateOverride: templateOverride, + CiBuildConfig: ciPipeline.DockerConfigOverride.CiBuildConfig, + UserId: createRequest.UserId, + } + err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { - impl.logger.Errorw("error in saving template override", "err", err, "templateOverrideConfig", templateOverride) return nil, err } } diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 14aab4c1ea..274b04e94b 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -113,14 +114,14 @@ type PipelineBuilder interface { } type PipelineBuilderImpl struct { - logger *zap.SugaredLogger - dbPipelineOrchestrator DbPipelineOrchestrator - dockerArtifactStoreRepository repository.DockerArtifactStoreRepository - materialRepo pipelineConfig.MaterialRepository - appRepo app2.AppRepository - pipelineRepository pipelineConfig.PipelineRepository - propertiesConfigService PropertiesConfigService - ciTemplateRepository pipelineConfig.CiTemplateRepository + logger *zap.SugaredLogger + dbPipelineOrchestrator DbPipelineOrchestrator + dockerArtifactStoreRepository repository.DockerArtifactStoreRepository + materialRepo pipelineConfig.MaterialRepository + appRepo app2.AppRepository + pipelineRepository pipelineConfig.PipelineRepository + propertiesConfigService PropertiesConfigService + //ciTemplateRepository pipelineConfig.CiTemplateRepository ciPipelineRepository pipelineConfig.CiPipelineRepository application application.ServiceClient chartRepository chartRepoRepository.ChartRepository @@ -152,8 +153,10 @@ type PipelineBuilderImpl struct { helmAppService client.HelmAppService deploymentGroupRepository repository.DeploymentGroupRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - userService user.UserService - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + //ciBuildConfigService CiBuildConfigService + ciTemplateService CiTemplateService + userService user.UserService } func NewPipelineBuilderImpl(logger *zap.SugaredLogger, @@ -192,17 +195,17 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, deploymentGroupRepository repository.DeploymentGroupRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, userService user.UserService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository) *PipelineBuilderImpl { + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, ciTemplateService CiTemplateService) *PipelineBuilderImpl { return &PipelineBuilderImpl{ - logger: logger, - dbPipelineOrchestrator: dbPipelineOrchestrator, - dockerArtifactStoreRepository: dockerArtifactStoreRepository, - materialRepo: materialRepo, - appService: appService, - appRepo: pipelineGroupRepo, - pipelineRepository: pipelineRepository, - propertiesConfigService: propertiesConfigService, - ciTemplateRepository: ciTemplateRepository, + logger: logger, + dbPipelineOrchestrator: dbPipelineOrchestrator, + dockerArtifactStoreRepository: dockerArtifactStoreRepository, + materialRepo: materialRepo, + appService: appService, + appRepo: pipelineGroupRepo, + pipelineRepository: pipelineRepository, + propertiesConfigService: propertiesConfigService, + //ciTemplateRepository: ciTemplateRepository, ciPipelineRepository: ciPipelineRepository, application: application, chartRepository: chartRepository, @@ -233,8 +236,10 @@ func NewPipelineBuilderImpl(logger *zap.SugaredLogger, helmAppService: helmAppService, deploymentGroupRepository: deploymentGroupRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, - userService: userService, - ciTemplateOverrideRepository: ciTemplateOverrideRepository, + ciTemplateService: ciTemplateService, + //ciTemplateOverrideRepository: ciTemplateOverrideRepository, + //ciBuildConfigService: ciBuildConfigService, + userService: userService, } } @@ -286,13 +291,16 @@ func (impl PipelineBuilderImpl) DeleteMaterial(request *bean.UpdateMaterialDTO) if len(pipelines) > 0 { //pipelines are present, in this case we will check if this material is used in docker config //if it is used, then we won't delete - ciTemplate, err := impl.ciTemplateRepository.FindByAppId(request.AppId) + ciTemplateBean, err := impl.ciTemplateService.FindByAppId(request.AppId) if err != nil && err == errors.NotFoundf(err.Error()) { impl.logger.Errorw("err in getting docker registry", "appId", request.AppId, "err", err) return err } - if ciTemplate != nil && ciTemplate.GitMaterialId == request.Material.Id { - return fmt.Errorf("cannot delete git material, is being used in docker config") + if ciTemplateBean != nil { + ciTemplate := ciTemplateBean.CiTemplate + if ciTemplate != nil && ciTemplate.GitMaterialId == request.Material.Id { + return fmt.Errorf("cannot delete git material, is being used in docker config") + } } } existingMaterial, err := impl.materialRepo.FindById(request.Material.Id) @@ -348,8 +356,8 @@ func (impl PipelineBuilderImpl) GetMaterialsForAppId(appId int) []*bean.GitMater } /* -1. create pipelineGroup -2. save material (add credential provider support) + 1. create pipelineGroup + 2. save material (add credential provider support) */ @@ -365,7 +373,8 @@ func (impl PipelineBuilderImpl) getDefaultArtifactStore(id string) (store *repos } func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bean.CiConfigRequest, err error) { - template, err := impl.ciTemplateRepository.FindByAppId(appId) + //template, err := impl.ciTemplateRepository.FindByAppId(appId) + ciTemplateBean, err := impl.ciTemplateService.FindByAppId(appId) if err != nil && !errors.IsNotFound(err) { impl.logger.Errorw("error in fetching ci pipeline", "appId", appId, "err", err) return nil, err @@ -375,6 +384,7 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea err = &util.ApiError{Code: "404", HttpStatusCode: 200, UserMessage: "no ci pipeline exists"} return nil, err } + template := ciTemplateBean.CiTemplate gitMaterials, err := impl.materialRepo.FindByAppId(appId) if err != nil && err != pg.ErrNoRows { @@ -396,19 +406,11 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea materials = append(materials, m) } - dockerArgs := map[string]string{} - if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { - impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) - return nil, err - } - if template.DockerBuildOptions == "" { - template.DockerBuildOptions = "{}" - } - dockerBuildOptions := map[string]string{} - if err := json.Unmarshal([]byte(template.DockerBuildOptions), &dockerBuildOptions); err != nil { - impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) - return nil, err - } + //dockerArgs := map[string]string{} + //if err := json.Unmarshal([]byte(template.Args), &dockerArgs); err != nil { + // impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) + // return nil, err + //} regHost, err := template.DockerRegistry.GetRegistryLocation() if err != nil { impl.logger.Errorw("invalid reg url", "err", err) @@ -421,19 +423,15 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea DockerRepository: template.DockerRepository, DockerRegistry: template.DockerRegistry.Id, DockerRegistryUrl: regHost, - DockerBuildConfig: &bean.DockerBuildConfig{ - DockerfilePath: template.DockerfilePath, - Args: dockerArgs, - GitMaterialId: template.GitMaterialId, - TargetPlatform: template.TargetPlatform, - DockerBuildOptions: dockerBuildOptions, - }, + CiBuildConfig: ciTemplateBean.CiBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform}, Version: template.Version, CiTemplateName: template.TemplateName, Materials: materials, } return ciConfig, err } + func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfigRequest, err error) { ciConfig, err = impl.getCiTemplateVariables(appId) if err != nil { @@ -459,14 +457,15 @@ func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfi } } //map of ciPipelineId and their templateOverrideConfig - templateOverrideMap := make(map[int]*pipelineConfig.CiTemplateOverride) - templateOverrides, err := impl.ciTemplateOverrideRepository.FindByAppId(appId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting ciTemplateOverrides by appId", "err", err, "appId", appId) + ciOverrideTemplateMap := make(map[int]*bean3.CiTemplateBean) + ciTemplateBeanOverrides, err := impl.ciTemplateService.FindTemplateOverrideByAppId(appId) + if err != nil { return nil, err } - for _, templateOverride := range templateOverrides { - templateOverrideMap[templateOverride.CiPipelineId] = templateOverride + + for _, templateBeanOverride := range ciTemplateBeanOverrides { + ciTemplateOverride := templateBeanOverride.CiTemplateOverride + ciOverrideTemplateMap[ciTemplateOverride.CiPipelineId] = templateBeanOverride } var ciPipelineResp []*bean.CiPipeline for _, pipeline := range pipelines { @@ -533,14 +532,12 @@ func (impl PipelineBuilderImpl) GetCiPipeline(appId int) (ciConfig *bean.CiConfi ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, } - if templateOverride, ok := templateOverrideMap[pipeline.Id]; ok { + if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { + templateOverride := ciTemplateBean.CiTemplateOverride ciPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: templateOverride.GitMaterialId, - DockerfilePath: templateOverride.DockerfilePath, - }, + CiBuildConfig: ciTemplateBean.CiBuildConfig, } } for _, material := range pipeline.CiPipelineMaterials { @@ -648,15 +645,15 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq originalCiConf.AfterDockerBuild = updateRequest.AfterDockerBuild originalCiConf.BeforeDockerBuild = updateRequest.BeforeDockerBuild - originalCiConf.DockerBuildConfig = updateRequest.DockerBuildConfig + //originalCiConf.CiBuildConfigBean = updateRequest.CiBuildConfigBean originalCiConf.DockerRegistry = updateRequest.DockerRegistry originalCiConf.DockerRepository = updateRequest.DockerRepository originalCiConf.DockerRegistryUrl = regHost - argByte, err := json.Marshal(originalCiConf.DockerBuildConfig.Args) - if err != nil { - return nil, err - } + //argByte, err := json.Marshal(originalCiConf.DockerBuildConfig.Args) + //if err != nil { + // return nil, err + //} afterByte, err := json.Marshal(originalCiConf.AfterDockerBuild) if err != nil { return nil, err @@ -665,31 +662,38 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq if err != nil { return nil, err } - buildOptionsByte, err := json.Marshal(originalCiConf.DockerBuildConfig.DockerBuildOptions) - if err != nil { - impl.logger.Errorw("error in marshaling dockerBuildOptions", "err", err) - return nil, err - } + //buildOptionsByte, err := json.Marshal(originalCiConf.DockerBuildConfig.DockerBuildOptions) + //if err != nil { + // impl.logger.Errorw("error in marshaling dockerBuildOptions", "err", err) + // return nil, err + //} + ciBuildConfig := updateRequest.CiBuildConfig + originalCiBuildConfig := originalCiConf.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ - DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, - GitMaterialId: originalCiConf.DockerBuildConfig.GitMaterialId, - Args: string(argByte), - DockerBuildOptions: string(buildOptionsByte), - TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, - BeforeDockerBuild: string(beforeByte), - AfterDockerBuild: string(afterByte), - Version: originalCiConf.Version, - Id: originalCiConf.Id, - DockerRepository: originalCiConf.DockerRepository, - DockerRegistryId: originalCiConf.DockerRegistry, - Active: true, - } - - err = impl.ciTemplateRepository.Update(ciTemplate) - if err != nil { - impl.logger.Errorw("error in updating ci template in db", "template", ciTemplate, "err", err) + //DockerfilePath: originalCiConf.DockerBuildConfig.DockerfilePath, + GitMaterialId: ciBuildConfig.GitMaterialId, + //Args: string(argByte), + //TargetPlatform: originalCiConf.DockerBuildConfig.TargetPlatform, + BeforeDockerBuild: string(beforeByte), + AfterDockerBuild: string(afterByte), + Version: originalCiConf.Version, + Id: originalCiConf.Id, + DockerRepository: originalCiConf.DockerRepository, + DockerRegistryId: originalCiConf.DockerRegistry, + Active: true, + } + + ciBuildConfig.Id = originalCiBuildConfig.Id + ciTemplateBean := &bean3.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: ciBuildConfig, + UserId: updateRequest.UserId, + } + err = impl.ciTemplateService.Update(ciTemplateBean) + if err != nil { return nil, err } + originalCiConf.CiBuildConfig = ciBuildConfig return originalCiConf, nil } @@ -736,10 +740,10 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq //--ecr config end //-- template config start - argByte, err := json.Marshal(createRequest.DockerBuildConfig.Args) - if err != nil { - return nil, err - } + //argByte, err := json.Marshal(createRequest.DockerBuildConfig.Args) + //if err != nil { + // return nil, err + //} afterByte, err := json.Marshal(createRequest.AfterDockerBuild) if err != nil { return nil, err @@ -748,34 +752,32 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq if err != nil { return nil, err } - buildOptionsByte, err := json.Marshal(createRequest.DockerBuildConfig.DockerBuildOptions) - if err != nil { - impl.logger.Errorw("error in marshaling dockerBuildOptions", "err", err) - return nil, err - } + buildConfig := createRequest.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ - DockerRegistryId: createRequest.DockerRegistry, - DockerRepository: createRequest.DockerRepository, - GitMaterialId: createRequest.DockerBuildConfig.GitMaterialId, - DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, - Args: string(argByte), - DockerBuildOptions: string(buildOptionsByte), - TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, - Active: true, - TemplateName: createRequest.CiTemplateName, - Version: createRequest.Version, - AppId: createRequest.AppId, - AfterDockerBuild: string(afterByte), - BeforeDockerBuild: string(beforeByte), - AuditLog: sql.AuditLog{CreatedOn: time.Now(), UpdatedOn: time.Now(), CreatedBy: createRequest.UserId, UpdatedBy: createRequest.UserId}, - } - - err = impl.ciTemplateRepository.Save(ciTemplate) - if err != nil { - impl.logger.Errorw("error in saving ci template in db ", "template", ciTemplate, "err", err) - //TODO delete template from gocd otherwise dangling+ no create in future + DockerRegistryId: createRequest.DockerRegistry, + DockerRepository: createRequest.DockerRepository, + GitMaterialId: buildConfig.GitMaterialId, + //DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, + //Args: string(argByte), + //TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, + Active: true, + TemplateName: createRequest.CiTemplateName, + Version: createRequest.Version, + AppId: createRequest.AppId, + AfterDockerBuild: string(afterByte), + BeforeDockerBuild: string(beforeByte), + AuditLog: sql.AuditLog{CreatedOn: time.Now(), UpdatedOn: time.Now(), CreatedBy: createRequest.UserId, UpdatedBy: createRequest.UserId}, + } + + ciTemplateBean := &bean3.CiTemplateBean{ + CiTemplate: ciTemplate, + CiBuildConfig: createRequest.CiBuildConfig, + } + err = impl.ciTemplateService.Save(ciTemplateBean) + if err != nil { return nil, err } + //-- template config end createRequest.Id = ciTemplate.Id createRequest.CiTemplateName = ciTemplate.TemplateName @@ -2426,18 +2428,20 @@ func (impl PipelineBuilderImpl) GetCiPipelineById(pipelineId int) (ciPipeline *b IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, } if !ciPipeline.IsExternal && ciPipeline.IsDockerConfigOverridden { - templateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipeline.Id) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipeline.IsDockerConfigOverridden) + ciTemplateBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(ciPipeline.Id) + if err != nil { return nil, err } + templateOverride := ciTemplateBean.CiTemplateOverride + ciBuildConfig := ciTemplateBean.CiBuildConfig ciPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: templateOverride.DockerRegistryId, DockerRepository: templateOverride.DockerRepository, - DockerBuildConfig: &bean.DockerBuildConfig{ - GitMaterialId: templateOverride.GitMaterialId, - DockerfilePath: templateOverride.DockerfilePath, - }, + CiBuildConfig: ciBuildConfig, + //DockerBuildConfig: &bean.DockerBuildConfig{ + // GitMaterialId: templateOverride.GitMaterialId, + // DockerfilePath: templateOverride.DockerfilePath, + //}, } } for _, material := range pipeline.CiPipelineMaterials { diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 2d4d85e8fc..3c10eb2d47 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -71,10 +71,8 @@ type WorkflowRequest struct { DockerRegistryURL string `json:"dockerRegistryURL"` DockerConnection string `json:"dockerConnection"` DockerCert string `json:"dockerCert"` - DockerBuildArgs string `json:"dockerBuildArgs"` - DockerBuildTargetPlatform string `json:"dockerBuildTargetPlatform"` DockerRepository string `json:"dockerRepository"` - DockerFileLocation string `json:"dockerfileLocation"` + CheckoutPath string `json:"checkoutPath"` DockerUsername string `json:"dockerUsername"` DockerPassword string `json:"dockerPassword"` AwsRegion string `json:"awsRegion"` @@ -110,7 +108,8 @@ type WorkflowRequest struct { RefPlugins []*bean2.RefPluginObject `json:"refPlugins"` AppName string `json:"appName"` TriggerByAuthor string `json:"triggerByAuthor"` - DockerBuildOptions string `json:"dockerBuildOptions"` + CiBuildConfig *bean2.CiBuildConfigBean `json:"ciBuildConfig"` + CiBuildDockerMtuValue int `json:"ciBuildDockerMtuValue"` IgnoreDockerCachePush bool `json:"ignoreDockerCachePush"` IgnoreDockerCachePull bool `json:"ignoreDockerCachePull"` } diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go new file mode 100644 index 0000000000..bd3c7b0b84 --- /dev/null +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -0,0 +1,161 @@ +package bean + +import ( + "encoding/json" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/sql" + "time" +) + +type CiBuildType string + +const ( + SELF_DOCKERFILE_BUILD_TYPE CiBuildType = "self-dockerfile-build" + MANAGED_DOCKERFILE_BUILD_TYPE CiBuildType = "managed-dockerfile-build" + SKIP_BUILD_BUILD_TYPE CiBuildType = "skip-build" + BUILDPACK_BUILD_TYPE CiBuildType = "buildpack-build" +) + +type CiBuildConfigBean struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` + CiBuildType CiBuildType `json:"ciBuildType"` + DockerBuildConfig *DockerBuildConfig `json:"dockerBuildConfig,omitempty"` + BuildPackConfig *BuildPackConfig `json:"buildPackConfig"` +} + +type DockerBuildConfig struct { + DockerfilePath string `json:"dockerfileRelativePath,omitempty"` + DockerfileContent string `json:"dockerfileContent"` + Args map[string]string `json:"args,omitempty"` + TargetPlatform string `json:"targetPlatform,omitempty"` + Language string `json:"language,omitempty"` + LanguageFramework string `json:"languageFramework,omitempty"` + DockerBuildOptions map[string]string `json:"dockerBuildOptions,omitempty"` +} + +type BuildPackConfig struct { + BuilderId string `json:"builderId"` + Language string `json:"language"` + LanguageVersion string `json:"languageVersion"` + BuildPacks []string `json:"buildPacks"` + Args map[string]string `json:"args"` + ProjectPath string `json:"projectPath,omitempty"` +} + +func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean, userId int32) (*pipelineConfig.CiBuildConfig, error) { + buildMetadata := "" + ciBuildType := ciBuildConfigBean.CiBuildType + if ciBuildType == BUILDPACK_BUILD_TYPE { + buildPackConfigMetadataBytes, err := json.Marshal(ciBuildConfigBean.BuildPackConfig) + if err != nil { + return nil, err + } + buildMetadata = string(buildPackConfigMetadataBytes) + } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildMetadataBytes, err := json.Marshal(ciBuildConfigBean.DockerBuildConfig) + if err != nil { + return nil, err + } + buildMetadata = string(dockerBuildMetadataBytes) + } + ciBuildConfigEntity := &pipelineConfig.CiBuildConfig{ + Id: ciBuildConfigBean.Id, + Type: string(ciBuildType), + CiTemplateId: templateId, + CiTemplateOverrideId: overrideTemplateId, + BuildMetadata: buildMetadata, + AuditLog: sql.AuditLog{UpdatedOn: time.Now(), UpdatedBy: userId}, + } + return ciBuildConfigEntity, nil +} + +func ConvertDbBuildConfigToBean(dbBuildConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { + var buildPackConfig *BuildPackConfig + var dockerBuildConfig *DockerBuildConfig + var err error + if dbBuildConfig == nil { + return nil, nil + } + ciBuildType := CiBuildType(dbBuildConfig.Type) + if ciBuildType == BUILDPACK_BUILD_TYPE { + buildPackConfig, err = convertMetadataToBuildPackConfig(dbBuildConfig.BuildMetadata) + if err != nil { + return nil, err + } + } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbBuildConfig.BuildMetadata) + if err != nil { + return nil, err + } + } + ciBuildConfigBean := &CiBuildConfigBean{ + Id: dbBuildConfig.Id, + CiBuildType: ciBuildType, + BuildPackConfig: buildPackConfig, + DockerBuildConfig: dockerBuildConfig, + } + return ciBuildConfigBean, nil +} + +func convertMetadataToBuildPackConfig(buildConfMetadata string) (*BuildPackConfig, error) { + buildPackConfig := &BuildPackConfig{} + err := json.Unmarshal([]byte(buildConfMetadata), buildPackConfig) + return buildPackConfig, err +} + +func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuildConfig, error) { + dockerBuildConfig := &DockerBuildConfig{} + err := json.Unmarshal([]byte(dockerBuildMetadata), dockerBuildConfig) + return dockerBuildConfig, err +} + +func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, dockerBuildOptions string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { + oldDockerArgs := map[string]string{} + ciLevelDockerArgs := map[string]string{} + dockerBuildOptionsMap := map[string]string{} + if oldArgs != "" { + if err := json.Unmarshal([]byte(oldArgs), &oldDockerArgs); err != nil { + return nil, err + } + } + if ciLevelArgs != "" { + if err := json.Unmarshal([]byte(ciLevelArgs), &ciLevelDockerArgs); err != nil { + return nil, err + } + } + if dockerBuildOptions != "" { + if err := json.Unmarshal([]byte(dockerBuildOptions), &dockerBuildOptionsMap); err != nil { + return nil, err + } + } + if ciBuildConfigBean == nil { + dockerArgs := mergeMap(oldDockerArgs, ciLevelDockerArgs) + ciBuildConfigBean = &CiBuildConfigBean{ + CiBuildType: SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &DockerBuildConfig{ + DockerfilePath: dockerfilePath, + Args: dockerArgs, + TargetPlatform: targetPlatform, + DockerBuildOptions: dockerBuildOptionsMap, + }, + } + } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) + //dockerBuildConfig.DockerfilePath = dockerfilePath + dockerBuildConfig.Args = dockerArgs + } + return ciBuildConfigBean, nil +} + +func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]string) map[string]string { + dockerArgs := make(map[string]string) + for key, value := range oldDockerArgs { + dockerArgs[key] = value + } + for key, value := range ciLevelDockerArgs { + dockerArgs[key] = value + } + return dockerArgs +} diff --git a/pkg/pipeline/bean/CiTemplateBean.go b/pkg/pipeline/bean/CiTemplateBean.go new file mode 100644 index 0000000000..d49e4b2634 --- /dev/null +++ b/pkg/pipeline/bean/CiTemplateBean.go @@ -0,0 +1,10 @@ +package bean + +import "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + +type CiTemplateBean struct { + CiTemplate *pipelineConfig.CiTemplate + CiTemplateOverride *pipelineConfig.CiTemplateOverride + CiBuildConfig *CiBuildConfigBean + UserId int32 +} diff --git a/pkg/pipeline/mocks/CiBuildConfigService.go b/pkg/pipeline/mocks/CiBuildConfigService.go new file mode 100644 index 0000000000..4064426ce4 --- /dev/null +++ b/pkg/pipeline/mocks/CiBuildConfigService.go @@ -0,0 +1,95 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + bean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + mock "github.com/stretchr/testify/mock" +) + +// CiBuildConfigService is an autogenerated mock type for the CiBuildConfigService type +type CiBuildConfigService struct { + mock.Mock +} + +// Delete provides a mock function with given fields: ciBuildConfigId +func (_m *CiBuildConfigService) Delete(ciBuildConfigId int) error { + ret := _m.Called(ciBuildConfigId) + + var r0 error + if rf, ok := ret.Get(0).(func(int) error); ok { + r0 = rf(ciBuildConfigId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetCountByBuildType provides a mock function with given fields: +func (_m *CiBuildConfigService) GetCountByBuildType() map[bean.CiBuildType]int { + ret := _m.Called() + + var r0 map[bean.CiBuildType]int + if rf, ok := ret.Get(0).(func() map[bean.CiBuildType]int); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[bean.CiBuildType]int) + } + } + + return r0 +} + +// Save provides a mock function with given fields: templateId, overrideTemplateId, ciBuildConfigBean, userId +func (_m *CiBuildConfigService) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { + ret := _m.Called(templateId, overrideTemplateId, ciBuildConfigBean, userId) + + var r0 error + if rf, ok := ret.Get(0).(func(int, int, *bean.CiBuildConfigBean, int32) error); ok { + r0 = rf(templateId, overrideTemplateId, ciBuildConfigBean, userId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateOrSave provides a mock function with given fields: templateId, overrideTemplateId, ciBuildConfig, userId +func (_m *CiBuildConfigService) UpdateOrSave(templateId int, overrideTemplateId int, ciBuildConfig *bean.CiBuildConfigBean, userId int32) (*bean.CiBuildConfigBean, error) { + ret := _m.Called(templateId, overrideTemplateId, ciBuildConfig, userId) + + var r0 *bean.CiBuildConfigBean + if rf, ok := ret.Get(0).(func(int, int, *bean.CiBuildConfigBean, int32) *bean.CiBuildConfigBean); ok { + r0 = rf(templateId, overrideTemplateId, ciBuildConfig, userId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bean.CiBuildConfigBean) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int, int, *bean.CiBuildConfigBean, int32) error); ok { + r1 = rf(templateId, overrideTemplateId, ciBuildConfig, userId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewCiBuildConfigService interface { + mock.TestingT + Cleanup(func()) +} + +// NewCiBuildConfigService creates a new instance of CiBuildConfigService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCiBuildConfigService(t mockConstructorTestingTNewCiBuildConfigService) *CiBuildConfigService { + mock := &CiBuildConfigService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/sample-docker-templates/django/Dockerfile b/sample-docker-templates/django/Dockerfile index db241a3995..b84c90ebe6 100644 --- a/sample-docker-templates/django/Dockerfile +++ b/sample-docker-templates/django/Dockerfile @@ -31,7 +31,10 @@ RUN pip install -r requirements.txt # install nginx RUN apt-get update && apt-get install nginx vim -y --no-install-recommends + +#Refer https://github.com/devtron-labs/devtron/blob/main/sample-docker-templates/django/nginx.default for sample nginx.default file COPY nginx.default /etc/nginx/sites-available/default + RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log @@ -41,4 +44,5 @@ EXPOSE 8000 STOPSIGNAL SIGTERM +# Refer https://github.com/devtron-labs/devtron/blob/main/sample-docker-templates/django/start-server.sh for sample start-server.sh file CMD ["/app/start-server.sh"] \ No newline at end of file diff --git a/sample-docker-templates/flask/Dockerfile b/sample-docker-templates/flask/Dockerfile index a2cb7c3085..ad20d787cc 100644 --- a/sample-docker-templates/flask/Dockerfile +++ b/sample-docker-templates/flask/Dockerfile @@ -29,8 +29,10 @@ ADD . /app/ #Installing dependencies RUN pip3 install -r requirements.txt +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/nginx.default for sample nginx.default file COPY nginx.default /etc/nginx/sites-available/default +#Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/start.sh for sample start.sh file #Making start.sh executable RUN chmod +x ./start.sh diff --git a/sample-docker-templates/flask/start.sh b/sample-docker-templates/flask/start.sh index a1e9b76869..0909a01c2c 100644 --- a/sample-docker-templates/flask/start.sh +++ b/sample-docker-templates/flask/start.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash service nginx start +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/flask/uwsgi.ini for sample uwsgi.ini file uwsgi --ini uwsgi.ini diff --git a/sample-docker-templates/node/Dockerfile b/sample-docker-templates/node/Dockerfile index 8d05a864a7..123a903771 100644 --- a/sample-docker-templates/node/Dockerfile +++ b/sample-docker-templates/node/Dockerfile @@ -24,6 +24,7 @@ WORKDIR /app # Adding complete files and dirs in app dir in container ADD . /app/ +# Refer https://raw.githubusercontent.com/devtron-labs/devtron/main/sample-docker-templates/node/nginx.default for sample nginx.default COPY nginx.default /etc/nginx/sites-available/default # Installing dependencies diff --git a/scripts/sql/90_ci_build_config.down.sql b/scripts/sql/90_ci_build_config.down.sql new file mode 100644 index 0000000000..6cfec2dff0 --- /dev/null +++ b/scripts/sql/90_ci_build_config.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE ci_template DROP COLUMN IF EXISTS ci_build_config_id; +ALTER TABLE ci_template_override DROP COLUMN IF EXISTS ci_build_config_id; + +DROP TABLE IF EXISTS "public"."ci_build_config"; + +ALTER TABLE ci_workflow + DROP COLUMN IF EXISTS ci_build_type; \ No newline at end of file diff --git a/scripts/sql/90_ci_build_config.up.sql b/scripts/sql/90_ci_build_config.up.sql new file mode 100644 index 0000000000..43db192323 --- /dev/null +++ b/scripts/sql/90_ci_build_config.up.sql @@ -0,0 +1,33 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_ci_build_config; + +-- Table Definition +CREATE TABLE IF NOT EXISTS public.ci_build_config +( + "id" integer NOT NULL DEFAULT nextval('id_seq_ci_build_config'::regclass), + "type" varchar(100), + "ci_template_id" integer, + "ci_template_override_id" integer, + "build_metadata" text, + "created_on" timestamptz, + "created_by" integer, + "updated_on" timestamptz, + "updated_by" integer, + PRIMARY KEY ("id") +); + + +ALTER TABLE ci_template + ADD COLUMN IF NOT EXISTS ci_build_config_id integer; + +ALTER TABLE ONLY public.ci_template + ADD CONSTRAINT IF NOT EXISTS ci_template_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + + +ALTER TABLE ci_template_override + ADD COLUMN IF NOT EXISTS ci_build_config_id integer; + +ALTER TABLE ONLY public.ci_template_override + ADD CONSTRAINT IF NOT EXISTS ci_template_override_ci_build_config_id_fkey FOREIGN KEY (ci_build_config_id) REFERENCES public.ci_build_config(id); + +ALTER TABLE ci_workflow + ADD COLUMN IF NOT EXISTS ci_build_type varchar(100); diff --git a/wire_gen.go b/wire_gen.go index 366c6e0baa..f2afde3313 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate wire -//+build !wireinject +//go:generate go run github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject package main @@ -350,19 +351,22 @@ func InitializeApp() (*App, error) { globalPluginRepositoryImpl := repository8.NewGlobalPluginRepository(sugaredLogger, db) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) - dbPipelineOrchestratorImpl := pipeline.NewDbPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl) - propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, dbPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl) + ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) + ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + dbPipelineOrchestratorImpl := pipeline.NewDbPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) + propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, dbPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, dbPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) globalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(sugaredLogger, db) globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig, globalCMCSServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateOverrideRepositoryImpl, appCrudOperationServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl) ciLogServiceImpl := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, ciConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, gitSensorClientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciConfig, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, gitSensorClientImpl) @@ -374,7 +378,7 @@ func InitializeApp() (*App, error) { cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, cdConfig, userServiceImpl, cdWorkflowRepositoryImpl, cdWorkflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, dbPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) - appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl) + appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl)